Support new Hyper plan format with "inputs" array

DEVELOPMENT.md

@@ -42,6 +42,44 @@ To do so, run `yarn dev-server` inside the `standalone-app` folder and connect t
 Changes inside the `query-graphs` directory are not immediately picked up by the `standalone-app`. 
 For your changes to take effect, you will have to re-run `yarn build` inside the `query-graphs` folder.
 
+## Testing with Hyper Query Plans
+
+When making changes to the query plan visualization (e.g., supporting a new plan format), follow these steps to verify everything works end-to-end.
+
+### 1. Dump fresh plans from Hyper
+
+The `plan-dumper/dump-plans.py` script runs SQL queries against a Hyper instance and writes the resulting `EXPLAIN` output as JSON files into `standalone-app/examples/`.
+
+**Prerequisites:**
+
+```shell
+pip3 install tableauhyperapi
+```
+
+**Running with the default (pip-installed) Hyper:**
+
+```shell
+cd plan-dumper
+python3 dump-plans.py
+```
+
+**Running with a custom Hyper build:**
+
+```shell
+cd plan-dumper
+python3 dump-plans.py --hyper-path ~/workspace/hyper-db/bazel-bin/hyper/tools/hyperd
+```
+
+The `--hyper-path` argument should point to the directory containing the `hyperd` binary.
+When omitted, the script uses the `hyperd` bundled with the pip-installed `tableauhyperapi` package.
+
+If you also have a Postgres instance running on port 5433, install `psycopg2` to dump Postgres plans as well. Otherwise, the Postgres section is skipped automatically.
+
+### 2. Visually verify plans in the local server
+
+Build and start the app (if not already running).
+Then open [localhost:8080/examples.html](http://localhost:8080/examples.html) in your browser. This page lists all example plans. Click through the Hyper plans to verify they render correctly.
+
 ## Linting
 
 We use eslint as our code linter.

plan-dumper/dump-plans.py

@@ -1,5 +1,9 @@
 from tableauhyperapi import HyperProcess, Telemetry, Connection
-import psycopg2
+try:
+    import psycopg2
+except ImportError:
+    psycopg2 = None
+import argparse
 import shutil
 import os
 import json
@@ -51,38 +55,47 @@ def dump_plans(name, exec_stmt, get_plan):
 
 
 # Postgres
-with psycopg2.connect("port=5433") as conn:
-    def exec_postgres(sql):
-        with conn.cursor() as cur:
-            if sql.strip() != "":
-                cur.execute(sql)
-
-    def get_postgres_plan(sql, mode):
-        if mode == "steps":
-            return None
-        elif mode == "analyze":
-            explain = "EXPLAIN (VERBOSE, ANALYZE, FORMAT JSON) "
-        elif mode is None:
-            explain = "EXPLAIN (VERBOSE, FORMAT JSON) "
-        else:
-            return None
-        with conn.cursor() as cur:
-            cur.execute(explain + sql)
-            records = cur.fetchall()
-            return json.dumps(records[0][0])
-
-    dump_plans("postgres", exec_postgres, get_postgres_plan)
+if psycopg2 is not None:
+    with psycopg2.connect("port=5433") as conn:
+        def exec_postgres(sql):
+            with conn.cursor() as cur:
+                if sql.strip() != "":
+                    cur.execute(sql)
+
+        def get_postgres_plan(sql, mode):
+            if mode == "steps":
+                return None
+            elif mode == "analyze":
+                explain = "EXPLAIN (VERBOSE, ANALYZE, FORMAT JSON) "
+            elif mode is None:
+                explain = "EXPLAIN (VERBOSE, FORMAT JSON) "
+            else:
+                return None
+            with conn.cursor() as cur:
+                cur.execute(explain + sql)
+                records = cur.fetchall()
+                return json.dumps(records[0][0])
+
+        dump_plans("postgres", exec_postgres, get_postgres_plan)
+else:
+    print("Skipping Postgres: psycopg2 not installed")
+
 
+parser = argparse.ArgumentParser()
+parser.add_argument("--hyper-path", type=Path, default=None,
+                    help="Path to a directory containing the hyperd binary. "
+                         "Uses the pip-installed Hyper by default.")
+args = parser.parse_args()
 
 # Hyper
-with HyperProcess(telemetry=Telemetry.SEND_USAGE_DATA_TO_TABLEAU, parameters=hyper_params) as hyper:
+with HyperProcess(telemetry=Telemetry.SEND_USAGE_DATA_TO_TABLEAU, parameters=hyper_params, hyper_path=args.hyper_path) as hyper:
     with Connection(endpoint=hyper.endpoint) as connection:
         def exec_hyper(sql):
             connection.execute_command(sql)
 
         def get_hyper_plan(sql, mode):
             if mode == "steps":
-                explain = "EXPLAIN (FORMAT JSON, OPTIMIZERSTEPS) "
+                explain = "EXPLAIN (FORMAT JSON, OPTIMIZE STEPS) "
             elif mode == "analyze":
                 explain = "EXPLAIN (FORMAT JSON, ANALYZE) "
             elif mode is None:

query-graphs/src/hyper.ts

@@ -152,7 +152,7 @@ function convertHyperNode(rawNode: Json, parentKey, conversionState: ConversionS
         // Determine the order in which other keys are displayed.
         // For some keys, we enforce a specific order here (e.g., "left" comes before "right").
         // For all other keys, we use alphabetic order.
-        const fixedChildOrder = ["input", "left", "right", "value", "valueForComparison"];
+        const fixedChildOrder = ["inputs", "input", "left", "right", "value", "valueForComparison"];
         const orderedKeys = Object.getOwnPropertyNames(rawNode)
             .filter((k) => {
                 // `propertyKeys` and `operator`/`expression` were already handled

standalone-app/examples/hyper/cte-recursive.plan.json

@@ -3,45 +3,46 @@
   "operatorId": 1,
   "cardinality": 11,
   "producesRows": true,
-  "output": [{"expression": "iuref", "iu": ["v", ["Integer"]]}],
+  "output": [{"expression": "iuref", "iu": ["map", ["Integer"]]}],
   "outputNames": ["i"],
-  "input": {
+  "inputs": [{
     "operator": "iteration",
     "operatorId": 2,
     "cardinality": 11,
-    "resultIUs": [["v", ["Integer"]]],
-    "leftValues": [{"expression": "iuref", "iu": ["v2", ["Integer"]]}],
-    "rightValues": [{"expression": "iuref", "iu": ["v3", ["Integer"]]}],
+    "resultIUs": [["map", ["Integer"]]],
+    "leftValues": [{"expression": "iuref", "iu": ["map2", ["Integer"]]}],
+    "rightValues": [{"expression": "iuref", "iu": ["map3", ["Integer"]]}],
     "unionAll": true,
-    "left": {
+    "inputs": [{
       "operator": "tableconstruction",
       "operatorId": 3,
       "cardinality": 1,
-      "output": [["v2", ["Integer"]]],
-      "values": [[{"expression": "const", "value": {"type": ["Integer"], "value": 1}}]]
-    },
-    "right": {
+      "output": [["map2", ["Integer"]]],
+      "values": [[{"expression": "const", "value": {"type": ["Integer"], "value": 1}}]],
+      "inputs": []
+    }, {
       "operator": "map",
       "operatorId": 4,
       "sqlpos": [[85, 90]],
       "cardinality": 1,
-      "input": {
-        "operator": "select",
+      "inputs": [{
+        "operator": "filter",
         "operatorId": 5,
         "sqlpos": [[104, 111]],
         "cardinality": 1,
-        "input": {
+        "inputs": [{
           "operator": "iterationincrement",
           "operatorId": 6,
           "sqlpos": [[96, 97]],
           "cardinality": 1,
-          "values": [["v4", ["Integer"]]],
+          "values": [["map_increment", ["Integer"]]],
           "source": 2
-        },
-        "condition": {"expression": "comparison", "mode": "<", "left": {"expression": "iuref", "iu": "v4"}, "right": {"expression": "const", "value": {"type": ["Integer"], "value": 100}}}
-      },
-      "values": [{"iu": ["v3", ["Integer"]], "value": {"expression": "add", "left": {"expression": "iuref", "iu": "v4"}, "right": {"expression": "const", "value": {"type": ["Integer"], "value": 1}}}}]
-    },
-    "collates": [null]
-  }
+        }],
+        "condition": {"expression": "comparison", "mode": "<", "left": {"expression": "iuref", "iu": "map_increment"}, "right": {"expression": "const", "value": {"type": ["Integer"], "value": 100}}}
+      }],
+      "values": [{"iu": ["map3", ["Integer"]], "value": {"expression": "add", "left": {"expression": "iuref", "iu": "map_increment"}, "right": {"expression": "const", "value": {"type": ["Integer"], "value": 1}}}}]
+    }],
+    "collates": [null],
+    "algorithm": "materializing"
+  }]
 }

standalone-app/examples/hyper/cte.plan.json

@@ -3,49 +3,51 @@
   "operatorId": 1,
   "cardinality": 200,
   "producesRows": true,
-  "output": [{"expression": "iuref", "iu": ["v", ["Integer"]]}, {"expression": "iuref", "iu": ["v2", ["BigInt"]]}, {"expression": "iuref", "iu": ["v3", ["Numeric", 16, 6]]}],
+  "output": [{"expression": "iuref", "iu": ["union", ["Integer"]]}, {"expression": "iuref", "iu": ["union2", ["BigInt"]]}, {"expression": "iuref", "iu": ["union3", ["Numeric", 16, 6]]}],
   "outputNames": ["a1", "sum", "avg"],
-  "input": {
+  "inputs": [{
     "operator": "unionall",
     "operatorId": 2,
     "sqlpos": [[109, 118]],
     "cardinality": 200,
-    "input": [{
+    "inputs": [{
       "operator": "explicitscan",
       "operatorId": 3,
       "cardinality": 100,
-      "mapping": [{"source": {"expression": "iuref", "iu": ["v4", ["Integer"]]}, "target": ["v5", ["Integer"]]}, {"source": {"expression": "iuref", "iu": ["v6", ["Numeric", 16, 6]]}, "target": ["v7", ["Numeric", 16, 6]]}, {"source": {"expression": "iuref", "iu": ["v8", ["BigInt"]]}, "target": ["v9", ["BigInt"]]}],
+      "mapping": [{"source": {"expression": "iuref", "iu": ["GroupByKey", ["Integer"]]}, "target": ["GroupByKey2", ["Integer"]]}, {"source": {"expression": "iuref", "iu": ["avg", ["Numeric", 16, 6]]}, "target": ["avg2", ["Numeric", 16, 6]]}, {"source": {"expression": "iuref", "iu": ["sum", ["BigInt"]]}, "target": ["sum2", ["BigInt"]]}],
       "input": {
         "operator": "groupby",
         "operatorId": 4,
         "sqlpos": [[77, 88], [46, 53], [55, 62]],
         "cardinality": 100,
-        "input": {
+        "inputs": [{
           "operator": "tablescan",
           "operatorId": 5,
           "sqlpos": [[71, 73]],
           "cardinality": 2000,
+          "volatility": "stable",
           "relationId": 0,
           "schema": {"type":"sessionschema"},
-          "values": [{"name": "a1", "type": ["Integer"], "iu": ["v10", ["Integer"]]}, {"name": "b1", "type": ["Integer"], "iu": ["v11", ["Integer"]]}, {"name": "c1", "type": ["Integer"], "iu": ["v12", ["Integer"]]}],
+          "attributeCount": 3,
+          "attributes": [{"colId": 0, "name": "a1", "type": ["Integer"], "iu": ["scan_a1", ["Integer"]]},{"colId": 1, "name": "b1", "type": ["Integer"], "iu": ["scan_b1", ["Integer"]]},{"colId": 2, "name": "c1", "type": ["Integer"], "iu": ["scan_c1", ["Integer"]]}],
           "debugName": {"classification": "nonsensitive", "value": "t1"},
           "selectivity": 1
-        },
-        "keyExpressions": [{"expression": {"value": {"expression": "iuref", "iu": "v10"}}, "iu": ["v4", ["Integer"]]}],
+        }],
+        "keyExpressions": [{"expression": {"value": {"expression": "iuref", "iu": "scan_a1"}}, "iu": ["GroupByKey", ["Integer"]]}],
         "groupingSets": [{"keyIndices": [0], "coreIndices": [0], "behavior": "regular"}],
         "emptyGroups": false,
-        "aggExpressions": [{"value": {"expression": "iuref", "iu": "v11"}}, {"value": {"expression": "iuref", "iu": "v12"}}],
-        "aggregates": [{"source": 1, "operation": {"aggregate": "avg"}, "iu": ["v6", ["Numeric", 16, 6]]}, {"source": 0, "operation": {"aggregate": "sum"}, "iu": ["v8", ["BigInt"]]}]
+        "aggExpressions": [{"value": {"expression": "iuref", "iu": "scan_b1"}}, {"value": {"expression": "iuref", "iu": "scan_c1"}}],
+        "aggregates": [{"source": 1, "operation": {"aggregate": "avg"}, "iu": ["avg", ["Numeric", 16, 6]]}, {"source": 0, "operation": {"aggregate": "sum"}, "iu": ["sum", ["BigInt"]]}]
       }
     }, {
       "operator": "explicitscan",
       "operatorId": 6,
       "cardinality": 100,
-      "mapping": [{"source": {"expression": "iuref", "iu": "v4"}, "target": ["v13", ["Integer"]]}, {"source": {"expression": "iuref", "iu": "v6"}, "target": ["v14", ["Numeric", 16, 6]]}, {"source": {"expression": "iuref", "iu": "v8"}, "target": ["v15", ["BigInt"]]}],
+      "mapping": [{"source": {"expression": "iuref", "iu": "GroupByKey"}, "target": ["GroupByKey3", ["Integer"]]}, {"source": {"expression": "iuref", "iu": "avg"}, "target": ["avg3", ["Numeric", 16, 6]]}, {"source": {"expression": "iuref", "iu": "sum"}, "target": ["sum3", ["BigInt"]]}],
       "input": 4
     }],
-    "ius": [["v", ["Integer"]], ["v2", ["BigInt"]], ["v3", ["Numeric", 16, 6]]],
-    "values": [[{"expression": "iuref", "iu": "v5"}, {"expression": "iuref", "iu": "v9"}, {"expression": "iuref", "iu": "v7"}], [{"expression": "iuref", "iu": "v13"}, {"expression": "iuref", "iu": "v15"}, {"expression": "iuref", "iu": "v14"}]],
+    "ius": [["union", ["Integer"]], ["union2", ["BigInt"]], ["union3", ["Numeric", 16, 6]]],
+    "values": [[{"expression": "iuref", "iu": "GroupByKey2"}, {"expression": "iuref", "iu": "sum2"}, {"expression": "iuref", "iu": "avg2"}], [{"expression": "iuref", "iu": "GroupByKey3"}, {"expression": "iuref", "iu": "sum3"}, {"expression": "iuref", "iu": "avg3"}]],
     "collates": [null, null, null]
-  }
+  }]
 }

standalone-app/examples/hyper/groupby.plan.json

@@ -3,28 +3,30 @@
   "operatorId": 1,
   "cardinality": 100,
   "producesRows": true,
-  "output": [{"expression": "iuref", "iu": ["v", ["Integer"]]}, {"expression": "iuref", "iu": ["v2", ["BigInt"]]}, {"expression": "iuref", "iu": ["v3", ["Numeric", 16, 6]]}],
+  "output": [{"expression": "iuref", "iu": ["GroupByKey", ["Integer"]]}, {"expression": "iuref", "iu": ["sum", ["BigInt"]]}, {"expression": "iuref", "iu": ["avg", ["Numeric", 16, 6]]}],
   "outputNames": ["a1", "sum", "avg"],
-  "input": {
+  "inputs": [{
     "operator": "groupby",
     "operatorId": 2,
     "sqlpos": [[58, 69], [33, 40], [42, 49]],
     "cardinality": 100,
-    "input": {
+    "inputs": [{
       "operator": "tablescan",
       "operatorId": 3,
       "sqlpos": [[55, 57]],
       "cardinality": 2000,
+      "volatility": "stable",
       "relationId": 0,
       "schema": {"type":"sessionschema"},
-      "values": [{"name": "a1", "type": ["Integer"], "iu": ["v4", ["Integer"]]}, {"name": "b1", "type": ["Integer"], "iu": ["v5", ["Integer"]]}, {"name": "c1", "type": ["Integer"], "iu": ["v6", ["Integer"]]}],
+      "attributeCount": 3,
+      "attributes": [{"colId": 0, "name": "a1", "type": ["Integer"], "iu": ["scan_a1", ["Integer"]]},{"colId": 1, "name": "b1", "type": ["Integer"], "iu": ["scan_b1", ["Integer"]]},{"colId": 2, "name": "c1", "type": ["Integer"], "iu": ["scan_c1", ["Integer"]]}],
       "debugName": {"classification": "nonsensitive", "value": "t1"},
       "selectivity": 1
-    },
-    "keyExpressions": [{"expression": {"value": {"expression": "iuref", "iu": "v4"}}, "iu": ["v", ["Integer"]]}],
+    }],
+    "keyExpressions": [{"expression": {"value": {"expression": "iuref", "iu": "scan_a1"}}, "iu": ["GroupByKey", ["Integer"]]}],
     "groupingSets": [{"keyIndices": [0], "coreIndices": [0], "behavior": "regular"}],
     "emptyGroups": false,
-    "aggExpressions": [{"value": {"expression": "iuref", "iu": "v5"}}, {"value": {"expression": "iuref", "iu": "v6"}}],
-    "aggregates": [{"source": 1, "operation": {"aggregate": "avg"}, "iu": ["v3", ["Numeric", 16, 6]]}, {"source": 0, "operation": {"aggregate": "sum"}, "iu": ["v2", ["BigInt"]]}]
-  }
+    "aggExpressions": [{"value": {"expression": "iuref", "iu": "scan_b1"}}, {"value": {"expression": "iuref", "iu": "scan_c1"}}],
+    "aggregates": [{"source": 1, "operation": {"aggregate": "avg"}, "iu": ["avg", ["Numeric", 16, 6]]}, {"source": 0, "operation": {"aggregate": "sum"}, "iu": ["sum", ["BigInt"]]}]
+  }]
 }

standalone-app/examples/hyper/insert.plan.json

@@ -5,50 +5,53 @@
   "producesRows": false,
   "output": [],
   "outputNames": [],
-  "input": {
+  "inputs": [{
     "operator": "insert",
     "operatorId": 2,
     "cardinality": 4000,
+    "volatility": "volatile",
     "relationId": 1,
     "schema": {"type":"sessionschema"},
-    "output": [["v", ["Integer", "nullable"]], ["v2", ["Integer", "nullable"]], ["v3", ["Integer", "nullable"]]],
-    "values": [{"expression": "cast", "value": {"expression": "iuref", "iu": ["v4", ["Integer"]]}, "type": ["Integer", "nullable"], "errorHandling": "trap"}, {"expression": "cast", "value": {"expression": "iuref", "iu": ["v5", ["Integer"]]}, "type": ["Integer", "nullable"], "errorHandling": "trap"}, {"expression": "cast", "value": {"expression": "iuref", "iu": ["v6", ["Integer"]]}, "type": ["Integer", "nullable"], "errorHandling": "trap"}],
-    "input": {
+    "output": [["cast", ["Integer", "nullable"]], ["cast2", ["Integer", "nullable"]], ["cast3", ["Integer", "nullable"]]],
+    "values": [{"expression": "cast", "value": {"expression": "iuref", "iu": ["map", ["Integer"]]}, "type": ["Integer", "nullable"], "errorHandling": "trap", "allowTrustedCasts": false}, {"expression": "cast", "value": {"expression": "iuref", "iu": ["map2", ["Integer"]]}, "type": ["Integer", "nullable"], "errorHandling": "trap", "allowTrustedCasts": false}, {"expression": "cast", "value": {"expression": "iuref", "iu": ["map3", ["Integer"]]}, "type": ["Integer", "nullable"], "errorHandling": "trap", "allowTrustedCasts": false}],
+    "inputs": [{
       "operator": "map",
       "operatorId": 3,
       "sqlpos": [[44, 48]],
       "cardinality": 4000,
-      "input": {
+      "inputs": [{
         "operator": "join",
         "operatorId": 4,
         "cardinality": 4000,
         "method": "bnl",
-        "left": {
+        "inputs": [{
           "operator": "tableconstruction",
           "operatorId": 5,
           "sqlpos": [[71, 80]],
           "cardinality": 2,
-          "output": [["v7", ["Integer"]]],
-          "values": [[{"expression": "const", "value": {"type": ["Integer"], "value": 1}}], [{"expression": "const", "value": {"type": ["Integer"], "value": 2}}]]
-        },
-        "right": {
+          "output": [["tableconstruction", ["Integer"]]],
+          "values": [[{"expression": "const", "value": {"type": ["Integer"], "value": 1}}], [{"expression": "const", "value": {"type": ["Integer"], "value": 2}}]],
+          "inputs": []
+        }, {
           "operator": "tablescan",
           "operatorId": 6,
           "sqlpos": [[66, 68]],
           "cardinality": 2000,
+          "volatility": "stable",
           "relationId": 0,
           "schema": {"type":"sessionschema"},
-          "values": [{"name": "a1", "type": ["Integer"], "iu": ["v8", ["Integer"]]}, {"name": "b1", "type": ["Integer"], "iu": ["v9", ["Integer"]]}, {"name": "c1", "type": ["Integer"], "iu": ["v10", ["Integer"]]}],
+          "attributeCount": 3,
+          "attributes": [{"colId": 0, "name": "a1", "type": ["Integer"], "iu": ["scan_a1", ["Integer"]]},{"colId": 1, "name": "b1", "type": ["Integer"], "iu": ["scan_b1", ["Integer"]]},{"colId": 2, "name": "c1", "type": ["Integer"], "iu": ["scan_c1", ["Integer"]]}],
           "debugName": {"classification": "nonsensitive", "value": "t1"},
           "selectivity": 1
-        },
+        }],
         "condition": {"expression": "const", "value": {"type": ["Bool"], "value": true}}
-      },
-      "values": [{"iu": ["v4", ["Integer"]], "value": {"expression": "mul", "left": {"expression": "iuref", "iu": "v8"}, "right": {"expression": "iuref", "iu": "v7"}}}, {"iu": ["v5", ["Integer"]], "value": {"expression": "mul", "left": {"expression": "iuref", "iu": "v9"}, "right": {"expression": "iuref", "iu": "v7"}}}, {"iu": ["v6", ["Integer"]], "value": {"expression": "mul", "left": {"expression": "iuref", "iu": "v10"}, "right": {"expression": "iuref", "iu": "v7"}}}]
-    },
-    "tid": ["v11", ["Numeric", 18]],
-    "tableOid": ["v12", ["RegClass"]],
-    "tupleFlags": ["v13", ["BigInt"]],
+      }],
+      "values": [{"iu": ["map", ["Integer"]], "value": {"expression": "mul", "left": {"expression": "iuref", "iu": "scan_a1"}, "right": {"expression": "iuref", "iu": "tableconstruction"}}}, {"iu": ["map2", ["Integer"]], "value": {"expression": "mul", "left": {"expression": "iuref", "iu": "scan_b1"}, "right": {"expression": "iuref", "iu": "tableconstruction"}}}, {"iu": ["map3", ["Integer"]], "value": {"expression": "mul", "left": {"expression": "iuref", "iu": "scan_c1"}, "right": {"expression": "iuref", "iu": "tableconstruction"}}}]
+    }],
+    "tid": ["tid", ["Numeric", 18]],
+    "tableOid": ["tableOid", ["RegClass"]],
+    "tupleFlags": ["tupleFlags", ["BigInt"]],
     "bulkInsert": false
-  }
+  }]
 }