Drill-XXXX: Refactor User Interface

.github/workflows/sqllab-frontend.yml

@@ -0,0 +1,64 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: SQL Lab Frontend CI
+
+on:
+  push:
+    paths:
+      - 'exec/java-exec/src/main/resources/webapp/**'
+  pull_request:
+    paths:
+      - 'exec/java-exec/src/main/resources/webapp/**'
+
+defaults:
+  run:
+    working-directory: exec/java-exec/src/main/resources/webapp
+
+jobs:
+  build-and-test:
+    name: Build & Test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+          cache-dependency-path: exec/java-exec/src/main/resources/webapp/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: TypeScript type check
+        run: npx tsc --noEmit
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Run tests
+        run: npx vitest run --passWithNoTests
+
+      - name: Build
+        run: npm run build

.gitignore

@@ -33,4 +33,31 @@ target/
 tools/venv/
 venv/
 .vscode/*
-exec/java-exec/src/main/resources/webapp/
+/exec/java-exec/src/main/resources/python/.venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+.coverage
+
+# Frontend (webapp)
+node_modules/
+.vite/
+.env
+.env.local
+.env.*.local
+*.swo
+Thumbs.db
+coverage/
+/exec/java-exec/src/main/resources/webapp/dist/
+/exec/java-exec/src/main/resources/webapp/node/
+result-cache/
+
+# Large GeoJSON files (for local development only, not committed to git)
+# Exception: us-zipcodes.topojson (merged all-states, 91 MB) is committed to git
+# Individual state files (us-al-zipcodes.json, us-ca-zipcodes.json, etc.) are git-ignored
+# These can be regenerated locally using scripts/process-zipcodes.py
+/exec/java-exec/src/main/resources/webapp/public/geojson/us-*-zipcodes.json
+/exec/java-exec/src/main/resources/webapp/public/geojson/us-zipcodes.json
+!/exec/java-exec/src/main/resources/webapp/public/geojson/us-zipcodes.topojson
+/exec/java-exec/src/main/resources/webapp/public/geojson/*.zip
+/exec/java-exec/src/main/resources/webapp/public/geojson/cb_*

TODO.md

@@ -0,0 +1 @@
+4.  Force Directed Graph Visualization

build-frontend.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Build the SQL Lab frontend and update the distribution.
+# Usage: ./build-frontend.sh
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+WEBAPP_DIR="$SCRIPT_DIR/exec/java-exec/src/main/resources/webapp"
+DIST_BASE="$SCRIPT_DIR/distribution/target"
+
+echo "=== Building SQL Lab frontend ==="
+cd "$WEBAPP_DIR"
+npm run build
+
+echo ""
+echo "=== Building java-exec module ==="
+cd "$SCRIPT_DIR"
+mvn package -pl exec/java-exec -DskipTests -Dcheckstyle.skip=true -q
+
+# Find the distribution directory and copy the jar
+DIST_DIR=$(find "$DIST_BASE" -name "jars" -type d 2>/dev/null | head -1)
+if [ -n "$DIST_DIR" ]; then
+  echo ""
+  echo "=== Copying jar to distribution ==="
+  cp "$SCRIPT_DIR/exec/java-exec/target/drill-java-exec-"*"-SNAPSHOT.jar" "$DIST_DIR/"
+  echo "Updated: $DIST_DIR/"
+else
+  echo ""
+  echo "WARNING: Distribution directory not found under $DIST_BASE"
+  echo "Run a full 'mvn package' first to create the distribution, or copy the jar manually:"
+  echo "  cp exec/java-exec/target/drill-java-exec-*-SNAPSHOT.jar <drill-distribution>/jars/"
+fi
+
+echo ""
+echo "=== Done! Restart Drill and hard-refresh your browser (Cmd+Shift+R) ==="

contrib/format-pdf/src/main/java/org/apache/drill/exec/store/pdf/PdfFormatConfig.java

@@ -100,6 +100,13 @@ public List<String> extensions() {
     return this.extensions;
   }
 
+  /**
+   * Standard getter for extensions, consistent with other FormatPluginConfig implementations.
+   */
+  public List<String> getExtensions() {
+    return this.extensions;
+  }
+
   public boolean combinePages() {
     return this.combinePages;
   }

contrib/format-xml/src/main/java/org/apache/drill/exec/store/xml/XMLReader.java

@@ -161,8 +161,8 @@ public void close() {
 
   /**
    * This function processes the XML elements.  This function stops reading when the
-   * limit (if any) which came from the query has been reached or the Iterator runs out of
-   * elements.
+   * limit (if any) which came from the query has been reached, a complete row has been
+   * read, or the Iterator runs out of elements.
    * @return True if there are more elements to parse, false if not
    */
   private boolean processElements() {
@@ -197,6 +197,13 @@ private boolean processElements() {
 
         // Process the event
         processEvent(currentEvent, lastEvent, reader.peek());
+
+        // After completing a row, return to let next() check batch capacity.
+        // This prevents batch overflow errors that occur when rows accumulate
+        // beyond what the batch can hold without the isFull() check running.
+        if (currentState == xmlState.ROW_ENDED) {
+          return true;
+        }
       } catch (XMLStreamException e) {
         throw UserException
           .dataReadError(e)

contrib/storage-cassandra/src/test/java/org/apache/drill/exec/store/cassandra/TestCassandraSuite.java

@@ -27,6 +27,7 @@
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.CassandraContainer;
 
 @Category(SlowTest.class)
@@ -42,6 +43,10 @@ public class TestCassandraSuite extends BaseTest {
 
   @BeforeClass
   public static void initCassandra() {
+    org.junit.Assume.assumeTrue(
+      "Docker is not available, skipping container tests",
+      DockerClientFactory.instance().isDockerAvailable()
+    );
     synchronized (TestCassandraSuite.class) {
       if (initCount.get() == 0) {
         startCassandra();

contrib/storage-elasticsearch/src/test/java/org/apache/drill/exec/store/elasticsearch/TestElasticsearchSuite.java

@@ -39,6 +39,7 @@
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.elasticsearch.ElasticsearchContainer;
 
 import com.google.api.client.util.SslUtils;
@@ -54,6 +55,8 @@
 import java.time.Duration;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.junit.Assume.assumeTrue;
+
 
 @Category(SlowTest.class)
 @RunWith(Suite.class)
@@ -76,6 +79,10 @@ public class TestElasticsearchSuite extends BaseTest {
 
   @BeforeClass
   public static void initElasticsearch() throws IOException, GeneralSecurityException {
+    assumeTrue(
+      "Docker is not available, skipping container tests",
+      DockerClientFactory.instance().isDockerAvailable()
+    );
     synchronized (TestElasticsearchSuite.class) {
       if (initCount.get() == 0) {
         startElasticsearch();

contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpSchemaFactory.java

@@ -20,6 +20,7 @@
 import java.util.Collections;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.schema.Table;
@@ -112,6 +113,16 @@ public Table getTable(String name) {
       }
     }
 
+    @Override
+    public Set<String> getTableNames() {
+      return tables.keySet();
+    }
+
+    @Override
+    public Set<String> getSubSchemaNames() {
+      return subSchemas.keySet();
+    }
+
     @Override
     public String getTypeName() {
       return HttpStoragePluginConfig.NAME;

contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithMySQL.java

@@ -28,6 +28,7 @@
 import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.containers.MySQLContainer;
 import org.testcontainers.ext.ScriptUtils;
@@ -41,6 +42,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
 
 @Category(JdbcStorageTest.class)
 public class TestJdbcInsertWithMySQL extends ClusterTest {
@@ -51,6 +53,10 @@ public class TestJdbcInsertWithMySQL extends ClusterTest {
 
   @BeforeClass
   public static void initMysql() throws Exception {
+    assumeTrue(
+      "Docker is not available, skipping container tests",
+      DockerClientFactory.instance().isDockerAvailable()
+    );
     startCluster(ClusterFixture.builder(dirTestWatcher));
     dirTestWatcher.copyResourceToRoot(Paths.get(""));
 

contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcInsertWithPostgres.java

@@ -26,6 +26,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.containers.PostgreSQLContainer;
 import org.testcontainers.utility.DockerImageName;
@@ -37,6 +38,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
 
 @Category(JdbcStorageTest.class)
 public class TestJdbcInsertWithPostgres extends ClusterTest {
@@ -46,6 +48,10 @@ public class TestJdbcInsertWithPostgres extends ClusterTest {
 
   @BeforeClass
   public static void initPostgres() throws Exception {
+    assumeTrue(
+      "Docker is not available, skipping container tests",
+      DockerClientFactory.instance().isDockerAvailable()
+    );
     startCluster(ClusterFixture.builder(dirTestWatcher));
     dirTestWatcher.copyResourceToRoot(Paths.get(""));
 

contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithClickhouse.java

@@ -33,6 +33,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.ClickHouseContainer;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.utility.DockerImageName;
@@ -42,6 +43,7 @@
 import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeTrue;
 
 /**
  * JDBC storage plugin tests against Clickhouse.
@@ -56,6 +58,10 @@ public class TestJdbcPluginWithClickhouse extends ClusterTest {
 
   @BeforeClass
   public static void initClickhouse() throws Exception {
+    assumeTrue(
+      "Docker is not available, skipping container tests",
+      DockerClientFactory.instance().isDockerAvailable()
+    );
     startCluster(ClusterFixture.builder(dirTestWatcher));
     String osName = System.getProperty("os.name").toLowerCase();
     DockerImageName imageName;

contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMSSQL.java

@@ -35,6 +35,7 @@
 import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.MSSQLServerContainer;
 import org.testcontainers.utility.DockerImageName;
 
@@ -55,6 +56,10 @@ public class TestJdbcPluginWithMSSQL extends ClusterTest {
 
   @BeforeClass
   public static void initMSSQL() throws Exception {
+    Assume.assumeTrue(
+      "Docker is not available, skipping container tests",
+      DockerClientFactory.instance().isDockerAvailable()
+    );
     Assume.assumeTrue(System.getProperty("os.arch").matches("(amd64|x86_64)"));
 
     startCluster(ClusterFixture.builder(dirTestWatcher));

contrib/storage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java

@@ -34,6 +34,7 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
+import org.testcontainers.DockerClientFactory;
 import org.testcontainers.containers.JdbcDatabaseContainer;
 import org.testcontainers.containers.MySQLContainer;
 import org.testcontainers.ext.ScriptUtils;
@@ -58,6 +59,10 @@ public class TestJdbcPluginWithMySQLIT extends ClusterTest {
 
   @BeforeClass
   public static void initMysql() throws Exception {
+    Assume.assumeTrue(
+      "Docker is not available, skipping container tests",
+      DockerClientFactory.instance().isDockerAvailable()
+    );
     startCluster(ClusterFixture.builder(dirTestWatcher));
     String osName = System.getProperty("os.name").toLowerCase();
     String mysqlDBName = "drill_mysql_test";