DO NOT MERGE - Test Bench

.github/workflows/test-bench-vectorize.yaml

@@ -0,0 +1,158 @@
+name: Test Bench - Vectorize Run
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to test against'
+        required: true
+        type: environment
+  pull_request:
+
+# needed when a workflow wants to use OIDC (OpenID Connect) to authenticate to cloud
+permissions:
+  id-token: write
+  contents: write
+  packages: write
+  pull-requests: write
+
+# global env vars, available in all jobs and steps
+env:
+  MAVEN_OPTS: '-Xmx4g'
+  DS_ARTIFACTORY_USERNAME: ${{ secrets.DS_ARTIFACTORY_USERNAME }}
+  DS_ARTIFACTORY_PASSWORD: ${{ secrets.DS_ARTIFACTORY_PASSWORD }}
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.environment || 'PROD' }}
+
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }}
+    steps:
+      - id: set-matrix
+        env:
+          TEST_TARGETS: ${{ vars.TEST_TARGETS }}
+        run: |
+          {
+            echo "TEST_TARGETS<<EOF"
+            echo "$TEST_TARGETS"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - id: write-setup-summary
+        name: Write setup summary
+        env:
+          TEST_TARGETS: ${{ vars.TEST_TARGETS }}
+        run: |
+          FORMATTED_TARGETS=$(echo "$TEST_TARGETS" | jq '.')
+
+          cat >> $GITHUB_STEP_SUMMARY << 'EOF'
+          ## Test Run Setup
+
+          **Environment:** ${{ github.event.inputs.environment || 'PROD' }}
+          **Triggered by:** ${{ github.actor }}
+
+          ### Test Targets
+          EOF
+
+          echo '```json' >> $GITHUB_STEP_SUMMARY
+          echo "$FORMATTED_TARGETS" >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+
+  # runs unit tests
+  build:
+    name: Test Bench- ENV-${{ github.event.inputs.environment || 'PROD' }} Target- ${{ matrix.name }}
+    needs: setup
+    runs-on: ubuntu-latest
+    environment: ${{ github.event.inputs.environment || 'PROD' }}
+
+    timeout-minutes: 120
+
+    strategy:
+      # do not fail fast, we want to run on all the different target dbs
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.setup.outputs.matrix) }}
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+          cache: maven
+
+      - name: Setup Maven
+        run: |
+          mkdir -p ~/.m2
+          cat <<EOF > ~/.m2/settings.xml
+          <settings>
+            <servers>
+              <server>
+                <id>stargate-central</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+              </server>
+              <server>
+                <id>stargate-snapshots</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+             </server>
+              <server>
+                <id>artifactory</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+              </server>
+              <server>
+                <id>artifactory-snapshots</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+              </server>
+              <server>
+                <id>artifactory-releases</id>
+                <username>${DS_ARTIFACTORY_USERNAME}</username>
+                <password>${DS_ARTIFACTORY_PASSWORD}</password>
+             </server>
+           </servers>
+          </settings>
+          EOF
+
+      - name: Set env
+        # Sanitize the name of the target, it could be things like "smoketest-prod-gcp-us-east4 (astra-serverless-prod-49)"
+        # TEST_BENCH_REPORT path to where the summary report will be written so we can include in the GH summary
+        run: |
+          SAFE_NAME=$(echo "${{ matrix.name }}" | sed 's/[^a-zA-Z0-9._-]/_/g')
+          echo "TEST_BENCH_REPORT=test-bench-report-${SAFE_NAME}.md" >> $GITHUB_ENV
+
+      - name: Build & Test
+        env:
+          TEST_PLAN_FILE: classpath:testbench/testplans/test-plan-astra-vectorize.yaml
+          DATA_API_TOKEN: ${{ secrets.DATA_API_TOKEN }}
+          TARGET_NAME: ${{ matrix.name }}
+          ENDPOINT: ${{ matrix.endpoint }}
+          HUGGINGFACE_KEY: ${{ secrets.HUGGINGFACE_KEY }}
+          JINA_AI_KEY: ${{ secrets.JINA_AI_KEY }}
+          MISTRAL_KEY: ${{ secrets.MISTRAL_KEY }}
+          OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }}
+          UPSTAGE_AI_KEY: ${{ secrets.UPSTAGE_AI_KEY }}
+          VOYAGE_AI_KEY: ${{ secrets.VOYAGE_AI_KEY }}
+
+        # reducing extra output with the -Dfailsafe.printSummary=false -Dfailsafe.trimStackTrace=true
+        # -Dorg.fusesource.jansi.Ansi.disable=true - disable ansi because GH action summary does not handle
+        run: |
+          ./mvnw -B -ntp verify \
+            -Dfmt.skip \
+            -DskipUnitTests \
+            -Dorg.fusesource.jansi.Ansi.disable=true \
+            -Dfailsafe.printSummary=false \
+            -Dfailsafe.trimStackTrace=true \
+            -Dit.test=TestBenchByTestPlan \
+            -Dtest-bench-report-path=$TEST_BENCH_REPORT
+
+      - name: Write summary
+        if: always()
+        run: |
+          cat "$TEST_BENCH_REPORT" >> $GITHUB_STEP_SUMMARY

AGENTS.md

@@ -0,0 +1,116 @@
+# Agent Guidelines
+
+## Code Comment Standards
+
+### Purpose of Comments
+
+Comments should explain **WHY** code exists or **WHY** a particular approach was chosen, NOT simply describe **WHAT** the code does.
+
+### What to Avoid
+
+- Redundant comments that restate obvious code behavior
+- Comments that merely describe what a line or block of code does
+- Stating the obvious (e.g., `// increment counter` above `counter++`)
+
+### What to Include
+
+- **Intent and reasoning**: Why this approach was chosen over alternatives
+- **Business logic context**: Why certain rules or constraints exist
+- **Edge cases**: Non-obvious scenarios the code handles
+- **Non-obvious behavior**: Subtle interactions or side effects
+- **Workarounds**: Why a particular workaround was necessary
+- **Performance considerations**: Why certain optimizations were made
+- **Security implications**: Why certain checks or patterns are used
+
+## Documentation Hierarchy
+
+### Class-Level Documentation
+
+Class-level Javadoc should provide comprehensive context and explain the design intent. Include:
+
+- **Purpose**: What problem this class/interface solves
+- **Design rationale**: Why this approach was chosen
+- **Usage context**: How and when this should be used
+- **Constraints and rules**: What implementations should NOT do
+- **Integration points**: How this fits into the larger system
+
+**Avoid repeating this information at the method level.**
+
+### Method-Level Documentation
+
+Documentation requirements vary based on the method's visibility and purpose:
+
+#### Public API, Interface, and Abstract Methods
+
+These require formal Javadoc with:
+- **When the method is called**: Brief statement of why this should be called and what it does
+- **Parameter documentation**: Standard `@param` tags for all parameters
+- **Return value**: Standard `@return` tag
+
+**Do NOT repeat class-level context or add implementation examples at the method level.**
+
+#### Private Methods
+
+Private implementation methods typically only need a brief comment explaining **WHY** the method exists or **WHY** a particular approach was taken. Formal `@param` and `@return` tags are not required unless:
+- The method is complex with non-obvious parameter usage
+- The method has subtle behavior that needs explanation
+
+**Example (simple private method):**
+```java
+// Normalize user input to prevent injection attacks
+private String sanitizeInput(String raw) {
+    return raw.replaceAll("[^a-zA-Z0-9]", "");
+}
+```
+
+**Example (complex private method needing params):**
+```java
+/**
+ * Merges overlapping time ranges to optimize query performance.
+ * Adjacent ranges within the tolerance window are combined to reduce
+ * the number of database queries.
+ *
+ * @param ranges List of time ranges, may contain overlaps
+ * @param toleranceMs Milliseconds of gap allowed between ranges to still merge them
+ * @return Consolidated list with overlapping ranges merged
+ */
+private List<TimeRange> mergeTimeRanges(List<TimeRange> ranges, long toleranceMs) {
+```
+
+### Documentation Example: Interface
+
+**Class-level (comprehensive):**
+```java
+/**
+ * Defines the stages in the lifecycle of a test bench run.
+ * <p>
+ * Designed to be implemented by a Backend so that it can make changes 
+ * to the data environment so tests can run in a common environment. 
+ * For example, when we use Cassandra as a backend we need to create 
+ * a keyspace but for Astra we use the default one.
+ * </p>
+ * <p>
+ * There should not be any test logic within the implementations, 
+ * that should all be in the test definitions.
+ * </p>
+ */
+public interface TestPlanLifecycle {
+```
+
+**Method-level (minimal):**
+```java
+  /**
+   * Called to optionally add a node to execute before the workflow starts.
+   *
+   * @param testNodeFactory Factory to use to create test nodes
+   * @param uriBuilder Builder to use to create URIs
+   * @param workflow The workflow about to execute
+   * @return Optional DynamicNode to run before the workflow
+   */
+  default Optional<DynamicNode> beforeWorkflow(...) {
+```
+
+**What to avoid at method level:**
+- Repeating "useful for cleanup" or "allows setting up resources" (already covered at class level)
+- Adding specific implementation examples (violates the "no test logic" constraint stated at class level)
+- Restating the overall purpose of the interface

pom.xml

@@ -54,11 +54,7 @@
     <!-- Testing -->
     <skipTests>false</skipTests>
     <skipUnitTests>${skipTests}</skipUnitTests>
-    <!-- 09-Apr-2026, tatu: Wrt [data-api#2447] need to use not-latest
-        (specifically, 3.5.3; latest being 3.5.5)
-        See: https://github.com/fabriciorby/maven-surefire-junit5-tree-reporter?tab=readme-ov-file#important
-      -->
-    <surefire-plugin.version>3.5.3</surefire-plugin.version>
+    <surefire-plugin.version>3.5.5</surefire-plugin.version>
     <skipITs>false</skipITs>
   </properties>
   <dependencyManagement>
@@ -102,6 +98,11 @@
     </repository>
   </repositories>
   <dependencies>
+    <dependency>
+      <groupId>com.jayway.jsonpath</groupId>
+      <artifactId>json-path</artifactId>
+      <version>2.10.0</version>
+    </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-arc</artifactId>
@@ -345,6 +346,18 @@
       <artifactId>quarkus-mcp-server-test</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.maven.plugins</groupId>
+      <artifactId>maven-surefire-plugin</artifactId>
+      <version>${surefire-plugin.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>me.fabriciorby</groupId>
+      <artifactId>maven-surefire-junit5-tree-reporter</artifactId>
+      <version>1.5.1</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>
@@ -466,6 +479,7 @@
               <goal>verify</goal>
             </goals>
             <configuration>
+              <statelessTestsetReporter implementation="org.apache.maven.plugin.surefire.extensions.junit5.JUnit5Xml30StatelessReporter"/>
               <systemPropertyVariables>
                 <!-- Pass JaCoCo agent args to Quarkus application JVM for @QuarkusIntegrationTest coverage -->
                 <quarkus.test.arg-line>${argLine}</quarkus.test.arg-line>

src/main/resources/application.yaml

@@ -16,7 +16,7 @@ stargate:
 
   database:
     limits:
-      max-collections: 5
+      max-collections: 15
 
   debug:
     enabled: false

src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java

@@ -87,7 +87,7 @@ public class ResponseAssertions {
             FieldMatcher.errors(hasErrors));
 
     final String msg =
-        "%s: Response fields %s:%s, %s:%s, %s:%s"
+        "%s: %s:%s, %s:%s, %s:%s"
             .formatted(
                 message,
                 Presence.REQUIRED,

src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java

@@ -0,0 +1,51 @@
+package io.stargate.sgv2.jsonapi.testbench;
+
+import io.stargate.sgv2.jsonapi.testbench.testspec.SpecFiles;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+import org.junit.jupiter.api.DynamicNode;
+import org.junit.jupiter.api.TestFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Entry point for running a Test Bench from a Test Plan file.
+ *
+ * <p>Put the name of the test plan file in the <code>TEST_PLAN_FILE</code> env var, this can set
+ * the target to hit, and the workflows to run. See {@link TestPlanFile}
+ * </p>
+ * <p>
+ *  This will look like a unit test, so we only run when the env var is set.
+ * </p>
+ */
+public class TestBenchByTestPlan {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchByTestPlan.class);
+
+  @TestFactory
+  public Stream<DynamicNode> runTestPlanFile() {
+
+    var rawPath = System.getenv("TEST_PLAN_FILE");
+    if (rawPath == null) {
+      return Stream.empty();
+    }
+    LOGGER.info("runTestPlanFile() - getting TEST_PLAN_FILE from ENV, rawPath={}", rawPath);
+
+    var path =
+        rawPath.startsWith("classpath:")
+            ? SpecFiles.resourceDir(rawPath.substring("classpath:".length()))
+            : Path.of(rawPath);
+
+    var testPlan = TestBenchPlan.fromFile(path);
+    LOGGER.info("runTestPlanFile() - building test plan tree");
+    var testPlanNodeTree = testPlan.testNode();
+
+    LOGGER.info(
+        "runTestPlanFile() - test plan tree build, totalNodeCount={}",
+        testPlanNodeTree.totalNodeCount());
+    System.setProperty(
+        TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY,
+        String.valueOf(testPlanNodeTree.totalNodeCount()));
+    return Stream.of(testPlanNodeTree.root());
+  }
+}