Skip to content

Commit d9bded0

Browse files
ostinruostinru
authored
Run JDBC tests in Testcontainers (MVP) (#91)
* Run JDBC test in Testcontainers * both Ubuntu 22.04 and Rocky Linux 9
1 parent b968bbe commit d9bded0

16 files changed

Lines changed: 2098 additions & 38 deletions

File tree

‎.github/workflows/pxf-ci.yml‎

Lines changed: 147 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,39 @@ jobs:
198198
path: /tmp/singlecluster-rocky9-image.tar
199199
retention-days: 1
200200

201+
build-pxf-cbdb-testcontainer-image:
202+
name: Build PXF-CBDB Testcontainer Image (${{ matrix.distro }})
203+
runs-on: ubuntu-latest
204+
strategy:
205+
matrix:
206+
include:
207+
- distro: ubuntu
208+
base_image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest
209+
image_tag: pxf/cbdb-testcontainer:1
210+
- distro: rocky9
211+
base_image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
212+
image_tag: pxf/cbdb-testcontainer-rocky9:1
213+
steps:
214+
- name: Checkout PXF source
215+
uses: actions/checkout@v4
216+
with:
217+
fetch-depth: 1
218+
219+
- name: Build pxf-cbdb testcontainer image
220+
run: |
221+
docker build \
222+
--build-arg BASE_IMAGE=${{ matrix.base_image }} \
223+
-t ${{ matrix.image_tag }} \
224+
automation/src/main/resources/testcontainers/pxf-cbdb
225+
docker save ${{ matrix.image_tag }} > /tmp/pxf-cbdb-testcontainer-${{ matrix.distro }}.tar
226+
227+
- name: Upload pxf-cbdb testcontainer image
228+
uses: actions/upload-artifact@v4
229+
with:
230+
name: pxf-cbdb-testcontainer-image-${{ matrix.distro }}
231+
path: /tmp/pxf-cbdb-testcontainer-${{ matrix.distro }}.tar
232+
retention-days: 1
233+
201234
# Stage 2: Parallel test jobs using matrix strategy
202235
pxf-test:
203236
name: Test PXF - ${{ matrix.test_group }}
@@ -541,10 +574,123 @@ jobs:
541574
exit 1
542575
fi
543576
577+
578+
# Stage 2c: Testcontainers-based tests
579+
pxf-testcontainer-test:
580+
name: "TC Test - ${{ matrix.tc_group }} (${{ matrix.use_fdw == 'true' && 'fdw' || 'external-table' }}, ${{ matrix.distro }})"
581+
needs: [build-pxf-cbdb-testcontainer-image]
582+
runs-on: ubuntu-latest
583+
strategy:
584+
fail-fast: false
585+
matrix:
586+
tc_group:
587+
- 'pxf-jdbc'
588+
use_fdw:
589+
- 'false'
590+
- 'true'
591+
distro:
592+
- 'ubuntu'
593+
- 'rocky9'
594+
steps:
595+
- name: Checkout PXF source
596+
uses: actions/checkout@v4
597+
with:
598+
fetch-depth: 1
599+
submodules: true
600+
601+
- name: Set up JDK ${{ env.JAVA_VERSION }}
602+
uses: actions/setup-java@v4
603+
with:
604+
distribution: temurin
605+
java-version: ${{ env.JAVA_VERSION }}
606+
607+
- name: Download pxf-cbdb testcontainer image
608+
uses: actions/download-artifact@v4
609+
with:
610+
name: pxf-cbdb-testcontainer-image-${{ matrix.distro }}
611+
path: /tmp
612+
613+
- name: Load pxf-cbdb testcontainer image
614+
run: |
615+
docker load < /tmp/pxf-cbdb-testcontainer-${{ matrix.distro }}.tar
616+
617+
- name: Build PXF stage artifacts (no unit tests)
618+
run: |
619+
make -C server stage-notest
620+
621+
- name: Run Testcontainers tests - ${{ matrix.tc_group }} (${{ matrix.use_fdw == 'true' && 'fdw' || 'external-table' }}, ${{ matrix.distro }})
622+
id: run_test
623+
continue-on-error: true
624+
timeout-minutes: 120
625+
working-directory: automation
626+
env:
627+
PXF_HOME: ${{ github.workspace }}/server/build/stage
628+
run: |
629+
make test-tc TC_GROUP=${{ matrix.tc_group }} USE_FDW=${{ matrix.use_fdw }} DISTRO=${{ matrix.distro }}
630+
631+
- name: Collect artifacts and generate stats
632+
if: always()
633+
id: collect_artifacts
634+
run: |
635+
mkdir -p artifacts/logs
636+
TC_GROUP="${{ matrix.tc_group }}"
637+
TEST_MODE="${{ matrix.use_fdw == 'true' && 'fdw' || 'external-table' }}"
638+
TEST_RESULT="${{ steps.run_test.outcome }}"
639+
640+
# Copy test artifacts
641+
cp -r automation/automation_logs/* artifacts/ 2>/dev/null || true
642+
643+
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
644+
for xml in automation/target/surefire-reports/TEST-*.xml; do
645+
if [ -f "$xml" ]; then
646+
tests=$(grep -oP 'tests="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
647+
failures=$(grep -oP 'failures="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
648+
errors=$(grep -oP 'errors="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
649+
skipped=$(grep -oP 'skipped="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
650+
TOTAL=$((TOTAL + tests))
651+
FAILED=$((FAILED + failures + errors))
652+
SKIPPED=$((SKIPPED + skipped))
653+
fi
654+
done
655+
PASSED=$((TOTAL - FAILED - SKIPPED))
656+
657+
cat > artifacts/test_stats.json <<EOF
658+
{
659+
"group": "$TC_GROUP:$TEST_MODE:${{ matrix.distro }}",
660+
"result": "$TEST_RESULT",
661+
"total": $TOTAL,
662+
"passed": $PASSED,
663+
"failed": $FAILED,
664+
"skipped": $SKIPPED
665+
}
666+
EOF
667+
668+
echo "failed_count=$FAILED" >> $GITHUB_OUTPUT
669+
echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT
670+
echo "Test stats for tc:$TC_GROUP ($TEST_MODE, ${{ matrix.distro }}): total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED"
671+
672+
- name: Upload test artifacts
673+
if: always()
674+
uses: actions/upload-artifact@v4
675+
with:
676+
name: test-results-tc-${{ matrix.tc_group }}-${{ matrix.use_fdw == 'true' && 'fdw' || 'external-table' }}-${{ matrix.distro }}
677+
path: artifacts/**
678+
if-no-files-found: ignore
679+
retention-days: 7
680+
681+
- name: Check test result
682+
if: always()
683+
run: |
684+
FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}"
685+
if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then
686+
echo "Testcontainer test ${{ matrix.tc_group }} (${{ matrix.test_mode }}, ${{ matrix.distro }}) failed (Failures: $FAILED_COUNT)"
687+
exit 1
688+
fi
689+
544690
# Stage 3: Summary job
545691
test-summary:
546692
name: Test Summary
547-
needs: [pxf-test, pxf-test-rocky9]
693+
needs: [pxf-test, pxf-test-rocky9, pxf-testcontainer-test]
548694
if: always()
549695
runs-on: ubuntu-latest
550696
steps:

‎automation/Makefile‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ ifneq "$(GROUP)" ""
2222
MAVEN_TEST_OPTS+= -Dgroups=$(GROUP)
2323
endif
2424

25+
EXCLUDED_GROUPS ?=
26+
2527
MAVEN_TEST_OPTS+= -Djava.awt.headless=true -DuseFDW=$(USE_FDW) -Duser.timezone=UTC
2628

2729
ifneq "$(OFFLINE)" "true"
@@ -94,7 +96,11 @@ MVN=mvn
9496
all: test
9597

9698
check-env:
97-
@if [ -z "$(PXF_HOME)" ]; then echo 'ERROR: PXF_HOME must be set'; exit 1; fi
99+
@if [ -z "$(PXF_HOME)" ]; then \
100+
echo 'ERROR: PXF_HOME must be set'; \
101+
echo 'Example: export PXF_HOME="$(abspath ../server/build/stage)"'; \
102+
exit 1; \
103+
fi
98104

99105
symlink_pxf_jars: check-env
100106
@if [ -d "$(PXF_HOME)/application" ]; then \
@@ -118,7 +124,7 @@ symlink_pxf_jars: check-env
118124
fi
119125

120126
test: check-env clean-logs symlink_pxf_jars sync_cloud_configs sync_jdbc_config pxf_regress
121-
$(MVN) $(MAVEN_TEST_OPTS) ${MAVEN_DEBUG_OPTS} test
127+
$(MVN) $(MAVEN_TEST_OPTS) ${MAVEN_DEBUG_OPTS} -DexcludedGroups=testcontainers$${EXCLUDED_GROUPS:+,$$EXCLUDED_GROUPS} test
122128

123129
clean: clean-logs
124130
$(MVN) $(MAVEN_TEST_OPTS) clean
@@ -245,6 +251,20 @@ else
245251
@ls src/test/java/org/apache/cloudberry/pxf/automation/features/*/*Test.java | sed 's/.*\///g' | sed 's/\.java//g' | awk '{print "* ", $$1}'
246252
endif
247253

254+
# Run Testcontainers-based tests.
255+
# Usage:
256+
# make test-tc => run all testcontainers tests (Ubuntu)
257+
# make test-tc TC_GROUP=pxf-jdbc => run only pxf-jdbc group
258+
# make test-tc DISTRO=rocky9 => run with Rocky Linux 9 base image
259+
.PHONY: test-tc
260+
test-tc: check-env symlink_pxf_jars pxf_regress
261+
$(MVN) -B -e -Djava.awt.headless=true -Duser.timezone=UTC \
262+
-DuseFDW=$(USE_FDW) \
263+
-Dpxf.test.distro=$(or $(DISTRO),$(PXF_TEST_DISTRO),ubuntu) \
264+
-Dgroups=$(or $(TC_GROUP),testcontainers) \
265+
-DexcludedGroups= \
266+
test
267+
248268
.PHONY: pxf_regress
249269
pxf_regress:
250270
$(MAKE) -C pxf_regress

‎automation/pom.xml‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@
7676
</plugins>
7777

7878
<resources>
79+
<resource>
80+
<directory>src/main/resources</directory>
81+
<includes>
82+
<include>**/*</include>
83+
</includes>
84+
</resource>
7985
<resource>
8086
<directory>src/test/resources</directory>
8187
<includes>
@@ -177,6 +183,30 @@
177183
<version>4.2.0</version>
178184
</dependency>
179185

186+
<dependency>
187+
<groupId>org.testcontainers</groupId>
188+
<artifactId>testcontainers</artifactId>
189+
<version>2.0.3</version>
190+
</dependency>
191+
192+
<dependency>
193+
<groupId>org.apache.commons</groupId>
194+
<artifactId>commons-lang3</artifactId>
195+
<version>3.17.0</version>
196+
</dependency>
197+
198+
<dependency>
199+
<groupId>commons-io</groupId>
200+
<artifactId>commons-io</artifactId>
201+
<version>2.17.0</version>
202+
</dependency>
203+
204+
<dependency>
205+
<groupId>org.apache.commons</groupId>
206+
<artifactId>commons-compress</artifactId>
207+
<version>1.26.2</version>
208+
</dependency>
209+
180210
<dependency>
181211
<groupId>org.jsystemtest</groupId>
182212
<artifactId>jsystemCore</artifactId>
@@ -262,19 +292,19 @@
262292
<dependency>
263293
<groupId>com.fasterxml.jackson.core</groupId>
264294
<artifactId>jackson-core</artifactId>
265-
<version>2.14.3</version>
295+
<version>2.20.2</version>
266296
</dependency>
267297

268298
<dependency>
269299
<groupId>com.fasterxml.jackson.core</groupId>
270300
<artifactId>jackson-databind</artifactId>
271-
<version>2.14.3</version>
301+
<version>2.20.2</version>
272302
</dependency>
273303

274304
<dependency>
275305
<groupId>com.fasterxml.jackson.core</groupId>
276306
<artifactId>jackson-annotations</artifactId>
277-
<version>2.14.3</version>
307+
<version>2.20</version>
278308
</dependency>
279309

280310
<dependency>

0 commit comments

Comments
 (0)