Skip to content

ci: optimize verify for increased parallelization and early results #1171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
joaodinissf wants to merge 6 commits into from
Closed

ci: optimize verify for increased parallelization and early results #1171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a1cfdb3
ci: experiment with new verify
joaodinissf Sep 19, 2025
2fff550
try further parallelization
joaodinissf Sep 19, 2025
d0f8724
split up static checks for early results
joaodinissf Sep 20, 2025
9193b6f
fix: set actions/download-artifact@v4 path
joaodinissf Sep 20, 2025
fbd7a15
fix spotbugs classpath
joaodinissf Sep 20, 2025
fa0ae86
use spotbugs cli to solve classpath issue
joaodinissf Sep 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
297 changes: 279 additions & 18 deletions .github/workflows/verify.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,49 +1,310 @@
name: verify
on: [push, pull_request]
env:
JAVA_VERSION: '21'
MAVEN_VERSION: 3.9.11
PMD_VERSION: '7.15.0'
jobs:
pmd:
runs-on: ubuntu-22.04
compile:
runs-on: ubuntu-24.04
steps:
- name: Set up Maven
uses: stCarolas/setup-maven@v5
with:
maven-version: ${{ env.MAVEN_VERSION }}
- uses: actions/checkout@v5
- name: Set up JDK 21
uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: ${{ env.JAVA_VERSION }}
- name: Set up Workspace Environment Variable
run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV
- name: Cache Maven and p2 dependencies
uses: actions/cache@v4
with:
path: |
/home/runner/.m2/repository
/home/runner/.p2
key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml', 'ddk-target/*.target') }}
- name: Compile with Maven
run: mvn clean compile -f ./ddk-parent/pom.xml --batch-mode -T3C
- name: Upload compiled artifacts
uses: actions/upload-artifact@v4
with:
name: compiled-classes
path: |
**/target/classes
**/target/generated-sources
retention-days: 1
- name: Upload p2 pool cache for SpotBugs
if: always()
uses: actions/upload-artifact@v4
with:
name: p2-pool
path: /home/runner/.p2/pool
if-no-files-found: ignore
retention-days: 1

# test:
# needs: compile
# runs-on: ubuntu-24.04
# steps:
# - name: Set up Maven
# uses: stCarolas/setup-maven@v5
# with:
# maven-version: ${{ env.MAVEN_VERSION }}
# - uses: actions/checkout@v5
# - name: Set up JDK 21
# uses: actions/setup-java@v5
# with:
# distribution: 'temurin'
# java-version: ${{ env.JAVA_VERSION }}
# - name: Set up Workspace Environment Variable
# run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV
# - name: Cache Maven dependencies
# uses: actions/cache@v4
# with:
# path: /home/runner/.m2/repository
# key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml') }}
# - name: Download compiled artifacts
# uses: actions/download-artifact@v4
# with:
# name: compiled-classes
# - name: Run tests with Maven within a virtual X Server Environment
# run: xvfb-run mvn test -f ./ddk-parent/pom.xml --batch-mode -T1C
# - name: Archive Tycho Surefire Plugin
# if: ${{ failure() }}
# uses: actions/upload-artifact@v4
# with:
# name: tycho-surefire-plugin-test
# path: ${{ env.GITHUB_WORKSPACE }}/com.avaloq.tools.ddk.xtext.test/target/work/data/.metadata/.log

pmd-github:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v5
- uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: '21'
java-version: ${{ env.JAVA_VERSION }}
- name: PMD
uses: pmd/pmd-github-action@v2.0.0
id: pmd
with:
version: '7.15.0'
version: ${{ env.PMD_VERSION }}
rulesets: 'ddk-configuration/pmd/ruleset.xml'
analyzeModifiedFilesOnly: false
- name: Fail build if there are violations
if: steps.pmd.outputs.violations != 0
run: exit 1
maven-verify:
runs-on: ubuntu-22.04

early-static-analysis:
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
tool: [checkstyle, pmd]
steps:
- name: Set up Maven
uses: stCarolas/setup-maven@v5
with:
maven-version: ${{ env.MAVEN_VERSION }}
- uses: actions/checkout@v5
- name: Set up JDK 21
uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: ${{ env.JAVA_VERSION }}
- name: Set up Workspace Environment Variable
run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV
- name: Cache Maven and p2 dependencies
uses: actions/cache@v4
with:
path: |
/home/runner/.m2/repository
/home/runner/.p2
key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml', 'ddk-target/*.target') }}
- name: Run ${{ matrix.tool }}
run: |
case "${{ matrix.tool }}" in
checkstyle)
mvn checkstyle:check -f ./ddk-parent/pom.xml --batch-mode -T2C
;;
pmd)
mvn pmd:pmd pmd:check pmd:cpd-check -f ./ddk-parent/pom.xml --batch-mode -T2C
;;
esac

spotbugs:
needs: compile
runs-on: ubuntu-24.04
steps:
- name: Set up Maven
uses: stCarolas/setup-maven@v5
with:
maven-version: ${{ env.MAVEN_VERSION }}
- uses: actions/checkout@v5
- name: Set up JDK 21
uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: ${{ env.JAVA_VERSION }}
- name: Set up Workspace Environment Variable
run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV
- name: Cache Maven and p2 dependencies
uses: actions/cache@v4
with:
path: |
/home/runner/.m2/repository
/home/runner/.p2
key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml', 'ddk-target/*.target') }}
- name: Download compiled artifacts
uses: actions/download-artifact@v4
with:
name: compiled-classes
path: .
- name: Download p2 pool cache
continue-on-error: true
uses: actions/download-artifact@v4
with:
name: p2-pool
path: /home/runner/.p2
- name: Run SpotBugs
run: |
set -eo pipefail
mkdir -p target/spotbugs

# Collect all compiled class directories
mapfile -t CLASS_DIRS < <(find . -type d -path '*/target/classes' -print | sort)
if [ "${#CLASS_DIRS[@]}" -eq 0 ]; then
echo "No class directories found. Did the compile artifacts download?"
exit 1
fi

# Build classpath from all class directories
CLASS_CP=$(IFS=:; echo "${CLASS_DIRS[*]}")

# Also include Tycho cache location used by some setups
TYCHO_CACHE_ROOT="$HOME/.m2/repository/.cache/tycho"
TYCHO_CP=""
if [ -d "$TYCHO_CACHE_ROOT" ]; then
echo "Collecting Tycho cache bundle jars from $TYCHO_CACHE_ROOT"
mapfile -t TYCHO_JARS < <(find "$TYCHO_CACHE_ROOT" -type f \( -path '*/plugins/*.jar' -o -path '*/bundles/*.jar' \) -print | sort -u)
if [ "${#TYCHO_JARS[@]}" -gt 0 ]; then
TYCHO_CP=$(IFS=:; echo "${TYCHO_JARS[*]}")
fi
fi

# Include Eclipse p2 shared pool used by Tycho on GitHub runners
P2_POOL_ROOT="$HOME/.p2/pool"
P2_POOL_CP=""
if [ -d "$P2_POOL_ROOT" ]; then
echo "Collecting p2 pool bundle jars from $P2_POOL_ROOT"
mapfile -t P2_POOL_JARS < <(find "$P2_POOL_ROOT" -type f -path '*/plugins/*.jar' -print | sort -u)
if [ "${#P2_POOL_JARS[@]}" -gt 0 ]; then
P2_POOL_CP=$(IFS=:; echo "${P2_POOL_JARS[*]}")
fi
fi

# Include any built bundle jars (if present) for extra coverage
mapfile -t BUNDLE_JARS < <(find . -type f -path '*/target/*.jar' -print | sort -u)
BUNDLE_CP=""
if [ "${#BUNDLE_JARS[@]}" -gt 0 ]; then
BUNDLE_CP=$(IFS=:; echo "${BUNDLE_JARS[*]}")
fi

# Combine to final AUX_CP
AUX_CP="$CLASS_CP"
if [ -n "$TYCHO_CP" ]; then
AUX_CP="$AUX_CP:$TYCHO_CP"
fi
if [ -n "$P2_POOL_CP" ]; then
AUX_CP="$AUX_CP:$P2_POOL_CP"
fi
if [ -n "$BUNDLE_CP" ]; then
AUX_CP="$AUX_CP:$BUNDLE_CP"
fi

# Fetch SpotBugs distribution ZIP (includes all dependencies)
mvn -q dependency:copy \
-Dartifact=com.github.spotbugs:spotbugs:4.9.6:zip \
-DoutputDirectory=target/spotbugs

unzip -q -o target/spotbugs/spotbugs-4.9.6.zip -d target/spotbugs/dist
SPOTBUGS_HOME=$(find target/spotbugs/dist -maxdepth 1 -type d -name 'spotbugs-*' | head -n1)
echo "Using SpotBugs home: $SPOTBUGS_HOME"

# Run SpotBugs CLI via main class with full lib classpath
set +e
java -Xmx6g -cp "$SPOTBUGS_HOME/lib/*" edu.umd.cs.findbugs.LaunchAppropriateUI \
-textui -effort:max -low -maxRank 15 -exitcode \
-exclude ddk-configuration/findbugs/exclusion-filter.xml \
-auxclasspath "$AUX_CP" \
-xml:withMessages \
-output target/spotbugs/spotbugs.xml \
"${CLASS_DIRS[@]}" 2>&1 | tee target/spotbugs/spotbugs.log
SPOTBUGS_RC=${PIPESTATUS[0]}
set -e

# Optional: fail if any missing classes remain
MISSING=0
if grep -q "needed for analysis were missing" target/spotbugs/spotbugs.log; then
echo "ERROR: SpotBugs reported missing classes. Aux classpath is incomplete."
MISSING=1
fi

# Preserve SpotBugs exit status but still report missing-classes
if [ "$SPOTBUGS_RC" -ne 0 ]; then
exit "$SPOTBUGS_RC"
fi
if [ "$MISSING" -ne 0 ]; then
exit 2
fi

- name: Upload SpotBugs report
if: always()
uses: actions/upload-artifact@v4
with:
name: spotbugs-report
path: |
target/spotbugs/spotbugs.xml
target/spotbugs/spotbugs.log
retention-days: 7

integration-tests:
needs: compile
runs-on: ubuntu-24.04
steps:
- name: Set up Maven
uses: stCarolas/setup-maven@v5
with:
maven-version: 3.9.9
maven-version: ${{ env.MAVEN_VERSION }}
- uses: actions/checkout@v5
- name: Set up JDK 21
uses: actions/setup-java@v5
with:
distribution: 'temurin'
java-version: '21'
- name: Set up Workspace Enviroment Variable
run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV
- name: Cache Maven dependencies
uses: actions/cache@v4
with:
path: /home/runner/.m2/repository
key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml') }}
- name: Build with Maven within a virtual X Server Environment
run: xvfb-run mvn clean verify checkstyle:check pmd:pmd pmd:check pmd:cpd-check spotbugs:check -f ./ddk-parent/pom.xml --batch-mode --fail-at-end
java-version: ${{ env.JAVA_VERSION }}
- name: Set up Workspace Environment Variable
run: echo "WORKSPACE=${{ github.workspace }}" >> $GITHUB_ENV
- name: Cache Maven and p2 dependencies
uses: actions/cache@v4
with:
path: |
/home/runner/.m2/repository
/home/runner/.p2
key: ${{ runner.os }}-maven-0-${{ hashFiles('**/pom.xml', 'ddk-target/*.target') }}
- name: Download compiled artifacts
uses: actions/download-artifact@v4
with:
name: compiled-classes
path: .
- name: Run integration tests with Maven within a virtual X Server Environment
run: xvfb-run mvn verify -f ./ddk-parent/pom.xml --batch-mode --fail-at-end -T1C
- name: Archive Tycho Surefire Plugin
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: tycho-surefire-plugin
name: tycho-surefire-plugin-integration
path: ${{ env.GITHUB_WORKSPACE }}/com.avaloq.tools.ddk.xtext.test/target/work/data/.metadata/.log
10 changes: 6 additions & 4 deletions ddk-parent/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>com.avaloq.tools.ddk</groupId>
Expand Down Expand Up @@ -50,14 +50,15 @@
<checkstyle.version>11.0.1</checkstyle.version>
<clean.version>3.5.0</clean.version>
<deploy.version>3.1.4</deploy.version>
<spotbugs.plugin.version>4.9.4.2</spotbugs.plugin.version>
<spotbugs.version>4.9.5</spotbugs.version>
<spotbugs.plugin.version>4.9.6.0</spotbugs.plugin.version>
<spotbugs.version>4.9.6</spotbugs.version>
<pmd.plugin.version>3.27.0</pmd.plugin.version>
<pmd.version>7.17.0</pmd.version>
<tycho.version>5.0.0</tycho.version>
<xtend.version>2.40.0</xtend.version>
</properties>


<modules>
<module>../ddk-target</module>
<module>../ddk-repository</module>
Expand Down Expand Up @@ -359,6 +360,7 @@
<threshold>Low</threshold>
<excludeFilterFile>${spotbugs.excludeFilterFile}</excludeFilterFile>
<maxHeap>1024</maxHeap>
<classFilesDirectory>${project.build.directory}/classes</classFilesDirectory>
</configuration>
</plugin>
</plugins>
Expand Down
Loading