Improve CI: add caching for Bookkeeper old release downloads and improve maven caching and error logs

.github/workflows/bk-ci.yml

@@ -72,6 +72,8 @@ jobs:
             !~/.m2/repository/org/apache/bookkeeper
             !~/.m2/repository/org/apache/distributedlog
           key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
 
       - name: Set up JDK 17
         if: steps.check_changes.outputs.docs_only != 'true'
@@ -118,7 +120,7 @@ jobs:
           - step_name: Replication Tests
             module: bookkeeper-server
             flag: replication
-            test_args: "-Dtest='org.apache.bookkeeper.replication.**'"
+            test_args: "-DreuseForks=false -Dtest='org.apache.bookkeeper.replication.**'"
           - step_name: Remaining Tests
             module: bookkeeper-server
             flag: remaining
@@ -150,6 +152,8 @@ jobs:
             !~/.m2/repository/org/apache/bookkeeper
             !~/.m2/repository/org/apache/distributedlog
           key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
@@ -193,7 +197,7 @@ jobs:
 
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v4
-        if: failure()
+        if: ${{ !success() }}
         continue-on-error: true
         with:
           name: unit-${{ matrix.step_name }}-reports
@@ -231,6 +235,8 @@ jobs:
             !~/.m2/repository/org/apache/bookkeeper
             !~/.m2/repository/org/apache/distributedlog
           key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
@@ -251,7 +257,7 @@ jobs:
           $GITHUB_WORKSPACE/dev/ci-tool pick_ubuntu_mirror
 
       - name: Build with Maven
-        run: mvn -B -nsu clean install -Pdocker -DskipTests
+        run: mvn -B -nsu clean install -Pdocker,currentVersionOnly -DskipTests
 
       - name: Run metadata driver tests
         # Exclude jetcd-core-shaded from integration tests, as it’s a POM-only project used internally,
@@ -290,7 +296,7 @@ jobs:
 
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v4
-        if: failure()
+        if: ${{ !success() }}
         continue-on-error: true
         with:
           name: integration-tests-reports
@@ -326,13 +332,23 @@ jobs:
             !~/.m2/repository/org/apache/bookkeeper
             !~/.m2/repository/org/apache/distributedlog
           key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: 17
 
+      - name: Restore released versions cache
+        uses: actions/cache/restore@v4
+        with:
+          path: tests/docker-images/all-released-versions-image/.released-versions
+          key: released-versions-${{ hashFiles('tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh') }}
+          restore-keys: |
+            released-versions-
+
       - name: Pick ubuntu mirror for the docker image build
         run: |
           # pick the closest ubuntu mirror and set it to UBUNTU_MIRROR environment variable
@@ -351,6 +367,10 @@ jobs:
         run: |
           mvn -B -nsu -DbackwardCompatTests -DfailIfNoTests -pl :bc-non-fips,:hierarchical-ledger-manager,:hostname-bookieid,:old-cookie-new-cluster,:recovery-no-password,:upgrade-direct test
 
+      - name: print JVM thread dumps when cancelled
+        if: cancelled()
+        run: ./dev/ci-tool print_thread_dumps
+
       - name: Upload container logs on failure
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}
@@ -375,7 +395,7 @@ jobs:
 
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v4
-        if: failure()
+        if: ${{ !success() }}
         continue-on-error: true
         with:
           name: backward-compatibility-tests-reports
@@ -408,6 +428,8 @@ jobs:
             !~/.m2/repository/org/apache/bookkeeper
             !~/.m2/repository/org/apache/distributedlog
           key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
@@ -440,6 +462,8 @@ jobs:
             !~/.m2/repository/org/apache/bookkeeper
             !~/.m2/repository/org/apache/distributedlog
           key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
@@ -481,6 +505,8 @@ jobs:
             !~/.m2/repository/org/apache/bookkeeper
             !~/.m2/repository/org/apache/distributedlog
           key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
 
       - name: Set up JDK ${{ matrix.jdk_version }}
         uses: actions/setup-java@v4

.github/workflows/bk-streamstorage-python.yml

@@ -66,6 +66,8 @@ jobs:
             !~/.m2/repository/org/apache/bookkeeper
             !~/.m2/repository/org/apache/distributedlog
           key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:

.github/workflows/ci-all-released-versions-update.yaml

@@ -0,0 +1,68 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Keeps the GitHub Actions cache of released BookKeeper version tarballs
+# up-to-date so that backward-compatibility test containers don't need to
+# re-download everything on every run.
+#
+# The cache key is derived from the hash of the download script.  When a new
+# version is added to the script the hash changes, the previous cache is
+# restored as a starting point (via restore-keys), and only the newly added
+# files are downloaded.  The schedule trigger ensures the cache is accessed
+# regularly so that it never expires due to the GitHub Actions 7-day inactivity
+# eviction policy.
+#
+
+name: CI - Released Versions Cache Update
+on:
+  push:
+    branches:
+      - master
+      - branch-*
+    paths:
+      - 'tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh'
+      - '.github/workflows/ci-all-released-versions-update.yaml'
+  schedule:
+    - cron: '35 */12 * * *'
+  workflow_dispatch:
+
+jobs:
+  update-released-versions-cache:
+    name: Update released versions cache
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: Cache released versions
+        id: cache
+        uses: actions/cache@v4
+        timeout-minutes: 10
+        with:
+          path: tests/docker-images/all-released-versions-image/.released-versions
+          key: released-versions-${{ hashFiles('tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh') }}
+          restore-keys: |
+            released-versions-
+
+      - name: Download released versions
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          bash tests/docker-images/all-released-versions-image/scripts/download-released-versions.sh \
+            tests/docker-images/all-released-versions-image/.released-versions

.github/workflows/java21-daily-build.yml

@@ -27,6 +27,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-bookkeeper-all-
       - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
@@ -35,12 +45,12 @@ jobs:
       - name: Build with Maven
         run: mvn -B clean install
       - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure
-        if: failure()
+        if: ${{ !success() }}
         continue-on-error: true
         uses: ./.github/actions/copy-test-reports
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v4
-        if: failure()
+        if: ${{ !success() }}
         continue-on-error: true
         with:
           name: jdk21-tests-reports

.github/workflows/windows-daily-build.yml

@@ -27,6 +27,17 @@ jobs:
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/bookkeeper
+            !~/.m2/repository/org/apache/distributedlog
+          key: ${{ runner.os }}-bookkeeper-all-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            Linux-bookkeeper-all-
+            ${{ runner.os }}-bookkeeper-all-
       - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
@@ -35,12 +46,12 @@ jobs:
       - name: Build with Maven
         run: mvn -B clean install
       - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories If failure
-        if: failure()
+        if: ${{ !success() }}
         continue-on-error: true
         uses: ./.github/actions/copy-test-reports
       - name: Upload Surefire reports
         uses: actions/upload-artifact@v4
-        if: failure()
+        if: ${{ !success() }}
         continue-on-error: true
         with:
           name: windows-tests-reports

.gitignore

@@ -47,4 +47,7 @@ package-lock.json
 build/
 .gradle/
 *.log
-*.dat
+*.dat
+
+# Downloaded released BookKeeper versions (cached by CI, not committed)
+.released-versions/

metadata-drivers/etcd/pom.xml

@@ -57,30 +57,6 @@
       <groupId>io.grpc</groupId>
       <artifactId>grpc-netty-shaded</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.arquillian.cube</groupId>
-      <artifactId>arquillian-cube-docker</artifactId>
-      <version>${arquillian-cube.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>com.github.docker-java</groupId>
-          <artifactId>*</artifactId>
-        </exclusion>
-      </exclusions>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.jboss.arquillian.junit</groupId>
-      <artifactId>arquillian-junit-standalone</artifactId>
-      <version>${arquillian-junit.version}</version>
-      <exclusions>
-        <exclusion>
-          <groupId>com.github.docker-java</groupId>
-          <artifactId>*</artifactId>
-        </exclusion>
-      </exclusions>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>testcontainers</artifactId>

tests/README.md

@@ -77,7 +77,8 @@ Remember to start the unix socket proxy for docker as described in the previous
 # remove possible remaining python client versions from previous builds
 git clean -fdx -- stream/clients/python
 # build the project and docker images
-mvn -B -nsu clean install -Pdocker -DskipTests
+# currentVersionOnly profile avoids downloading old Bookkeeper releases.
+mvn -B -nsu clean install -Pdocker,currentVersionOnly -DskipTests
 docker images | grep apachebookkeeper
 ```
 
@@ -93,6 +94,9 @@ mvn -f tests/pom.xml test -DintegrationTests -DredirectTestOutputToFile=false -D
 
 ### Running backward compatibility tests
 
+Building docker images with `-Pdocker` is required to build the docker images for backward compatibility tests.
+The `currentVersionOnly` profile shouldn't be active.
+
 ```bash
 # Test current server with old clients
 mvn -DbackwardCompatTests -pl :backward-compat-current-server-old-clients test -DredirectTestOutputToFile=false -DtestRetryCount=0

tests/docker-images/all-released-versions-image/Dockerfile

@@ -17,6 +17,28 @@
 # under the License.
 #
 
+# Stage 1: download and install all released BookKeeper versions.
+# Pre-downloaded tarballs from .released-versions (populated and cached by CI)
+# are copied into the build; the download script fills any gaps, then all
+# versions are extracted and installed into /opt/bookkeeper/.
+FROM alpine:3 AS installer
+
+ARG CURRENT_VERSION_ONLY=false
+
+RUN apk add --no-cache bash curl gnupg ca-certificates coreutils
+
+RUN mkdir -p /opt/bookkeeper /etc/supervisord/conf.d
+
+# Copy pre-downloaded tarballs and key files from the .released-versions cache.
+# The download script will fetch anything that is still missing.
+COPY .released-versions/ /released-versions/
+ADD scripts/download-released-versions.sh /download-released-versions.sh
+ADD scripts/install-all-tarballs.sh /install-all-tarballs.sh
+ADD scripts/install-tarball.sh /install-tarball.sh
+
+RUN if [ "$CURRENT_VERSION_ONLY" != "true" ]; then bash /download-released-versions.sh && bash /install-all-tarballs.sh; fi
+
+# Stage 2: runtime image.
 FROM eclipse-temurin:17-jdk-jammy
 MAINTAINER Apache BookKeeper <dev@bookkeeper.apache.org>
 
@@ -39,29 +61,12 @@ RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://arc
     && echo networkaddress.cache.ttl=1 >> $JAVA_HOME/conf/security/java.security \
     && echo networkaddress.cache.negative.ttl=1 >> $JAVA_HOME/conf/security/java.security
 
-RUN mkdir /tarballs
-WORKDIR /tarballs
-RUN wget -nv https://archive.apache.org/dist/zookeeper/bookkeeper/bookkeeper-4.1.0/bookkeeper-server-4.1.0-bin.tar.gz{,.sha1,.md5,.asc}
-RUN wget -nv https://archive.apache.org/dist/zookeeper/bookkeeper/bookkeeper-4.2.0/bookkeeper-server-4.2.0-bin.tar.gz{,.sha1,.md5,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.8.2/bookkeeper-server-4.8.2-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.9.2/bookkeeper-server-4.9.2-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.10.0/bookkeeper-server-4.10.0-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.11.1/bookkeeper-server-4.11.1-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.12.1/bookkeeper-server-4.12.1-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.13.0/bookkeeper-server-4.13.0-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.14.8/bookkeeper-server-4.14.8-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.15.5/bookkeeper-server-4.15.5-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.16.7/bookkeeper-server-4.16.7-bin.tar.gz{,.sha512,.asc}
-RUN wget -nv https://archive.apache.org/dist/bookkeeper/bookkeeper-4.17.2/bookkeeper-server-4.17.2-bin.tar.gz{,.sha512,.asc}
-
-RUN wget -nv https://dist.apache.org/repos/dist/release/bookkeeper/KEYS
-RUN wget -nv http://svn.apache.org/repos/asf/zookeeper/bookkeeper/dist/KEYS?p=1620552 -O KEYS.old
+COPY --from=installer /opt/bookkeeper /opt/bookkeeper
 
 RUN mkdir -p /etc/supervisord/conf.d && mkdir -p /var/run/supervisor && mkdir -p /var/log/bookkeeper
 ADD conf/supervisord.conf /etc/supervisord.conf
-ADD scripts/install-all-tarballs.sh /install-all-tarballs.sh
+COPY --from=installer /etc/supervisord/conf.d /etc/supervisord/conf.d
 ADD scripts/install-tarball.sh /install-tarball.sh
-RUN bash /install-all-tarballs.sh && rm -rf /tarballs
 
 WORKDIR /
 ADD scripts/update-conf-and-boot.sh /update-conf-and-boot.sh