tests: add nightly workflow

.github/workflows/lxc-test.yml

@@ -43,8 +43,9 @@ jobs:
         run: |
           i=1
           while IFS= read -r line || [ -n "$line" ]; do
-            [[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue
-            echo "cmd${i}=${line}" >> "$GITHUB_OUTPUT"
+            trimmed=$(echo "$line" | xargs)
+            [[ -z "$trimmed" || "$trimmed" == "#"* ]] && continue
+            echo "cmd${i}=${trimmed}" >> "$GITHUB_OUTPUT"
             i=$((i+1))
           done <<'EOF'
           ${{ inputs.cmlxc_commands }}
@@ -68,22 +69,25 @@ jobs:
           repository: chatmail/cmlxc
           ref: ${{ inputs.cmlxc_version }}
           path: cmlxc
-          fetch-depth: 0
 
       - name: Install Incus (Zabbly)
         run: |
           sudo mkdir -p /etc/apt/keyrings
           sudo curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc
           echo "deb [signed-by=/etc/apt/keyrings/zabbly.asc] https://pkgs.zabbly.com/incus/stable $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/zabbly-incus.list
           sudo apt-get update
-          sudo apt-get install -y incus
+          sudo apt-get install -y incus-base
 
       - name: Initialise Incus
         run: |
           sudo systemctl stop docker.socket docker || true
           sudo iptables -P FORWARD ACCEPT
           sudo sysctl -w fs.inotify.max_user_instances=65535
           sudo sysctl -w fs.inotify.max_user_watches=65535
+          # Disable AppArmor restrictions so Docker-in-LXC containers
+          # can run systemd (needs cgroup notification socket access).
+          sudo systemctl stop apparmor || true
+          sudo apparmor_parser -R /etc/apparmor.d/* 2>/dev/null || true
           sudo incus admin init --auto
           sudo chmod 666 /var/lib/incus/unix.socket
 
@@ -97,17 +101,19 @@ jobs:
           python -m pip install --upgrade pip
           pip install ./cmlxc
 
-      - name: Cache Incus images
+      - name: Restore Incus image cache
         id: cache-images
-        uses: actions/cache@v5
+        uses: actions/cache/restore@v5
         with:
           path: /tmp/incus-cache
-          key: incus-v4-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }}
+          key: incus-v5-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }}
+          restore-keys: |
+            incus-v5-${{ runner.os }}-
 
       - name: Import cached images
         run: |
           mkdir -p /tmp/incus-cache
-          for alias in localchat-base localchat-builder; do
+          for alias in localchat-base localchat-builder localchat-cmdeploy localchat-docker; do
             if [ -f /tmp/incus-cache/$alias.tar.gz ]; then
               echo "Importing: $alias"
               incus image import /tmp/incus-cache/$alias.tar.gz --alias $alias || true
@@ -170,12 +176,12 @@ jobs:
           set -eu
           i=0
           while IFS= read -r cmd || [ -n "$cmd" ]; do
-            [[ "$cmd" =~ ^[[:space:]]*(#|$) ]] && continue
+            trimmed=$(echo "$cmd" | xargs)
+            [[ -z "$trimmed" || "$trimmed" == "#"* ]] && continue
             i=$((i+1))
             if [ $i -le 12 ]; then continue; fi
-
-            echo "::group::Run: $cmd"
-            eval "$cmd" || { echo "::endgroup::"; exit 1; }
+            echo "::group::Run: $trimmed"
+            eval "$trimmed" || { echo "::endgroup::"; exit 1; }
             echo "::endgroup::"
           done <<< "$CMLXC_COMMANDS"
 
@@ -184,26 +190,73 @@ jobs:
         run: |
           for c in $(incus list -c n --format csv); do
             echo "::group::Logs for $c"
-            incus exec "$c" -- journalctl -p warning --no-pager -n 100 || true
+            incus exec "$c" -- journalctl --no-pager -n 200 || true
+            # Dump Docker container logs if present
+            svc=chatmail
+            if incus exec "$c" -- docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "$svc"; then
+              echo "--- docker logs $svc ---"
+              incus exec "$c" -- docker logs "$svc" --tail 200 2>&1 || true
+              echo "--- dovecot journal ---"
+              incus exec "$c" -- docker exec "$svc" journalctl -u dovecot --no-pager -n 50 2>&1 || true
+              echo "--- postfix journal ---"
+              incus exec "$c" -- docker exec "$svc" journalctl -u postfix --no-pager -n 50 2>&1 || true
+              echo "--- failed units ---"
+              incus exec "$c" -- docker exec "$svc" systemctl --failed --no-pager 2>&1 || true
+              echo "--- dovecot -n (effective config) ---"
+              incus exec "$c" -- docker exec "$svc" dovecot -n 2>&1 | tail -40 || true
+              echo "--- ssl cert check ---"
+              incus exec "$c" -- docker exec "$svc" ls -la /etc/ssl/certs/mailserver.pem /etc/ssl/private/mailserver.key 2>&1 || true
+            fi
             echo "::endgroup::"
           done
 
       - name: Export images for cache
         if: always() && steps.cache-images.outputs.cache-hit != 'true'
         run: |
           mkdir -p /tmp/incus-cache
+          # Publish the builder LXC container as a cached image (the Docker
+          # container inside gets recreated on compose up, so the LXC is clean).
+          # Only skip localchat-cmdeploy on failure -- it bakes deploy state
+          # directly into the LXC and would carry broken config into the next run.
           if incus list -c n --format csv | grep -q builder-localchat; then
-            incus exec builder-localchat -- rm -rf /root/relays /root/minitest-venv /root/.ssh/config*
+            echo "Cleaning up builder container before publishing ..."
+            incus exec builder-localchat -- bash -c 'rm -rf /root/relays/* /root/.cache/* /root/.npm /root/.bun'
             echo "Publishing builder container as image ..."
             incus publish builder-localchat --alias localchat-builder --force || true
           fi
-          for alias in localchat-base localchat-builder; do
+          # Publish Docker relay container with engine only (strip images to keep cache small)
+          for ct in $(incus list -c n --format csv | grep -v builder); do
+            if incus exec "$ct" -- docker info >/dev/null 2>&1; then
+              echo "Stripping Docker images from $ct ..."
+              incus exec "$ct" -- docker system prune -af --volumes 2>/dev/null || true
+              echo "Publishing $ct as localchat-docker ..."
+              incus publish "$ct" --alias localchat-docker --force || true
+              break
+            fi
+          done
+          exported=0
+          if [ "${{ job.status }}" = "success" ]; then
+            aliases="localchat-base localchat-builder localchat-cmdeploy localchat-docker"
+          else
+            aliases="localchat-base localchat-builder localchat-docker"
+          fi
+          for alias in $aliases; do
             if incus image list --format csv -c l | grep -q "^$alias$"; then
               echo "Exporting: $alias"
               incus image export $alias /tmp/incus-cache/$alias || true
               if [ -f /tmp/incus-cache/$alias ] && [ ! -f /tmp/incus-cache/$alias.tar.gz ]; then
                 mv /tmp/incus-cache/$alias /tmp/incus-cache/$alias.tar.gz
               fi
+              exported=$((exported+1))
             fi
           done
+          echo "exported=$exported" >> "$GITHUB_OUTPUT"
+        id: export-images
+
+      - name: Save Incus image cache
+        if: always() && steps.export-images.outputs.exported > 0
+        uses: actions/cache/save@v5
+        with:
+          path: /tmp/incus-cache
+          key: incus-v5-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }}
 

.github/workflows/nightly.yml

@@ -0,0 +1,36 @@
+name: Nightly relay integration
+
+on:
+  schedule:
+    - cron: "17 2 * * *"
+  workflow_dispatch:
+
+jobs:
+  nightly:
+    uses: ./.github/workflows/lxc-test.yml
+    with:
+      cmlxc_commands: |
+        cmlxc init
+        cmlxc deploy-cmdeploy --source @main fulltest0
+        cmlxc deploy-cmdeploy --source @main fulltest1
+        cmlxc test-mini fulltest0
+        cmlxc test-cmdeploy fulltest0 fulltest1
+        cmlxc stop fulltest0 fulltest1
+        cmlxc destroy fulltest1
+        cmlxc deploy-cmdeploy --type ipv4 --source @main fulltest-ip0
+        cmlxc test-mini fulltest-ip0
+        cmlxc test-cmdeploy fulltest-ip0
+        cmlxc deploy-madmail --source @main fulltest-mad0
+        cmlxc test-mini fulltest-mad0
+        cmlxc test-madmail fulltest-mad0
+        cmlxc test-mini fulltest0 fulltest-mad0
+        cmlxc test-mini fulltest-mad0 fulltest0
+        cmlxc test-mini fulltest-ip0 fulltest-mad0
+        cmlxc test-mini fulltest-mad0 fulltest-ip0
+        cmlxc test-mini fulltest-ip0 fulltest0
+        cmlxc test-mini fulltest0 fulltest-ip0
+        cmlxc docker deploy fulltest-dock0 --source ghcr:main
+        cmlxc docker ps fulltest-dock0
+        cmlxc docker logs fulltest-dock0
+        cmlxc test-cmdeploy fulltest-dock0
+        cmlxc destroy fulltest-dock0

.github/workflows/test.yml

@@ -28,7 +28,7 @@ jobs:
         sudo curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc
         echo "deb [signed-by=/etc/apt/keyrings/zabbly.asc] https://pkgs.zabbly.com/incus/stable $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/zabbly-incus.list
         sudo apt-get update
-        sudo apt-get install -y incus
+        sudo apt-get install -y incus-base
 
     - name: Initialise Incus
       run: |