test server configuration

caldav/compatibility_hints.py

@@ -883,39 +883,13 @@ def dotted_feature_set_list(self, compact=False):
 
 }
 
-## This is for Xandikos 0.2.12.
-## Lots of development going on as of summer 2025, so expect the list to become shorter soon!
-xandikos_v0_2_12 = {
-    ## this only applies for very simple installations
-    "auto-connect.url": {"domain": "localhost", "scheme": "http", "basepath": "/"},
-    'search.recurrences.includes-implicit': {'support': 'unsupported'},
-    'search.recurrences.expanded': {'support': 'unsupported'},
-    'search.time-range.todo': {'support': 'unsupported'},
-    'search.time-range.alarm': {'support': 'ungraceful', 'behaviour': '500 internal server error'},
-    'search.comp-type.optional': {'support': 'ungraceful'},
-    "search.text.substring": {"support": "unsupported"},
-    "search.text.category.substring": {"support": "unsupported"},
-    'principal-search': {'support': 'unsupported'},
-    'freebusy-query': {'support': 'ungraceful', 'behaviour': '500 internal server error'},
-    "scheduling": {"support": "unsupported"},
-    ## https://github.com/jelmer/xandikos/issues/8
-    'search.time-range.open.start.duration': {'support': 'unsupported'},
-    'search.time-range.open.start': {'support': 'broken', 'behaviour': 'future tasks are returned when only an end bound is given'},
-}
-
 xandikos = {
     ## Principal property search returns 403 (not implemented)
     "principal-search": "ungraceful",
 
     ## VTODO RRULE expansion was fixed in xandikos PR #627 (released in 0.3.7).
     ## Exception expansion (CALDAV:expand with EXDATE/RECURRENCE-ID) is now also supported.
 
-    ## Open-start time-range searches (no lower bound) crash xandikos 0.3.7 with a
-    ## 500 Internal Server Error (OverflowError: date value out of range in icalendar.py
-    ## _expand_rrule_component when computing adjusted_start = start - duration).
-    "search.time-range.open.start": {"support": "ungraceful", "behaviour": "500 Internal Server Error (OverflowError in rrule expansion)"},
-    "search.time-range.open.start.duration": True,
-
     ## this only applies for very simple installations
     "auto-connect.url": {"domain": "localhost", "scheme": "http", "basepath": "/"},
 

caldav/jmap/session.py

@@ -57,14 +57,18 @@ def _parse_session_data(url: str, data: dict) -> Session:
     # return a relative path. Resolve it against the session endpoint URL.
     api_url = urljoin(url, api_url)
 
-    # Some servers (e.g. Stalwart behind a port-remapping proxy) advertise an
-    # api_url whose host matches ours but whose port reflects the internal
-    # listener rather than the port we actually connected through. Rewrite to
-    # match the session endpoint's authority so subsequent calls succeed.
+    # Some servers (e.g. Stalwart) advertise an api_url whose host matches ours
+    # but with a different scheme (https vs http) and/or port than the one we
+    # actually connected through. Rewrite both scheme and netloc to match the
+    # session endpoint so that subsequent calls succeed without TLS errors.
     session_parsed = urlparse(url)
     api_parsed = urlparse(api_url)
-    if api_parsed.hostname == session_parsed.hostname and api_parsed.port != session_parsed.port:
-        api_url = urlunparse(api_parsed._replace(netloc=session_parsed.netloc))
+    if api_parsed.hostname == session_parsed.hostname and (
+        api_parsed.port != session_parsed.port or api_parsed.scheme != session_parsed.scheme
+    ):
+        api_url = urlunparse(
+            api_parsed._replace(scheme=session_parsed.scheme, netloc=session_parsed.netloc)
+        )
 
     state = data.get("state", "")
     server_capabilities = data.get("capabilities", {})

tests/caldav_test_servers.yaml.example

@@ -128,8 +128,9 @@ test-servers:
     enabled: ${TEST_STALWART:-false}
     host: ${STALWART_HOST:-localhost}
     port: ${STALWART_PORT:-8809}
-    username: ${STALWART_USERNAME:-testuser}
-    password: ${STALWART_PASSWORD:-testpass}
+    # v0.16+: username is a full email address; password must not be in zxcvbn common-word list.
+    username: ${STALWART_USERNAME:-testuser@example.org}
+    password: ${STALWART_PASSWORD:-testcaldav}
 
   # OX App Suite requires a locally built Docker image — run build.sh first.
   ox:

tests/docker-test-servers/davical/setup_davical.sh

@@ -15,7 +15,7 @@ TEST_USER="testuser"
 TEST_PASSWORD="testpass"
 
 run_sql() {
-    docker exec "$DB_CONTAINER" psql -U "$DB_USER" "$DB_NAME" -tAc "$1" 2>&1
+    docker exec "$DB_CONTAINER" psql -U "$DB_USER" "$DB_NAME" -tAc "$1" 2>/dev/null
 }
 
 echo "Waiting for DAViCal to be accessible..."

tests/docker-test-servers/stalwart/config/config.json

@@ -0,0 +1,4 @@
+{
+  "@type": "Sqlite",
+  "path": "/opt/stalwart/data/stalwart.db"
+}

tests/docker-test-servers/stalwart/docker-compose.yml

@@ -6,14 +6,20 @@ services:
     ports:
       - "8809:8080"
     environment:
-      - STALWART_ADMIN_PASSWORD=adminpass
+      # v0.16.6+: STALWART_ADMIN_PASSWORD is ignored when no config exists (bootstrap mode).
+      # STALWART_RECOVERY_ADMIN pins a bootstrap credential so setup scripts can authenticate.
+      - STALWART_RECOVERY_ADMIN=admin:adminpass
+    volumes:
+      # config.json tells Stalwart where to store its database (avoids bootstrap mode).
+      - ./config/config.json:/etc/stalwart/config.json:ro
     tmpfs:
       - /opt/stalwart/data:size=500m
       - /opt/stalwart/logs:size=50m
-      - /opt/stalwart/etc:size=10m
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8080/"]
+      # /admin/ requires a GitHub web-UI download that can be slow; /dav/cal/ (401) is
+      # available as soon as the database is initialised, so use that instead.
+      test: ["CMD-SHELL", "curl -s -o /dev/null http://localhost:8080/dav/cal/"]
       interval: 5s
       timeout: 3s
       retries: 15
-      start_period: 25s
+      start_period: 30s

tests/docker-test-servers/stalwart/setup_stalwart.sh

@@ -1,12 +1,18 @@
 #!/bin/bash
 # Setup script for Stalwart test server.
-# Creates the test domain and user via the management REST API.
+# Creates the test domain and user via the JMAP management API.
 #
-# Stalwart requires:
-#   1. A domain principal before a user can be created with that email.
-#   2. A user principal with a plain-text secret (Stalwart hashes it internally).
+# Stalwart v0.16+ architecture:
+#   - REST /api/ endpoints are gone; management is done via JMAP (POST /jmap).
+#   - x:Domain/set creates a domain principal.
+#   - x:Account/set creates a user account (name = local part, domainId = domain id).
+#   - Authentication uses full email: testuser@example.org.
+#   - CalDAV URL encodes the @ as %40: /dav/cal/testuser%40example.org/
+#   - config.json (mounted read-only) points Stalwart at the SQLite database,
+#     preventing bootstrap mode from activating.
+#   - STALWART_RECOVERY_ADMIN pins the admin credential during bootstrap.
 #
-# CalDAV is served at /dav/cal/<username>/ over plain HTTP on port 8080.
+# Default passwords avoid zxcvbn's common-password blacklist ("testpass" is rejected).
 
 set -e
 
@@ -16,45 +22,22 @@ ADMIN_USER="admin"
 ADMIN_PASSWORD="adminpass"
 DOMAIN="example.org"
 TEST_USER="${STALWART_USERNAME:-testuser}"
-TEST_PASSWORD="${STALWART_PASSWORD:-testpass}"
-API_BASE="http://localhost:${HOST_PORT}/api"
+TEST_PASSWORD="${STALWART_PASSWORD:-testcaldav}"
+JMAP_URL="http://localhost:${HOST_PORT}/jmap"
 
-api_post() {
-    local endpoint="$1"
-    local body="$2"
-    curl -s -X POST "${API_BASE}${endpoint}" \
+jmap_call() {
+    local body="$1"
+    curl -s -u "${ADMIN_USER}:${ADMIN_PASSWORD}" \
+        -X POST "${JMAP_URL}" \
         -H "Content-Type: application/json" \
-        -H "Accept: application/json" \
-        -u "${ADMIN_USER}:${ADMIN_PASSWORD}" \
         -d "${body}"
 }
 
-create_user() {
-    local username="$1"
-    local password="$2"
-    local result
-    result=$(api_post "/principal" "{
-        \"type\": \"individual\",
-        \"name\": \"${username}\",
-        \"secrets\": [\"${password}\"],
-        \"emails\": [\"${username}@${DOMAIN}\"],
-        \"roles\": [\"user\"]
-    }")
-    if echo "$result" | grep -q '"error"'; then
-        if echo "$result" | grep -q '"fieldAlreadyExists"'; then
-            echo "User '${username}' already exists (OK)"
-        else
-            echo "Warning: user '${username}' creation returned: $result"
-        fi
-    else
-        echo "User '${username}' created"
-    fi
-}
-
 echo "Waiting for Stalwart HTTP endpoint to be ready..."
 max_attempts=60
 for i in $(seq 1 $max_attempts); do
-    if curl -s -o /dev/null -w "%{http_code}" "http://localhost:${HOST_PORT}/" 2>/dev/null | grep -q "200"; then
+    # /dav/cal/ returns 401 once the database is initialised — no GitHub download needed.
+    if curl -s -o /dev/null -w "%{http_code}" "http://localhost:${HOST_PORT}/dav/cal/" 2>/dev/null | grep -q "401"; then
         echo "Stalwart is ready"
         break
     fi
@@ -68,48 +51,124 @@ for i in $(seq 1 $max_attempts); do
 done
 
 echo ""
-echo "Creating domain '${DOMAIN}'..."
-RESULT=$(api_post "/principal" "{\"type\": \"domain\", \"name\": \"${DOMAIN}\"}")
-if echo "$RESULT" | grep -q '"error"'; then
-    if echo "$RESULT" | grep -q '"fieldAlreadyExists"'; then
-        echo "Domain already exists (OK)"
-    else
-        echo "Warning: domain creation returned: $RESULT"
-    fi
+echo "Disabling password strength check for test environment..."
+# Allow simple test passwords; zxcvbn by default rejects common words like "testpass".
+RESULT=$(jmap_call '{
+    "using": ["urn:ietf:params:jmap:core"],
+    "methodCalls": [["x:Authentication/set", {
+        "accountId": "d333333",
+        "update": {"singleton": {"passwordMinStrength": "zero"}}
+    }, "0"]]
+}')
+if echo "$RESULT" | grep -q '"updated"'; then
+    echo "Password strength check disabled"
+else
+    echo "Warning: could not disable password strength check: $RESULT"
 fi
 
-echo "Creating test user '${TEST_USER}'..."
-RESULT=$(api_post "/principal" "{
-    \"type\": \"individual\",
-    \"name\": \"${TEST_USER}\",
-    \"secrets\": [\"${TEST_PASSWORD}\"],
-    \"emails\": [\"${TEST_USER}@${DOMAIN}\"],
-    \"roles\": [\"user\"]
+echo ""
+echo "Disabling rate limiting for test environment..."
+# Stalwart applies HTTP rate limits by default; disable them to avoid 429s during tests.
+# The Http object is a singleton with id "singleton". period is milliseconds (integer).
+# max count per Stalwart validation is 1,000,000.  Try create first; if it already
+# exists (primaryKeyViolation), update the singleton instead.
+RATE_BODY='{"rateLimitAnonymous":{"count":1000000,"period":60000},"rateLimitAuthenticated":{"count":1000000,"period":60000}}'
+RESULT=$(jmap_call "{
+    \"using\": [\"urn:ietf:params:jmap:core\"],
+    \"methodCalls\": [[\"x:Http/set\", {
+        \"accountId\": \"d333333\",
+        \"create\": {\"h1\": ${RATE_BODY}}
+    }, \"0\"]]
 }")
-if echo "$RESULT" | grep -q '"error"'; then
-    if echo "$RESULT" | grep -q '"fieldAlreadyExists"'; then
-        echo "User already exists (OK)"
-    else
-        echo "Error creating user: $RESULT"
-        exit 1
-    fi
+if echo "$RESULT" | grep -q '"primaryKeyViolation"'; then
+    RESULT=$(jmap_call "{
+        \"using\": [\"urn:ietf:params:jmap:core\"],
+        \"methodCalls\": [[\"x:Http/set\", {
+            \"accountId\": \"d333333\",
+            \"update\": {\"singleton\": ${RATE_BODY}}
+        }, \"0\"]]
+    }")
+fi
+if echo "$RESULT" | grep -q '"notCreated"\|"notUpdated"\|"error"'; then
+    echo "Warning: rate limit update returned: $RESULT"
+else
+    echo "Rate limiting disabled"
+fi
+
+echo ""
+echo "Creating domain '${DOMAIN}'..."
+RESULT=$(jmap_call "{
+    \"using\": [\"urn:ietf:params:jmap:core\"],
+    \"methodCalls\": [[\"x:Domain/set\", {
+        \"accountId\": \"d333333\",
+        \"create\": {\"d1\": {\"name\": \"${DOMAIN}\"}}
+    }, \"0\"]]
+}")
+if echo "$RESULT" | grep -q '"created"'; then
+    DOMAIN_ID=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d['methodResponses'][0][1]['created']['d1']['id'])" 2>/dev/null)
+    echo "Domain created (id=${DOMAIN_ID})"
+elif echo "$RESULT" | grep -q '"alreadyExists"\|"primaryKeyViolation"'; then
+    echo "Domain already exists, fetching id..."
+    DOMAIN_ID=$(jmap_call '{"using":["urn:ietf:params:jmap:core"],"methodCalls":[["x:Domain/get",{"accountId":"d333333","ids":null},"0"]]}' | \
+        python3 -c "import json,sys; l=json.load(sys.stdin)['methodResponses'][0][1]['list']; d=[x for x in l if x['name']=='${DOMAIN}']; print(d[0]['id'] if d else '')" 2>/dev/null)
+    echo "Existing domain id=${DOMAIN_ID}"
 else
-    echo "User created: $RESULT"
+    echo "Warning: domain creation returned: $RESULT"
+    DOMAIN_ID=$(jmap_call '{"using":["urn:ietf:params:jmap:core"],"methodCalls":[["x:Domain/get",{"accountId":"d333333","ids":null},"0"]]}' | \
+        python3 -c "import json,sys; l=json.load(sys.stdin)['methodResponses'][0][1]['list']; d=[x for x in l if x['name']=='${DOMAIN}']; print(d[0]['id'] if d else '')" 2>/dev/null)
 fi
 
-# Additional users for RFC6638 scheduling tests
-create_user "user1" "testpass1"
-create_user "user2" "testpass2"
-create_user "user3" "testpass3"
+if [ -z "$DOMAIN_ID" ]; then
+    echo "Error: could not determine domain id"
+    exit 1
+fi
+
+create_user() {
+    local username="$1"
+    local password="$2"
+    local result
+    result=$(jmap_call "{
+        \"using\": [\"urn:ietf:params:jmap:core\"],
+        \"methodCalls\": [[\"x:Account/set\", {
+            \"accountId\": \"d333333\",
+            \"create\": {\"u1\": {
+                \"@type\": \"User\",
+                \"name\": \"${username}\",
+                \"domainId\": \"${DOMAIN_ID}\",
+                \"credentials\": {\"0\": {\"@type\": \"Password\", \"secret\": \"${password}\"}}
+            }}
+        }, \"0\"]]
+    }")
+    if echo "$result" | grep -q '"created"'; then
+        echo "User '${username}@${DOMAIN}' created"
+    elif echo "$result" | grep -q '"primaryKeyViolation"'; then
+        echo "User '${username}@${DOMAIN}' already exists (OK)"
+    else
+        echo "Warning: user '${username}' creation returned: $result"
+    fi
+}
+
+echo ""
+echo "Creating test user '${TEST_USER}'..."
+create_user "${TEST_USER}" "${TEST_PASSWORD}"
+
+echo ""
+echo "Creating additional users for RFC6638 scheduling tests..."
+# Passwords avoid the common-word blacklist: "caldavtest{N}" passes, "testpass{N}" does not.
+create_user "user1" "caldavtest1"
+create_user "user2" "caldavtest2"
+create_user "user3" "caldavtest3"
 
 echo ""
 echo "Verifying CalDAV access..."
+# v0.16+: CalDAV path encodes the @ in the email address as %40.
+CALDAV_PATH="/dav/cal/${TEST_USER}%40${DOMAIN}/"
 max_caldav_attempts=15
 for i in $(seq 1 $max_caldav_attempts); do
     RESPONSE=$(curl -s -X PROPFIND \
         -H "Depth: 0" \
-        -u "${TEST_USER}:${TEST_PASSWORD}" \
-        "http://localhost:${HOST_PORT}/dav/cal/${TEST_USER}/" 2>/dev/null)
+        -u "${TEST_USER}@${DOMAIN}:${TEST_PASSWORD}" \
+        "http://localhost:${HOST_PORT}${CALDAV_PATH}" 2>/dev/null)
     if echo "$RESPONSE" | grep -qi "multistatus\|collection"; then
         echo "CalDAV is accessible"
         break
@@ -124,28 +183,10 @@ for i in $(seq 1 $max_caldav_attempts); do
     sleep 2
 done
 
-echo ""
-echo "Disabling rate limiting for test environment..."
-# Stalwart applies HTTP and authentication rate limits by default, which causes
-# 429 responses during rapid test runs. Append generous limits to config inside
-# the container, then reload.
-docker exec "$CONTAINER_NAME" sh -c 'cat >> /opt/stalwart/etc/config.toml << '"'"'EOF'"'"'
-
-[http]
-rate-limit-anonymous = { count = 999999999, period = "1m" }
-rate-limit-authenticated = { count = 999999999, period = "1m" }
-EOF'
-RELOAD_RESULT=$(curl -s -u "${ADMIN_USER}:${ADMIN_PASSWORD}" "${API_BASE}/reload")
-if echo "$RELOAD_RESULT" | grep -q '"errors":{}'; then
-    echo "Rate limiting disabled (config reloaded)"
-else
-    echo "Warning: config reload result: $RELOAD_RESULT"
-fi
-
 echo ""
 echo "Stalwart setup complete!"
 echo ""
 echo "Credentials:"
-echo "  Admin:     ${ADMIN_USER} / ${ADMIN_PASSWORD}  (web UI: http://localhost:${HOST_PORT})"
-echo "  Test user: ${TEST_USER} / ${TEST_PASSWORD}"
-echo "  CalDAV URL: http://localhost:${HOST_PORT}/dav/cal/${TEST_USER}/"
+echo "  Admin:     ${ADMIN_USER} / ${ADMIN_PASSWORD}  (web UI: http://localhost:${HOST_PORT}/admin/)"
+echo "  Test user: ${TEST_USER}@${DOMAIN} / ${TEST_PASSWORD}"
+echo "  CalDAV URL: http://localhost:${HOST_PORT}${CALDAV_PATH}"

tests/test_jmap_integration.py

@@ -39,8 +39,8 @@
 STALWART_HOST = "localhost"
 STALWART_PORT = 8809
 STALWART_JMAP_URL = f"http://{STALWART_HOST}:{STALWART_PORT}/.well-known/jmap"
-STALWART_USERNAME = "testuser"
-STALWART_PASSWORD = "testpass"
+STALWART_USERNAME = "testuser@example.org"
+STALWART_PASSWORD = "testcaldav"
 
 
 def _reachable(host: str, port: int) -> bool: