feat: rate limits, migration CI, event E2E tests, and production deploy docs

.env.example

@@ -41,13 +41,20 @@ DB_NAME=postgres
 DB_PASSWORD=password
 DB_CONTAINER_NAME=neurowealth_db
 
-# Security
+# Security — Rate limiting
+# Global limiter
 RATE_LIMIT_WINDOW_MS=900000
 RATE_LIMIT_MAX=100
+# Auth endpoints — stricter to resist credential stuffing (15 min window, 20 req)
 AUTH_RATE_LIMIT_WINDOW_MS=900000
 AUTH_RATE_LIMIT_MAX=20
-
-# Admin API (required in production)
-# Generate with: openssl rand -hex 32
-# Used to authenticate requests to /api/admin/* endpoints
-ADMIN_API_TOKEN=your_admin_api_token_here
+# Admin endpoints — tightest limits (15 min window, 10 req)
+ADMIN_RATE_LIMIT_WINDOW_MS=900000
+ADMIN_RATE_LIMIT_MAX=10
+# Internal/agent service endpoints — higher throughput (1 min window, 500 req)
+INTERNAL_RATE_LIMIT_WINDOW_MS=60000
+INTERNAL_RATE_LIMIT_MAX=500
+# Trusted-IP bypass: comma-separated IPs that skip all rate limits
+TRUSTED_IPS=
+# Internal service token: value expected in X-Internal-Token request header
+INTERNAL_SERVICE_TOKEN=

.github/workflows/node-ci.yml

@@ -28,6 +28,7 @@ jobs:
     env:
       NODE_ENV: test
       DATABASE_URL: postgresql://user:pass@localhost:5432/db
+      WALLET_ENCRYPTION_KEY: a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2
 
     steps:
       # Checkout repository
@@ -65,3 +66,74 @@ jobs:
       # Run tests
       - name: Run tests
         run: npm test
+
+  # ── Issue #100: migration smoke gate ──────────────────────────────────────
+  # Isolated job that re-applies migrations against a clean DB and runs the
+  # smoke test.  Fails the pipeline if either step errors, preventing a broken
+  # migration from reaching production.
+  migration-smoke:
+    name: Migration smoke test
+    runs-on: ubuntu-latest
+    needs: ci
+    services:
+      postgres:
+        image: postgres:14.4
+        env:
+          POSTGRES_USER: smoke_user
+          POSTGRES_PASSWORD: smoke_pass
+          POSTGRES_DB: smoke_db
+        ports:
+          - 5433:5432
+        options: >-
+          --health-cmd="pg_isready -U smoke_user"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
+    env:
+      NODE_ENV: test
+      DATABASE_URL: postgresql://smoke_user:smoke_pass@localhost:5433/smoke_db
+      # Non-secret stubs required by env.ts at module-load time
+      STELLAR_NETWORK: testnet
+      STELLAR_RPC_URL: https://rpc.example.com
+      STELLAR_AGENT_SECRET_KEY: SXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+      VAULT_CONTRACT_ID: CDUMMYVAULTCONTRACTID
+      USDC_TOKEN_ADDRESS: CDUMMYUSDC
+      ANTHROPIC_API_KEY: smoke-anthropic-key
+      JWT_SEED: smoke-jwt-seed
+      JWT_SESSION_TTL_HOURS: '24'
+      JWT_NONCE_TTL_MS: '300000'
+      JWT_CLEANUP_INTERVAL_MS: '86400000'
+      WALLET_ENCRYPTION_KEY: a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Prisma generate
+        run: npx prisma generate
+
+      - name: Check migration status (staging gate)
+        run: npx prisma migrate status
+
+      - name: Apply migrations to smoke DB
+        run: npx prisma migrate deploy
+
+      - name: Confirm no pending migrations after deploy
+        run: |
+          npx prisma migrate status | tee /tmp/migrate-status.txt
+          if grep -q "following migration have not yet been applied" /tmp/migrate-status.txt; then
+            echo "::error::Pending migrations remain after migrate deploy"
+            exit 1
+          fi
+
+      - name: Run smoke test
+        run: npm run smoke

docs/DEPLOYMENT_GUIDE.md

@@ -1,5 +1,7 @@
 # Deployment Guide - Vault Events Persistence
 
+> For production secrets, migration roll-forward/rollback, and the full release checklist, see **[PRODUCTION_DEPLOYMENT.md](./PRODUCTION_DEPLOYMENT.md)**.
+
 ## Pre-Deployment Checklist
 
 - [x] Code review completed

docs/PRODUCTION_DEPLOYMENT.md

@@ -0,0 +1,159 @@
+# Production deployment, secrets, and migrations
+
+This guide covers secret management, CI/CD injection, database migrations, health/readiness checks, and rollback for the NeuroWealth backend.
+
+## Secret managers (recommended)
+
+| Provider | Best for | Notes |
+|----------|----------|-------|
+| **AWS Secrets Manager** | AWS-hosted production | Automatic rotation hooks; inject via ECS task secrets or Lambda env |
+| **HashiCorp Vault** | Multi-cloud / on-prem | Dynamic secrets, audit trail; use AppRole or K8s auth |
+| **GitHub Actions secrets** | CI/CD and staging | Store `DATABASE_URL`, `JWT_SEED`, etc. per environment; never log values |
+
+Never commit raw secrets. Use `.env.example` as a template only.
+
+## Required production secrets
+
+| Variable | Purpose | Rotation |
+|----------|---------|----------|
+| `JWT_SEED` | Signs session JWTs (64-hex) | Every 90 days; invalidate all sessions on rotate |
+| `WALLET_ENCRYPTION_KEY` | Encrypts stored wallet material (32-byte hex) | Coordinated re-encryption migration required |
+| `STELLAR_AGENT_SECRET_KEY` | On-chain agent signing (56-char `S…` key) | Generate new keypair, fund, update env, drain old key |
+| `DATABASE_URL` | PostgreSQL connection | Rotate DB password in provider; update URL; restart app |
+| `ANTHROPIC_API_KEY` | AI agent | Rotate in Anthropic console; update secret store |
+
+Generate locally (development only):
+
+```bash
+openssl rand -hex 64   # JWT_SEED
+openssl rand -hex 32   # WALLET_ENCRYPTION_KEY
+```
+
+### JWT_SEED rotation
+
+1. Generate a new 64-hex value and store it in your secret manager.
+2. Deploy with the new `JWT_SEED` during a maintenance window.
+3. All existing sessions become invalid; users re-authenticate via Stellar challenge.
+4. Monitor auth error rates and `/api/auth` traffic.
+
+### WALLET_ENCRYPTION_KEY rotation
+
+1. Provision `WALLET_ENCRYPTION_KEY_NEW` alongside the current key.
+2. Run a one-off migration job that decrypts with the old key and re-encrypts with the new key.
+3. Swap env to the new key only after verification.
+4. Remove the old key from the secret store.
+
+### STELLAR_AGENT_SECRET_KEY rotation
+
+1. Create and fund a new Stellar keypair on the target network.
+2. Update contract/agent permissions if your vault requires an allowlist.
+3. Set `STELLAR_AGENT_SECRET_KEY` in the secret manager and redeploy.
+4. Verify agent loop and deposit/withdraw paths on testnet before mainnet.
+
+## CI/CD secret injection
+
+- Map GitHub Environment secrets (`staging`, `production`) to job `env` blocks.
+- Use OIDC to AWS/GCP where possible instead of long-lived access keys.
+- Restrict `workflow_dispatch` and production deploy jobs to protected branches.
+- The `migration-smoke` CI job validates `npx prisma migrate deploy` + `npm run smoke` before release promotion.
+
+Example (GitHub Actions):
+
+```yaml
+env:
+  DATABASE_URL: ${{ secrets.DATABASE_URL }}
+  JWT_SEED: ${{ secrets.JWT_SEED }}
+  WALLET_ENCRYPTION_KEY: ${{ secrets.WALLET_ENCRYPTION_KEY }}
+  STELLAR_AGENT_SECRET_KEY: ${{ secrets.STELLAR_AGENT_SECRET_KEY }}
+```
+
+## Deploy checklist (use at every release)
+
+### Pre-deploy
+
+- [ ] Review pending Prisma migrations (`npx prisma migrate status`)
+- [ ] Confirm migration SQL is non-destructive or has a documented data backfill
+- [ ] Take a database backup/snapshot (provider console or `pg_dump`)
+- [ ] Schedule during low traffic; notify on-call
+- [ ] Staging deploy passed CI (`migration-smoke` job green)
+
+### Deploy (roll-forward)
+
+1. **Backup** — snapshot or `pg_dump` of production DB.
+2. **Apply migrations** — use the safe script (non-interactive in CI):
+
+   ```bash
+   export DATABASE_URL="postgresql://..."
+   CI=1 bash scripts/apply-migration.sh
+   ```
+
+   Or manually:
+
+   ```bash
+   npx prisma migrate deploy
+   npm run smoke
+   ```
+
+3. **Smoke test** — `npm run smoke` must exit 0 (connectivity + core tables).
+4. **Deploy application** — roll out new containers/instances with updated image.
+5. **Promote traffic** — only after health checks pass (see below).
+
+### Post-deploy verification
+
+- [ ] `GET /health` returns 200
+- [ ] Readiness subsystems show `database`, `eventListener`, `agentLoop` ready
+- [ ] No spike in DLQ size (`dead_letter_events` table)
+- [ ] Monitor logs (Winston → CloudWatch/Datadog/etc.)
+
+## Health and readiness
+
+| Endpoint | Purpose |
+|----------|---------|
+| `GET /health` | Liveness; returns subsystem readiness from `src/config/readiness.ts` |
+| Load balancer | Use readiness: return 503 until `database` (and optionally `eventListener`) are marked ready |
+
+If the event listener fails to start, the API may still serve read-only routes but on-chain ingestion will lag—treat sustained DLQ growth as a rollback trigger.
+
+## Migration rollback
+
+Prisma does not auto-reverse `migrate deploy`. Plan rollbacks explicitly:
+
+### When to rollback
+
+- `migrate deploy` or `npm run smoke` fails in CI or production
+- Application errors correlated with a specific migration
+- Data integrity issues in `processed_events`, `transactions`, or `positions`
+
+### Roll-forward vs rollback
+
+| Situation | Action |
+|-----------|--------|
+| Migration applied, app bug only | Roll back **application** image to previous tag; DB unchanged |
+| Bad migration, no data loss yet | Restore DB from pre-deploy snapshot; redeploy previous app + migration set |
+| Bad migration with partial writes | Restore snapshot; replay DLQ after fix; document manual reconciliation |
+
+### Rollback steps
+
+1. Stop traffic to new instances (drain load balancer).
+2. Restore database from the pre-deploy backup/snapshot.
+3. Deploy the **previous** application image (matching the restored schema).
+4. Run `npm run smoke` against the restored DB.
+5. Re-enable traffic; post-mortem and fix-forward migration in a new release.
+
+## Automated checks in CI
+
+The `migration-smoke` workflow job:
+
+1. Spins up an isolated Postgres service
+2. Runs `npx prisma migrate status` and `npx prisma migrate deploy`
+3. Fails if pending migrations remain after deploy
+4. Runs `npm run smoke`
+
+A failing job blocks merge/deploy—treat it as the migration alert for staging.
+
+## Related files
+
+- `scripts/apply-migration.sh` — interactive checklist + migrate + smoke (for operators)
+- `scripts/smoke-test.ts` — minimal schema connectivity test
+- `.github/workflows/node-ci.yml` — `migration-smoke` job
+- `docs/DEPLOYMENT_GUIDE.md` — vault event listener operational notes

jest.config.ts

@@ -5,7 +5,7 @@ const config: Config = {
   testEnvironment: "node",
   roots: ["<rootDir>/src", "<rootDir>/tests"],
   testMatch: ["**/?(*.)+(test).ts"],
-  setupFiles: ["<rootDir>/tests/setupEnv.ts"],
+  setupFiles: ["<rootDir>/tests/setupEnv.ts", "<rootDir>/tests/unit/config/jest.setup.env.ts"],
   transform: {
     "^.+\\.ts$": ["ts-jest", { tsconfig: "<rootDir>/tsconfig.test.json" }],
   },

package.json

@@ -15,7 +15,8 @@
     "prisma:generate": "npx prisma generate",
     "pretest": "npm run prisma:generate",
     "test": "jest",
-    "test:coverage": "jest --coverage"
+    "test:coverage": "jest --coverage",
+    "smoke": "ts-node --transpile-only scripts/smoke-test.ts"
   },
   "prisma": {
     "schema": "prisma/schema.prisma",

readme.md

@@ -93,6 +93,21 @@ npm run build
 npm start
 ```
 
+Rate limiting
+-------------
+The API applies layered rate limits (all configurable via `.env`):
+
+| Limiter | Routes | Default | Env vars |
+|---------|--------|---------|----------|
+| Global | All routes | 100 req / 15 min | `RATE_LIMIT_MAX`, `RATE_LIMIT_WINDOW_MS` |
+| Auth | `/api/auth/*` | 20 req / 15 min | `AUTH_RATE_LIMIT_MAX`, `AUTH_RATE_LIMIT_WINDOW_MS` |
+| Admin | `/api/admin/*` | 10 req / 15 min | `ADMIN_RATE_LIMIT_MAX`, `ADMIN_RATE_LIMIT_WINDOW_MS` |
+| Internal | `/api/agent/*` | 500 req / 1 min | `INTERNAL_RATE_LIMIT_MAX`, `INTERNAL_RATE_LIMIT_WINDOW_MS` |
+
+**Bypass (trusted services only):** set `TRUSTED_IPS` to a comma-separated allowlist of IPs, or send the shared secret in the `X-Internal-Token` header (`INTERNAL_SERVICE_TOKEN`). Mount order matters: the bypass middleware runs before limiters in `src/index.ts`.
+
+For production secret handling, migrations, and rollback steps see `docs/PRODUCTION_DEPLOYMENT.md`.
+
 Testing
 -------
 Run unit tests (Jest):
@@ -101,6 +116,12 @@ Run unit tests (Jest):
 npm test
 ```
 
+Post-migration smoke check (DB connectivity + core tables):
+
+```bash
+npm run smoke
+```
+
 Auth overview (short)
 ---------------------
 - `POST /api/auth/challenge` — client posts `stellarPubKey`, server returns a one-time `nonce`.

scripts/apply-migration.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# apply-migration.sh — Safe migration runner for CI/CD and manual deployments.
+#
+# Usage:
+#   DATABASE_URL=postgresql://... bash scripts/apply-migration.sh
+#
+# What it does:
+#   1. Prints a pre-flight checklist as a reminder of manual steps.
+#   2. Applies pending Prisma migrations (migrate deploy — non-destructive).
+#   3. Runs the smoke test to verify schema and connectivity.
+#   4. Exits 1 on any failure so CI/CD pipelines detect the breakage.
+#
+# References: Issue #99 — Secrets management and production deployment checklist
+
+set -euo pipefail
+
+echo ""
+echo "╔══════════════════════════════════════════════════════════════╗"
+echo "║          NeuroWealth — Safe Migration Checklist              ║"
+echo "╠══════════════════════════════════════════════════════════════╣"
+echo "║  Before proceeding, confirm the following (Ctrl-C to abort): ║"
+echo "║                                                               ║"
+echo "║  [ ] DATABASE_URL points to the correct environment          ║"
+echo "║  [ ] A database backup/snapshot was taken in the last 1 hr   ║"
+echo "║  [ ] The migration has been reviewed and is non-destructive   ║"
+echo "║  [ ] Deployment is scheduled during a low-traffic window      ║"
+echo "║  [ ] Rollback plan is documented and ready                    ║"
+echo "╚══════════════════════════════════════════════════════════════╝"
+echo ""
+
+if [[ -z "${CI:-}" ]]; then
+  # Interactive: give operator 5 s to abort
+  echo "Proceeding in 5 seconds... (Ctrl-C to abort)"
+  sleep 5
+fi
+
+echo "[migration] → Applying Prisma migrations..."
+npx prisma migrate deploy
+echo "[migration] ✓ Migrations applied successfully"
+
+echo "[migration] → Running smoke test..."
+npm run smoke
+echo "[migration] ✓ Smoke test passed"
+
+echo ""
+echo "[migration] ✓ Deployment migration complete."
+echo "  Next steps:"
+echo "    • Monitor application logs for errors (winston / CloudWatch / Datadog)"
+echo "    • Verify /health and /api readiness endpoints respond 200"
+echo "    • If errors appear, execute rollback plan immediately"
+echo ""