feat: pgbouncer health check

.github/workflows/deploy.yaml

@@ -65,6 +65,110 @@ jobs:
           uv run npx cdk deploy --ci --all --require-approval never
           cd -
 
+      - name: Get stack outputs and run operational checks
+        id: operational_checks
+        env:
+          PROJECT_ID: ${{ steps.short-sha.outputs.sha }}
+        run: |
+          cd integration_tests/cdk
+
+          echo "=== Retrieving Stack Outputs ==="
+
+          # Get list of deployed stacks
+          DEPLOYED_STACKS=$(uv run npx cdk list)
+          echo "Deployed stacks: $DEPLOYED_STACKS"
+
+          # Create outputs file
+          echo "{}" > stack_outputs.json
+
+          # Get outputs from each stack
+          for STACK_NAME in $DEPLOYED_STACKS; do
+            echo "Processing stack: $STACK_NAME"
+
+            # Try to get outputs using AWS CLI
+            STACK_OUTPUTS=$(aws cloudformation describe-stacks \
+              --stack-name "$STACK_NAME" \
+              --query 'Stacks[0].Outputs' \
+              --output json 2>/dev/null || echo "[]")
+
+            if [ "$STACK_OUTPUTS" != "[]" ] && [ "$STACK_OUTPUTS" != "null" ]; then
+              echo "Found outputs for $STACK_NAME:"
+              echo "$STACK_OUTPUTS" | jq .
+
+              # Convert to key-value format and merge with existing outputs
+              TEMP_OUTPUTS=$(echo "$STACK_OUTPUTS" | jq -r 'if . then [.[] | select(.OutputKey and .OutputValue) | {(.OutputKey): .OutputValue}] | add // {} else {} end')
+              echo "$TEMP_OUTPUTS" > temp_stack_outputs.json
+
+              # Merge with existing outputs
+              jq -s '.[0] * .[1]' stack_outputs.json temp_stack_outputs.json > merged.json
+              mv merged.json stack_outputs.json
+            else
+              echo "No outputs found for $STACK_NAME"
+            fi
+          done
+
+          echo "=== Final Combined Outputs ==="
+          cat stack_outputs.json | jq .
+
+          echo "=== Running Operational Checks ==="
+
+          echo "Available CloudFormation output keys:"
+          cat stack_outputs.json | jq -r 'keys[]'
+
+          STAC_API_URL=$(cat stack_outputs.json | jq -r '
+            to_entries[] |
+            select(.key | test("pgstacapistacapioutput"; "i")) |
+            .value' | head -1)
+
+          TITILER_PGSTAC_API_URL=$(cat stack_outputs.json | jq -r '
+            to_entries[] |
+            select(.key | test("titilerpgstacapioutput"; "i")) |
+            .value' | head -1)
+
+          TIPG_API_URL=$(cat stack_outputs.json | jq -r '
+            to_entries[] |
+            select(.key | test("tipgapioutput"; "i")) |
+            .value' | head -1)
+
+          echo "Extracted URLs:"
+          echo "STAC_API_URL: $STAC_API_URL"
+          echo "TITILER_PGSTAC_API_URL: $TITILER_PGSTAC_API_URL"
+          echo "TIPG_API_URL: $TIPG_API_URL"
+
+          # Array of API URLs to check
+          declare -a API_HEALTH_ENDPOINTS=(
+            "STAC_API_URL:${STAC_API_URL}_mgmt/health"
+            "TITILER_PGSTAC_API_URL:${TITILER_PGSTAC_API_URL}healthz"
+            "TIPG_API_URL:${TIPG_API_URL}healthz"
+          )
+
+          # Check each API
+          echo "=== Sleeping for 5 minutes to ensure everything is running ==="
+          sleep 300
+
+          for API_INFO in "${API_HEALTH_ENDPOINTS[@]}"; do
+            API_NAME=$(echo "$API_INFO" | cut -d: -f1)
+            API_URL=$(echo "$API_INFO" | cut -d: -f2-)
+
+            if [ -n "$API_URL" ] && [ "$API_URL" != "null" ]; then
+              echo "Checking $API_NAME at: $API_URL"
+
+              HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 30 "$API_URL" || echo "000")
+
+              if [ "$HTTP_STATUS" = "200" ]; then
+                echo "✅ $API_NAME returned 200"
+              else
+                echo "❌ $API_NAME returned $HTTP_STATUS"
+                exit 1
+              fi
+            else
+              echo "⚠️ $API_NAME URL not found in stack outputs"
+            fi
+          done
+
+          echo "=== Operational Checks Complete ==="
+          cd -
+
       - name: Tear down any infrastructure
         if: always()
         env:

.gitignore

@@ -12,3 +12,5 @@ tests/*.egg*
 tests/*venv*
 tests/__pycache__
 integration_tests/cdk/cdk.out
+integration_tests/cdk/stack_outputs.json
+integration_tests/cdk/temp_stack_outputs.json

lib/database/PgBouncer.ts

@@ -4,6 +4,7 @@ import {
   aws_lambda as lambda,
   aws_secretsmanager as secretsmanager,
   CustomResource,
+  Duration,
   Stack,
 } from "aws-cdk-lib";
 import { Construct } from "constructs";
@@ -66,6 +67,7 @@ export class PgBouncer extends Construct {
   public readonly pgbouncerSecret: secretsmanager.Secret;
   public readonly securityGroup: ec2.SecurityGroup;
   public readonly secretUpdateComplete: CustomResource;
+  public readonly healthCheck: CustomResource;
 
   // The max_connections parameter in PgBouncer determines the maximum number of
   // connections to open on the actual database instance. We want that number to
@@ -220,6 +222,47 @@ export class PgBouncer extends Construct {
         },
       }
     );
+
+    // Add health check custom resource
+    const healthCheckFunction = new lambda.Function(
+      this,
+      "HealthCheckFunction",
+      {
+        runtime: lambda.Runtime.NODEJS_20_X,
+        handler: "index.handler",
+        timeout: Duration.minutes(10),
+        code: lambda.Code.fromAsset(
+          path.join(__dirname, "lambda/pgbouncer-health-check")
+        ),
+        description: "PgBouncer health check function",
+      }
+    );
+
+    // Grant SSM permissions for health check
+    healthCheckFunction.addToRolePolicy(
+      new iam.PolicyStatement({
+        actions: [
+          "ssm:SendCommand",
+          "ssm:GetCommandInvocation",
+          "ssm:DescribeInstanceInformation",
+          "ssm:ListCommandInvocations",
+        ],
+        resources: ["*"],
+      })
+    );
+
+    this.healthCheck = new CustomResource(this, "PgBouncerHealthCheck", {
+      serviceToken: healthCheckFunction.functionArn,
+      properties: {
+        InstanceId: this.instance.instanceId,
+        // Add timestamp to force re-execution on stack updates
+        Timestamp: new Date().toISOString(),
+      },
+    });
+
+    // Ensure health check runs after instance is created but before secret update
+    this.healthCheck.node.addDependency(this.instance);
+    this.secretUpdateComplete.node.addDependency(this.healthCheck);
   }
 
   private loadUserDataScript(

lib/database/index.ts

@@ -27,9 +27,47 @@ function hasVpc(
 }
 
 /**
- * An RDS instance with pgSTAC installed. This is a wrapper around the
- * `rds.DatabaseInstance` higher-level construct making use
- * of the BootstrapPgStac construct.
+ * An RDS instance with pgSTAC installed and PgBouncer connection pooling.
+ *
+ * This construct creates an optimized pgSTAC database setup that includes:
+ * - RDS PostgreSQL instance with pgSTAC extension
+ * - PgBouncer connection pooler (enabled by default)
+ * - Automated health monitoring system
+ * - Optimized database parameters for the selected instance type
+ *
+ * ## Connection Pooling with PgBouncer
+ *
+ * By default, this construct deploys PgBouncer as a connection pooler running on
+ * a dedicated EC2 instance. PgBouncer provides several benefits:
+ *
+ * - **Connection Management**: Pools and reuses database connections to reduce overhead
+ * - **Performance**: Optimizes connection handling for high-traffic applications
+ * - **Scalability**: Allows more concurrent connections than the RDS instance alone
+ * - **Health Monitoring**: Includes comprehensive health checks to ensure availability
+ *
+ * ### PgBouncer Configuration
+ * - Pool mode: Transaction-level pooling (default)
+ * - Maximum client connections: 1000
+ * - Default pool size: 20 connections per database/user combination
+ * - Instance type: t3.micro EC2 instance
+ *
+ * ### Health Check System
+ * The construct includes an automated health check system that validates:
+ * - PgBouncer service is running and listening on port 5432
+ * - Connection tests to ensure accessibility
+ * - Cloud-init setup completion before validation
+ * - Detailed diagnostics for troubleshooting
+ *
+ * ### Connection Details
+ * When PgBouncer is enabled, applications connect through the PgBouncer instance
+ * rather than directly to RDS. The `pgstacSecret` contains connection information
+ * pointing to PgBouncer, and the `connectionTarget` property refers to the
+ * PgBouncer EC2 instance.
+ *
+ * To disable PgBouncer and connect directly to RDS, set `addPgbouncer: false`.
+ *
+ * This is a wrapper around the `rds.DatabaseInstance` higher-level construct
+ * making use of the BootstrapPgStac construct.
  */
 export class PgStacDatabase extends Construct {
   db: rds.DatabaseInstance;
@@ -40,6 +78,7 @@ export class PgStacDatabase extends Construct {
   public readonly connectionTarget: rds.IDatabaseInstance | ec2.Instance;
   public readonly securityGroup?: ec2.SecurityGroup;
   public readonly secretBootstrapper?: CustomResource;
+  public readonly pgbouncerHealthCheck?: CustomResource;
 
   constructor(scope: Construct, id: string, props: PgStacDatabaseProps) {
     super(scope, id);
@@ -186,6 +225,7 @@ export class PgStacDatabase extends Construct {
       this.connectionTarget = this._pgBouncerServer.instance;
       this.securityGroup = this._pgBouncerServer.securityGroup;
       this.secretBootstrapper = this._pgBouncerServer.secretUpdateComplete;
+      this.pgbouncerHealthCheck = this._pgBouncerServer.healthCheck;
     } else {
       this.connectionTarget = this.db;
     }