refactor: consolidate ClickHouse configuration into single CLICKHOUSE_URL variable

Author: LeoKaynan

.env.example

@@ -21,6 +21,9 @@ REDIS_HOST="localhost"
 REDIS_PORT="6379"
 REDIS_TLS_DISABLED="true"
 
+# ClickHouse is used for runs replication and analytics queries
+CLICKHOUSE_URL=http://default:password@localhost:8123
+
 DEV_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3030/otel"
 DEV_OTEL_BATCH_PROCESSING_ENABLED="0"
 

apps/webapp/app/env.server.ts

@@ -1077,8 +1077,7 @@ const EnvironmentSchema = z
       .string()
       .default(process.env.REDIS_TLS_DISABLED ?? "false"),
 
-    RUN_REPLICATION_CLICKHOUSE_URL: z.string().optional(),
-    RUN_REPLICATION_ENABLED: z.string().default("0"),
+    RUN_REPLICATION_ENABLED: z.string().default("1"),
     RUN_REPLICATION_SLOT_NAME: z.string().default("task_runs_to_clickhouse_v1"),
     RUN_REPLICATION_PUBLICATION_NAME: z.string().default("task_runs_to_clickhouse_v1_publication"),
     RUN_REPLICATION_MAX_FLUSH_CONCURRENCY: z.coerce.number().int().default(2),

apps/webapp/app/routes/admin.api.v1.runs-replication.create.ts

@@ -78,7 +78,7 @@ export async function action({ request }: ActionFunctionArgs) {
 
 function createRunReplicationService(params: CreateRunReplicationServiceParams) {
   const clickhouse = new ClickHouse({
-    url: env.RUN_REPLICATION_CLICKHOUSE_URL,
+    url: env.CLICKHOUSE_URL,
     name: params.name,
     keepAlive: {
       enabled: params.keepAliveEnabled,

apps/webapp/app/services/runsReplicationInstance.server.ts

@@ -15,15 +15,15 @@ function initializeRunsReplicationInstance() {
   const { DATABASE_URL } = process.env;
   invariant(typeof DATABASE_URL === "string", "DATABASE_URL env var not set");
 
-  if (!env.RUN_REPLICATION_CLICKHOUSE_URL) {
-    console.log("🗃️  Runs replication service not enabled");
+  if (env.RUN_REPLICATION_ENABLED !== "1") {
+    console.log("🗃️  Runs replication service disabled (RUN_REPLICATION_ENABLED=0)");
     return;
   }
 
-  console.log("🗃️  Runs replication service enabled");
+  console.log("🗃️  Runs replication service starting");
 
   const clickhouse = new ClickHouse({
-    url: env.RUN_REPLICATION_CLICKHOUSE_URL,
+    url: env.CLICKHOUSE_URL,
     name: "runs-replication",
     keepAlive: {
       enabled: env.RUN_REPLICATION_KEEP_ALIVE_ENABLED === "1",
@@ -68,21 +68,19 @@ function initializeRunsReplicationInstance() {
     insertStrategy: env.RUN_REPLICATION_INSERT_STRATEGY,
   });
 
-  if (env.RUN_REPLICATION_ENABLED === "1") {
-    service
-      .start()
-      .then(() => {
-        console.log("🗃️ Runs replication service started");
-      })
-      .catch((error) => {
-        console.error("🗃️ Runs replication service failed to start", {
-          error,
-        });
+  service
+    .start()
+    .then(() => {
+      console.log("🗃️  Runs replication service started");
+    })
+    .catch((error) => {
+      console.error("🗃️  Runs replication service failed to start", {
+        error,
       });
+    });
 
-    signalsEmitter.on("SIGTERM", service.shutdown.bind(service));
-    signalsEmitter.on("SIGINT", service.shutdown.bind(service));
-  }
+  signalsEmitter.on("SIGTERM", service.shutdown.bind(service));
+  signalsEmitter.on("SIGINT", service.shutdown.bind(service));
 
   return service;
 }

hosting/docker/.env.example

@@ -63,7 +63,6 @@ DEV_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8030/otel
 CLICKHOUSE_USER=default
 CLICKHOUSE_PASSWORD=password
 CLICKHOUSE_URL=http://default:password@clickhouse:8123?secure=false
-RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@clickhouse:8123
 
 # Docker Registry
 # - When testing locally, the default values should be fine

hosting/docker/webapp/docker-compose.yml

@@ -66,7 +66,6 @@ services:
       CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info}
       # Run replication
       RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1}
-      RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:password@clickhouse:8123}
       RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info}
       # Limits
       # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB

hosting/k8s/helm/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: trigger
 description: The official Trigger.dev Helm chart
 type: application
-version: 4.0.4
+version: 4.0.5
 appVersion: v4.0.4
 home: https://trigger.dev
 sources:

hosting/k8s/helm/templates/webapp.yaml

@@ -381,8 +381,6 @@ spec:
               value: {{ .Values.webapp.clickhouse.logLevel | quote }}
             - name: RUN_REPLICATION_ENABLED
               value: "1"
-            - name: RUN_REPLICATION_CLICKHOUSE_URL
-              value: {{ include "trigger-v4.clickhouse.replication.url" . | quote }}
             - name: RUN_REPLICATION_LOG_LEVEL
               value: {{ .Values.webapp.runReplication.logLevel | quote }}
             {{- if not .Values.telemetry.enabled }}