Refactor DBC file handling and path resolution

installer/.env.example

@@ -1,3 +1,8 @@
+# ------------------------------------------------------------
+# DBC file path
+# ------------------------------------------------------------
+DBC_FILE_PATH=example.dbc
+
 # ------------------------------------------------------------
 # InfluxDB credentials
 # ------------------------------------------------------------

installer/docker-compose.yml

@@ -150,6 +150,7 @@ services:
     networks:
       - datalink
 
+
   startup-data-loader:
     build: ./startup-data-loader
     container_name: startup-data-loader
@@ -159,7 +160,11 @@ services:
       INFLUXDB_URL: "${INFLUXDB_URL:-http://influxdb3:8181}"
       BACKFILL: "1"
       CSV_RESTART_INTERVAL: "${CSV_RESTART_INTERVAL:-10}"
+      # Fixed container path, host path remains flexible
+      DBC_FILE_PATH: "/installer/example.dbc"
     volumes:
+      # Host can name DBC file anything they want
+      - ./${DBC_FILE_PATH:-example.dbc}:/installer/example.dbc:ro
       - ./startup-data-loader/data:/data:ro
       - telegraf-data:/var/lib/telegraf
     working_dir: /app
@@ -174,26 +179,32 @@ services:
              python load_data.py &&
              echo 'Telegraf input file created successfully'"
 
+
   file-uploader:
     build: ./file-uploader
     container_name: file-uploader
     ports:
       - "8084:8084"
     volumes:
       - ./file-uploader:/app
+      # Host DBC file → fixed container location
+      - ./${DBC_FILE_PATH:-example.dbc}:/installer/example.dbc:ro
     restart: unless-stopped
     networks:
       - datalink
     environment:
       - INFLUXDB_TOKEN=${INFLUXDB_ADMIN_TOKEN:-apiv3_dev-influxdb-admin-token}
       - INFLUXDB_URL=${INFLUXDB_URL:-http://influxdb3:8181}
       - FILE_UPLOADER_WEBHOOK_URL=${FILE_UPLOADER_WEBHOOK_URL:-}
+      # Fixed container path only
+      - DBC_FILE_PATH=/installer/example.dbc
     deploy:
       resources:
         limits:
           cpus: "1"
           memory: 1024M
 
+
   data-downloader-api:
     build:
       context: ./data-downloader

installer/file-uploader/helper.py

@@ -3,6 +3,7 @@
 from typing import List, Optional, IO, Callable, Generator
 from zoneinfo import ZoneInfo
 from dataclasses import dataclass
+from pathlib import Path
 import cantools
 from influxdb_client.client.influxdb_client import InfluxDBClient
 from influxdb_client.client.write.point import Point
@@ -11,6 +12,9 @@
 
 # Global list to track temp directories for emergency cleanup
 _temp_directories = []
+DBC_ENV_VAR = "DBC_FILE_PATH"
+DBC_FILENAME = "example.dbc"
+INSTALLER_ROOT = Path(__file__).resolve().parent.parent
 
 def _rolling_cleanup():
     """Clean up temp files older than 6 hours."""
@@ -39,6 +43,37 @@ def _rolling_cleanup():
 atexit.register(_rolling_cleanup)
 _rolling_cleanup()  # Clean up any old temp files from previous runs
 
+
+def _resolve_dbc_path() -> Path:
+    """Find the DBC file via env override, shared volume, or local fallback."""
+    env_override = os.getenv(DBC_ENV_VAR)
+    if env_override:
+        env_path = Path(env_override).expanduser()
+        if env_path.exists():
+            return env_path
+        print(f"⚠️  {DBC_ENV_VAR}={env_override} not found; falling back to default lookup.")
+
+    for candidate in (
+        INSTALLER_ROOT / DBC_FILENAME,
+        Path("/installer") / DBC_FILENAME,
+    ):
+        if candidate.exists():
+            return candidate
+
+    # Final fallback: search nearby for compatibility with older layouts
+    local_candidates = sorted(
+        Path(__file__).resolve().parent.glob("*.dbc"),
+        key=lambda file_path: file_path.stat().st_mtime,
+        reverse=True,
+    )
+    if local_candidates:
+        return local_candidates[0]
+
+    raise FileNotFoundError(
+        f"Could not locate {DBC_FILENAME}. Place it in the repository root or set "
+        f"{DBC_ENV_VAR} to the desired path."
+    )
+
 if os.getenv("DEBUG") is None:
     from dotenv import load_dotenv
 
@@ -78,14 +113,9 @@ def __init__(
         self.tz_toronto = ZoneInfo("America/Toronto")
         self.url = os.getenv("INFLUXDB_URL", "http://influxdb3:8181")
 
-        # finding dbc file in the current directory
-        self.db = cantools.database.load_file(
-            [
-                file
-                for file in os.listdir(os.path.dirname(os.path.abspath(__file__)))
-                if file.endswith(".dbc")
-            ][0]
-        )
+        dbc_path = _resolve_dbc_path()
+        self.db = cantools.database.load_file(str(dbc_path))
+        print(f"📁 Loaded DBC file: {dbc_path}")
 
         self.client = InfluxDBClient(
             url=self.url,

installer/startup-data-loader/Dockerfile

@@ -14,8 +14,6 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Copy application files (exclude the runtime `data/` directory)
 # Keep the build context small by only copying the files needed to run the loader.
 COPY load_data.py requirements.txt ./
-# Copy any .dbc files found at the build context root into the image
-COPY *.dbc ./
 
 # Create Telegraf output directory
 RUN mkdir -p /var/lib/telegraf

installer/startup-data-loader/README.md

@@ -6,7 +6,7 @@ This container pre-loads InfluxDB 3 with small CAN datasets whenever the compose
 
 1. Waits for InfluxDB 3 to pass its health check.
 2. Reads CSV files from the mounted `/data` directory (copy `2024-01-01-00-00-00.csv.md` to `2024-01-01-00-00-00.csv` for the sample dataset).
-3. Uses `example.dbc` to decode each CAN frame into human-readable signals.
+3. Uses the shared `/installer/example.dbc` file (or the path specified by `DBC_FILE_PATH`) to decode each CAN frame into human-readable signals.
 4. Writes the decoded metrics to InfluxDB 3 (`WFR25` bucket, `WFR` organisation) and emits line protocol for Telegraf.
 5. Exits once all files finish processing.
 
@@ -30,8 +30,8 @@ relative_ms,protocol,can_id,byte0,byte1,byte2,byte3,byte4,byte5,byte6,byte7
 ## Adding real data
 
 1. Drop additional CSV files into `data/` using the naming convention `YYYY-MM-DD-HH-MM-SS.csv`.
-2. Replace `example.dbc` with your production CAN database.
-3. Rebuild the image (`docker compose build startup-data-loader`) and restart the stack.
+2. Replace the repository-level `example.dbc` (or set `DBC_FILE_PATH`) with your production CAN database.
+3. Restart the stack so the new DBC is picked up by the container.
 
 ## Monitoring
 

installer/startup-data-loader/load_data.py

@@ -28,6 +28,7 @@
 from typing import List, Optional, IO, Callable, Dict, Set
 from zoneinfo import ZoneInfo
 from dataclasses import dataclass, asdict
+from pathlib import Path
 import cantools
 from influxdb_client import InfluxDBClient, WriteOptions
 
@@ -53,6 +54,9 @@ def _env_int(var_name: str, default: int) -> int:
 INFLUX_TOKEN = os.getenv("INFLUXDB_TOKEN", "apiv3_dev-influxdb-admin-token")
 INFLUX_ORG = "WFR"
 INFLUX_BUCKET = "WFR25"
+DBC_ENV_VAR = "DBC_FILE_PATH"
+DBC_FILENAME = "example.dbc"
+INSTALLER_ROOT = Path(__file__).resolve().parent.parent
 
 # Mode switch
 BACKFILL_MODE = os.getenv("BACKFILL", "0") == "1"
@@ -169,6 +173,39 @@ def compute_file_hash(file_path: str) -> str:
     return hash_md5.hexdigest()
 
 
+def _resolve_dbc_path() -> Path:
+    """Resolve the DBC path using env override, shared installer copy or local fallback."""
+    env_override = os.getenv(DBC_ENV_VAR)
+    if env_override:
+        env_path = Path(env_override).expanduser()
+        if env_path.exists():
+            return env_path
+        print(f"⚠️  {DBC_ENV_VAR}={env_override} not found; falling back to default lookup.")
+
+    shared_candidates = [
+        INSTALLER_ROOT / DBC_FILENAME,
+        Path("/installer") / DBC_FILENAME,
+    ]
+    for candidate in shared_candidates:
+        if candidate.exists():
+            return candidate
+
+    # Final fallback: look for local .dbc files (maintains backwards compatibility)
+    current_dir = Path(__file__).resolve().parent
+    dbc_candidates = sorted(
+        current_dir.glob("*.dbc"),
+        key=lambda file_path: file_path.stat().st_mtime,
+        reverse=True
+    )
+    if dbc_candidates:
+        return dbc_candidates[0]
+
+    raise FileNotFoundError(
+        f"Could not locate {DBC_FILENAME}. Place it in the installer root "
+        f"or set {DBC_ENV_VAR} to the desired path."
+    )
+
+
 class CANLineProtocolWriter:
     def __init__(self, output_path: str, batch_size: int = 1000, progress_state: Optional[ProgressState] = None):
         self.batch_size = batch_size
@@ -177,13 +214,10 @@ def __init__(self, output_path: str, batch_size: int = 1000, progress_state: Opt
         self.tz_toronto = ZoneInfo("America/Toronto")
         self.progress_state = progress_state or ProgressState.load()
 
-        # Find DBC file in current directory
-        dbc_files = [f for f in os.listdir(".") if f.endswith(".dbc")]
-        if not dbc_files:
-            raise FileNotFoundError("No DBC file found in container")
-
-        self.db = cantools.database.load_file(dbc_files[0])
-        print(f"📁 Loaded DBC file: {dbc_files[0]}")
+        # Load the shared DBC file
+        dbc_path = _resolve_dbc_path()
+        self.db = cantools.database.load_file(str(dbc_path))
+        print(f"📁 Loaded DBC file: {dbc_path}")
 
         # Influx client setup (only if in backfill mode)
         if BACKFILL_MODE: