Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions installer/data-downloader/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ SCANNER_INCLUDE_COUNTS=true
SCANNER_INITIAL_CHUNK_DAYS=31
SENSOR_WINDOW_DAYS=7
SENSOR_LOOKBACK_DAYS=30
SENSOR_FALLBACK_START=2025-06-19T00:00:00
SENSOR_FALLBACK_END=2025-07-10T00:00:00
SCAN_INTERVAL_SECONDS=3600
VITE_API_BASE_URL=http://localhost:8000
ALLOWED_ORIGINS=http://localhost:3000,http://localhost:5173
2 changes: 1 addition & 1 deletion installer/data-downloader/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,4 @@ Both JSON files are shared through the `./data` directory so every service (fron

Set `INFLUX_SCHEMA`/`INFLUX_TABLE` to the same values used in the legacy scripts (e.g. `iox` + `WFR25`) so the SQL sent from `backend/server_scanner.py` and `backend/sql.py` matches the proven queries.

All services mount `./data` inside the container and the FastAPI layer manages file I/O with atomic writes to keep data consistent between the worker and UI actions. If the rolling lookback produces no sensors, the collector automatically falls back to the historic 2025-06-19 -> 2025-07-10 window (tune via `SENSOR_FALLBACK_START` / `SENSOR_FALLBACK_END`).
All services mount `./data` inside the container and the FastAPI layer manages file I/O with atomic writes to keep data consistent between the worker and UI actions. If the rolling lookback produces no sensors, the collector now falls back to the oldest/newest run windows discovered by the date scanner, so no manual date tuning is required.
2 changes: 0 additions & 2 deletions installer/data-downloader/backend/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ class Settings(BaseModel):

sensor_window_days: int = Field(default_factory=lambda: int(os.getenv("SENSOR_WINDOW_DAYS", "7")))
sensor_lookback_days: int = Field(default_factory=lambda: int(os.getenv("SENSOR_LOOKBACK_DAYS", "30")))
sensor_fallback_start: str | None = Field(default_factory=lambda: os.getenv("SENSOR_FALLBACK_START", "2025-06-19T00:00:00"))
sensor_fallback_end: str | None = Field(default_factory=lambda: os.getenv("SENSOR_FALLBACK_END", "2025-07-10T00:00:00"))

periodic_interval_seconds: int = Field(default_factory=lambda: int(os.getenv("SCAN_INTERVAL_SECONDS", "3600")))

Expand Down
23 changes: 20 additions & 3 deletions installer/data-downloader/backend/services.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from datetime import datetime, timezone
import logging
from pathlib import Path
from typing import Dict, Optional
from typing import Dict, List, Optional

from influxdb_client_3 import InfluxDBClient3

Expand Down Expand Up @@ -68,6 +68,7 @@ def run_full_scan(self, source: str = "manual") -> Dict[str, dict]:
initial_chunk_days=self.settings.scanner_initial_chunk_days,
)
)
fallback_start, fallback_end = self._build_sensor_fallback_range(runs)
runs_payload = self.runs_repo.merge_scanned_runs(runs)

sensors = fetch_unique_sensors(
Expand All @@ -79,8 +80,8 @@ def run_full_scan(self, source: str = "manual") -> Dict[str, dict]:
table=self.settings.influx_table,
window_days=self.settings.sensor_window_days,
lookback_days=self.settings.sensor_lookback_days,
fallback_start=_parse_iso(self.settings.sensor_fallback_start),
fallback_end=_parse_iso(self.settings.sensor_fallback_end),
fallback_start=fallback_start,
fallback_end=fallback_end,
)
)
sensors_payload = self.sensors_repo.write_sensors(sensors)
Expand Down Expand Up @@ -118,3 +119,19 @@ def _log_influx_connectivity(self) -> None:
logger.info("InfluxDB connectivity OK")
except Exception:
logger.exception("InfluxDB connectivity check failed")

@staticmethod
def _build_sensor_fallback_range(runs: List[dict]) -> tuple[Optional[datetime], Optional[datetime]]:
"""Use scanner output to determine the oldest/newest windows for sensor fallback."""
fallback_start: Optional[datetime] = None
fallback_end: Optional[datetime] = None
for run in runs:
start_dt = _parse_iso(run.get("start_utc"))
end_dt = _parse_iso(run.get("end_utc"))
if start_dt is None or end_dt is None:
continue
fallback_start = start_dt if fallback_start is None else min(fallback_start, start_dt)
fallback_end = end_dt if fallback_end is None else max(fallback_end, end_dt)
if fallback_start and fallback_end and fallback_start > fallback_end:
return None, None
return fallback_start, fallback_end
Loading