Add build timeout handling and TIMED_OUT state

build_manager/manager.py

@@ -15,6 +15,7 @@ class BuildState(Enum):
     SUCCESS = 2
     FAILURE = 3
     ERROR = 4
+    TIMED_OUT = 5
 
 
 class BuildProgress:
@@ -71,6 +72,7 @@ def __init__(self,
             percent=0
         )
         self.time_created = time.time()
+        self.time_started = None # when build state becomes RUNNING
 
     def to_dict(self) -> dict:
         return {
@@ -81,6 +83,7 @@ def to_dict(self) -> dict:
             'selected_features': list(self.selected_features),
             'progress': self.progress.to_dict(),
             'time_created': self.time_created,
+            'time_started': getattr(self, 'time_started', None),
         }
 
 
@@ -353,6 +356,27 @@ def __update_build_info(self,
             keepttl=True
         )
 
+    def update_build_time_started(self,
+                              build_id: str,
+                              time_started: float) -> None:
+        """
+        Update the build's time_started timestamp.
+
+        Parameters:
+            build_id (str): The ID of the build to update.
+            time_started (float): The timestamp when the build started running.
+        """
+        build_info = self.get_build_info(build_id=build_id)
+
+        if build_info is None:
+            raise ValueError(f"Build with id {build_id} not found.")
+
+        build_info.time_started = time_started
+        self.__update_build_info(
+            build_id=build_id,
+            build_info=build_info
+        )
+
     def update_build_progress_percent(self,
                                       build_id: str,
                                       percent: int) -> None:

build_manager/progress_updater.py

@@ -6,7 +6,9 @@
     BuildManager as bm,
     BuildState
 )
+import time
 
+CBS_BUILD_TIMEOUT_SEC = int(os.getenv('CBS_BUILD_TIMEOUT_SEC', 900))  # 15 minutes default
 
 class BuildProgressUpdater:
     """
@@ -157,6 +159,28 @@ def __refresh_running_build_state(self, build_id: str) -> BuildState:
             raise RuntimeError(
                 "This method should only be called for running builds."
             )
+        # Set time_started if not already set
+        if build_info.time_started is None:
+            start_time = time.time()
+            bm.get_singleton().update_build_time_started(
+                build_id=build_id,
+                time_started=start_time
+            )
+            self.logger.info(
+                f"Build {build_id} started running at {start_time}"
+            )
+            build_info.time_started = start_time
+
+        # Check for timeout
+        elapsed = time.time() - build_info.time_started
+        if elapsed > CBS_BUILD_TIMEOUT_SEC:
+            self.logger.warning(
+                f"Build {build_id} timed out after {elapsed:.0f} seconds"
+            )
+            build_info.error_message = (
+                f"Build exceeded {CBS_BUILD_TIMEOUT_SEC // 60} minute timeout"
+            )
+            return BuildState.TIMED_OUT
 
         # Builder ships the archive post completion
         # This is irrespective of SUCCESS or FAILURE
@@ -213,6 +237,9 @@ def __update_build_percent(self, build_id: str) -> None:
         elif current_state == BuildState.ERROR:
             # Keep existing percentage
             pass
+        elif current_state == BuildState.TIMED_OUT:
+            # Keep existing percentage
+            pass
         else:
             raise Exception("Unhandled BuildState.")
 
@@ -259,6 +286,9 @@ def __update_build_state(self, build_id: str) -> None:
         elif current_state == BuildState.ERROR:
             # ERROR is a conclusive state
             pass
+        elif current_state == BuildState.TIMED_OUT:
+            # TIMED_OUT is a conclusive state
+            pass
         else:
             raise Exception("Unhandled BuildState.")
 

builder/builder.py

@@ -14,6 +14,7 @@
 )
 from pathlib import Path
 
+CBS_BUILD_TIMEOUT_SEC = int(os.getenv('CBS_BUILD_TIMEOUT_SEC', 900))  # 15 minutes default
 
 class Builder:
     """
@@ -377,52 +378,65 @@ def __build(self, build_id: str) -> None:
             )
             build_log.flush()
 
-            # Run the build steps
-            self.logger.info("Running waf configure")
-            build_log.write("Running waf configure\n")
-            build_log.flush()
-            subprocess.run(
-                [
-                    "python3",
-                    "./waf",
-                    "configure",
-                    "--board",
-                    build_info.board,
-                    "--out",
-                    self.__get_path_to_build_dir(build_id),
-                    "--extra-hwdef",
-                    self.__get_path_to_extra_hwdef(build_id),
-                ],
-                cwd=self.__get_path_to_build_src(build_id),
-                stdout=build_log,
-                stderr=build_log,
-                shell=False,
-            )
-
-            self.logger.info("Running clean")
-            build_log.write("Running clean\n")
-            build_log.flush()
-            subprocess.run(
-                ["python3", "./waf", "clean"],
-                cwd=self.__get_path_to_build_src(build_id),
-                stdout=build_log,
-                stderr=build_log,
-                shell=False,
-            )
-
-            self.logger.info("Running build")
-            build_log.write("Running build\n")
-            build_log.flush()
-            build_command = vehicle.waf_build_command
-            subprocess.run(
-                ["python3", "./waf", build_command],
-                cwd=self.__get_path_to_build_src(build_id),
-                stdout=build_log,
-                stderr=build_log,
-                shell=False,
-            )
-            build_log.write("done build\n")
-            build_log.flush()
+            try:
+                # Run the build steps
+                self.logger.info("Running waf configure")
+                build_log.write("Running waf configure\n")
+                build_log.flush()
+                subprocess.run(
+                    [
+                        "python3",
+                        "./waf",
+                        "configure",
+                        "--board",
+                        build_info.board,
+                        "--out",
+                        self.__get_path_to_build_dir(build_id),
+                        "--extra-hwdef",
+                        self.__get_path_to_extra_hwdef(build_id),
+                    ],
+                    cwd=self.__get_path_to_build_src(build_id),
+                    stdout=build_log,
+                    stderr=build_log,
+                    shell=False,
+                    timeout=CBS_BUILD_TIMEOUT_SEC,
+                )
+
+                self.logger.info("Running clean")
+                build_log.write("Running clean\n")
+                build_log.flush()
+                subprocess.run(
+                    ["python3", "./waf", "clean"],
+                    cwd=self.__get_path_to_build_src(build_id),
+                    stdout=build_log,
+                    stderr=build_log,
+                    shell=False,
+                    timeout=CBS_BUILD_TIMEOUT_SEC,
+                )
+
+                self.logger.info("Running build")
+                build_log.write("Running build\n")
+                build_log.flush()
+                build_command = vehicle.waf_build_command
+                subprocess.run(
+                    ["python3", "./waf", build_command],
+                    cwd=self.__get_path_to_build_src(build_id),
+                    stdout=build_log,
+                    stderr=build_log,
+                    shell=False,
+                    timeout=CBS_BUILD_TIMEOUT_SEC,
+                )
+                build_log.write("done build\n")
+                build_log.flush()
+            except subprocess.TimeoutExpired:
+                self.logger.error(
+                    f"Build {build_id} timed out after "
+                    f"{CBS_BUILD_TIMEOUT_SEC} seconds."
+                )
+                build_log.write(
+                    f"Build timed out after {CBS_BUILD_TIMEOUT_SEC} seconds.\n"
+                )
+                build_log.flush()
 
     def shutdown(self) -> None:
         """

docker-compose.yml

@@ -21,6 +21,7 @@ services:
       CBS_GITHUB_ACCESS_TOKEN: ${CBS_GITHUB_ACCESS_TOKEN}
       PYTHONPATH: /app
       GUNICORN_CMD_ARGS: --bind=0.0.0.0:80 --timeout=300
+      CBS_BUILD_TIMEOUT_SEC: ${CBS_BUILD_TIMEOUT_SEC}
     volumes:
       - ./base:/base:rw
     depends_on:
@@ -40,6 +41,7 @@ services:
       CBS_BASEDIR: /base
       CBS_LOG_LEVEL: ${CBS_LOG_LEVEL:-INFO}
       PYTHONPATH: /app
+      CBS_BUILD_TIMEOUT_SEC: ${CBS_BUILD_TIMEOUT_SEC}
     volumes:
       - ./base:/base:rw
     depends_on:

web/static/js/index.js

@@ -67,7 +67,7 @@ function updateBuildsTable(builds) {
             status_color = 'success';
         } else if (build_info['progress']['state'] == 'PENDING') {
             status_color = 'warning';
-        } else if (build_info['progress']['state'] == 'FAILURE' || build_info['progress']['state'] == 'ERROR') {
+        } else if (build_info['progress']['state'] == 'FAILURE' || build_info['progress']['state'] == 'ERROR' || build_info['progress']['state'] == 'TIMED_OUT') {
             status_color = 'danger';
         }
 
@@ -216,7 +216,7 @@ async function tryAutoDownload(buildId) {
         }
 
         // Stop running if the build is in a terminal state
-        if (["FAILURE", "SUCCESS", "ERROR"].includes(currentState)) {
+        if (["FAILURE", "SUCCESS", "ERROR", "TIMED_OUT"].includes(currentState)) {
             clearInterval(autoDownloadIntervalId);
             return;
         }