From 19a4808d74c8ecdf20ea91c17635b9afd354f165 Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 21 Jan 2026 01:34:13 +0530 Subject: [PATCH 1/4] web: add red color for TIMED_OUT state in UI --- web/static/js/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/static/js/index.js b/web/static/js/index.js index 719e3dd..0c62ebc 100644 --- a/web/static/js/index.js +++ b/web/static/js/index.js @@ -67,7 +67,7 @@ function updateBuildsTable(builds) { status_color = 'success'; } else if (build_info['progress']['state'] == 'PENDING') { status_color = 'warning'; - } else if (build_info['progress']['state'] == 'FAILURE' || build_info['progress']['state'] == 'ERROR') { + } else if (build_info['progress']['state'] == 'FAILURE' || build_info['progress']['state'] == 'ERROR' || build_info['progress']['state'] == 'TIMED_OUT') { status_color = 'danger'; } @@ -216,7 +216,7 @@ async function tryAutoDownload(buildId) { } // Stop running if the build is in a terminal state - if (["FAILURE", "SUCCESS", "ERROR"].includes(currentState)) { + if (["FAILURE", "SUCCESS", "ERROR", "TIMED_OUT"].includes(currentState)) { clearInterval(autoDownloadIntervalId); return; } From 8c11bda7721ce4516854568cf920d0eaf2794178 Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 21 Jan 2026 01:35:02 +0530 Subject: [PATCH 2/4] builder: add timeout handling using CBS_BUILD_TIMEOUT_SEC --- builder/builder.py | 106 +++++++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 46 deletions(-) diff --git a/builder/builder.py b/builder/builder.py index 5aaa0aa..b97b92d 100644 --- a/builder/builder.py +++ b/builder/builder.py @@ -14,6 +14,7 @@ ) from pathlib import Path +CBS_BUILD_TIMEOUT_SEC = int(os.getenv('CBS_BUILD_TIMEOUT_SEC', 900)) # 15 minutes default class Builder: """ @@ -377,52 +378,65 @@ def __build(self, build_id: str) -> None: ) build_log.flush() - # Run the build steps - self.logger.info("Running waf configure") - build_log.write("Running waf configure\n") - build_log.flush() - subprocess.run( - [ - "python3", - "./waf", - "configure", - "--board", - build_info.board, - "--out", - self.__get_path_to_build_dir(build_id), - "--extra-hwdef", - self.__get_path_to_extra_hwdef(build_id), - ], - cwd=self.__get_path_to_build_src(build_id), - stdout=build_log, - stderr=build_log, - shell=False, - ) - - self.logger.info("Running clean") - build_log.write("Running clean\n") - build_log.flush() - subprocess.run( - ["python3", "./waf", "clean"], - cwd=self.__get_path_to_build_src(build_id), - stdout=build_log, - stderr=build_log, - shell=False, - ) - - self.logger.info("Running build") - build_log.write("Running build\n") - build_log.flush() - build_command = vehicle.waf_build_command - subprocess.run( - ["python3", "./waf", build_command], - cwd=self.__get_path_to_build_src(build_id), - stdout=build_log, - stderr=build_log, - shell=False, - ) - build_log.write("done build\n") - build_log.flush() + try: + # Run the build steps + self.logger.info("Running waf configure") + build_log.write("Running waf configure\n") + build_log.flush() + subprocess.run( + [ + "python3", + "./waf", + "configure", + "--board", + build_info.board, + "--out", + self.__get_path_to_build_dir(build_id), + "--extra-hwdef", + self.__get_path_to_extra_hwdef(build_id), + ], + cwd=self.__get_path_to_build_src(build_id), + stdout=build_log, + stderr=build_log, + shell=False, + timeout=CBS_BUILD_TIMEOUT_SEC, + ) + + self.logger.info("Running clean") + build_log.write("Running clean\n") + build_log.flush() + subprocess.run( + ["python3", "./waf", "clean"], + cwd=self.__get_path_to_build_src(build_id), + stdout=build_log, + stderr=build_log, + shell=False, + timeout=CBS_BUILD_TIMEOUT_SEC, + ) + + self.logger.info("Running build") + build_log.write("Running build\n") + build_log.flush() + build_command = vehicle.waf_build_command + subprocess.run( + ["python3", "./waf", build_command], + cwd=self.__get_path_to_build_src(build_id), + stdout=build_log, + stderr=build_log, + shell=False, + timeout=CBS_BUILD_TIMEOUT_SEC, + ) + build_log.write("done build\n") + build_log.flush() + except subprocess.TimeoutExpired: + self.logger.error( + f"Build {build_id} timed out after " + f"{CBS_BUILD_TIMEOUT_SEC} seconds." + ) + build_log.write( + f"Build timed out after {CBS_BUILD_TIMEOUT_SEC} seconds.\n" + ) + build_log.flush() def shutdown(self) -> None: """ From d12faa8451e7635f08b0a14d00b0472cb58b7c8c Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 21 Jan 2026 01:36:08 +0530 Subject: [PATCH 3/4] build_manager: track time_started and handle timeout state --- build_manager/manager.py | 24 ++++++++++++++++++++++++ build_manager/progress_updater.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/build_manager/manager.py b/build_manager/manager.py index 4143f64..3069bb1 100644 --- a/build_manager/manager.py +++ b/build_manager/manager.py @@ -15,6 +15,7 @@ class BuildState(Enum): SUCCESS = 2 FAILURE = 3 ERROR = 4 + TIMED_OUT = 5 class BuildProgress: @@ -71,6 +72,7 @@ def __init__(self, percent=0 ) self.time_created = time.time() + self.time_started = None # when build state becomes RUNNING def to_dict(self) -> dict: return { @@ -81,6 +83,7 @@ def to_dict(self) -> dict: 'selected_features': list(self.selected_features), 'progress': self.progress.to_dict(), 'time_created': self.time_created, + 'time_started': getattr(self, 'time_started', None), } @@ -353,6 +356,27 @@ def __update_build_info(self, keepttl=True ) + def update_build_time_started(self, + build_id: str, + time_started: float) -> None: + """ + Update the build's time_started timestamp. + + Parameters: + build_id (str): The ID of the build to update. + time_started (float): The timestamp when the build started running. + """ + build_info = self.get_build_info(build_id=build_id) + + if build_info is None: + raise ValueError(f"Build with id {build_id} not found.") + + build_info.time_started = time_started + self.__update_build_info( + build_id=build_id, + build_info=build_info + ) + def update_build_progress_percent(self, build_id: str, percent: int) -> None: diff --git a/build_manager/progress_updater.py b/build_manager/progress_updater.py index c6ddd35..6a9e538 100644 --- a/build_manager/progress_updater.py +++ b/build_manager/progress_updater.py @@ -6,7 +6,9 @@ BuildManager as bm, BuildState ) +import time +CBS_BUILD_TIMEOUT_SEC = int(os.getenv('CBS_BUILD_TIMEOUT_SEC', 900)) # 15 minutes default class BuildProgressUpdater: """ @@ -157,6 +159,28 @@ def __refresh_running_build_state(self, build_id: str) -> BuildState: raise RuntimeError( "This method should only be called for running builds." ) + # Set time_started if not already set + if build_info.time_started is None: + start_time = time.time() + bm.get_singleton().update_build_time_started( + build_id=build_id, + time_started=start_time + ) + self.logger.info( + f"Build {build_id} started running at {start_time}" + ) + build_info.time_started = start_time + + # Check for timeout + elapsed = time.time() - build_info.time_started + if elapsed > CBS_BUILD_TIMEOUT_SEC: + self.logger.warning( + f"Build {build_id} timed out after {elapsed:.0f} seconds" + ) + build_info.error_message = ( + f"Build exceeded {CBS_BUILD_TIMEOUT_SEC // 60} minute timeout" + ) + return BuildState.TIMED_OUT # Builder ships the archive post completion # This is irrespective of SUCCESS or FAILURE @@ -213,6 +237,9 @@ def __update_build_percent(self, build_id: str) -> None: elif current_state == BuildState.ERROR: # Keep existing percentage pass + elif current_state == BuildState.TIMED_OUT: + # Keep existing percentage + pass else: raise Exception("Unhandled BuildState.") @@ -259,6 +286,9 @@ def __update_build_state(self, build_id: str) -> None: elif current_state == BuildState.ERROR: # ERROR is a conclusive state pass + elif current_state == BuildState.TIMED_OUT: + # TIMED_OUT is a conclusive state + pass else: raise Exception("Unhandled BuildState.") From 02b8f7b887e97563263cf369b64433a6dfb573b7 Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 21 Jan 2026 01:36:31 +0530 Subject: [PATCH 4/4] docker-compose: add CBS_BUILD_TIMEOUT_SEC env var --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index cb55812..db78c31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ services: CBS_GITHUB_ACCESS_TOKEN: ${CBS_GITHUB_ACCESS_TOKEN} PYTHONPATH: /app GUNICORN_CMD_ARGS: --bind=0.0.0.0:80 --timeout=300 + CBS_BUILD_TIMEOUT_SEC: ${CBS_BUILD_TIMEOUT_SEC} volumes: - ./base:/base:rw depends_on: @@ -40,6 +41,7 @@ services: CBS_BASEDIR: /base CBS_LOG_LEVEL: ${CBS_LOG_LEVEL:-INFO} PYTHONPATH: /app + CBS_BUILD_TIMEOUT_SEC: ${CBS_BUILD_TIMEOUT_SEC} volumes: - ./base:/base:rw depends_on: