diff --git a/bsp/stm32/stm32f407-rt-spark/.ci/attachconfig/ci.attachconfig.yml b/bsp/stm32/stm32f407-rt-spark/.ci/attachconfig/ci.attachconfig.yml index 206761fcc2c..2cafba63f66 100644 --- a/bsp/stm32/stm32f407-rt-spark/.ci/attachconfig/ci.attachconfig.yml +++ b/bsp/stm32/stm32f407-rt-spark/.ci/attachconfig/ci.attachconfig.yml @@ -203,3 +203,13 @@ component.cherryusb_cdc: - CONFIG_RT_CHERRYUSB_DEVICE_DWC2_ST=y - CONFIG_RT_CHERRYUSB_DEVICE_CDC_ACM=y - CONFIG_RT_CHERRYUSB_DEVICE_TEMPLATE_CDC_ACM=y +ci-guard-test: + env: + RTT_CI_GUARD_TEST_INJECT: "1" + pre_build: | + python -c "import sys; sys.exit(23)" + post_build: | + python -c "import sys; sys.exit(24)" + buildcheckresult: RTT_CI_GUARD_TEST_EXPECTED_OUTPUT_NOT_FOUND + kconfig: + - CONFIG_RT_USING_FINSH=y diff --git a/tools/ci/bsp_buildings.py b/tools/ci/bsp_buildings.py index eb0de0434d1..91cd718d4cb 100644 --- a/tools/ci/bsp_buildings.py +++ b/tools/ci/bsp_buildings.py @@ -18,12 +18,68 @@ import multiprocessing import yaml +SCONS_FATAL_PATTERNS = ( + re.compile(r'No SConstruct file found', re.IGNORECASE), + re.compile(r'SConscript.*No such file or directory', re.IGNORECASE), + re.compile(r'No such file or directory.*SConscript', re.IGNORECASE), + re.compile(r"(can'?t|cannot|unable to)\s+(read|open|find).*SConscript", re.IGNORECASE), + re.compile(r'FileNotFoundError:.*SConscript', re.IGNORECASE), + re.compile(r'scons:\s+\*\*\*', re.IGNORECASE), + re.compile(r'RTT_CI_GUARD_TEST_SCONS_FATAL', re.IGNORECASE), +) + def add_summary(text): """ add summary to github action. """ - os.system(f'echo "{text}" >> $GITHUB_STEP_SUMMARY ;') + summary_file = os.getenv('GITHUB_STEP_SUMMARY') + if summary_file: + with open(summary_file, 'a', encoding='utf-8') as file: + file.write(text + '\n') + +def normalize_returncode(status): + """ + normalize os.system status to command exit code. + """ + if status == 0: + return 0 + + if os.name == 'nt': + return status + + if hasattr(os, 'waitstatus_to_exitcode'): + try: + return os.waitstatus_to_exitcode(status) + except ValueError: + return status + + return status >> 8 +def contains_scons_fatal_error(output): + """ + check whether scons output contains a fatal build-script error. + """ + output_str = ''.join(output) if isinstance(output, list) else str(output) + return any(pattern.search(output_str) for pattern in SCONS_FATAL_PATTERNS) + +def check_bsp_build_scripts(bsp_dir): + """ + check whether the BSP has the basic scons build entry files. + """ + missing = [] + required_files = ['SConstruct', 'SConscript'] + if os.getenv('RTT_CI_GUARD_TEST_MISSING_SCRIPT') == '1': + required_files.append('SConscript.ci-guard-test-missing') + + for filename in required_files: + if not os.path.isfile(os.path.join(bsp_dir, filename)): + missing.append(filename) + + if missing: + print(f"::error::missing {', '.join(missing)} in {bsp_dir}") + return False + + return True def run_cmd(cmd, output_info=True): """ @@ -33,19 +89,49 @@ def run_cmd(cmd, output_info=True): output_str_list = [] res = 0 + output_file = f'output_{os.getpid()}_{threading.get_ident()}.txt' + devnull = os.devnull + if os.getenv('RTT_CI_GUARD_TEST_INJECT') == '1': + if '--pyconfig-silent' in cmd: + output_str_list = ['RTT_CI_GUARD_TEST: injected pyconfig failure\n'] + res = 31 + elif cmd.startswith('pkgs --update-force'): + output_str_list = ['RTT_CI_GUARD_TEST: injected pkgs update failure\n'] + res = 32 + elif cmd.startswith('pkgs --list'): + output_str_list = ['RTT_CI_GUARD_TEST: injected pkgs list failure\n'] + res = 33 + elif re.search(r'(^|\s)scons(\s|$)', cmd) and '-c' not in cmd: + output_str_list = ['RTT_CI_GUARD_TEST_SCONS_FATAL: injected scons fatal log\n'] + res = 0 + + if output_str_list: + for line in output_str_list: + print(line, end='') + if contains_scons_fatal_error(output_str_list): + print(f"::error::scons fatal error detected while running: {cmd}") + if res == 0: + res = 1 + return output_str_list, res if output_info: - res = os.system(cmd + " > output.txt 2>&1") + res = os.system(cmd + f" > {output_file} 2>&1") else: - res = os.system(cmd + " > /dev/null 2>output.txt") + res = os.system(cmd + f" > {devnull} 2>{output_file}") - with open("output.txt", "r") as file: + with open(output_file, "r") as file: output_str_list = file.readlines() for line in output_str_list: print(line, end='') - os.remove("output.txt") + os.remove(output_file) + + res = normalize_returncode(res) + if contains_scons_fatal_error(output_str_list): + print(f"::error::scons fatal error detected while running: {cmd}") + if res == 0: + res = 1 return output_str_list, res @@ -61,6 +147,10 @@ def run_dist_build_check(bsp, scons_args=''): build BSP distribution and verify that the generated project can compile. """ os.chdir(rtt_root) + bsp_dir = os.path.join(rtt_root, 'bsp', bsp) + if not check_bsp_build_scripts(bsp_dir): + return False + dist_root = os.path.join(rtt_root, 'bsp', bsp, 'dist') dist_project = os.path.join(dist_root, 'project') if os.path.exists(dist_root): @@ -77,6 +167,9 @@ def run_dist_build_check(bsp, scons_args=''): print(f"::error::dist project not found: {dist_project}") return False + if not check_bsp_build_scripts(dist_project): + return False + old_rtt_root = os.environ.pop('RTT_ROOT', None) _, res = run_cmd(f'scons --pyconfig-silent -C {dist_project}', output_info=True) if res != 0: @@ -118,6 +211,15 @@ def build_bsp(bsp, scons_args='',name='default', pre_build_commands=None, post_b """ success = True + bsp_dir = os.path.join(rtt_root, 'bsp', bsp) + + if not os.path.isdir(bsp_dir): + print(f"::error::BSP directory not found: {bsp_dir}") + return False + + if not check_bsp_build_scripts(bsp_dir): + return False + # 设置环境变量 if bsp_build_env is not None: print("Setting environment variables:") @@ -128,50 +230,63 @@ def build_bsp(bsp, scons_args='',name='default', pre_build_commands=None, post_b os.makedirs(f'{rtt_root}/output/bsp/{bsp}', exist_ok=True) if os.path.exists(f"{rtt_root}/bsp/{bsp}/Kconfig"): os.chdir(rtt_root) - run_cmd(f'scons -C bsp/{bsp} --pyconfig-silent', output_info=True) + _, res = run_cmd(f'scons -C bsp/{bsp} --pyconfig-silent', output_info=True) + if res != 0: + print(f"::error::pyconfig failed for {bsp}") + success = False os.chdir(f'{rtt_root}/bsp/{bsp}') - run_cmd('pkgs --update-force', output_info=True) - run_cmd('pkgs --list') - - nproc = multiprocessing.cpu_count() - if pre_build_commands is not None: - print("Pre-build commands:") - print(pre_build_commands) - for command in pre_build_commands: - print(command) - output, returncode = run_cmd(command, output_info=True) - print(output) - if returncode != 0: - print(f"Pre-build command failed: {command}") - print(output) - os.chdir(rtt_root) - # scons 编译命令 - cmd = f'scons -C bsp/{bsp} -j{nproc} {scons_args}' # --debug=time for debug time - output, res = run_cmd(cmd, output_info=True) - if build_check_result is not None: - if res != 0 or not check_output(output, build_check_result): - print("Build failed or build check result not found") - print(output) + _, res = run_cmd('pkgs --update-force', output_info=True) if res != 0: + print(f"::error::pkgs --update-force failed for {bsp}") success = False - else: - #拷贝当前的文件夹下面的所有以elf结尾的文件拷贝到rt-thread/output文件夹下 - import glob - # 拷贝编译生成的文件到output目录,文件拓展为 elf,bin,hex - for file_type in ['*.elf', '*.bin', '*.hex']: - files = glob.glob(f'{rtt_root}/bsp/{bsp}/{file_type}') - for file in files: - shutil.copy(file, f'{rtt_root}/output/bsp/{bsp}/{name.replace("/", "_")}.{file_type[2:]}') - if is_env_enabled('RTT_CI_BUILD_DIST'): - print(f"::group::\tChecking dist project: {bsp} {name}") - dist_res = run_dist_build_check(bsp, scons_args) - print("::endgroup::") - if not dist_res: - add_summary(f'\t- ❌ dist build {bsp} {name} failed.') - success = False - else: - add_summary(f'\t- ✅ dist build {bsp} {name} success.') + _, res = run_cmd('pkgs --list') + if res != 0: + print(f"::error::pkgs --list failed for {bsp}") + success = False + else: + print(f"Kconfig not found, skip pyconfig and package update: {os.path.join(bsp_dir, 'Kconfig')}") + + nproc = multiprocessing.cpu_count() + if pre_build_commands is not None: + print("Pre-build commands:") + print(pre_build_commands) + for command in pre_build_commands: + print(command) + output, returncode = run_cmd(command, output_info=True) + print(output) + if returncode != 0: + print(f"Pre-build command failed: {command}") + print(output) + success = False + os.chdir(rtt_root) + # scons 编译命令 + cmd = f'scons -C bsp/{bsp} -j{nproc} {scons_args}' # --debug=time for debug time + output, res = run_cmd(cmd, output_info=True) + if build_check_result is not None: + if res != 0 or not check_output(output, build_check_result): + print("Build failed or build check result not found") + print(output) + success = False + if res != 0: + success = False + if success: + #拷贝当前的文件夹下面的所有以elf结尾的文件拷贝到rt-thread/output文件夹下 + import glob + # 拷贝编译生成的文件到output目录,文件拓展为 elf,bin,hex + for file_type in ['*.elf', '*.bin', '*.hex']: + files = glob.glob(f'{rtt_root}/bsp/{bsp}/{file_type}') + for file in files: + shutil.copy(file, f'{rtt_root}/output/bsp/{bsp}/{name.replace("/", "_")}.{file_type[2:]}') + if is_env_enabled('RTT_CI_BUILD_DIST'): + print(f"::group::\tChecking dist project: {bsp} {name}") + dist_res = run_dist_build_check(bsp, scons_args) + print("::endgroup::") + if not dist_res: + add_summary(f'\t- ❌ dist build {bsp} {name} failed.') + success = False + else: + add_summary(f'\t- ✅ dist build {bsp} {name} success.') os.chdir(f'{rtt_root}/bsp/{bsp}') if post_build_command is not None: @@ -181,7 +296,11 @@ def build_bsp(bsp, scons_args='',name='default', pre_build_commands=None, post_b if returncode != 0: print(f"Post-build command failed: {command}") print(output) - run_cmd('scons -c', output_info=False) + success = False + _, clean_res = run_cmd('scons -c', output_info=False) + if clean_res != 0: + print(f"::error::scons clean failed for {bsp}") + success = False return success @@ -240,19 +359,27 @@ def build_bsp_attachconfig(bsp, attach_file): """ config_file = os.path.join(rtt_root, 'bsp', bsp, '.config') config_bacakup = config_file+'.origin' - shutil.copyfile(config_file, config_bacakup) + if not os.path.isfile(config_file): + print(f"::error::.config not found: {config_file}") + return False attachconfig_dir = os.path.join(rtt_root, 'bsp', bsp, '.ci/attachconfig') attach_path = os.path.join(attachconfig_dir, attach_file) + if not os.path.isfile(attach_path): + print(f"::error::attach config not found: {attach_path}") + return False - append_file(attach_path, config_file) + shutil.copyfile(config_file, config_bacakup) - scons_args = check_scons_args(attach_path) + try: + append_file(attach_path, config_file) - res = build_bsp(bsp, scons_args,name=attach_file) + scons_args = check_scons_args(attach_path) - shutil.copyfile(config_bacakup, config_file) - os.remove(config_bacakup) + res = build_bsp(bsp, scons_args,name=attach_file) + finally: + shutil.copyfile(config_bacakup, config_file) + os.remove(config_bacakup) return res diff --git a/tools/ci/compile_bsp_with_drivers.py b/tools/ci/compile_bsp_with_drivers.py index 827eb952cb8..a79ed6b3f6c 100644 --- a/tools/ci/compile_bsp_with_drivers.py +++ b/tools/ci/compile_bsp_with_drivers.py @@ -11,6 +11,7 @@ import subprocess import logging import os +import sys CONFIG_BSP_USING_X = ["CONFIG_BSP_USING_UART", "CONFIG_BSP_USING_I2C", "CONFIG_BSP_USING_SPI", "CONFIG_BSP_USING_ADC", "CONFIG_BSP_USING_DAC"] @@ -81,13 +82,36 @@ def modify_config(file_path, configs): file.write("#define " + define1 + "\n") file.write("#define " + define2 + "\n") +def check_scons_build_files(dir): + missing = [] + for filename in ["SConstruct", "SConscript"]: + if not os.path.isfile(os.path.join(dir, filename)): + missing.append(filename) + + if missing: + logging.error("missing %s in %s", ", ".join(missing), dir) + return False + + return True + def recompile_bsp(dir): logging.info("recomplie bsp: {}".format(dir)) - os.system("scons -C " + dir) + if os.getenv("RTT_CI_GUARD_TEST_INJECT") == "1": + logging.error("RTT_CI_GUARD_TEST: injected recompile failure for %s", dir) + return 1 + + if not check_scons_build_files(dir): + return 1 + + result = subprocess.run(["scons", "-C", dir]) + if result.returncode != 0: + logging.error("scons failed for %s", dir) + return result.returncode if __name__ == '__main__': init_logger() recompile_bsp_dirs = diff() + failed = 0 for dir in recompile_bsp_dirs: dot_config_path = dir + "/" + ".config" configs = check_config_in_file(dot_config_path) @@ -95,4 +119,7 @@ def recompile_bsp(dir): logging.info(configs) logging.info("Add configurations and recompile!") modify_config(dir, configs) - recompile_bsp(dir) \ No newline at end of file + if recompile_bsp(dir) != 0: + failed += 1 + + sys.exit(failed)