From 79fd209f1ddc12a863cdf3a44d8bfbfa40a948a7 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 18:09:37 -0400 Subject: [PATCH 01/14] DAOS-19111 build: Enable code coverage Enable the option for generating Bullseye code coverage reports. Signed-off-by: Phil Henderson --- Jenkinsfile | 1304 ++++++++++++----- ci/bullseye_generate_report.sh | 23 - ci/provisioning/post_provision_config.sh | 1 + .../post_provision_config_common_functions.sh | 45 +- .../post_provision_config_nodes.sh | 2 +- ci/rpm/build_deps.sh | 14 + ci/rpm/build_success.sh | 12 +- ci/rpm/gen_rpms.sh | 29 +- ci/rpm/install_deps.sh | 7 + ci/summary/bullseye_report.sh | 33 + ci/summary/install_pkgs.sh | 37 + ci/unit/required_packages.sh | 75 +- ci/unit/test_main.sh | 10 - ci/unit/test_main_node.sh | 36 +- ci/unit/test_nlt.sh | 2 +- ci/unit/test_nlt_node.sh | 18 + ci/unit/test_nlt_post.sh | 16 +- ci/unit/test_post_always.sh | 16 +- site_scons/prereq_tools/base.py | 34 +- src/tests/ftest/control/version.py | 10 +- src/tests/ftest/dfuse/bash.py | 13 +- src/tests/ftest/io/unaligned_io.yaml | 2 - src/tests/ftest/util/apricot/apricot/test.py | 3 +- src/tests/ftest/util/code_coverage_utils.py | 48 +- src/tests/ftest/util/collection_utils.py | 9 +- src/tests/ftest/util/command_utils.py | 5 +- src/tests/ftest/util/environment_utils.py | 23 +- utils/node_local_test.py | 8 +- utils/rpms/build_packages.sh | 6 +- utils/rpms/bullseye.changelog | 9 + utils/rpms/bullseye.sh | 80 + utils/rpms/bullseye_build.sh | 30 + utils/rpms/daos.sh | 73 +- utils/rpms/package_info.sh | 6 + utils/run_utest.py | 18 - 35 files changed, 1473 insertions(+), 584 deletions(-) delete mode 100755 ci/bullseye_generate_report.sh create mode 100644 ci/summary/bullseye_report.sh create mode 100644 ci/summary/install_pkgs.sh create mode 100644 utils/rpms/bullseye.changelog create mode 100644 utils/rpms/bullseye.sh create mode 100644 utils/rpms/bullseye_build.sh diff --git a/Jenkinsfile b/Jenkinsfile index 9c0232492e9..47f6a0d0e3e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -16,9 +16,13 @@ * LICENSE file. */ +import groovy.transform.Field +import org.jenkinsci.plugins.pipeline.modeldefinition.Utils + // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value='pipeline-lib@your_branch') _ +@Library(value='pipeline-lib@hendersp/DAOS-18348') _ // The trusted-pipeline-lib daosLatestVersion() method will convert this into a number /* groovylint-disable-next-line CompileStatic, VariableName */ @@ -29,6 +33,294 @@ String next_version() { /* groovylint-disable-next-line CompileStatic */ job_status_internal = [:] +// Keys and values updated by the updateRunStage() function using the parameters. +@Field +Map runStage = [:] + +// Update the runStage map +void updateRunStage() { + Map reasons = [:] + + // Ordered list of stage names as params.keySet() does not guarantee order + List stageOrder = [ + 'Cancel Previous Builds', + 'Pre-build', + 'Python Bandit check', + 'Build', + 'Build on EL 8', + 'Build on EL 9', + 'Build on Leap 15', + 'Build on EL 9 with Bullseye', + 'Unit Tests', + 'Unit Test', + 'Unit Test bdev', + 'NLT', + 'NLT with Bullseye', + 'Unit Test with memcheck', + 'Unit Test bdev with memcheck', + 'Test', + 'Functional on EL 8.8 with Valgrind', + 'Functional on EL 8', + 'Functional on EL 9', + 'Functional on Leap 15', + 'Functional on SLES 15', + 'Functional on Ubuntu 20.04', + 'Fault injection testing', + 'Test RPMs on EL 9.6', + 'Test RPMs on Leap 15.5', + 'Test Hardware', + 'Functional Hardware Medium', + 'Functional Hardware Medium MD on SSD', + 'Functional Hardware Medium VMD', + 'Functional Hardware Medium Verbs Provider', + 'Functional Hardware Medium Verbs Provider MD on SSD', + 'Functional Hardware Medium UCX Provider', + 'Functional Hardware Large', + 'Functional Hardware Large MD on SSD' + ] + + // Initialize the run state of each stage using the parameter stage keys + for (name in stageOrder) { + value = params.get(name, null) + if (value instanceof Boolean && !name.startsWith('CI_')) { + runStage[name] = value + reasons[name] = "parameter selection/default" + } + } + + // Handle doc-only changes: Only run default or selected build stages + if (docOnlyChange(target_branch)) { + println("updateRunStage: Detected doc-only change, skipping testing") + for (stage in runStage.keySet()) { + if (stage in ['Unit Tests', 'Test', 'Test Hardware']) { + runStage[stage] = false + reasons[stage] = "doc-only change" + } + } + displayRunStage(reasons) + return + } + + // Handle landing builds + if (startedByLanding()) { + println("updateRunStage: Detected landing build, overwriting defaults") + for (stage in runStage.keySet()) { + runStage[stage] = false + if (stage in ['Pre-build', 'Python Bandit check', 'Build', 'Unit Tests', 'Test'] + || stage.contains('Build on') + || stage.contains('Unit Test') + || stage.contains('NLT') + || stage.contains('Fault injection') + || stage.contains('Test RPMs') + || (stage.contains('Functional on') && !stage.contains('Ubuntu'))) { + runStage[stage] = true + } + reasons[stage] = "landing build" + } + displayRunStage(reasons) + return + } + + // Handle user setting CI_RPM_TEST_VERSION + if (params.CI_RPM_TEST_VERSION) { + println("updateRunStage: Detected specific RPM version, skipping build/RPM test stages") + for (stage in runStage.keySet()) { + if (stage.contains('Build') + || stage.contains('Unit Tests') + || stage.contains('Test RPMs')) { + runStage[stage] = false + reasons[stage] = "specific RPM version" + } + } + displayRunStage(reasons) + return + } + + // Handle user setting CI_FULL_BULLSEYE_REPORT + if (params.CI_FULL_BULLSEYE_REPORT) { + println("updateRunStage: Detected CI_FULL_BULLSEYE_REPORT, skipping unrelated stages") + for (stage in runStage.keySet()) { + runStage[stage] = false + if (stage in ['Build on EL 9 with Bullseye', + 'Unit Test', 'Unit Test bdev', + 'NLT with Bullseye', + 'Functional on EL 9']) { + runStage[stage] = true + } else if (stage.contains('Functional Hardware')) { + runStage[stage] = true + } + reasons[stage] = "CI_FULL_BULLSEYE_REPORT" + } + displayRunStage(reasons) + return + } + + // Handle user setting CI_BUILD_PACKAGES_ONLY + if (params.CI_BUILD_PACKAGES_ONLY) { + println("updateRunStage: Detected CI_BUILD_PACKAGES_ONLY, skipping unit test stages") + for (stage in runStage.keySet()) { + if (stage.contains('Unit Tests')) { + runStage[stage] = false + reasons[stage] = "CI_BUILD_PACKAGES_ONLY" + } else if (stage.contains('Build')) { + runStage[stage] = true + reasons[stage] = "CI_BUILD_PACKAGES_ONLY" + } + } + displayRunStage(reasons) + return + } + + // Handle builds started by the user + if (startedByUser()) { + println("updateRunStage: Build started by the user, skipping commit pragma checks") + displayRunStage(reasons) + return + } + + // Update stage running based on commit pragmas + println("updateRunStage: Converting env.pragmas string back into a Map: ${env.pragmas}") + Map commitPragmas = envToPragmas() + println("updateRunStage: Checking skip commit pragmas from commit message:") + commitPragmas.each { key, value -> + println(" ${key}: ${value}") + } + for (stage in runStage.keySet()) { + List skipPragmas = getStageNameSkipPragmas(stage) + for (pragma in skipPragmas) { + // commitPragmas will already contain lower case keys from pragmasToMap() + println("updateRunStage: ${stage} checking for a ${pragma} commit pragma") + if (commitPragmas.get(pragma, '').toLowerCase() == 'true') { + runStage[stage] = false + reasons[stage] = "commit pragma ${pragma}: true" + break + } else if (commitPragmas.get(pragma, '').toLowerCase() == 'false') { + runStage[stage] = true + reasons[stage] = "commit pragma ${pragma}: false" + break + } + } + } + displayRunStage(reasons) +} + +// Log which stages will be run and why based on the current state of the runStage map +void displayRunStage(Map reasons = [:]) { + println("Stage run conditions:") + for (stage in runStage.keySet()) { + String reason = reasons.get(stage, 'default') + if (runStage[stage]) { + echo("Running: ${stage} (reason: ${reason})") + } else { + echo("Skipping: ${stage} (reason: ${reason})") + } + } +} + +// Get a list of skip commit pragmas to check for a given stage name +List getStageNameSkipPragmas(String stageName) { + String stagePragma = "skip-${stageName.replaceAll(' ', '-').toLowerCase()}" + List pragmas = [] + + // Build up a priority list of pragmas to check based on the stage name. + if (stageName in ['Cancel Previous Builds', 'Pre-build']) { + // Add skip pragma for this stage + pragmas.add(stagePragma) + + } else if (stageName == 'Python Bandit check') { + // Add skip pragma for this stage + pragmas.add(stagePragma) + // Compatibility with existing commit pragmas + pragmas.add(stagePragma.replace('-bandit-check', '-bandit')) + + } else if (stageName.contains('Build')) { + // Add skip pragma for parent stage + if (stageName != 'Build') { + pragmas.add('skip-build') + } + // Add skip pragma for this stage + pragmas.add(stagePragma) + // Compatibility with existing commit pragmas + if (stagePragma.contains('build-on-')) { + pragmas.add(stagePragma.replace('build-on-', 'build-')) + } + + } else if (stageName.contains('Unit Test') || stageName.contains('NLT')) { + // Add skip pragma for parent stage + if (stageName != 'Unit Tests') { + pragmas.add('skip-unit-tests') + } + // Add skip pragma for this stage + pragmas.add(stagePragma) + // Compatibility with existing commit pragmas + if (stagePragma.contains('-with-')) { + pragmas.add(stagePragma.replace('-with-', '-')) + } + + } else if (stageName == 'Test' || stageName.contains('Functional on') + || stageName.contains('Fault injection') || stageName.contains('Test RPMs')) { + // Add skip pragma for parent stage + if (stageName != 'Test') { + pragmas.add('skip-test') + } + if (stageName.contains('Functional on')) { + // Add skip pragma alias for all functional tests + pragmas.add('skip-func-test') + // Add skip pragma alias for all functional VM tests + pragmas.add('skip-func-test-vm') + pragmas.add('skip-func-vm-test') + // Compatibility with existing commit pragmas + pragmas.add(stagePragma.replace('functional-on-', 'func-test-')) + } else if (stageName.contains('Test RPMs on')) { + // Add skip pragma alias for all RPM tests + pragmas.add('skip-test-rpms') + } else if (stageName.contains('Fault injection')) { + // Compatibility with existing commit pragmas + pragmas.add('skip-fault-injection-test') + } + // Add skip pragma for this stage + pragmas.add(stagePragma) + + } else if (stageName.contains('Hardware')) { + if (stageName != 'Test Hardware') { + pragmas.add('skip-test-hardware') + pragmas.add('skip-test-hw') + } + if (stageName.contains('Functional')) { + // Add skip pragma alias for all functional tests + pragmas.add('skip-func-test') + // Add skip pragma alias for all functional HW tests + pragmas.add('skip-func-test-hw') + pragmas.add('skip-func-hw-test') + // Compatibility with existing commit pragmas + if (stagePragma.contains('functional-hardware-')) { + pragmas.add(stagePragma.replace('functional-hardware-', 'func-hw-test-')) + pragmas.add(stagePragma.replace('functional-hardware-', 'func-hw-')) + } + } + // Add skip pragma for this stage + pragmas.add(stagePragma) + + // Support shortening hardware to hw + if (stagePragma.contains('hardware-')) { + pragmas.add(stagePragma.replace('hardware-', 'hw-')) + } + } + + // Compatibility with existing commit pragmas using distro versions + List distros = ['el', 'leap', 'sles', 'ubuntu'] + List copyPragmas = pragmas.clone() + for (distro in distros) { + for (_pragma in copyPragmas) { + if (_pragma.contains("-${distro}-")) { + pragmas.add(_pragma.replace("-${distro}-", "-${distro}")) + } + } + } + + return pragmas +} + void get_rpm_relval() { env.DAOS_RELVAL = sh(label: 'get git tag', script: '''if [ -n "$GIT_CHECKOUT_DIR" ] && [ -d "$GIT_CHECKOUT_DIR" ]; then @@ -55,9 +347,6 @@ void job_step_update(def value=currentBuild.currentResult) { // Don't define this as a type or it loses it's global scope target_branch = env.CHANGE_TARGET ? env.CHANGE_TARGET : env.BRANCH_NAME -String sanitized_JOB_NAME() { - return JOB_NAME.toLowerCase().replaceAll('/', '-').replaceAll('%2f', '-') -} // bail out of branch builds that are not on a whitelist if (!env.CHANGE_ID && @@ -107,7 +396,10 @@ void fixup_rpmlintrc() { } void uploadNewRPMs(String target, String stage) { - buildRpmPost target: target, condition: stage, rpmlint: false, new_rpm: true + buildRpmPost target: target, + condition: stage, + rpmlint: false, + productArtifacts: ['daos', 'deps', 'bullseye'] } String vm9_label(String distro) { @@ -151,49 +443,262 @@ Map update_default_commit_pragmas() { } } -Boolean skip_pragma_set(String name, String def_val='false') { - // Return whether or not the skip pragma is set - return cachedCommitPragma("Skip-${name}", def_val).toLowerCase() == 'true' +/** + * getScriptOutput + * + * Run a script and return the trimmed output. + * + * @param script the script to run + * @param args optional arguments to pass to the script + * @return the trimmed output from the script + */ +String getScriptOutput(String script, String args='') { + return sh(script: "${script} ${args}", returnStdout: true).trim() } -Boolean skip_build_stage(String distro='', String compiler='gcc') { - // Skip the stage if the CI__NOBUILD parameter is set - if (distro) { - if (startedByUser() && paramsValue("CI_${distro}_NOBUILD", false)) { - println("[${env.STAGE_NAME}] Skipping build stage due to CI_${distro}_NOBUILD") - return true - } +/** + * scriptedBuildStage + * + * Get a build stage in scripted syntax. + * + * @param kwargs Map containing the following optional arguments (empty strings yield defaults): + * name the build stage name + * distro the shorthand distro name; defaults to 'el8' + * rpmDistro the distro to use for rpm building; defaults to distro + * compiler the compiler to use; defaults to 'gcc' + * runCondition optional additional condition to determine if the stage runs, used + * in conjunction with runBuildStage(distro, compiler); defaults + * to true + * buildRpms whether or not to build rpms; defaults to true + * release the DAOS RPM release value to use; defaults to env.DAOS_RELVAL + * dockerBuildArgs optional docker build arguments + * sconsBuildArgs optional scons build arguments + * artifacts optional artifacts name to archive; defaults to + * "config.log-${distro}-${compiler}" + * uploadTarget the distro to use when uploading rpms; defaults to distro + * @return a scripted stage to run in a pipeline + */ +def scriptedBuildStage(Map kwargs = [:]) { + String name = kwargs.get('name', 'Unknown Build Stage') + String distro = kwargs.get('distro', 'el8') + String rpmDistro = kwargs.get('rpmDistro', distro) + String compiler = kwargs.get('compiler', 'gcc') + Boolean runCondition = kwargs.get('runCondition', true) + Boolean buildRpms = kwargs.get('buildRpms', true) + String release = kwargs.get('release', env.DAOS_RELVAL) + String dockerBuildArgs = kwargs.get('dockerBuildArgs', '') + Map sconsBuildArgs = kwargs.get('sconsBuildArgs', [:]) + String artifacts = kwargs.get('artifacts', "config.log-${distro}-${compiler}") + String uploadTarget = kwargs.get('uploadTarget', distro) + String dockerTag = jobStatusKey("build-${uploadTarget}-${compiler}").toLowerCase() + String bullseye = 'false' + if (compiler == 'covc') { + bullseye = 'true' } + return { + stage("${name}") { + if (runBuildStage(distro, compiler, runCondition)) { + node('docker_runner') { + println("[${name}] Check out from version control") + checkoutScm(pruneStaleBranch: true) - // Skip the stage if any Skip-build[--] pragmas are true - List pragma_names = ['build'] - if (distro && compiler) { - pragma_names << "build-${distro}-${compiler}" + def dockerImage = docker.build(dockerTag, dockerBuildArgs) + try { + dockerImage.inside() { + if (buildRpms) { + sh label: 'Install RPMs', + script: "./ci/rpm/install_deps.sh ${rpmDistro} ${release} ${bullseye}" + // Avoid interpolation of sensitive environment variables + sh label: 'Build deps', + script: "./ci/rpm/build_deps.sh ${bullseye}" + ' ${BULLSEYE_KEY}' + } + job_step_update(sconsBuild(sconsBuildArgs)) + if (buildRpms) { + sh label: 'Generate RPMs', + script: "./ci/rpm/gen_rpms.sh ${rpmDistro} ${release} ${bullseye}" + // Success actions + uploadNewRPMs(uploadTarget, 'success') + } + } + } catch (Exception e) { + // Unsuccessful actions + sh """if [ -f config.log ]; then + mv config.log ${artifacts} + fi""" + archiveArtifacts artifacts: "${artifacts}", allowEmptyArchive: true + throw e + } finally { + // Cleanup actions + if (buildRpms) { + uploadNewRPMs(uploadTarget, 'cleanup') + } + jobStatusUpdate(job_status_internal, name) + } + } + } + else { + println("[${name}] Marking build stage as skipped") + Utils.markStageSkippedForConditional("${name}") + } + println("[${name}] Finished with ${job_status_internal}") + } } - Boolean any_pragma_skip = pragma_names.any { name -> - if (skip_pragma_set(name)) { - println("[${env.STAGE_NAME}] Skipping build stage due to \"Skip-${name}: true\" pragma") - return true +} + +/** + * scriptedSummaryStage + * + * Get a summary stage in scripted syntax. + * + * @param kwargs Map containing the following optional arguments (empty strings yield defaults): + * name the summary stage name + * distro the shorthand distro name; defaults to 'el8' + * compiler the compiler to use; defaults to 'gcc' + * runCondition Optional additional condition to determine if the stage runs + * dockerBuildArgs optional docker build arguments + * installScript optional script to install RPMs + * runScriptArgs Map of arguments to pass to runScriptWithStashes() + * archiveArtifactsArgs Map of arguments to pass to archiveArtifacts() + * publishHtmlArgs Map of arguments to pass to publishHTML() + * @return a scripted stage to run in a pipeline + */ +def scriptedSummaryStage(Map kwargs = [:]) { + String name = kwargs.get('name', 'Unknown Summary Stage') + String distro = kwargs.get('distro', 'el8') + String compiler = kwargs.get('compiler', 'gcc') + Boolean runCondition = kwargs.get('runCondition', true) + String dockerBuildArgs = kwargs.get('dockerBuildArgs', '') + String installScript = kwargs.get('installScript', '') + Map runScriptArgs = kwargs.get('runScriptArgs', [:]) + Map archiveArtifactsArgs = kwargs.get('archiveArtifactsArgs', [:]) + Map publishHtmlArgs = kwargs.get('publishHtmlArgs', [:]) + String dockerTag = jobStatusKey("${name}-${distro}-${compiler}").toLowerCase() + + return { + stage("${name}") { + if (runCondition) { + node('docker_runner') { + println("[${name}] Check out from version control") + checkoutScm(pruneStaleBranch: true) + + def dockerImage = docker.build(dockerTag, dockerBuildArgs) + try { + dockerImage.inside() { + if (installScript) { + sh label: 'Install RPMs', + script: "${installScript} ${distro}" + } + job_step_update(runScriptWithStashes(runScriptArgs)) + } + } finally { + // Cleanup actions + if (publishHtmlArgs) { + publishHTML(publishHtmlArgs) + } + if (archiveArtifactsArgs) { + archiveArtifacts(archiveArtifactsArgs) + } + jobStatusUpdate(job_status_internal, name) + } + } + } + else { + println("[${name}] Marking summary stage as skipped") + Utils.markStageSkippedForConditional("${name}") + } + println("[${name}] Finished with ${job_status_internal}") } } - if (any_pragma_skip) { - return true +} + +// Determine if the Build with Bullseye was run and successful +Boolean withBullseye() { + if (runStage['Build on EL 9 with Bullseye'] == false) { + println("withBullseye: Build on EL 9 with Bullseye stage was not selected to run") + return false } + Map status = job_status_internal['Build_on_EL_9_with_Bullseye'] ?: [:] + println("withBullseye: status=${status}, status.result=${status.result}") + return status.result == 'SUCCESS' +} - // Skip the stage if a specific DAOS RPM version is specified - if (rpmTestVersion() != '') { - println("[${env.STAGE_NAME}] Skipping build stage for due to specific DAOS RPM version") - return true +// Get the inst_rpms argument for the unitTest method +String unitTestInstRpms(String distro='el9') { + if (withBullseye()) { + return getScriptOutput("ci/unit/required_packages.sh ${distro} true") } + return getScriptOutput("ci/unit/required_packages.sh ${distro}") +} - // Otherwise run the build stage - return false +// Get the compiler argument for the unitTest method +String unitTestCompiler() { + if (withBullseye()) { + return 'covc' + } + return 'gcc' +} + +// Get the packages to install for functional testing +String functionalInstRpms(String otherPackages, Boolean bullseye=false, String rpmDistro=null) { + String packages = functionalPackages( + clientVersion: 1, + nextVersion: next_version(), + addDaosPkgs: 'tests-internal', + rpmDistribution: rpmDistro) + if (bullseye) { + packages = packages.replace('daos', 'daos-bullseye') + packages += ' bullseye' + } + if (otherPackages) { + packages += " ${otherPackages}" + } + return packages } +// Boolean skip_pragma_set(String name, String def_val='false') { +// // Return whether or not the skip pragma is set +// return cachedCommitPragma("Skip-${name}", def_val).toLowerCase() == 'true' +// } + +// Boolean skip_build_stage(String distro='', String compiler='gcc') { +// // Skip the stage if the CI__NOBUILD parameter is set +// if (distro) { +// if (startedByUser() && paramsValue("CI_${distro}_NOBUILD", false)) { +// println("[${env.STAGE_NAME}] Skipping build stage due to CI_${distro}_NOBUILD") +// return true +// } +// } + +// // Skip the stage if any Skip-build[--] pragmas are true +// List pragma_names = ['build'] +// if (distro && compiler) { +// pragma_names << "build-${distro}-${compiler}" +// } +// Boolean any_pragma_skip = pragma_names.any { name -> +// if (skip_pragma_set(name)) { +// println("[${env.STAGE_NAME}] Skipping build stage due to \"Skip-${name}: true\" pragma") +// return true +// } +// } +// if (any_pragma_skip) { +// return true +// } + +// // Skip the stage if a specific DAOS RPM version is specified +// if (rpmTestVersion() != '') { +// println("[${env.STAGE_NAME}] Skipping build stage for due to specific DAOS RPM version") +// return true +// } + +// // Otherwise run the build stage +// return false +// } + pipeline { agent { label 'lightweight' } environment { + BULLSEYE_KEY = credentials('bullseye_license_key') GITHUB_USER = credentials('daos-jenkins-review-posting') SSH_KEY_ARGS = '-ici_key' CLUSH_ARGS = "-o$SSH_KEY_ARGS" @@ -237,12 +742,15 @@ pipeline { 'stages. Specifies the default provider to use the daos_server ' + 'config file when running functional tests (the launch.py ' + '--provider argument; i.e. "ucx+dc_x", "ofi+verbs", "ofi+tcp")') - booleanParam(name: 'CI_CANCEL_PREV_BUILD_SKIP', - defaultValue: false, - description: 'Do not cancel previous build.') booleanParam(name: 'CI_BUILD_PACKAGES_ONLY', defaultValue: false, description: 'Build RPM and DEB packages, Skip unit tests.') + booleanParam(name: 'CI_ALLOW_UNSTABLE_TEST', + defaultValue: false, + description: 'Continue testing if a previous stage is Unstable') + booleanParam(name: 'CI_FULL_BULLSEYE_REPORT', + defaultValue: false, + description: 'Use this build to generate a full Bullseye code coverage report') string(name: 'CI_SCONS_ARGS', defaultValue: '', description: 'Arguments for scons when building DAOS') @@ -269,84 +777,108 @@ pipeline { string(name: 'CI_UBUNTU20.04_TARGET', defaultValue: '', description: 'Image to used for Ubuntu 20 CI tests. I.e. ubuntu20.04, etc.') - booleanParam(name: 'CI_el8_NOBUILD', - defaultValue: false, - description: 'Do not build sources and RPMs on EL 8') - booleanParam(name: 'CI_el9_NOBUILD', - defaultValue: false, - description: 'Do not build sources and RPMs on EL 9') - booleanParam(name: 'CI_leap15_NOBUILD', - defaultValue: false, - description: 'Do not build sources and RPMs on Leap 15') - booleanParam(name: 'CI_ALLOW_UNSTABLE_TEST', - defaultValue: false, - description: 'Continue testing if a previous stage is Unstable') - booleanParam(name: 'CI_UNIT_TEST', + booleanParam(name: 'Cancel Previous Builds', defaultValue: true, - description: 'Run the Unit Test test stage') - booleanParam(name: 'CI_NLT_TEST', + description: 'Run the Cancel Previous Builds stage.') + booleanParam(name: 'Pre-build', defaultValue: true, - description: 'Run the NLT test stage') - booleanParam(name: 'CI_UNIT_TEST_MEMCHECK', + description: 'Run the pre-build stage.') + booleanParam(name: 'Python Bandit check', defaultValue: true, - description: 'Run the Unit Test with memcheck test stage') - booleanParam(name: 'CI_FI_TEST', + description: 'Run the Python Bandit check stage.') + booleanParam(name: 'Build', defaultValue: true, - description: 'Run the Fault injection testing test stage') - booleanParam(name: 'CI_TEST_EL_RPMs', + description: 'Run the build stage.') + booleanParam(name: 'Build on EL 8', defaultValue: true, - description: 'Run the Test RPMs on EL stage') - booleanParam(name: 'CI_TEST_LEAP_RPMs', + description: 'Run the build on EL 8 stage.') + booleanParam(name: 'Build on EL 9', defaultValue: true, - description: 'Run the Test RPMs on Leap test stage') - booleanParam(name: 'CI_FUNCTIONAL_TEST_SKIP', - defaultValue: false, - description: 'Skip all functional test stages (Test)') - booleanParam(name: 'CI_MORE_FUNCTIONAL_PR_TESTS', - defaultValue: false, - description: 'Enable more distros for functional CI tests') - booleanParam(name: 'CI_FUNCTIONAL_el8_VALGRIND_TEST', + description: 'Run the build on EL 9 stage.') + booleanParam(name: 'Build on Leap 15', + defaultValue: true, + description: 'Run the build on Leap 15 stage.') + booleanParam(name: 'Build on EL 9 with Bullseye', + defaultValue: true, + description: 'Run the build on EL 9 with Bullseye stage.') + booleanParam(name: 'Unit Tests', + defaultValue: true, + description: 'Run the Unit Tests stage.') + booleanParam(name: 'Unit Test', + defaultValue: true, + description: 'Run the Unit Test stage.') + booleanParam(name: 'Unit Test bdev', + defaultValue: true, + description: 'Run the Unit Test bdev stage.') + booleanParam(name: 'NLT', + defaultValue: true, + description: 'Run the NLT stage.') + booleanParam(name: 'NLT with Bullseye', + defaultValue: true, + description: 'Run the NLT with Bullseye stage.') + booleanParam(name: 'Unit Test with memcheck', + defaultValue: true, + description: 'Run the Unit Test with memcheck stage.') + booleanParam(name: 'Unit Test bdev with memcheck', + defaultValue: true, + description: 'Run the Unit Test bdev with memcheck stage.') + booleanParam(name: 'Test', + defaultValue: true, + description: 'Run the Test stage.') + booleanParam(name: 'Functional on EL 8.8 with Valgrind', defaultValue: false, - description: 'Run the Functional on EL 8 with Valgrind test stage') - booleanParam(name: 'CI_FUNCTIONAL_el8_TEST', + description: 'Run the Functional on EL 8.8 with Valgrind stage.') + booleanParam(name: 'Functional on EL 8', defaultValue: false, - description: 'Run the Functional on EL 8 test stage') - booleanParam(name: 'CI_FUNCTIONAL_el9_TEST', + description: 'Run the Functional on EL 8 stage.') + booleanParam(name: 'Functional on EL 9', defaultValue: true, - description: 'Run the Functional on EL 9 test stage') - booleanParam(name: 'CI_FUNCTIONAL_leap15_TEST', + description: 'Run the Functional on EL 9 stage.') + booleanParam(name: 'Functional on Leap 15', defaultValue: false, - description: 'Run the Functional on Leap 15 test stage') - booleanParam(name: 'CI_FUNCTIONAL_ubuntu20_TEST', + description: 'Run the Functional on Leap 15 stage.') + booleanParam(name: 'Functional on SLES 15', defaultValue: false, - description: 'Run the Functional on Ubuntu 20.04 test stage') - booleanParam(name: 'CI_FUNCTIONAL_HARDWARE_TEST_SKIP', + description: 'Run the Functional on SLES 15 stage.') + booleanParam(name: 'Functional on Ubuntu 20.04', defaultValue: false, - description: 'Skip Functional Hardware (Test Hardware) stage') - booleanParam(name: 'CI_medium_TEST', + description: 'Run the Functional on Ubuntu 20.04 stage.') + booleanParam(name: 'Fault injection testing', + defaultValue: true, + description: 'Run the Fault injection testing stage.') + booleanParam(name: 'Test RPMs on EL 9.6', + defaultValue: true, + description: 'Run the Test RPMs on EL 9.6 stage.') + booleanParam(name: 'Test RPMs on Leap 15.5', + defaultValue: true, + description: 'Run the Test RPMs on Leap 15.5 stage.') + booleanParam(name: 'Test Hardware', + defaultValue: true, + description: 'Run the Test Hardware stage.') + booleanParam(name: 'Functional Hardware Medium', defaultValue: false, - description: 'Run the Functional Hardware Medium test stage') - booleanParam(name: 'CI_medium_md_on_ssd_TEST', + description: 'Run the Functional Hardware Medium stage.') + booleanParam(name: 'Functional Hardware Medium MD on SSD', defaultValue: true, - description: 'Run the Functional Hardware Medium MD on SSD test stage') - booleanParam(name: 'CI_medium_vmd_TEST', + description: 'Run the Functional Hardware Medium MD on SSD stage.') + booleanParam(name: 'Functional Hardware Medium VMD', defaultValue: false, - description: 'Run the Functional Hardware Medium VMD test stage') - booleanParam(name: 'CI_medium_verbs_provider_TEST', + description: 'Run the Functional Hardware Medium VMD stage.') + booleanParam(name: 'Functional Hardware Medium Verbs Provider', defaultValue: false, - description: 'Run the Functional Hardware Medium Verbs Provider test stage') - booleanParam(name: 'CI_medium_verbs_provider_md_on_ssd_TEST', + description: 'Run the Functional Hardware Medium Verbs Provider stage.') + booleanParam(name: 'Functional Hardware Medium Verbs Provider MD on SSD', defaultValue: true, - description: 'Run the Functional Hardware Medium Verbs Provider MD on SSD test stage') - booleanParam(name: 'CI_medium_ucx_provider_TEST', + description: 'Run the Functional Hardware Medium Verbs Provider MD on SSD stage.') + booleanParam(name: 'Functional Hardware Medium UCX Provider', defaultValue: false, - description: 'Run the Functional Hardware Medium UCX Provider test stage') - booleanParam(name: 'CI_large_TEST', + description: 'Run the Functional Hardware Medium UCX Provider stage.') + booleanParam(name: 'Functional Hardware Large', defaultValue: false, - description: 'Run the Functional Hardware Large test stage') - booleanParam(name: 'CI_large_md_on_ssd_TEST', + description: 'Run the Functional Hardware Large stage.') + booleanParam(name: 'Functional Hardware Large MD on SSD', defaultValue: true, - description: 'Run the Functional Hardware Large MD on SSD test stage') + description: 'Run the Functional Hardware Large MD on SSD stage.') string(name: 'CI_UNIT_VM1_LABEL', defaultValue: 'ci_vm1', description: 'Label to use for 1 VM node unit and RPM tests') @@ -389,22 +921,23 @@ pipeline { } stages { - stage('Set Description') { - steps { - script { - if (params.CI_BUILD_DESCRIPTION) { - buildDescription params.CI_BUILD_DESCRIPTION - } - } - } - } stage('Prepare Environment Variables') { // TODO: Could/should these be moved to the environment block? parallel { - stage('Get Commit Message') { + stage('Set Description') { + steps { + script { + if (params.CI_BUILD_DESCRIPTION) { + buildDescription params.CI_BUILD_DESCRIPTION + } + } + } + } + stage('Setup Stages') { steps { pragmasToEnv() update_default_commit_pragmas() + updateRunStage() } } stage('Get RPM relval') { @@ -445,7 +978,7 @@ pipeline { stage('Cancel Previous Builds') { when { beforeAgent true - expression { !paramsValue('CI_CANCEL_PREV_BUILD_SKIP', false) && !skipStage() } + expression { runStage['Cancel Previous Builds'] } } steps { cancelPreviousBuilds() @@ -454,13 +987,13 @@ pipeline { stage('Pre-build') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Pre-build'] } } parallel { stage('Python Bandit check') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Python Bandit check'] } } agent { dockerfile { @@ -493,198 +1026,144 @@ pipeline { //failFast true when { beforeAgent true - expression { !skip_build_stage() } - } - parallel { - stage('Build on EL 8') { - when { - beforeAgent true - expression { !skip_build_stage('el8') } - } - agent { - dockerfile { - filename 'utils/docker/Dockerfile.el.8' - label 'docker_runner' - additionalBuildArgs dockerBuildArgs(repo_type: 'stable', - deps_build: false, - parallel_build: true) + - " -t ${sanitized_JOB_NAME()}-el8 " + - ' --build-arg DAOS_PACKAGES_BUILD=no ' + - ' --build-arg DAOS_KEEP_SRC=yes ' + - ' --build-arg REPOS="' + prRepos() + '"' + - ' --build-arg POINT_RELEASE=.10 ' + - " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" - } - } - steps { - script { - sh label: 'Install RPMs', - script: './ci/rpm/install_deps.sh el8 "' + env.DAOS_RELVAL + '"' - sh label: 'Build deps', - script: './ci/rpm/build_deps.sh' - job_step_update( - sconsBuild(parallel_build: true, - stash_files: 'ci/test_files_to_stash.txt', - build_deps: 'no', - stash_opt: true, - scons_args: sconsArgs() + - ' PREFIX=/opt/daos TARGET_TYPE=release')) - sh label: 'Generate RPMs', - script: './ci/rpm/gen_rpms.sh el8 "' + env.DAOS_RELVAL + '"' - } - } - post { - success { - uploadNewRPMs('el8', 'success') - } - unsuccessful { - sh '''if [ -f config.log ]; then - mv config.log config.log-el8-gcc - fi''' - archiveArtifacts artifacts: 'config.log-el8-gcc', - allowEmptyArchive: true - } - cleanup { - uploadNewRPMs('el8', 'cleanup') - job_status_update() - } - } - } - stage('Build on EL 9') { - when { - beforeAgent true - expression { !skip_build_stage('el9') } - } - agent { - dockerfile { - filename 'utils/docker/Dockerfile.el.9' - label 'docker_runner' - additionalBuildArgs dockerBuildArgs(repo_type: 'stable', - deps_build: false, - parallel_build: true) + - " -t ${sanitized_JOB_NAME()}-el9 " + - ' --build-arg DAOS_PACKAGES_BUILD=no ' + - ' --build-arg DAOS_KEEP_SRC=yes ' + - ' --build-arg REPOS="' + prRepos() + '"' + - ' --build-arg POINT_RELEASE=.7' + - " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" - } - } - steps { - script { - sh label: 'Install RPMs', - script: './ci/rpm/install_deps.sh el9 "' + env.DAOS_RELVAL + '"' - sh label: 'Build deps', - script: './ci/rpm/build_deps.sh' - job_step_update( - sconsBuild(parallel_build: true, - stash_files: 'ci/test_files_to_stash.txt', - build_deps: 'no', - stash_opt: true, - scons_args: sconsArgs() + - ' PREFIX=/opt/daos TARGET_TYPE=release')) - sh label: 'Generate RPMs', - script: './ci/rpm/gen_rpms.sh el9 "' + env.DAOS_RELVAL + '"' - } - } - post { - success { - uploadNewRPMs('el9', 'success') - } - unsuccessful { - sh '''if [ -f config.log ]; then - mv config.log config.log-el9-gcc - fi''' - archiveArtifacts artifacts: 'config.log-el9-gcc', - allowEmptyArchive: true - } - cleanup { - uploadNewRPMs('el9', 'cleanup') - job_status_update() - } - } - } - stage('Build on Leap 15') { - when { - beforeAgent true - expression { !skip_build_stage('leap15') } - } - agent { - dockerfile { - filename 'utils/docker/Dockerfile.leap.15' - label 'docker_runner' - additionalBuildArgs dockerBuildArgs(repo_type: 'stable', - parallel_build: true, - deps_build: false) + - ' --build-arg DAOS_PACKAGES_BUILD=no ' + - ' --build-arg DAOS_KEEP_SRC=yes ' + - " -t ${sanitized_JOB_NAME()}-leap15-gcc" + - " -t ${sanitized_JOB_NAME()}-leap15" + - ' --build-arg POINT_RELEASE=.6' + - " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" - } - } - steps { - script { - sh label: 'Install RPMs', - script: './ci/rpm/install_deps.sh suse.lp156 "' + env.DAOS_RELVAL + '"' - sh label: 'Build deps', - script: './ci/rpm/build_deps.sh' - job_step_update( - sconsBuild(parallel_build: true, - scons_args: sconsFaultsArgs() + - ' PREFIX=/opt/daos TARGET_TYPE=release', - build_deps: 'yes')) - sh label: 'Generate RPMs', - script: './ci/rpm/gen_rpms.sh suse.lp156 "' + env.DAOS_RELVAL + '"' - } - } - post { - success { - uploadNewRPMs('leap15', 'success') - } - unsuccessful { - sh '''if [ -f config.log ]; then - mv config.log config.log-leap15-gcc - fi''' - archiveArtifacts artifacts: 'config.log-leap15-gcc', - allowEmptyArchive: true - } - cleanup { - uploadNewRPMs('leap15', 'cleanup') - job_status_update() - } - } - } + expression { runStage['Build'] } } - } + steps { + script { + parallel( + 'Build on EL 8': scriptedBuildStage( + name: 'Build on EL 8', + distro:'el8', + compiler: 'gcc', + runCondition: !paramsValue('CI_FULL_BULLSEYE_REPORT', false), + buildRpms: true, + release: env.DAOS_RELVAL, + dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', + deps_build: false, + parallel_build: true) + + ' --build-arg DAOS_PACKAGES_BUILD=no' + + ' --build-arg DAOS_KEEP_SRC=yes' + + ' --build-arg REPOS="' + prRepos('el8') + '"' + + ' --build-arg POINT_RELEASE=.10 ' + + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + + ' -f utils/docker/Dockerfile.el.8 .', + sconsBuildArgs: [ + parallel_build: true, + stash_files: 'ci/test_files_to_stash.txt', + build_deps: 'no', + stash_opt: true, + scons_args: sconsArgs() + ' PREFIX=/opt/daos TARGET_TYPE=release' + ], + artifacts: "config.log-el8-gcc" + ), + 'Build on EL 9': scriptedBuildStage( + name: 'Build on EL 9', + distro:'el9', + compiler: 'gcc', + runCondition: !paramsValue('CI_FULL_BULLSEYE_REPORT', false), + buildRpms: true, + release: env.DAOS_RELVAL, + dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', + deps_build: false, + parallel_build: true) + + ' --build-arg DAOS_PACKAGES_BUILD=no' + + ' --build-arg DAOS_KEEP_SRC=yes' + + ' --build-arg REPOS="' + prRepos('el9') + '"' + + ' --build-arg POINT_RELEASE=.7' + + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + + ' -f utils/docker/Dockerfile.el.9 .', + sconsBuildArgs: [ + parallel_build: true, + stash_files: 'ci/test_files_to_stash.txt', + build_deps: 'no', + stash_opt: true, + scons_args: sconsArgs() + ' PREFIX=/opt/daos TARGET_TYPE=release' + ], + artifacts: "config.log-el9-gcc" + ), + 'Build on Leap 15': scriptedBuildStage( + name: 'Build on Leap 15', + distro:'leap15', + rpmDistro: 'suse.lp156', + compiler: 'gcc', + runCondition: !paramsValue('CI_FULL_BULLSEYE_REPORT', false), + buildRpms: true, + release: env.DAOS_RELVAL, + dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', + deps_build: false, + parallel_build: true) + + ' --build-arg DAOS_PACKAGES_BUILD=no' + + ' --build-arg DAOS_KEEP_SRC=yes' + + ' --build-arg POINT_RELEASE=.6' + + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + + ' -f utils/docker/Dockerfile.leap.15 .', + sconsBuildArgs: [ + parallel_build: true, + build_deps: 'yes', + scons_args: sconsArgs() + ' PREFIX=/opt/daos TARGET_TYPE=release' + ], + artifacts: "config.log-leap156-gcc", + ), + 'Build on EL 9 with Bullseye': scriptedBuildStage( + name: 'Build on EL 9 with Bullseye', + distro:'el9', + compiler: 'covc', + buildRpms: true, + release: env.DAOS_RELVAL, + dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', + deps_build: false, + parallel_build: true) + + ' --build-arg DAOS_PACKAGES_BUILD=no' + + ' --build-arg DAOS_KEEP_SRC=yes' + + ' --build-arg REPOS="' + prRepos('el9') + '"' + + ' --build-arg POINT_RELEASE=.7' + + " --build-arg PYTHON_VERSION=${env.PYTHON_VERSION}" + + ' --build-arg COMPILER=covc' + + ' --build-arg CODE_COVERAGE=true' + + ' -f utils/docker/Dockerfile.el.9 .', + sconsBuildArgs: [ + parallel_build: true, + stash_files: 'ci/test_files_to_stash.txt', + build_deps: 'no', + stash_opt: true, + scons_args: sconsArgs() + ' PREFIX=/opt/daos TARGET_TYPE=release' + + ' COMPILER=covc' + ], + artifacts: "config.log-el9-covc", + uploadTarget: 'el9-bullseye' + ) + ) // parallel + } // script + } // steps + } // stage('Build') stage('Unit Tests') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Unit Tests'] } } parallel { stage('Unit Test') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Unit Test'] } } agent { label cachedCommitPragma(pragma: 'VM1-label', def_val: params.CI_UNIT_VM1_LABEL) } steps { - job_step_update( - unitTest(timeout_time: 60, - unstash_opt: true, - inst_repos: daosRepos(), - inst_rpms: unitPackages(target: 'el9'), - image_version: 'el9.7', - ) - ) + job_step_update( + unitTest(timeout_time: 120, + unstash_opt: true, + inst_repos: daosRepos(), + inst_rpms: unitTestInstRpms('el9'), + image_version: 'el9.7', + compiler: unitTestCompiler(), + coverage_stash: 'unit_test_bullseye')) } post { always { - unitTestPost artifacts: ['unit_test_logs/'] + unitTestPost artifacts: ['unit_test_logs/'], + compiler: unitTestCompiler() job_status_update() } } @@ -692,22 +1171,25 @@ pipeline { stage('Unit Test bdev') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Unit Test bdev'] } } agent { label params.CI_UNIT_VM1_NVME_LABEL } steps { job_step_update( - unitTest(timeout_time: 60, + unitTest(timeout_time: 120, unstash_opt: true, inst_repos: daosRepos(), - inst_rpms: unitPackages(target: 'el9'), - image_version: 'el9.7')) + inst_rpms: unitTestInstRpms('el9'), + image_version: 'el9.7', + compiler: unitTestCompiler(), + coverage_stash: 'unit_test_bdev_bullseye')) } post { always { - unitTestPost artifacts: ['unit_test_bdev_logs/'] + unitTestPost artifacts: ['unit_test_bdev_logs/'], + compiler: unitTestCompiler() job_status_update() } } @@ -715,7 +1197,7 @@ pipeline { stage('NLT') { when { beforeAgent true - expression { params.CI_NLT_TEST && !skipStage() } + expression { runStage['NLT'] } } agent { label params.CI_NLT_1_LABEL @@ -724,21 +1206,23 @@ pipeline { job_step_update( unitTest(timeout_time: 60, inst_repos: daosRepos(), + inst_rpms: unitTestInstRpms('el9'), + image_version: 'el9.7', + compiler: 'gcc', test_script: 'ci/unit/test_nlt.sh' + ' --system-ram-reserved 4' + ' --max-log-size 1950MiB' + ' --dfuse-dir /localhome/jenkins/' + ' --log-usage-save nltir.xml' + ' --log-usage-export nltr.json' + + ' --log-base-dir nlt_logs' + ' --class-name nlt all', - with_valgrind: 'memcheck', - valgrind_pattern: '*memcheck.xml', - always_script: 'ci/unit/test_nlt_post.sh', + always_script: 'ci/unit/test_nlt_post.sh nlt_logs', testResults: 'nlt-junit.xml', unstash_opt: true, unstash_tests: false, - inst_rpms: unitPackages(target: 'el9'), - image_version: 'el9.7', + with_valgrind: 'memcheck', + valgrind_pattern: '*memcheck.xml', prov_env_vars: 'VM_CPUS=14')) // recordCoverage(tools: [[parser: 'COBERTURA', pattern:'nltir.xml']], // skipPublishingChecks: true, @@ -759,17 +1243,61 @@ pipeline { name: 'NLT server leaks', qualityGates: [[threshold: 1, type: 'TOTAL', unstable: true]], tool: issues(pattern: 'nlt-server-leaks.json', - name: 'NLT server results', - id: 'NLT_server'), + name: 'NLT server results', + id: 'NLT_server'), scm: 'daos-stack/daos' job_status_update() } } } + stage('NLT with Bullseye') { + when { + beforeAgent true + expression { runStage['NLT with Bullseye'] } + } + agent { + label params.CI_NLT_1_LABEL + } + steps { + job_step_update( + unitTest(timeout_time: 150, + inst_repos: daosRepos(), + inst_rpms: unitTestInstRpms('el9'), + image_version: 'el9.7', + compiler: 'covc', + test_script: 'ci/unit/test_nlt.sh' + + ' --system-ram-reserved 4' + + ' --max-log-size 1950MiB' + + ' --dfuse-dir /localhome/jenkins/' + + ' --log-usage-save nltir-bullseye.xml' + + ' --log-usage-export nltr-bullseye.json' + + ' --log-base-dir nlt_bullseye_logs' + + ' --memcheck no' + + ' --class-name nlt all', + always_script: 'ci/unit/test_nlt_post.sh nlt_bullseye_logs', + testResults: 'nlt-junit.xml', + unstash_opt: true, + unstash_tests: false, + image_version: 'el9.7', + prov_env_vars: 'VM_CPUS=14', + ignore_failure: true, + coverage_stash: 'nlt_bullseye')) + stash(name:'nltr-bullseye', includes:'nltr-bullseye.json', allowEmpty: true) + } + post { + always { + unitTestPost artifacts: ['nlt_bullseye_logs/'], + testResults: 'nlt-junit.xml', + NLT: true, + compiler: 'covc' + job_status_update() + } + } + } stage('Unit Test with memcheck') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Unit Test with memcheck'] } } agent { label cachedCommitPragma(pragma: 'VM1-label', def_val: params.CI_UNIT_VM1_LABEL) @@ -777,11 +1305,12 @@ pipeline { steps { job_step_update( unitTest(timeout_time: 160, - unstash_opt: true, - ignore_failure: true, inst_repos: daosRepos(), - inst_rpms: unitPackages(target: 'el9'), - image_version: 'el9.7')) + inst_rpms: unitTestInstRpms('el9'), + image_version: 'el9.7', + compiler: 'gcc', + unstash_opt: true, + ignore_failure: true)) } post { always { @@ -795,7 +1324,7 @@ pipeline { stage('Unit Test bdev with memcheck') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Unit Test bdev with memcheck'] } } agent { label params.CI_UNIT_VM1_NVME_LABEL @@ -803,11 +1332,12 @@ pipeline { steps { job_step_update( unitTest(timeout_time: 180, - unstash_opt: true, - ignore_failure: true, inst_repos: daosRepos(), - inst_rpms: unitPackages(target: 'el9'), - image_version: 'el9.7')) + inst_rpms: unitTestInstRpms('el9'), + image_version: 'el9.7', + compiler: 'gcc', + unstash_opt: true, + ignore_failure: true)) } post { always { @@ -823,15 +1353,13 @@ pipeline { stage('Test') { when { beforeAgent true - //expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) && !skipStage() } - // Above not working, always skipping functional VM tests. - expression { !paramsValue('CI_FUNCTIONAL_TEST_SKIP', false) } + expression { runStage['Test'] } } parallel { stage('Functional on EL 8.8 with Valgrind') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Functional on EL 8.8 with Valgrind'] } } agent { label vm9_label('EL8') @@ -854,7 +1382,7 @@ pipeline { stage('Functional on EL 8') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Functional on EL 8'] } } agent { label vm9_label('EL8') @@ -863,8 +1391,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', + inst_rpms: functionalInstRpms('mercury-libfabric', false), test_function: 'runTestFunctionalV2', image_version: 'el8.10')) } @@ -878,7 +1405,7 @@ pipeline { stage('Functional on EL 9') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Functional on EL 9'] } } agent { label vm9_label('EL9') @@ -887,10 +1414,13 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', + inst_rpms: functionalInstRpms( + 'mercury-libfabric', + paramsValue('CI_FULL_BULLSEYE_REPORT', false)), test_function: 'runTestFunctionalV2', - image_version: 'el9.7')) + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false), + coverage_stash: 'func_vm_bullseye')) } post { always { @@ -902,7 +1432,7 @@ pipeline { stage('Functional on Leap 15') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Functional on Leap 15'] } } agent { label vm9_label('Leap15') @@ -911,8 +1441,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', + inst_rpms: functionalInstRpms('mercury-libfabric', false), test_function: 'runTestFunctionalV2', image_version: 'leap15.6')) } @@ -926,7 +1455,7 @@ pipeline { stage('Functional on Ubuntu 20.04') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Functional on Ubuntu 20.04'] } } agent { label vm9_label('Ubuntu') @@ -935,8 +1464,7 @@ pipeline { job_step_update( functionalTest( inst_repos: daosRepos(), - inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + - ' mercury-libfabric', + inst_rpms: functionalInstRpms('mercury-libfabric', false), test_function: 'runTestFunctionalV2')) } post { @@ -949,7 +1477,7 @@ pipeline { stage('Fault injection testing') { when { beforeAgent true - expression { !skipStage() } + expression { runStage['Fault injection testing'] } } agent { label params.CI_FI_1_LABEL @@ -958,18 +1486,19 @@ pipeline { job_step_update( unitTest(timeout_time: 240, inst_repos: daosRepos(), + inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', + image_version: 'el9.7', test_script: 'ci/unit/test_nlt.sh --memcheck no' + ' --system-ram-reserved 4 --server-debug WARN' + ' --log-usage-import nltr.json' + ' --log-usage-save nltr.xml' + + ' --log-base-dir nlt_logs' + ' --class-name fault-injection fi', with_valgrind: '', - always_script: 'ci/unit/test_nlt_post.sh', + always_script: 'ci/unit/test_nlt_post.sh nlt_logs', testResults: 'nlt-junit.xml', unstash_opt: true, unstash_tests: false, - inst_rpms: unitPackages(target: 'el9') + ' daos-client-tests', - image_version: 'el9.7', prov_env_vars: 'VM_CPUS=14')) } post { @@ -990,7 +1519,7 @@ pipeline { stage('Test RPMs on EL 9.6') { when { beforeAgent true - expression { params.CI_TEST_EL_RPMs && !skipStage() } + expression { runStage['Test RPMs on EL 9.6'] } } agent { label params.CI_UNIT_VM1_LABEL @@ -1011,7 +1540,7 @@ pipeline { stage('Test RPMs on Leap 15.5') { when { beforeAgent true - expression { params.CI_TEST_LEAP_RPMs && !skipStage() } + expression { runStage['Test RPMs on Leap 15.5'] } } agent { label params.CI_UNIT_VM1_LABEL @@ -1086,119 +1615,186 @@ pipeline { stage('Test Hardware') { when { beforeAgent true - expression { !paramsValue('CI_FUNCTIONAL_HARDWARE_TEST_SKIP', false) && !skipStage() } + expression { runStage['Functional Hardware'] } } steps { script { parallel( 'Functional Hardware Medium': getFunctionalTestStage( name: 'Functional Hardware Medium', + runStage: runStage['Functional Hardware Medium'], pragma_suffix: '-hw-medium', label: params.FUNCTIONAL_HARDWARE_MEDIUM_LABEL, - next_version: next_version(), + inst_rpms: functionalInstRpms( + 'mercury-libfabric', paramsValue('CI_FULL_BULLSEYE_REPORT', false)), stage_tags: 'hw,medium,-provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto', - run_if_pr: false, - run_if_landing: false, - job_status: job_status_internal + job_status: job_status_internal, + coverage_stash: 'func_hw_medium_bullseye', + image_version: 'el9.7' ), 'Functional Hardware Medium MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Medium MD on SSD', + runStage: runStage['Functional Hardware Medium MD on SSD'], pragma_suffix: '-hw-medium-md-on-ssd', label: params.FUNCTIONAL_HARDWARE_MEDIUM_LABEL, - next_version: next_version(), + inst_rpms: functionalInstRpms( + 'mercury-libfabric', paramsValue('CI_FULL_BULLSEYE_REPORT', false)), stage_tags: 'hw,medium,-provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto_md_on_ssd', - run_if_pr: true, - run_if_landing: false, - job_status: job_status_internal + job_status: job_status_internal, + coverage_stash: 'func_hw_medium_md_on_ssd_bullseye', + image_version: 'el9.7' ), 'Functional Hardware Medium VMD': getFunctionalTestStage( name: 'Functional Hardware Medium VMD', + runStage: runStage['Functional Hardware Medium VMD'], pragma_suffix: '-hw-medium-vmd', label: params.FUNCTIONAL_HARDWARE_MEDIUM_VMD_LABEL, - next_version: next_version(), + inst_rpms: functionalInstRpms( + 'mercury-libfabric', paramsValue('CI_FULL_BULLSEYE_REPORT', false)), stage_tags: 'hw_vmd,medium', /* groovylint-disable-next-line UnnecessaryGetter */ default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', nvme: 'auto', - run_if_pr: false, - run_if_landing: false, - job_status: job_status_internal + job_status: job_status_internal, + coverage_stash: 'func_hw_medium_vmd_bullseye', + image_version: 'el9.7' ), 'Functional Hardware Medium Verbs Provider': getFunctionalTestStage( name: 'Functional Hardware Medium Verbs Provider', + runStage: runStage['Functional Hardware Medium Verbs Provider'], pragma_suffix: '-hw-medium-verbs-provider', label: params.FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL, - next_version: next_version(), + inst_rpms: functionalInstRpms( + 'mercury-libfabric', paramsValue('CI_FULL_BULLSEYE_REPORT', false)), stage_tags: 'hw,medium,provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto', provider: 'ofi+verbs;ofi_rxm', - run_if_pr: false, - run_if_landing: false, job_status: job_status_internal, + coverage_stash: 'func_hw_medium_verbs_provider_bullseye', image_version: 'el9.7' ), 'Functional Hardware Medium Verbs Provider MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Medium Verbs Provider MD on SSD', + runStage: runStage['Functional Hardware Medium Verbs Provider MD on SSD'], pragma_suffix: '-hw-medium-verbs-provider-md-on-ssd', label: params.FUNCTIONAL_HARDWARE_MEDIUM_VERBS_PROVIDER_LABEL, - next_version: next_version(), + inst_rpms: functionalInstRpms( + 'mercury-libfabric', paramsValue('CI_FULL_BULLSEYE_REPORT', false)), stage_tags: 'hw,medium,provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto_md_on_ssd', provider: 'ofi+verbs;ofi_rxm', - run_if_pr: true, - run_if_landing: false, job_status: job_status_internal, + coverage_stash: 'func_hw_medium_verbs_provider_md_on_ssd_bullseye', image_version: 'el9.7' ), 'Functional Hardware Medium UCX Provider': getFunctionalTestStage( name: 'Functional Hardware Medium UCX Provider', + runStage: runStage['Functional Hardware Medium UCX Provider'], pragma_suffix: '-hw-medium-ucx-provider', label: params.FUNCTIONAL_HARDWARE_MEDIUM_UCX_PROVIDER_LABEL, - next_version: next_version(), + inst_rpms: functionalInstRpms( + 'mercury-libfabric', paramsValue('CI_FULL_BULLSEYE_REPORT', false)), stage_tags: 'hw,medium,provider', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto', provider: cachedCommitPragma('Test-provider-ucx', 'ucx+ud_x'), - run_if_pr: false, - run_if_landing: false, - job_status: job_status_internal + job_status: job_status_internal, + coverage_stash: 'func_hw_medium_ucx_provider_bullseye', + image_version: 'el9.7' ), 'Functional Hardware Large': getFunctionalTestStage( name: 'Functional Hardware Large', + runStage: runStage['Functional Hardware Large'], pragma_suffix: '-hw-large', label: params.FUNCTIONAL_HARDWARE_LARGE_LABEL, - next_version: next_version(), + inst_rpms: functionalInstRpms( + 'mercury-libfabric', paramsValue('CI_FULL_BULLSEYE_REPORT', false)), stage_tags: 'hw,large', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto', - run_if_pr: false, - run_if_landing: false, job_status: job_status_internal, + coverage_stash: 'func_hw_large_bullseye', image_version: 'el9.7' ), 'Functional Hardware Large MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Large MD on SSD', + runStage: runStage['Functional Hardware Large MD on SSD'], pragma_suffix: '-hw-large-md-on-ssd', label: params.FUNCTIONAL_HARDWARE_LARGE_LABEL, - next_version: next_version(), + inst_rpms: functionalInstRpms( + 'mercury-libfabric', paramsValue('CI_FULL_BULLSEYE_REPORT', false)), stage_tags: 'hw,large', default_tags: startedByTimer() ? 'pr daily_regression' : 'pr', default_nvme: 'auto_md_on_ssd', - run_if_pr: true, - run_if_landing: false, job_status: job_status_internal, + coverage_stash: 'func_hw_large_md_on_ssd_bullseye', image_version: 'el9.7' ), ) } } } // stage('Test Hardware') + stage('Test Summary') { + when { + beforeAgent true + expression { true } + } + steps { + script { + parallel( + 'Bullseye Report': scriptedSummaryStage( + name: 'Bullseye Report', + distro: 'el9', + compiler: 'covc', + runCondition: runStage(['CI_BUILD_BULLSEYE': true]), + nodeLabel: 'docker_runner', + dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', + deps_build: false, + parallel_build: true) + + ' --build-arg DAOS_PACKAGES_BUILD=no' + + ' --build-arg DAOS_KEEP_SRC=yes' + + ' --build-arg REPOS="' + prRepos('el9') + '"' + + ' --build-arg COMPILER=covc' + + ' --build-arg CODE_COVERAGE=true' + + ' -f utils/docker/Dockerfile.el.9 .', + installScript: './ci/summary/install_pkgs.sh el9 true', + runScriptArgs: [ + label: 'Generate Bullseye Report', + script: 'ci/summary/bullseye_report.sh', + stashes: ['unit_test_bullseye', + 'unit_test_bdev_bullseye', + 'nlt_bullseye', + 'func_vm_bullseye', + 'func_hw_medium_bullseye', + 'func_hw_medium_md_on_ssd_bullseye', + 'func_hw_medium_verbs_provider_bullseye', + 'func_hw_medium_verbs_provider_md_on_ssd_bullseye', + 'func_hw_medium_ucx_provider_bullseye', + 'func_hw_large_bullseye', + 'func_hw_large_md_on_ssd_bullseye'] + ], + archiveArtifactsArgs: [ + artifacts: 'bullseye_code_coverage_report/', + allowEmptyArchive: false + ], + publishHtmlArgs: [ + target: [ + reportDir: 'bullseye_code_coverage_report', + reportFiles: 'index.html', + reportName: 'Bullseye Coverage' + ] + ] + ) + ) // parallel + } // script + } // steps + } // stage('Test Summary') } // stages post { always { diff --git a/ci/bullseye_generate_report.sh b/ci/bullseye_generate_report.sh deleted file mode 100755 index d8e7421e90d..00000000000 --- a/ci/bullseye_generate_report.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -eux - -if [ ! -d '/opt/BullseyeCoverage/bin' ]; then - echo 'Bullseye not found.' - exit 1 -fi -export COVFILE="$WORKSPACE/test.cov" -export PATH="/opt/BullseyeCoverage/bin:$PATH" - -mv "$WORKSPACE/test.cov_1" "$COVFILE" -if [ -e "$WORKSPACE/test.cov_2" ]; then - covmerge --no-banner --file "$COVFILE" "$WORKSPACE"/test.cov_* -fi - -if [ ! -e "$COVFILE" ]; then - echo "Coverage file $COVFILE is missing" -else - ls -l "$COVFILE" -fi - -java -jar bullshtml.jar test_coverage diff --git a/ci/provisioning/post_provision_config.sh b/ci/provisioning/post_provision_config.sh index 6aab32f3ab1..c036601a08c 100755 --- a/ci/provisioning/post_provision_config.sh +++ b/ci/provisioning/post_provision_config.sh @@ -143,6 +143,7 @@ if ! retry_cmd 2400 clush -B -S -l root -w "$NODESTRING" \ DAOS_CI_INFO_DIR=\"${DAOS_CI_INFO_DIR:?DAOS_CI_INFO_DIR is missing. Can not continue with node(s) provisioning process}\" CI_SCONS_ARGS=\"${CI_SCONS_ARGS:-}\" PYTHON_VERSION=\"${PYTHON_VERSION}\" + CI_BULLSEYE=\"${CI_BULLSEYE:-false}\" $(cat ci/stacktrace.sh) $(cat ci/junit.sh) $(cat ci/provisioning/post_provision_config_common_functions.sh) diff --git a/ci/provisioning/post_provision_config_common_functions.sh b/ci/provisioning/post_provision_config_common_functions.sh index 46fba4b21c2..dcd10f177ff 100755 --- a/ci/provisioning/post_provision_config_common_functions.sh +++ b/ci/provisioning/post_provision_config_common_functions.sh @@ -60,13 +60,41 @@ add_repo() { fi } +add_inst_repo() { + local repo="$1" + local branch="$2" + local build_number="$3" + local distro_name="$4" + local bullseye="${5:-false}" + local sudo="${6:-false}" + local dnf_cmd=("dnf" "-y") + local repo_base="${ARTIFACTS_URL:-${JENKINS_URL}job/}daos-stack/job/${repo}/job/${branch//\//%252F}/${build_number}/artifact/artifacts" + local repo_url="${repo_base}/$distro_name/" + if [[ "$bullseye" == "true" ]]; then + repo_url="${repo_base}/${distro_name}-bullseye/" + fi + if [[ "$sudo" == "true" ]]; then + dnf_cmd=("sudo" "${dnf_cmd[@]}") + fi + "${dnf_cmd[@]}" config-manager --add-repo="$repo_url" + repo="$(url_to_repo "$repo_url")" + # PR-repos: should always be able to upgrade modular packages + "${dnf_cmd[@]}" config-manager --save --setopt "$repo.module_hotfixes=true" "$repo" + disable_gpg_check "$repo_url" "$sudo" +} + disable_gpg_check() { local url="$1" + local sudo="${2:-false}" + local dnf_cmd=("dnf" "-y") + if [[ "$sudo" == "true" ]]; then + dnf_cmd=("sudo" "${dnf_cmd[@]}") + fi repo="$(url_to_repo "$url")" # bug in EL7 DNF: this needs to be enabled before it can be disabled - dnf -y config-manager --save --setopt="$repo".gpgcheck=1 - dnf -y config-manager --save --setopt="$repo".gpgcheck=0 + "${dnf_cmd[@]}" config-manager --save --setopt="$repo".gpgcheck=1 + "${dnf_cmd[@]}" config-manager --save --setopt="$repo".gpgcheck=0 # but even that seems to be not enough, so just brute-force it if [ -d /etc/yum.repos.d ] && ! grep gpgcheck /etc/yum.repos.d/"$repo".repo; then @@ -304,6 +332,8 @@ update_repos() { } post_provision_config_nodes() { + local bullseye="${1:-false}" + # shellcheck disable=SC2154 if ! update_repos "$DISTRO_NAME"; then # need to use the image supplied repos @@ -379,16 +409,7 @@ post_provision_config_nodes() { branch="${branch%:*}" fi fi - local subdir - if ! $COVFN_DISABLED; then - subdir="bullseye/" - fi - local repo_url="${ARTIFACTS_URL:-${JENKINS_URL}job/}"daos-stack/job/"$repo"/job/"${branch//\//%252F}"/"$build_number"/artifact/artifacts/"${subdir:-}"$DISTRO_NAME/ - dnf -y config-manager --add-repo="$repo_url" - repo="$(url_to_repo "$repo_url")" - # PR-repos: should always be able to upgrade modular packages - dnf -y config-manager --save --setopt "$repo.module_hotfixes=true" "$repo" - disable_gpg_check "$repo_url" + add_inst_repo "${repo}" "${branch}" "${build_number}" "${DISTRO_NAME}" "${bullseye}" done # start with everything fully up-to-date diff --git a/ci/provisioning/post_provision_config_nodes.sh b/ci/provisioning/post_provision_config_nodes.sh index c62c7064cad..28296ed4724 100644 --- a/ci/provisioning/post_provision_config_nodes.sh +++ b/ci/provisioning/post_provision_config_nodes.sh @@ -63,7 +63,7 @@ fi # defined in ci/functional/post_provision_config_nodes_.sh # and catted to the remote node along with this script -if ! post_provision_config_nodes; then +if ! post_provision_config_nodes "${CI_BULLSEYE}"; then rc=${PIPESTATUS[0]} echo "post_provision_config_nodes failed with rc=$rc" exit "$rc" diff --git a/ci/rpm/build_deps.sh b/ci/rpm/build_deps.sh index f989a96332a..e56c4b85faa 100755 --- a/ci/rpm/build_deps.sh +++ b/ci/rpm/build_deps.sh @@ -1,3 +1,17 @@ #!/bin/bash +# +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP +# +# Build DAOS dependencies +code_coverage="${1:-false}" +bullseye_key="${2:-}" +mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + cd /home/daos/pre || exit 1 scons install --build-deps=only USE_INSTALLED=all PREFIX=/opt/daos TARGET_TYPE=release -j 32 + +if [[ "${code_coverage}" == "true" ]] ; then + pushd "${mydir}/../.." || exit 1 + utils/rpms/bullseye_build.sh "${bullseye_key}" + popd || exit 1 +fi diff --git a/ci/rpm/build_success.sh b/ci/rpm/build_success.sh index fa8e1828bd2..4ecb7d86b22 100755 --- a/ci/rpm/build_success.sh +++ b/ci/rpm/build_success.sh @@ -43,11 +43,13 @@ elif [ -d /var/cache/pbuilder/ ]; then fi if [ -d /home/daos/rpms/ ]; then - if [ -d /home/daos/rpms/deps ]; then - mkdir -p "$artdir/deps" - cp /home/daos/rpms/deps/*.rpm "${artdir}/deps" - fi - cp /home/daos/rpms/daos/*.rpm "${artdir}/daos" + # shellcheck disable=SC2044 + for dir in $(find /home/daos/rpms/ -maxdepth 1 -mindepth 1 -type d -exec basename {} \;); do + if [ -d "/home/daos/rpms/${dir}" ]; then + mkdir -p "${artdir}/${dir}" + cp "/home/daos/rpms/${dir}"/*.rpm "${artdir}/${dir}" + fi + done else mockroot="/var/lib/mock/${CHROOT_NAME}" cat "$mockroot"/result/{root,build}.log 2>/dev/null || true diff --git a/ci/rpm/gen_rpms.sh b/ci/rpm/gen_rpms.sh index 30a4e248952..e6f83e81c69 100755 --- a/ci/rpm/gen_rpms.sh +++ b/ci/rpm/gen_rpms.sh @@ -16,20 +16,29 @@ if [ -e "${ci_envs}" ]; then source "${ci_envs}" fi -env +function mv_rpms() { + local dir="${1}" + if ls -1 ./*.rpm; then + mkdir -p "/home/daos/rpms/${dir}" + cp ./*.rpm "/home/daos/rpms/${dir}" + rm -f ./*.rpm + fi +} + +env | sort -n pushd "${mydir}/../.." || exit 1 export DISTRO="${1}" export DAOS_RELVAL="${2}" +code_coverage="${3:-false}" +build_types="deps daos" +if [[ "${code_coverage}" == "true" ]]; then + build_types="deps bullseye" +fi rm -f ./*.rpm rm -rf /home/daos/rpms/* -utils/rpms/build_packages.sh deps -if ls -1 ./*.rpm; then - mkdir -p /home/daos/rpms/deps - cp ./*.rpm /home/daos/rpms/deps - rm -f ./*.rpm -fi -utils/rpms/build_packages.sh daos -mkdir -p /home/daos/rpms/daos -cp ./*.rpm /home/daos/rpms/daos +for build_type in ${build_types}; do + utils/rpms/build_packages.sh "${build_type}" + mv_rpms "${build_type}" +done popd || exit 1 diff --git a/ci/rpm/install_deps.sh b/ci/rpm/install_deps.sh index 7ac5ff26622..02806a845c3 100755 --- a/ci/rpm/install_deps.sh +++ b/ci/rpm/install_deps.sh @@ -21,6 +21,7 @@ env pushd "${mydir}/../.." || exit 1 export DISTRO="${1}" export DAOS_RELVAL="${2}" +code_coverage="${3:-false}" libfabric_pkg="$(utils/rpms/package_version.sh libfabric dev)" mercury_pkg="$(utils/rpms/package_version.sh mercury dev)" argobots_pkg="$(utils/rpms/package_version.sh argobots dev)" @@ -38,4 +39,10 @@ sudo dnf install --allowerasing -y "${fused_pkg}" || echo "${fused_pkg} not avai sudo dnf install --allowerasing -y "${pmdk_pkg}" || echo "${pmdk_pkg} not available" sudo dnf install --allowerasing -y "${isal_pkg}" || echo "${isal_pkg} not available" sudo dnf install --allowerasing -y "${isal_crypto_pkg}" || echo "${isal_crypto_pkg} not available" + +if [[ "${code_coverage}" == "true" ]] ; then + bullseye_pkg="$(utils/rpms/package_version.sh bullseye normal)" + sudo dnf install --allowerasing -y "${bullseye_pkg}" || echo "${bullseye_pkg} not available" +fi + popd || exit 1 diff --git a/ci/summary/bullseye_report.sh b/ci/summary/bullseye_report.sh new file mode 100644 index 00000000000..30e8360546e --- /dev/null +++ b/ci/summary/bullseye_report.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2026 Hewlett Packard Enterprise Development LP +# +# Script for generating a bullseye code coverage report summary +set -uex + +if [ ! -d '/opt/BullseyeCoverage/bin' ]; then + echo 'Bullseye not found.' + exit 1 +fi +export COVFILE="${WORKSPACE:-/tmp}/test.cov" +export PATH="/opt/BullseyeCoverage/bin:$PATH" + +# Merge all coverage files +cp /opt/BullseyeCoverage/daos/test.cov "${COVFILE}" +readarray -t cov_files < <(find "${WORKSPACE}" -name test.cov) +if [ ${#cov_files[@]} -gt 0 ]; then + covmerge --no-banner --file "${COVFILE}" "${cov_files[@]}" +fi + +if [ ! -e "$COVFILE" ]; then + echo "Coverage file ${COVFILE} is missing" + exit 1 +fi + +# Generate the html report +rm -fr bullseye_code_coverage_report || true +mkdir bullseye_code_coverage_report +cd bullseye_code_coverage_report +mkdir sources +tar -xf /opt/BullseyeCoverage/daos/bullseye_sources.tar.gz -C sources/ +covhtml --srcdir sources --file "${COVFILE}" . diff --git a/ci/summary/install_pkgs.sh b/ci/summary/install_pkgs.sh new file mode 100644 index 00000000000..d6d93d3f107 --- /dev/null +++ b/ci/summary/install_pkgs.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright 2026 Hewlett Packard Enterprise Development LP +# +# Script for installing packages used for CI summary steps +set -uex + +id +if [ "$(id -u)" = "0" ]; then + echo "Should not be run as root" + exit 1 +fi + +# Distro name for the repository path for accessing packages built by CI +distro_name="${1:-el9}" +code_coverage="${2:-false}" + +# Import provisioning functions to add the repo +mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +files=("$mydir/../parse_ci_envs.sh") +files+=("$mydir/../provisioning/post_provision_config_common_functions.sh") +for src_file in "${files[@]}"; do + if [ -e "${src_file}" ]; then + # shellcheck source=parse_ci_envs.sh disable=SC1091 + source "${src_file}" + fi +done + +env | sort -n + +# Add the repo for packages built by CI +add_inst_repo "daos" "${BRANCH_NAME}" "${BUILD_NUMBER}" "${distro_name}" "${code_coverage}" "true" + +# Install bullseye +export DISTRO="${distro_name}" +bullseye_pkg="$(utils/rpms/package_version.sh bullseye normal)" +sudo dnf install --allowerasing -y "${bullseye_pkg}" diff --git a/ci/unit/required_packages.sh b/ci/unit/required_packages.sh index 4790812779c..39b702e2004 100755 --- a/ci/unit/required_packages.sh +++ b/ci/unit/required_packages.sh @@ -7,47 +7,50 @@ # set -eu -# Provided by pipeline-lib -# distro="$1" -# quick_build="${2:-false}" +distro="${1:?ERROR: Missing distro argument. Usage: $0 }" +code_coverage="${2:-false}" OPENMPI_VER="" PY_MINOR_VER="" -DISTRO="${1:?ERROR: Missing distro argument. Usage: $0 }" -export DISTRO="${DISTRO%%.*}" +export DISTRO="${distro%%.*}" -pkgs="boost-python3$PY_MINOR_VER-devel \ - capstone \ - $(utils/rpms/package_version.sh argobots lib) \ - $(utils/rpms/package_version.sh argobots debug) \ - $(utils/rpms/package_version.sh daos_spdk dev) \ - $(utils/rpms/package_version.sh daos_spdk debug) \ - $(utils/rpms/package_version.sh isal dev) \ - $(utils/rpms/package_version.sh isal_crypto lib) \ - $(utils/rpms/package_version.sh isal_crypto debug) \ - $(utils/rpms/package_version.sh libfabric dev) \ - $(utils/rpms/package_version.sh libfabric debug) \ - $(utils/rpms/package_version.sh mercury dev) \ - $(utils/rpms/package_version.sh mercury debug) \ - $(utils/rpms/package_version.sh mercury lib mercury_libfabric) \ - $(utils/rpms/package_version.sh pmdk lib pmemobj) \ - $(utils/rpms/package_version.sh pmdk debug pmemobj) \ - $(utils/rpms/package_version.sh pmdk debug pmem) \ - fuse3 \ - gotestsum \ - hwloc-devel \ - libasan \ - libipmctl-devel \ - libyaml-devel \ - numactl \ - numactl-devel \ - openmpi$OPENMPI_VER \ - patchelf \ - pciutils-devel \ - protobuf-c \ - valgrind-devel" +pkgs=("boost-python3${PY_MINOR_VER}-devel") +pkgs+=("capstone") +pkgs+=("$(utils/rpms/package_version.sh argobots lib)") +pkgs+=("$(utils/rpms/package_version.sh argobots debug)") +pkgs+=("$(utils/rpms/package_version.sh daos_spdk dev)") +pkgs+=("$(utils/rpms/package_version.sh daos_spdk debug)") +pkgs+=("$(utils/rpms/package_version.sh isal dev)") +pkgs+=("$(utils/rpms/package_version.sh isal_crypto lib)") +pkgs+=("$(utils/rpms/package_version.sh isal_crypto debug)") +pkgs+=("$(utils/rpms/package_version.sh libfabric dev)") +pkgs+=("$(utils/rpms/package_version.sh libfabric debug)") +pkgs+=("$(utils/rpms/package_version.sh mercury dev)") +pkgs+=("$(utils/rpms/package_version.sh mercury debug)") +pkgs+=("$(utils/rpms/package_version.sh mercury lib mercury_libfabric)") +pkgs+=("$(utils/rpms/package_version.sh pmdk lib pmemobj)") +pkgs+=("$(utils/rpms/package_version.sh pmdk debug pmemobj)") +pkgs+=("$(utils/rpms/package_version.sh pmdk debug pmem)") +pkgs+=("fuse3") +pkgs+=("gotestsum") +pkgs+=("gperftools-devel") +pkgs+=("hwloc-devel") +pkgs+=("libasan") +pkgs+=("libipmctl-devel") +pkgs+=("libyaml-devel") +pkgs+=("numactl") +pkgs+=("numactl-devel") +pkgs+=("openmpi${OPENMPI_VER}") +pkgs+=("patchelf") +pkgs+=("pciutils-devel") +pkgs+=("protobuf-c") +pkgs+=("valgrind-devel") + +if [ "${code_coverage}" == "true" ] ; then + pkgs+=("$(utils/rpms/package_version.sh bullseye normal)") +fi # output with trailing newline suppressed -echo -e "$pkgs\c" +printf '%s' "${pkgs[*]}" exit 0 diff --git a/ci/unit/test_main.sh b/ci/unit/test_main.sh index 8cdb630233a..8d4836fd630 100755 --- a/ci/unit/test_main.sh +++ b/ci/unit/test_main.sh @@ -20,23 +20,13 @@ mkdir test_results chmod 777 test_results # Check if this is a Bulleye stage -USE_BULLSEYE=false BDEV_TEST=false case $STAGE_NAME in - *Bullseye**) - USE_BULLSEYE=true - ;; *bdev**) BDEV_TEST=true ;; esac -if $USE_BULLSEYE; then - rm -rf bullseye - mkdir -p bullseye - tar -C bullseye --strip-components=1 -xf bullseye.tar -fi - NODE=${NODELIST%%,*} : "${PYTHON_VERSION:=3.11}" diff --git a/ci/unit/test_main_node.sh b/ci/unit/test_main_node.sh index 38dace0cb3e..229d2c1043f 100755 --- a/ci/unit/test_main_node.sh +++ b/ci/unit/test_main_node.sh @@ -26,18 +26,12 @@ sudo mount --bind build "${SL_SRC_DIR}" log_prefix="unit_test" -: "${BULLSEYE:=}" -if [ -n "$BULLSEYE" ]; then - pushd "${SL_SRC_DIR}/bullseye" - set +x - echo + sudo ./install --quiet --key "**********" --prefix /opt/BullseyeCoverage - sudo ./install --quiet --key "${BULLSEYE}" --prefix /opt/BullseyeCoverage - set -x - popd - rm -rf bullseye +: "${BULLSEYE_DIR:=/opt/BullseyeCoverage}" +if [[ -d "${BULLSEYE_DIR}" ]]; then export COVFILE="${SL_SRC_DIR}/test.cov" - export PATH="/opt/BullseyeCoverage/bin:$PATH" - log_prefix="covc_test" + export PATH="${BULLSEYE_DIR}/bin:$PATH" + cp "${BULLSEYE_DIR}/daos/test.cov" "${COVFILE}" + ls -al "${COVFILE}" fi cd "${SL_SRC_DIR}" @@ -98,7 +92,21 @@ pip install --upgrade pip pip install --requirement requirements-utest.txt pip install /opt/daos/lib/daos/python/ +if [[ -n "${COVFILE:-}" ]]; then + echo "Code coverage before running unit tests:" + /opt/BullseyeCoverage/bin/covdir --file "${COVFILE}" || true +fi + HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" \ -NO_PROXY="${DAOS_NO_PROXY:-}" \ -utils/run_utest.py $RUN_TEST_VALGRIND \ - --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" $SUDO_ARG + NO_PROXY="${DAOS_NO_PROXY:-}" \ + utils/run_utest.py $RUN_TEST_VALGRIND \ + --no-fail-on-error $VDB_ARG --log_dir="$test_log_dir" $SUDO_ARG + +if [[ -n "${COVFILE:-}" ]]; then + echo "Code coverage after running unit tests:" + /opt/BullseyeCoverage/bin/covdir --file "${COVFILE}" || true + + # Copy bullseye file to expected location for stashing + cp "${COVFILE}" /tmp/test.cov + ls -al /tmp/test.cov || true +fi diff --git a/ci/unit/test_nlt.sh b/ci/unit/test_nlt.sh index 23e3bc8b549..346536454ef 100755 --- a/ci/unit/test_nlt.sh +++ b/ci/unit/test_nlt.sh @@ -3,7 +3,7 @@ # This is the script used for running utils/node_local_test.py (NLT) set -uex -rm -rf nlt_logs +rm -rf nlt_logs nlt_bullseye_logs # Remove any logs from a previous run rm -rf dnt.*.memcheck.xml vm_test/ diff --git a/ci/unit/test_nlt_node.sh b/ci/unit/test_nlt_node.sh index 53630da411a..f2c053dda9c 100755 --- a/ci/unit/test_nlt_node.sh +++ b/ci/unit/test_nlt_node.sh @@ -45,11 +45,29 @@ pip install /opt/daos/lib/daos/python/ sudo prlimit --nofile=1024:262144 --pid $$ prlimit -n +: "${BULLSEYE_DIR:=/opt/BullseyeCoverage}" +if [ -d "${BULLSEYE_DIR}" ]; then + export COVFILE="/tmp/test.cov" + export PATH="${BULLSEYE_DIR}/bin:$PATH" + cp "${BULLSEYE_DIR}/daos/test.cov" "${COVFILE}" + ls -al "${COVFILE}" +fi + mkdir -p nlt_logs sudo mount -t tmpfs tmpfs nlt_logs sudo chown jenkins:jenkins nlt_logs +if [ -e "${COVFILE:-}" ]; then + echo "Code coverage before running unit tests:" + /opt/BullseyeCoverage/bin/covdir --file "${COVFILE}" || true +fi + TMPDIR="$(pwd)/nlt_logs" \ HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" \ NO_PROXY="${DAOS_NO_PROXY:-}" \ exec ./utils/node_local_test.py "$@" + +if [ -e "${COVFILE:-}" ]; then + echo "Code coverage after running unit tests:" + /opt/BullseyeCoverage/bin/covdir --file "${COVFILE}" || true +fi diff --git a/ci/unit/test_nlt_post.sh b/ci/unit/test_nlt_post.sh index db39d9c7c3d..7e5e1b83e84 100755 --- a/ci/unit/test_nlt_post.sh +++ b/ci/unit/test_nlt_post.sh @@ -7,8 +7,14 @@ set -uex NODE="${NODELIST%%,*}" -rm -rf nlt_logs -mkdir nlt_logs +test_log_dir="${1:-}" +if [ -z "$test_log_dir" ]; then + echo "test_nlt_post: The test log directory argument is missing!" + exit 1 +fi + +rm -rf "$test_log_dir" +mkdir "$test_log_dir" # Copy any log files. Use rsync filters here to allow us to specify # all files we want to copy, as it's much more flexible than using @@ -16,14 +22,14 @@ mkdir nlt_logs # Assuming that node_local_test.py is run with --class-name, # the logs will be in build/nlt_logs/ on the node. -rsync -v -rlpt -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/nlt_logs/ \ +rsync -v -rlpt -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":"build/${test_log_dir}/" \ --filter="include dnt*.log" --filter="include dnt*.log.bz2" \ --filter="include dnt_fi_*_logs" --filter="include */" \ - --filter="exclude *" nlt_logs/ + --filter="include test.cov" --filter="exclude *" "${test_log_dir}/" rsync -v -dpt -z -e "ssh $SSH_KEY_ARGS" jenkins@"$NODE":build/ \ --filter="include nlt*.json" --filter="include dnt*.xml" \ - --filter="include nltir.xml" --filter="include nltr.json" \ + --filter="include nltir*.xml" --filter="include nltr*.json" \ --filter="include nlt-junit.xml" --filter="exclude *" ./ mkdir -p vm_test diff --git a/ci/unit/test_post_always.sh b/ci/unit/test_post_always.sh index 5d48ab5482d..7d3d570fe0c 100755 --- a/ci/unit/test_post_always.sh +++ b/ci/unit/test_post_always.sh @@ -25,17 +25,11 @@ ssh "$SSH_KEY_ARGS" jenkins@"$NODE" \ "DAOS_BASE=$DAOS_BASE \ $(cat "$mydir/test_post_always_node.sh")" -case $STAGE_NAME in - *Bullseye*) - test_log_dir="covc_test_logs" - ;; - *memcheck*) - test_log_dir="unit_test_memcheck_logs" - ;; - *Unit*) - test_log_dir="unit_test_logs" - ;; -esac +test_log_dir="${1:-}" +if [ -z "$test_log_dir" ]; then + echo "test_post_always: The test log directory argument is missing!" + exit 1 +fi mkdir -p "$test_log_dir" diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index aa96f4c941f..b268badf093 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -679,33 +679,31 @@ def _setup_compiler(self): os.remove(covfile) commands = [['$COV01', '-1'], ['$COV01', '-s'], - ['$CVS', '--add', '!**/src/cart/test/utest/'], + ['$CVS', '--add', '!**/src/bio/smd/tests/'], + ['$CVS', '--add', '!**/src/cart/crt_self_test.h'], + ['$CVS', '--add', '!**/src/cart/crt_self_test_client.c'], + ['$CVS', '--add', '!**/src/cart/crt_self_test_service.c'], + ['$CVS', '--add', '!**/src/client/api/tests/'], ['$CVS', '--add', '!**/src/common/tests/'], + ['$CVS', '--add', '!**/src/common/tests_dmg_helpers.c'], + ['$CVS', '--add', '!**/src/common/tests_lib.c'], + ['$CVS', '--add', '!**/src/dtx/tests/'], + ['$CVS', '--add', '!**/src/engine/tests/'], + ['$CVS', '--add', '!**/src/gurt/examples/'], ['$CVS', '--add', '!**/src/gurt/tests/'], - ['$CVS', '--add', '!**/src/iosrv/tests/'], ['$CVS', '--add', '!**/src/mgmt/tests/'], ['$CVS', '--add', '!**/src/object/tests/'], + ['$CVS', '--add', '!**/src/placement/ring_map.c'], ['$CVS', '--add', '!**/src/placement/tests/'], ['$CVS', '--add', '!**/src/rdb/tests/'], ['$CVS', '--add', '!**/src/security/tests/'], - ['$CVS', '--add', '!**/src/utils/self_test/'], - ['$CVS', '--add', '!**/src/utils/ctl/'], - ['$CVS', '--add', '!**/src/vea/tests/'], - ['$CVS', '--add', '!**/src/vos/tests/'], - ['$CVS', '--add', '!**/src/engine/tests/'], ['$CVS', '--add', '!**/src/tests/'], - ['$CVS', '--add', '!**/src/bio/smd/tests/'], - ['$CVS', '--add', '!**/src/cart/crt_self_test.h'], - ['$CVS', '--add', '!**/src/cart/crt_self_test_client.c'], - ['$CVS', '--add', '!**/src/cart/crt_self_test_service.c'], - ['$CVS', '--add', '!**/src/client/api/tests/'], - ['$CVS', '--add', '!**/src/client/dfuse/test/'], - ['$CVS', '--add', '!**/src/gurt/examples/'], - ['$CVS', '--add', '!**/src/utils/crt_launch/'], ['$CVS', '--add', '!**/src/utils/daos_autotest.c'], - ['$CVS', '--add', '!**/src/placement/ring_map.c'], - ['$CVS', '--add', '!**/src/common/tests_dmg_helpers.c'], - ['$CVS', '--add', '!**/src/common/tests_lib.c']] + ['$CVS', '--add', '!**/src/utils/crt_launch/'], + ['$CVS', '--add', '!**/src/utils/ctl/'], + ['$CVS', '--add', '!**/src/utils/self_test/'], + ['$CVS', '--add', '!**/src/vea/tests/'], + ['$CVS', '--add', '!**/src/vos/tests/']] if not RUNNER.run_commands(commands): raise BuildFailure("cov01") diff --git a/src/tests/ftest/control/version.py b/src/tests/ftest/control/version.py index 47ddd1f6752..bca1c9f220f 100644 --- a/src/tests/ftest/control/version.py +++ b/src/tests/ftest/control/version.py @@ -1,16 +1,18 @@ ''' (C) Copyright 2018-2023 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent ''' import json import re +from getpass import getuser from apricot import TestWithServers from ClusterShell.NodeSet import NodeSet +from command_utils_base import EnvironmentVariables from general_utils import append_error, report_errors -from run_utils import run_remote +from run_utils import command_as_user, run_remote from server_utils_base import DaosServerCommandRunner @@ -63,7 +65,9 @@ def test_version(self): # Get daos_agent version. daos_agent_version = None - daos_agent_cmd = "daos_agent --json version" + env = EnvironmentVariables() + self.test_env.add_to_env(env, 'bullseye_file') + daos_agent_cmd = command_as_user("daos_agent --json version", getuser(), env) result = run_remote(self.log, NodeSet(self.hostlist_servers[0]), daos_agent_cmd) if not result.passed: self.fail("Failed to get daos_agent version") diff --git a/src/tests/ftest/dfuse/bash.py b/src/tests/ftest/dfuse/bash.py index 69ab02adff1..301291d340e 100644 --- a/src/tests/ftest/dfuse/bash.py +++ b/src/tests/ftest/dfuse/bash.py @@ -7,6 +7,7 @@ import os from apricot import TestWithServers +from command_utils_base import EnvironmentVariables from dfuse_utils import get_dfuse, start_dfuse from host_utils import get_local_host from run_utils import run_remote @@ -47,16 +48,18 @@ def run_bashcmd(self, il_lib=None, compatible_mode=False, nobypass=False): dd_count = 512 dd_blocksize = 512 + env = EnvironmentVariables() if il_lib is not None: lib_path = os.path.join(self.prefix, "lib64", il_lib) if compatible_mode: - env_str = f"export LD_PRELOAD={lib_path}; export D_IL_COMPATIBLE=1; " + env["LD_PRELOAD"] = lib_path + env["D_IL_COMPATIBLE"] = "1" else: - env_str = f"export LD_PRELOAD={lib_path}; " + env["LD_PRELOAD"] = lib_path if nobypass: - env_str = env_str + "export D_IL_NO_BYPASS=1; " - else: - env_str = "" + env["D_IL_NO_BYPASS"] = "1" + self.test_env.add_to_env(env, 'bullseye_file') + env_str = env.to_export_str() # Create a pool if one does not already exist. self.log_step('Creating a single pool and container') diff --git a/src/tests/ftest/io/unaligned_io.yaml b/src/tests/ftest/io/unaligned_io.yaml index 3bcc450c8c5..8319203693c 100644 --- a/src/tests/ftest/io/unaligned_io.yaml +++ b/src/tests/ftest/io/unaligned_io.yaml @@ -19,7 +19,6 @@ server_config: - DAOS_MD_CAP=1024 - DD_MASK=mgmt,md,any - D_LOG_FILE_APPEND_PID=1 - - COVFILE=/tmp/test.cov storage: auto 1: targets: 8 @@ -33,7 +32,6 @@ server_config: - DAOS_MD_CAP=1024 - DD_MASK=mgmt,md,any - D_LOG_FILE_APPEND_PID=1 - - COVFILE=/tmp/test.cov storage: auto pool: diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 829ee0f7bfa..f231b3675cd 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -1,6 +1,6 @@ """ (C) Copyright 2020-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -1689,6 +1689,7 @@ def get_dmg_command(self, index=0): self.server_group, dmg_cert_dir, self.bin, dmg_config_file, dmg_config_temp, self.mgmt_svc_replicas_suffix) dmg_cmd.hostlist = self.mgmt_svc_replicas + self.test_env.add_to_env(dmg_cmd.env, 'bullseye_file') return dmg_cmd def get_daos_command(self): diff --git a/src/tests/ftest/util/code_coverage_utils.py b/src/tests/ftest/util/code_coverage_utils.py index 3a4042872b2..8743209f3e5 100644 --- a/src/tests/ftest/util/code_coverage_utils.py +++ b/src/tests/ftest/util/code_coverage_utils.py @@ -1,13 +1,16 @@ """ (C) Copyright 2022-2023 Intel Corporation. + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ +import glob import os +import shutil # pylint: disable=import-error,no-name-in-module from util.collection_utils import archive_files -from util.run_utils import run_remote +from util.run_utils import run_local, run_remote class CodeCoverage(): @@ -73,7 +76,7 @@ def setup(self, logger, result): logger.debug( "Updating %s bullseye code coverage file permissions", self.__test_env.bullseye_file) - command = ["chmod", "777", self.__test_env.bullseye_file] + command = ["chmod", "666", self.__test_env.bullseye_file] if not run_remote(logger, self.__hosts, " ".join(command)).passed: message = "Error updating bullseye code coverage file on at least one host" result.fail_test(logger, "Run", message, None) @@ -97,28 +100,29 @@ def finalize(self, logger, job_results_dir, result): return True logger.debug("-" * 80) - logger.debug("Collecting bullseye code coverage information on %s:", self.__hosts) + logger.debug("Collecting bullseye code coverage information from %s:", self.__hosts) bullseye_path, bullseye_file = os.path.split(self.__test_env.bullseye_file) bullseye_dir = os.path.join(job_results_dir, "bullseye_coverage_logs") status = archive_files( logger, "bullseye coverage log files", self.__hosts, bullseye_path, - "".join([bullseye_file, "*"]), bullseye_dir, 1, None, 900, result) - - # Rename bullseye_coverage_logs.host/test.cov.* to bullseye_coverage_logs/test.host.cov.* - for item in os.listdir(job_results_dir): - item_full = os.path.join(job_results_dir, item) - if os.path.isdir(item_full) and "bullseye_coverage_logs" in item: - host_ext = os.path.splitext(item) - if len(host_ext) > 1: - os.makedirs(bullseye_dir, exist_ok=True) - for name in os.listdir(item_full): - old_file = os.path.join(item_full, name) - if os.path.isfile(old_file): - new_name = name.split(".") - new_name.insert(1, host_ext[-1][1:]) - new_file_name = ".".join(new_name) - new_file = os.path.join(bullseye_dir, new_file_name) - logger.debug("Renaming %s to %s", old_file, new_file) - os.rename(old_file, new_file) - return status == 0 + "".join([bullseye_file, "*"]), bullseye_dir, 1, None, 900, result, compress=False) + if status != 0: + message = "Error retrieving bullseye code coverage files from at least one host" + result.fail_test(logger, "Run", message, None) + return False + + # Combine the bullseye code coverage files from each host into one file + logger.debug("Merging bullseye code coverage files") + os.makedirs(bullseye_dir, exist_ok=True) + shutil.copy(self.__test_env.bullseye_src, bullseye_dir) + command = ("/opt/BullseyeCoverage/bin/covmerge --no-banner --file " + f"{os.path.join(bullseye_dir, 'test.cov')} {bullseye_dir}.*/test.cov") + if not run_local(logger, command).passed: + message = "Error merging bullseye code coverage files" + result.fail_test(logger, "Run", message, None) + return False + for directory in glob.glob(f"{bullseye_dir}.*"): + if os.path.isdir(directory): + shutil.rmtree(directory, ignore_errors=True) + return True diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 7c0d3ccf08d..5c25f116510 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -225,7 +225,7 @@ def check_server_storage(logger, test, test_result, stage): def archive_files(logger, summary, hosts, source, pattern, destination, depth, threshold, timeout, - test_result, test=None): + test_result, test=None, compress=True): # pylint: disable=too-many-arguments """Archive the files from the source to the destination. @@ -241,6 +241,7 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t timeout (int): number of seconds to wait for the command to complete. test_result (TestResult): the test result used to update the status of the test test (TestInfo, optional): the test information. Defaults to None. + compress (bool, optional): compress files before transfer. Defaults to True Returns: int: status code: 0 = success, 16 = failure @@ -276,6 +277,10 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t # Remove any empty files return_code |= remove_empty_files(logger, file_hosts, source, pattern, depth, test_result) + if compress: + # Compress any files larger than 1 MB + return_code |= compress_files(logger, file_hosts, source, pattern, depth, test_result) + # Compress any files larger than 1 MB return_code |= compress_files(logger, file_hosts, source, pattern, depth, test_result) diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index f5d2885f94e..d8523c9b390 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2018-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -61,6 +61,9 @@ def __init__(self, namespace, command, path="", subprocess=False, check_results= self.output_check = "both" self.verbose = True self.env = EnvironmentVariables() + _cov_file = os.path.join(os.sep, "tmp", "test.cov") + if os.path.exists(_cov_file): + self.env["COVFILE"] = _cov_file # User to run the command as. "root" is equivalent to sudo self.run_user = run_user diff --git a/src/tests/ftest/util/environment_utils.py b/src/tests/ftest/util/environment_utils.py index 30c00204b2a..21fe4f67b02 100644 --- a/src/tests/ftest/util/environment_utils.py +++ b/src/tests/ftest/util/environment_utils.py @@ -95,6 +95,7 @@ def log_environment(logger): class TestEnvironment(): """Collection of test environment variables.""" + # pylint: disable=too-many-public-methods __ENV_VAR_MAP = { 'app_dir': 'DAOS_TEST_APP_DIR', @@ -180,8 +181,7 @@ def set_defaults(self, logger, servers=None, clients=None, provider=None, insecu if self.insecure_mode is None: self.insecure_mode = "True" if self.bullseye_src is None: - self.bullseye_src = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "..", "test.cov") + self.bullseye_src = os.path.join(os.sep, "opt", "BullseyeCoverage", "daos", "test.cov") if self.bullseye_file is None: self.bullseye_file = os.path.join(os.sep, "tmp", "test.cov") if self.daos_prefix is None: @@ -195,6 +195,25 @@ def set_defaults(self, logger, servers=None, clients=None, provider=None, insecu if self.server_config is None: self.server_config = os.path.join(self.log_dir, "configs", "daos_server.yml") + def add_to_env(self, env, name): + """Add the provided environment variable dictionary with the test environment value. + + If the environment variable dictionary already contains a value for the test environment + variable, do not override it. + + Args: + env (EnvironmentVariables): environment variable dictionary to update + name (str): test environment variable name to update in the provided dictionary + + Raises: + TestEnvironmentException: if the name is not a valid test environment variable name + """ + if name not in self.__ENV_VAR_MAP: + raise TestEnvironmentException(f"Invalid environment variable name: {name}") + + if self.__ENV_VAR_MAP[name] not in env: + env[self.__ENV_VAR_MAP[name]] = os.environ.get(self.__ENV_VAR_MAP[name]) + def __set_value(self, key, value): """Set the test environment variable. diff --git a/utils/node_local_test.py b/utils/node_local_test.py index c51b26ad33f..ade57f979e5 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -117,7 +117,7 @@ def __init__(self, json_file, args): prefix='dnt_dfuse_') self.tmp_dir = None if args.class_name: - self.tmp_dir = join('nlt_logs', args.class_name) + self.tmp_dir = join(args.log_base_dir, args.class_name) if os.path.exists(self.tmp_dir): for old_file in os.listdir(self.tmp_dir): os.unlink(join(self.tmp_dir, old_file)) @@ -446,6 +446,11 @@ def get_base_env(clean=False): if no_proxy: env['NO_PROXY'] = no_proxy + # If set, retain the COVFILE for bullseye code coverage + covfile = os.environ.get('COVFILE') + if covfile: + env['COVFILE'] = covfile + # Enable this to debug memory errors, it has a performance impact but will scan the heap # for corruption. See DAOS-12735 for why this can cause problems in practice. # env['MALLOC_CHECK_'] = '3' @@ -6862,6 +6867,7 @@ def main(): parser.add_argument('--exclude-test', action='append', help='space separated list of tests to exclude') parser.add_argument('--valgrind_verbose', action='store_true', help='Use --verbose w/ valgrind') + parser.add_argument('--log-base-dir', default='nlt_logs', help='Base directory for log files') parser.add_argument('mode', nargs='*') args = parser.parse_args() diff --git a/utils/rpms/build_packages.sh b/utils/rpms/build_packages.sh index 6c72cfd52e6..1148d6cbac0 100755 --- a/utils/rpms/build_packages.sh +++ b/utils/rpms/build_packages.sh @@ -13,5 +13,9 @@ if [[ "${build_type}" =~ deps|all ]]; then utils/rpms/daos-spdk.sh fi if [[ "${build_type}" =~ daos|all ]]; then - utils/rpms/daos.sh + utils/rpms/daos.sh false +fi +if [[ "${build_type}" =~ bullseye ]]; then + utils/rpms/bullseye.sh + utils/rpms/daos.sh true fi diff --git a/utils/rpms/bullseye.changelog b/utils/rpms/bullseye.changelog new file mode 100644 index 00000000000..5968300dee7 --- /dev/null +++ b/utils/rpms/bullseye.changelog @@ -0,0 +1,9 @@ +%changelog +* Fri Feb 6 2026 Phillip Henderson 9.24.0-1 +- Update to 9.24.0 + +* Fri Mar 31 2023 Brian J. Murrell - 9.1.1-1 +- Update to 9.1.1 + +* Fri Feb 19 2021 Brian J. Murrell - 8.21.4-1 +- First packaged version diff --git a/utils/rpms/bullseye.sh b/utils/rpms/bullseye.sh new file mode 100644 index 00000000000..e603964c980 --- /dev/null +++ b/utils/rpms/bullseye.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Copyright 2026 Hewlett Packard Enterprise Development LP +# +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +# Script for building the bullseye rpm package +set -eEuo pipefail + +root="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +. "${root}/fpm_common.sh" + +: "${SL_BULLSEYE_PREFIX:=/opt/BullseyeCoverage}" +if [ ! -d "${SL_BULLSEYE_PREFIX}" ]; then + echo "bullseye must be installed or built in ${SL_BULLSEYE_PREFIX}" + exit 0 +fi + +VERSION="${bullseye_version}" +RELEASE="${bullseye_release}" +LICENSE="Proprietary" +ARCH="${isa}" +DESCRIPTION="The BullseyeCoverage compiler" +URL="https://www.bullseye.com/index.html" +RPM_CHANGELOG="bullseye.changelog" +PACKAGE_TYPE="dir" +files=() + +# Add bullseye files +FILTER_LIST=("${SL_BULLSEYE_PREFIX}/sample") +readarray -t dir_list < <(find "${SL_BULLSEYE_PREFIX}" -mindepth 1 -maxdepth 1 -type d) +for dir in "${dir_list[@]}"; do + if filter_file "${dir}"; then + continue + fi + readarray -t dir_file_list < <(find "${dir}" -mindepth 1 -maxdepth 1 -type f) + TARGET_PATH="${dir}" + for dir_file in "${dir_file_list[@]}"; do + list_files files "${dir_file}" + append_install_list "${files[@]}" + done +done + +# Add test.cov file +TARGET_PATH="${SL_BULLSEYE_PREFIX}/daos" +list_files files "test.cov" +append_install_list "${files[@]}" + +# Create tar file containing all source files for the covhtml command +readarray -t src_file_list < <("${SL_BULLSEYE_PREFIX}/bin/covmgr" -l --file test.cov) +if [ ${#src_file_list[@]} -gt 0 ]; then + tar -czf "${tmp}/bullseye_sources.tar.gz" "${src_file_list[@]}" 2>/dev/null || { + echo "Warning: Some source files may not exist, creating tar with existing files only" + existing_files=() + for src_file in "${src_file_list[@]}"; do + if [ -f "${src_file}" ]; then + existing_files+=("${src_file}") + fi + done + if [ ${#existing_files[@]} -gt 0 ]; then + tar -czf "${tmp}/bullseye_sources.tar.gz" "${existing_files[@]}" + echo "Created tar file with ${#existing_files[@]} existing source files" + else + echo "No source files found to archive" + fi + } +else + echo "No source files found in src_file_list" +fi +list_files files "${tmp}/bullseye_sources.tar.gz" +append_install_list "${files[@]}" + +# Fix file permissions +cat << EOF > "${tmp}/post_install_bullseye" +chmod 666 ${SL_BULLSEYE_PREFIX}/daos/test.cov +chmod 666 ${SL_BULLSEYE_PREFIX}/daos/bullseye_sources.tar.gz +EOF +EXTRA_OPTS+=("--after-install" "${tmp}/post_install_bullseye") + +build_package "bullseye" diff --git a/utils/rpms/bullseye_build.sh b/utils/rpms/bullseye_build.sh new file mode 100644 index 00000000000..fb0eea2ab0e --- /dev/null +++ b/utils/rpms/bullseye_build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -uex + +bullseye_key="${1:-}" + +# Get the bullseye version +mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +pushd "${mydir}/../.." || exit 1 +source utils/rpms/package_info.sh +popd + +: "${SL_BULLSEYE_PREFIX:=/opt/BullseyeCoverage}" +bullseye_url="https://www.bullseye.com/download-archive/${bullseye_version%.*}" +bullseye_src="${bullseye_url}/BullseyeCoverage-${bullseye_version}-Linux-x64.tar.xz" +bullseye_out="bullseye.tar.xz" + +if [ -n "${DAOS_HTTPS_PROXY:-}" ]; then + curl --proxy "${DAOS_HTTPS_PROXY}" "${bullseye_src}" --retry 10 --retry-max-time 60 --silent --show-error -o "${bullseye_out}" +else + curl "${bullseye_src}" --retry 10 --retry-max-time 60 --silent --show-error -o "${bullseye_out}" +fi + +mkdir -p bullseye +tar -C bullseye --strip-components=1 -xf "${bullseye_out}" +pushd bullseye +set +x +echo + sudo ./install --quiet --key "**********" --prefix "${SL_BULLSEYE_PREFIX}" +sudo ./install --quiet --key "${bullseye_key}" --prefix "${SL_BULLSEYE_PREFIX}" +set -x +popd diff --git a/utils/rpms/daos.sh b/utils/rpms/daos.sh index 5ecfe36aefd..382121b6c5b 100755 --- a/utils/rpms/daos.sh +++ b/utils/rpms/daos.sh @@ -15,11 +15,16 @@ if [ -z "${SL_PREFIX:-}" ]; then exit 1 fi +code_coverage="${1:-false}" daoshome="${prefix}/lib/daos" server_svc_name="daos_server.service" agent_svc_name="daos_agent.service" sysctl_script_name="10-daos_server.conf" daos_log_dir="/var/log/daos" +base_name="daos" +if [ "$code_coverage" != "false" ]; then + base_name="${base_name}-bullseye" +fi VERSION=${daos_version} RELEASE=${daos_release} @@ -70,9 +75,23 @@ install_list+=("${tmp}${sysconfdir}/daos/certs=${sysconfdir}/daos") EXTRA_OPTS+=("--rpm-attr" "0755,root,root:${sysconfdir}/daos/certs") +# Code coverage +if [[ "${code_coverage}" != "false" ]]; then + code_coverage_prefix="/opt/BullseyeCoverage" + EXTERNAL_DEPENDS=("${bullseye_normal}") + TARGET_PATH="${code_coverage_prefix}/daos" + list_files files "test.cov" + append_install_list "${files[@]}" + + cat << EOF > "${tmp}/post_install_bullseye" +chmod 666 ${code_coverage_prefix}/daos/test.cov +EOF + EXTRA_OPTS+=("--after-install" "${tmp}/post_install_bullseye") +fi + DEPENDS=( "mercury >= ${mercury_version}" ) DEPENDS+=( "${isal_crypto_lib} >= ${isal_crypto_version}" ) -build_package "daos" +build_package "${base_name}" # Only build server RPMs if we built the server if [ -f "${SL_PREFIX}/bin/daos_server" ]; then @@ -187,10 +206,10 @@ EOF EXTRA_OPTS+=("--rpm-attr" "4750,root,daos_server:${bindir}/daos_server_helper") EXTRA_OPTS+=("--rpm-attr" "2755,root,daos_server:${bindir}/daos_server") - DEPENDS=( "daos = ${VERSION}-${RELEASE}" "daos-spdk = ${daos_spdk_full}" ) + DEPENDS=( "${base_name} = ${VERSION}-${RELEASE}" "daos-spdk = ${daos_spdk_full}" ) DEPENDS+=( "${pmemobj_lib} = ${pmdk_full}" "${argobots_lib} >= ${argobots_full}" ) DEPENDS+=( "${isal_crypto_lib} >= ${isal_crypto_version}" "numactl" "pciutils" ) - build_package "daos-server" + build_package "${base_name}-server" TARGET_PATH="${bindir}" list_files files "${SL_PREFIX}/bin/dtx_tests" \ @@ -209,8 +228,9 @@ EOF clean_bin "${files[@]}" append_install_list "${files[@]}" - DEPENDS=("daos-server = ${VERSION}-${RELEASE}" "daos-admin = ${VERSION}-${RELEASE}") - build_package "daos-server-tests" + DEPENDS=("${base_name}-server = ${VERSION}-${RELEASE}") + DEPENDS+=("${base_name}-admin = ${VERSION}-${RELEASE}") + build_package "${base_name}-server-tests" else echo "Skipping server packaging because server is not built" fi @@ -230,8 +250,8 @@ list_files files "${SL_PREFIX}/etc/daos_control.yml" append_install_list "${files[@]}" CONFIG_FILES+=("${TARGET_PATH}/daos_control.yml") -DEPENDS=( "daos = ${VERSION}-${RELEASE}" ) -build_package "daos-admin" +DEPENDS=( "${base_name} = ${VERSION}-${RELEASE}" ) +build_package "${base_name}-admin" # daos-client package TARGET_PATH="${bindir}" @@ -314,8 +334,8 @@ EOF fi EXTERNAL_DEPENDS=("fuse3") -DEPENDS=("daos = ${VERSION}-${RELEASE}") -build_package "daos-client" +DEPENDS=("${base_name} = ${VERSION}-${RELEASE}") +build_package "${base_name}-client" # client-tests TARGET_PATH="${daoshome}/TESTING" @@ -379,9 +399,10 @@ EXTERNAL_DEPENDS+=("${ndctl_dev}") if [[ "${DISTRO:-el8}" =~ el ]]; then EXTERNAL_DEPENDS+=("daxctl-devel") fi -DEPENDS=( "daos-client = ${VERSION}-${RELEASE}" "daos-admin = ${VERSION}-${RELEASE}") -DEPENDS+=("daos-devel = ${VERSION}-${RELEASE}") -build_package "daos-client-tests" +DEPENDS=( "${base_name}-client = ${VERSION}-${RELEASE}") +DEPENDS+=("${base_name}-admin = ${VERSION}-${RELEASE}") +DEPENDS+=("${base_name}-devel = ${VERSION}-${RELEASE}") +build_package "${base_name}-client-tests" TARGET_PATH="${includedir}" list_files files "${SL_PREFIX}/include/*" @@ -399,15 +420,15 @@ list_files files "${SL_PREFIX}/lib/daos/python/*" append_install_list "${files[@]}" EXTERNAL_DEPENDS=("${uuid_lib}") -DEPENDS=("daos-client = ${VERSION}-${RELEASE}") -build_package "${daos_dev}" +DEPENDS=("${base_name}-client = ${VERSION}-${RELEASE}") +build_package "${daos_dev//daos/${base_name}}" if [ "${OUTPUT_TYPE:-rpm}" = "rpm" ]; then EXTERNAL_DEPENDS=("${hdf5_lib}") TARGET_PATH="${libdir}" list_files files "${SL_PREFIX}/lib64/libdaos_serialize.so" append_install_list "${files[@]}" - build_package "daos-serialize" + build_package "${base_name}-serialize" fi if [ -f "${SL_PREFIX}/bin/daos_firmware_helper" ]; then @@ -417,8 +438,8 @@ if [ -f "${SL_PREFIX}/bin/daos_firmware_helper" ]; then EXTRA_OPTS+=("--rpm-attr" "4750,root,daos_server:${bindir}/daos_firmware_helper") - DEPENDS=("daos-server = ${VERSION}-${RELEASE}") - build_package "daos-firmware" + DEPENDS=("${base_name}-server = ${VERSION}-${RELEASE}") + build_package "${base_name}-firmware" fi TARGET_PATH="${libdir}" @@ -432,12 +453,12 @@ list_files files "${SL_PREFIX}/lib64/libdpar_mpi.so" clean_bin "${files[@]}" append_install_list "${files[@]}" # Don't do autoreq, we know we need OpenMPI so add it explicitly -build_package "daos-client-tests-openmpi" "noautoreq" +build_package "${base_name}-client-tests-openmpi" "noautoreq" #shim packages PACKAGE_TYPE="empty" ARCH="noarch" -DEPENDS=("daos-client-tests = ${VERSION}-${RELEASE}") +DEPENDS=("${base_name}-client-tests = ${VERSION}-${RELEASE}") DEPENDS+=("mpifileutils-mpich") DEPENDS+=("testmpio") DEPENDS+=("mpich = 4.1~a1") @@ -448,12 +469,12 @@ DEPENDS+=("MACSio-mpich") DEPENDS+=("simul-mpich") DEPENDS+=("romio-tests") DEPENDS+=("python3-mpi4py-tests >= 3.1.6") -build_package "daos-tests" +build_package "${base_name}-tests" -build_package "daos-client-tests-mpich" +build_package "${base_name}-client-tests-mpich" -DEPENDS=("daos-tests = ${VERSION}-${RELEASE}") -DEPENDS+=("daos-client-tests-openmpi = ${VERSION}-${RELEASE}") -DEPENDS+=("daos-client-tests-mpich = ${VERSION}-${RELEASE}") -DEPENDS+=("daos-serialize = ${VERSION}-${RELEASE}") -build_package "daos-tests-internal" +DEPENDS=("${base_name}-tests = ${VERSION}-${RELEASE}") +DEPENDS+=("${base_name}-client-tests-openmpi = ${VERSION}-${RELEASE}") +DEPENDS+=("${base_name}-client-tests-mpich = ${VERSION}-${RELEASE}") +DEPENDS+=("${base_name}-serialize = ${VERSION}-${RELEASE}") +build_package "${base_name}-tests-internal" diff --git a/utils/rpms/package_info.sh b/utils/rpms/package_info.sh index a2bb6ef8f2b..17c476b92e5 100644 --- a/utils/rpms/package_info.sh +++ b/utils/rpms/package_info.sh @@ -66,6 +66,9 @@ export daos_spdk_full="${daos_spdk_version}-${daos_spdk_release}" export fused_version="1.0.0" export fused_release="3${distro_name}" export fused_full="${fused_version}-${fused_release}" +export bullseye_version="9.23.8" +export bullseye_release="1${distro_name}" +export bullseye_full="${bullseye_version}-${bullseye_release}" set_lib_name openmpi lib openmpi openmpi3 openmpi export openmpi_lib @@ -133,6 +136,9 @@ export uuid_lib set_lib_name hdf5 lib hdf5 hdf5 hdf5 export hdf5_lib +set_lib_name bullseye normal bullseye bullseye bullseye +export bullseye_normal + lmod="Lmod" if [[ "${DISTRO:-el8}" =~ suse ]]; then lmod="lua-lmod" diff --git a/utils/run_utest.py b/utils/run_utest.py index f16288d4d79..329ce99ef8d 100755 --- a/utils/run_utest.py +++ b/utils/run_utest.py @@ -14,7 +14,6 @@ # pylint: disable=broad-except import os import re -import shutil import subprocess # nosec import sys import tempfile @@ -612,21 +611,6 @@ def run_suites(args, suites, results, aio): results.merge(suite.run_suite(args, aio)) -def move_codecov(base): - """Move any code coverage results""" - try: - target = "/tmp/test.cov" - if os.path.isfile(target): - os.unlink(target) - src = os.path.join(base, "test.cov") - if os.path.isfile(src): - print(f"Moving {src} to {target}") - shutil.move(src, target) - except Exception: - print("Exception trying to copy test.cov") - traceback.print_exc() - - def get_args(): """Parse the arguments""" parser = argparse.ArgumentParser(description='Run DAOS unit tests') @@ -709,8 +693,6 @@ def main(): results.create_junit() - move_codecov(path_info["DAOS_BASE"]) - if args.no_fail_on_error: return From 99444d952415e9affe5bb2fa673c58f4849ba421 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 18:12:28 -0400 Subject: [PATCH 02/14] Fix typo. Signed-off-by: Phil Henderson --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 47f6a0d0e3e..512911684b7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1278,7 +1278,6 @@ pipeline { testResults: 'nlt-junit.xml', unstash_opt: true, unstash_tests: false, - image_version: 'el9.7', prov_env_vars: 'VM_CPUS=14', ignore_failure: true, coverage_stash: 'nlt_bullseye')) From 615a57e67abbb248c263c016ae2fe159b5f35812 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 18:44:10 -0400 Subject: [PATCH 03/14] Update CI_FULL_BULLSEYE_REPORT handling Signed-off-by: Phil Henderson --- Jenkinsfile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 512911684b7..3921104863d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -140,14 +140,28 @@ void updateRunStage() { if (params.CI_FULL_BULLSEYE_REPORT) { println("updateRunStage: Detected CI_FULL_BULLSEYE_REPORT, skipping unrelated stages") for (stage in runStage.keySet()) { - runStage[stage] = false if (stage in ['Build on EL 9 with Bullseye', - 'Unit Test', 'Unit Test bdev', + 'Unit Test', + 'Unit Test bdev', 'NLT with Bullseye', 'Functional on EL 9']) { runStage[stage] = true } else if (stage.contains('Functional Hardware')) { runStage[stage] = true + } else if (stage in ['Build on EL 8', + 'Build on EL 9', + 'Build on Leap 15', + 'Unit Test with memcheck', + 'Unit Test bdev with memcheck', + 'Functional on EL 8.8 with Valgrind', + 'Functional on EL 8', + 'Functional on Leap 15', + 'Functional on SLES 15', + 'Functional on Ubuntu 20.04', + 'Fault injection testing', + 'Test RPMs on EL 9.6', + 'Test RPMs on Leap 15.5']) { + runStage[stage] = false } reasons[stage] = "CI_FULL_BULLSEYE_REPORT" } From 629ced7958d94652061030d129e4566fde5519cb Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 18:50:24 -0400 Subject: [PATCH 04/14] Replace runCondition Signed-off-by: Phil Henderson --- Jenkinsfile | 138 +++++++++++++++++++++++++--------------------------- 1 file changed, 67 insertions(+), 71 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3921104863d..0b8468a5159 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -480,9 +480,7 @@ String getScriptOutput(String script, String args='') { * distro the shorthand distro name; defaults to 'el8' * rpmDistro the distro to use for rpm building; defaults to distro * compiler the compiler to use; defaults to 'gcc' - * runCondition optional additional condition to determine if the stage runs, used - * in conjunction with runBuildStage(distro, compiler); defaults - * to true + * runStage Optional additional condition to determine if the stage runs * buildRpms whether or not to build rpms; defaults to true * release the DAOS RPM release value to use; defaults to env.DAOS_RELVAL * dockerBuildArgs optional docker build arguments @@ -497,7 +495,7 @@ def scriptedBuildStage(Map kwargs = [:]) { String distro = kwargs.get('distro', 'el8') String rpmDistro = kwargs.get('rpmDistro', distro) String compiler = kwargs.get('compiler', 'gcc') - Boolean runCondition = kwargs.get('runCondition', true) + Boolean runStage = kwargs.get('runStage', true) Boolean buildRpms = kwargs.get('buildRpms', true) String release = kwargs.get('release', env.DAOS_RELVAL) String dockerBuildArgs = kwargs.get('dockerBuildArgs', '') @@ -511,49 +509,48 @@ def scriptedBuildStage(Map kwargs = [:]) { } return { stage("${name}") { - if (runBuildStage(distro, compiler, runCondition)) { - node('docker_runner') { - println("[${name}] Check out from version control") - checkoutScm(pruneStaleBranch: true) + if (!runStage) { + println("[${name}] Marking build stage as skipped") + Utils.markStageSkippedForConditional("${name}") + return + } + node('docker_runner') { + println("[${name}] Check out from version control") + checkoutScm(pruneStaleBranch: true) - def dockerImage = docker.build(dockerTag, dockerBuildArgs) - try { - dockerImage.inside() { - if (buildRpms) { - sh label: 'Install RPMs', - script: "./ci/rpm/install_deps.sh ${rpmDistro} ${release} ${bullseye}" - // Avoid interpolation of sensitive environment variables - sh label: 'Build deps', - script: "./ci/rpm/build_deps.sh ${bullseye}" + ' ${BULLSEYE_KEY}' - } - job_step_update(sconsBuild(sconsBuildArgs)) - if (buildRpms) { - sh label: 'Generate RPMs', - script: "./ci/rpm/gen_rpms.sh ${rpmDistro} ${release} ${bullseye}" - // Success actions - uploadNewRPMs(uploadTarget, 'success') - } + def dockerImage = docker.build(dockerTag, dockerBuildArgs) + try { + dockerImage.inside() { + if (buildRpms) { + sh label: 'Install RPMs', + script: "./ci/rpm/install_deps.sh ${rpmDistro} ${release} ${bullseye}" + // Avoid interpolation of sensitive environment variables + sh label: 'Build deps', + script: "./ci/rpm/build_deps.sh ${bullseye}" + ' ${BULLSEYE_KEY}' } - } catch (Exception e) { - // Unsuccessful actions - sh """if [ -f config.log ]; then - mv config.log ${artifacts} - fi""" - archiveArtifacts artifacts: "${artifacts}", allowEmptyArchive: true - throw e - } finally { - // Cleanup actions + job_step_update(sconsBuild(sconsBuildArgs)) if (buildRpms) { - uploadNewRPMs(uploadTarget, 'cleanup') + sh label: 'Generate RPMs', + script: "./ci/rpm/gen_rpms.sh ${rpmDistro} ${release} ${bullseye}" + // Success actions + uploadNewRPMs(uploadTarget, 'success') } - jobStatusUpdate(job_status_internal, name) } + } catch (Exception e) { + // Unsuccessful actions + sh """if [ -f config.log ]; then + mv config.log ${artifacts} + fi""" + archiveArtifacts artifacts: "${artifacts}", allowEmptyArchive: true + throw e + } finally { + // Cleanup actions + if (buildRpms) { + uploadNewRPMs(uploadTarget, 'cleanup') + } + jobStatusUpdate(job_status_internal, name) } } - else { - println("[${name}] Marking build stage as skipped") - Utils.markStageSkippedForConditional("${name}") - } println("[${name}] Finished with ${job_status_internal}") } } @@ -568,7 +565,7 @@ def scriptedBuildStage(Map kwargs = [:]) { * name the summary stage name * distro the shorthand distro name; defaults to 'el8' * compiler the compiler to use; defaults to 'gcc' - * runCondition Optional additional condition to determine if the stage runs + * runStage Optional additional condition to determine if the stage runs * dockerBuildArgs optional docker build arguments * installScript optional script to install RPMs * runScriptArgs Map of arguments to pass to runScriptWithStashes() @@ -580,7 +577,7 @@ def scriptedSummaryStage(Map kwargs = [:]) { String name = kwargs.get('name', 'Unknown Summary Stage') String distro = kwargs.get('distro', 'el8') String compiler = kwargs.get('compiler', 'gcc') - Boolean runCondition = kwargs.get('runCondition', true) + Boolean runStage = kwargs.get('runStage', true) String dockerBuildArgs = kwargs.get('dockerBuildArgs', '') String installScript = kwargs.get('installScript', '') Map runScriptArgs = kwargs.get('runScriptArgs', [:]) @@ -590,36 +587,35 @@ def scriptedSummaryStage(Map kwargs = [:]) { return { stage("${name}") { - if (runCondition) { - node('docker_runner') { - println("[${name}] Check out from version control") - checkoutScm(pruneStaleBranch: true) + if (!runStage) { + println("[${name}] Marking summary stage as skipped") + Utils.markStageSkippedForConditional("${name}") + return + } + node('docker_runner') { + println("[${name}] Check out from version control") + checkoutScm(pruneStaleBranch: true) - def dockerImage = docker.build(dockerTag, dockerBuildArgs) - try { - dockerImage.inside() { - if (installScript) { - sh label: 'Install RPMs', - script: "${installScript} ${distro}" - } - job_step_update(runScriptWithStashes(runScriptArgs)) - } - } finally { - // Cleanup actions - if (publishHtmlArgs) { - publishHTML(publishHtmlArgs) - } - if (archiveArtifactsArgs) { - archiveArtifacts(archiveArtifactsArgs) + def dockerImage = docker.build(dockerTag, dockerBuildArgs) + try { + dockerImage.inside() { + if (installScript) { + sh label: 'Install RPMs', + script: "${installScript} ${distro}" } - jobStatusUpdate(job_status_internal, name) + job_step_update(runScriptWithStashes(runScriptArgs)) } + } finally { + // Cleanup actions + if (publishHtmlArgs) { + publishHTML(publishHtmlArgs) + } + if (archiveArtifactsArgs) { + archiveArtifacts(archiveArtifactsArgs) + } + jobStatusUpdate(job_status_internal, name) } } - else { - println("[${name}] Marking summary stage as skipped") - Utils.markStageSkippedForConditional("${name}") - } println("[${name}] Finished with ${job_status_internal}") } } @@ -1049,7 +1045,7 @@ pipeline { name: 'Build on EL 8', distro:'el8', compiler: 'gcc', - runCondition: !paramsValue('CI_FULL_BULLSEYE_REPORT', false), + runStage: runStage['Build on EL 8'], buildRpms: true, release: env.DAOS_RELVAL, dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', @@ -1074,7 +1070,7 @@ pipeline { name: 'Build on EL 9', distro:'el9', compiler: 'gcc', - runCondition: !paramsValue('CI_FULL_BULLSEYE_REPORT', false), + runStage: runStage['Build on EL 9'], buildRpms: true, release: env.DAOS_RELVAL, dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', @@ -1100,7 +1096,7 @@ pipeline { distro:'leap15', rpmDistro: 'suse.lp156', compiler: 'gcc', - runCondition: !paramsValue('CI_FULL_BULLSEYE_REPORT', false), + runStage: runStage['Build on Leap 15'], buildRpms: true, release: env.DAOS_RELVAL, dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', @@ -1765,7 +1761,7 @@ pipeline { name: 'Bullseye Report', distro: 'el9', compiler: 'covc', - runCondition: runStage(['CI_BUILD_BULLSEYE': true]), + runStage: withBullseye(), nodeLabel: 'docker_runner', dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', deps_build: false, From 915a5456877e5cce32d1e527e83f45b4087650aa Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 22:19:52 -0400 Subject: [PATCH 05/14] Set executable permissions. Signed-off-by: Phil Henderson --- Jenkinsfile | 1 + ci/summary/bullseye_report.sh | 0 ci/summary/install_pkgs.sh | 0 utils/rpms/bullseye.sh | 0 utils/rpms/bullseye_build.sh | 0 5 files changed, 1 insertion(+) mode change 100644 => 100755 ci/summary/bullseye_report.sh mode change 100644 => 100755 ci/summary/install_pkgs.sh mode change 100644 => 100755 utils/rpms/bullseye.sh mode change 100644 => 100755 utils/rpms/bullseye_build.sh diff --git a/Jenkinsfile b/Jenkinsfile index 0b8468a5159..588918180be 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1118,6 +1118,7 @@ pipeline { name: 'Build on EL 9 with Bullseye', distro:'el9', compiler: 'covc', + runStage: runStage['Build on EL 9 with Bullseye'], buildRpms: true, release: env.DAOS_RELVAL, dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', diff --git a/ci/summary/bullseye_report.sh b/ci/summary/bullseye_report.sh old mode 100644 new mode 100755 diff --git a/ci/summary/install_pkgs.sh b/ci/summary/install_pkgs.sh old mode 100644 new mode 100755 diff --git a/utils/rpms/bullseye.sh b/utils/rpms/bullseye.sh old mode 100644 new mode 100755 diff --git a/utils/rpms/bullseye_build.sh b/utils/rpms/bullseye_build.sh old mode 100644 new mode 100755 From 7a1c3be153c018bccdb0c3e8fc2b6b8910c1aa80 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 22:41:41 -0400 Subject: [PATCH 06/14] Fix stage skipping. Signed-off-by: Phil Henderson --- Jenkinsfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 588918180be..14fcfb649ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -84,7 +84,7 @@ void updateRunStage() { value = params.get(name, null) if (value instanceof Boolean && !name.startsWith('CI_')) { runStage[name] = value - reasons[name] = "parameter selection/default" + reasons[name] = "parameter selection or default" } } @@ -146,11 +146,14 @@ void updateRunStage() { 'NLT with Bullseye', 'Functional on EL 9']) { runStage[stage] = true + reasons[stage] = "CI_FULL_BULLSEYE_REPORT" } else if (stage.contains('Functional Hardware')) { runStage[stage] = true + reasons[stage] = "CI_FULL_BULLSEYE_REPORT" } else if (stage in ['Build on EL 8', 'Build on EL 9', 'Build on Leap 15', + 'NLT', 'Unit Test with memcheck', 'Unit Test bdev with memcheck', 'Functional on EL 8.8 with Valgrind', @@ -162,8 +165,8 @@ void updateRunStage() { 'Test RPMs on EL 9.6', 'Test RPMs on Leap 15.5']) { runStage[stage] = false + reasons[stage] = "CI_FULL_BULLSEYE_REPORT" } - reasons[stage] = "CI_FULL_BULLSEYE_REPORT" } displayRunStage(reasons) return From a6cb3aa8578d9b563623ec6d0a7a2c8911ff87d8 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 22:55:40 -0400 Subject: [PATCH 07/14] Enhancements. Full-bullseye-report: true Signed-off-by: Phil Henderson --- Jenkinsfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 14fcfb649ad..6c7d2cec183 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -121,23 +121,24 @@ void updateRunStage() { return } - // Handle user setting CI_RPM_TEST_VERSION - if (params.CI_RPM_TEST_VERSION) { - println("updateRunStage: Detected specific RPM version, skipping build/RPM test stages") + // Handle user setting CI_RPM_TEST_VERSION or RPM-test-version + if (rpmTestVersion()) { + println("updateRunStage: Detected RPM test version, skipping build/RPM test stages") for (stage in runStage.keySet()) { if (stage.contains('Build') || stage.contains('Unit Tests') || stage.contains('Test RPMs')) { runStage[stage] = false - reasons[stage] = "specific RPM version" + reasons[stage] = "RPM test version" } } displayRunStage(reasons) return } - // Handle user setting CI_FULL_BULLSEYE_REPORT - if (params.CI_FULL_BULLSEYE_REPORT) { + // Handle user setting CI_FULL_BULLSEYE_REPORT or Full-bullseye-report + if (params.CI_FULL_BULLSEYE_REPORT + || commitPragmas.get('full-bullseye-report', 'false').toLowerCase() == 'true') { println("updateRunStage: Detected CI_FULL_BULLSEYE_REPORT, skipping unrelated stages") for (stage in runStage.keySet()) { if (stage in ['Build on EL 9 with Bullseye', @@ -172,8 +173,9 @@ void updateRunStage() { return } - // Handle user setting CI_BUILD_PACKAGES_ONLY - if (params.CI_BUILD_PACKAGES_ONLY) { + // Handle user setting CI_BUILD_PACKAGES_ONLY or Build-packages-only + if (params.CI_BUILD_PACKAGES_ONLY + || commitPragmas.get('build-packages-only', 'false').toLowerCase() == 'true') { println("updateRunStage: Detected CI_BUILD_PACKAGES_ONLY, skipping unit test stages") for (stage in runStage.keySet()) { if (stage.contains('Unit Tests')) { From aba00daae7b4b3a72df543121322217f0bb98b80 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 22:59:52 -0400 Subject: [PATCH 08/14] Small revert. Signed-off-by: Phil Henderson --- Jenkinsfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6c7d2cec183..97270fae68f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -136,9 +136,8 @@ void updateRunStage() { return } - // Handle user setting CI_FULL_BULLSEYE_REPORT or Full-bullseye-report - if (params.CI_FULL_BULLSEYE_REPORT - || commitPragmas.get('full-bullseye-report', 'false').toLowerCase() == 'true') { + // Handle user setting CI_FULL_BULLSEYE_REPORT + if (params.CI_FULL_BULLSEYE_REPORT) { println("updateRunStage: Detected CI_FULL_BULLSEYE_REPORT, skipping unrelated stages") for (stage in runStage.keySet()) { if (stage in ['Build on EL 9 with Bullseye', @@ -173,9 +172,8 @@ void updateRunStage() { return } - // Handle user setting CI_BUILD_PACKAGES_ONLY or Build-packages-only - if (params.CI_BUILD_PACKAGES_ONLY - || commitPragmas.get('build-packages-only', 'false').toLowerCase() == 'true') { + // Handle user setting CI_BUILD_PACKAGES_ONLY + if (params.CI_BUILD_PACKAGES_ONLY) { println("updateRunStage: Detected CI_BUILD_PACKAGES_ONLY, skipping unit test stages") for (stage in runStage.keySet()) { if (stage.contains('Unit Tests')) { From 558a61164a7901819c7847210d293bd6970daf12 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Tue, 9 Jun 2026 23:54:58 -0400 Subject: [PATCH 09/14] Pass logs to unit test always script. Signed-off-by: Phil Henderson --- Jenkinsfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 97270fae68f..4019341b4d4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1172,6 +1172,8 @@ pipeline { inst_rpms: unitTestInstRpms('el9'), image_version: 'el9.7', compiler: unitTestCompiler(), + test_script: 'ci/unit/test_main.sh', + always_script: 'ci/unit/test_post_always.sh unit_test_logs', coverage_stash: 'unit_test_bullseye')) } post { @@ -1198,6 +1200,8 @@ pipeline { inst_rpms: unitTestInstRpms('el9'), image_version: 'el9.7', compiler: unitTestCompiler(), + test_script: 'ci/unit/test_main.sh', + always_script: 'ci/unit/test_post_always.sh unit_test_bdev_logs', coverage_stash: 'unit_test_bdev_bullseye')) } post { @@ -1322,6 +1326,8 @@ pipeline { inst_rpms: unitTestInstRpms('el9'), image_version: 'el9.7', compiler: 'gcc', + test_script: 'ci/unit/test_main.sh', + always_script: 'ci/unit/test_post_always.sh unit_test_memcheck_logs', unstash_opt: true, ignore_failure: true)) } @@ -1349,6 +1355,8 @@ pipeline { inst_rpms: unitTestInstRpms('el9'), image_version: 'el9.7', compiler: 'gcc', + test_script: 'ci/unit/test_main.sh', + always_script: 'ci/unit/test_post_always.sh unit_test_memcheck_bdev_logs', unstash_opt: true, ignore_failure: true)) } From 5a166752171c6701055efa320a606832a02e3fff Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 10 Jun 2026 13:50:44 -0400 Subject: [PATCH 10/14] Fix daos RPM dependency Signed-off-by: Phil Henderson --- utils/rpms/daos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/rpms/daos.sh b/utils/rpms/daos.sh index 382121b6c5b..f5a0509b5cc 100755 --- a/utils/rpms/daos.sh +++ b/utils/rpms/daos.sh @@ -443,7 +443,7 @@ if [ -f "${SL_PREFIX}/bin/daos_firmware_helper" ]; then fi TARGET_PATH="${libdir}" -DEPENDS=("daos-client-tests = ${VERSION}-${RELEASE}") +DEPENDS=("${base_name}-client-tests = ${VERSION}-${RELEASE}") DEPENDS+=("hdf5-${openmpi_lib}") DEPENDS+=("hdf5-vol-daos-${openmpi_lib}") DEPENDS+=("MACSio-${openmpi_lib}") From 9d2c8e21bb36cd2664524c8106f314fe4dbe08d1 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 10 Jun 2026 16:49:46 -0400 Subject: [PATCH 11/14] Cleanup. Signed-off-by: Phil Henderson --- Jenkinsfile | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4019341b4d4..fe5bfaff23e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -625,27 +625,20 @@ def scriptedSummaryStage(Map kwargs = [:]) { } // Determine if the Build with Bullseye was run and successful -Boolean withBullseye() { - if (runStage['Build on EL 9 with Bullseye'] == false) { - println("withBullseye: Build on EL 9 with Bullseye stage was not selected to run") - return false - } +Boolean bullseyeBuilt() { Map status = job_status_internal['Build_on_EL_9_with_Bullseye'] ?: [:] - println("withBullseye: status=${status}, status.result=${status.result}") + println("bullseyeBuilt: status=${status}, status.result=${status.result}") return status.result == 'SUCCESS' } // Get the inst_rpms argument for the unitTest method -String unitTestInstRpms(String distro='el9') { - if (withBullseye()) { - return getScriptOutput("ci/unit/required_packages.sh ${distro} true") - } - return getScriptOutput("ci/unit/required_packages.sh ${distro}") +String unitTestInstRpms(String distro='el9', Boolean bullseye=false) { + return getScriptOutput("ci/unit/required_packages.sh ${distro} ${bullseye.toString()}") } // Get the compiler argument for the unitTest method -String unitTestCompiler() { - if (withBullseye()) { +String unitTestCompiler(Boolean bullseye=false) { + if (bullseye) { return 'covc' } return 'gcc' @@ -1169,9 +1162,9 @@ pipeline { unitTest(timeout_time: 120, unstash_opt: true, inst_repos: daosRepos(), - inst_rpms: unitTestInstRpms('el9'), + inst_rpms: unitTestInstRpms('el9', bullseyeBuilt()), image_version: 'el9.7', - compiler: unitTestCompiler(), + compiler: unitTestCompiler(bullseyeBuilt()), test_script: 'ci/unit/test_main.sh', always_script: 'ci/unit/test_post_always.sh unit_test_logs', coverage_stash: 'unit_test_bullseye')) @@ -1179,7 +1172,7 @@ pipeline { post { always { unitTestPost artifacts: ['unit_test_logs/'], - compiler: unitTestCompiler() + compiler: unitTestCompiler(bullseyeBuilt()) job_status_update() } } @@ -1197,9 +1190,9 @@ pipeline { unitTest(timeout_time: 120, unstash_opt: true, inst_repos: daosRepos(), - inst_rpms: unitTestInstRpms('el9'), + inst_rpms: unitTestInstRpms('el9', bullseyeBuilt()), image_version: 'el9.7', - compiler: unitTestCompiler(), + compiler: unitTestCompiler(bullseyeBuilt()), test_script: 'ci/unit/test_main.sh', always_script: 'ci/unit/test_post_always.sh unit_test_bdev_logs', coverage_stash: 'unit_test_bdev_bullseye')) @@ -1207,7 +1200,7 @@ pipeline { post { always { unitTestPost artifacts: ['unit_test_bdev_logs/'], - compiler: unitTestCompiler() + compiler: unitTestCompiler(bullseyeBuilt()) job_status_update() } } @@ -1224,7 +1217,7 @@ pipeline { job_step_update( unitTest(timeout_time: 60, inst_repos: daosRepos(), - inst_rpms: unitTestInstRpms('el9'), + inst_rpms: unitTestInstRpms('el9', false), image_version: 'el9.7', compiler: 'gcc', test_script: 'ci/unit/test_nlt.sh' + @@ -1280,7 +1273,7 @@ pipeline { job_step_update( unitTest(timeout_time: 150, inst_repos: daosRepos(), - inst_rpms: unitTestInstRpms('el9'), + inst_rpms: unitTestInstRpms('el9', true), image_version: 'el9.7', compiler: 'covc', test_script: 'ci/unit/test_nlt.sh' + @@ -1323,7 +1316,7 @@ pipeline { job_step_update( unitTest(timeout_time: 160, inst_repos: daosRepos(), - inst_rpms: unitTestInstRpms('el9'), + inst_rpms: unitTestInstRpms('el9', false), image_version: 'el9.7', compiler: 'gcc', test_script: 'ci/unit/test_main.sh', @@ -1352,7 +1345,7 @@ pipeline { job_step_update( unitTest(timeout_time: 180, inst_repos: daosRepos(), - inst_rpms: unitTestInstRpms('el9'), + inst_rpms: unitTestInstRpms('el9', false), image_version: 'el9.7', compiler: 'gcc', test_script: 'ci/unit/test_main.sh', @@ -1773,7 +1766,7 @@ pipeline { name: 'Bullseye Report', distro: 'el9', compiler: 'covc', - runStage: withBullseye(), + runStage: bullseyeBuilt(), nodeLabel: 'docker_runner', dockerBuildArgs: dockerBuildArgs(repo_type: 'stable', deps_build: false, From 4b3b62b06aef42da75626681c762ded57fb3e928 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 10 Jun 2026 22:59:03 -0400 Subject: [PATCH 12/14] Test fixes. Signed-off-by: Phil Henderson --- src/tests/ftest/control/version.py | 9 ++++----- src/tests/ftest/process_core_files.py | 3 ++- src/tests/ftest/util/collection_utils.py | 3 --- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/tests/ftest/control/version.py b/src/tests/ftest/control/version.py index bca1c9f220f..6fbf0d36e3e 100644 --- a/src/tests/ftest/control/version.py +++ b/src/tests/ftest/control/version.py @@ -5,7 +5,6 @@ SPDX-License-Identifier: BSD-2-Clause-Patent ''' import json -import re from getpass import getuser from apricot import TestWithServers @@ -41,16 +40,16 @@ def test_version(self): :avocado: tags=DAOSVersion,test_version """ # Get RPM version. - rpm_command = "rpm -qa | grep daos-server" + rpm_command = ("rpm -q --queryformat '%{VERSION}\n' daos-server daos-bullseye-server " + "2>/dev/null | grep -v 'not installed'") result = run_remote(self.log, self.hostlist_servers, rpm_command) if not result.passed: self.fail("Failed to list daos-server RPMs") if not result.homogeneous: self.fail("Non-homogenous daos-server RPMs") - match = re.findall(r"daos-server-[tests-|tests_openmpi-]*([\d.]+)", result.joined_stdout) - if not match: + rpm_version = result.joined_stdout.strip() + if not rpm_version: self.fail("Failed to get version from daos-server RPMs") - rpm_version = match[0] self.log.info("RPM version = %s", rpm_version) # Get dmg version. diff --git a/src/tests/ftest/process_core_files.py b/src/tests/ftest/process_core_files.py index f5f68af1317..2260521dc77 100644 --- a/src/tests/ftest/process_core_files.py +++ b/src/tests/ftest/process_core_files.py @@ -303,7 +303,8 @@ def install_debuginfo_packages(self): else: raise RunException(f"Unsupported distro: {self.distro_info}") cmds.append(["sudo", "dnf", "-y", "install"] + dnf_args) - result = run_local(self.log, " ".join(["rpm", "-q", "--qf", "'%{evr}'", "daos"])) + result = run_local( + self.log, "rpm -q --qf '%{evr}' daos daos-bullseye | grep -v 'not installed'") rpm_version = result.joined_stdout cmds.append( ["sudo", "dnf", "debuginfo-install", "-y"] + dnf_args diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 5c25f116510..ec38c6c3c07 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -281,9 +281,6 @@ def archive_files(logger, summary, hosts, source, pattern, destination, depth, t # Compress any files larger than 1 MB return_code |= compress_files(logger, file_hosts, source, pattern, depth, test_result) - # Compress any files larger than 1 MB - return_code |= compress_files(logger, file_hosts, source, pattern, depth, test_result) - # Move the test files to the test-results directory on this host return_code |= move_files( logger, file_hosts, source, pattern, destination, depth, timeout, test_result) From b2e293840868695880921781f3727e6b2e6df8cf Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 11 Jun 2026 08:50:45 -0400 Subject: [PATCH 13/14] Fix Test Hardware Signed-off-by: Phil Henderson --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index fe5bfaff23e..148932897d7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1629,7 +1629,7 @@ pipeline { stage('Test Hardware') { when { beforeAgent true - expression { runStage['Functional Hardware'] } + expression { runStage['Test Hardware'] } } steps { script { From 47f5f0b3a2d152c42d7d21b530731c5c98e556ce Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 11 Jun 2026 16:34:41 -0400 Subject: [PATCH 14/14] Fix running HW satges w/ bullseye Signed-off-by: Phil Henderson --- Jenkinsfile | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 148932897d7..1451b9390e0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1646,7 +1646,8 @@ pipeline { nvme: 'auto', job_status: job_status_internal, coverage_stash: 'func_hw_medium_bullseye', - image_version: 'el9.7' + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false) ), 'Functional Hardware Medium MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Medium MD on SSD', @@ -1660,7 +1661,8 @@ pipeline { nvme: 'auto_md_on_ssd', job_status: job_status_internal, coverage_stash: 'func_hw_medium_md_on_ssd_bullseye', - image_version: 'el9.7' + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false) ), 'Functional Hardware Medium VMD': getFunctionalTestStage( name: 'Functional Hardware Medium VMD', @@ -1675,7 +1677,8 @@ pipeline { nvme: 'auto', job_status: job_status_internal, coverage_stash: 'func_hw_medium_vmd_bullseye', - image_version: 'el9.7' + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false) ), 'Functional Hardware Medium Verbs Provider': getFunctionalTestStage( name: 'Functional Hardware Medium Verbs Provider', @@ -1690,7 +1693,8 @@ pipeline { provider: 'ofi+verbs;ofi_rxm', job_status: job_status_internal, coverage_stash: 'func_hw_medium_verbs_provider_bullseye', - image_version: 'el9.7' + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false) ), 'Functional Hardware Medium Verbs Provider MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Medium Verbs Provider MD on SSD', @@ -1705,7 +1709,8 @@ pipeline { provider: 'ofi+verbs;ofi_rxm', job_status: job_status_internal, coverage_stash: 'func_hw_medium_verbs_provider_md_on_ssd_bullseye', - image_version: 'el9.7' + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false) ), 'Functional Hardware Medium UCX Provider': getFunctionalTestStage( name: 'Functional Hardware Medium UCX Provider', @@ -1720,7 +1725,8 @@ pipeline { provider: cachedCommitPragma('Test-provider-ucx', 'ucx+ud_x'), job_status: job_status_internal, coverage_stash: 'func_hw_medium_ucx_provider_bullseye', - image_version: 'el9.7' + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false) ), 'Functional Hardware Large': getFunctionalTestStage( name: 'Functional Hardware Large', @@ -1734,7 +1740,8 @@ pipeline { default_nvme: 'auto', job_status: job_status_internal, coverage_stash: 'func_hw_large_bullseye', - image_version: 'el9.7' + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false) ), 'Functional Hardware Large MD on SSD': getFunctionalTestStage( name: 'Functional Hardware Large MD on SSD', @@ -1748,7 +1755,8 @@ pipeline { default_nvme: 'auto_md_on_ssd', job_status: job_status_internal, coverage_stash: 'func_hw_large_md_on_ssd_bullseye', - image_version: 'el9.7' + image_version: 'el9.7', + bullseye: paramsValue('CI_FULL_BULLSEYE_REPORT', false) ), ) }