From 978e88f356f199a739e58808c573403b5b779f58 Mon Sep 17 00:00:00 2001 From: Shraddha Agrawal Date: Mon, 22 Jun 2026 12:00:39 +0530 Subject: [PATCH 1/2] ceph-dev-pipeline: extract matrix stage script bodies into helper funcs Problem: We were hitting JVM's 64KB bytecode limit when adding new distros to the matrix stage script. Solution: Add helper functions for each stage in the matrix stage script, such that the original block is significantly reduced, thus ensuring we do not hit the JVM's 64KB limit again. This will also ensure we can keep extending the matrix with more distros without worrying about this issue. Signed-off-by: Shraddha Agrawal --- ceph-dev-pipeline/build/Jenkinsfile | 692 ++++++++++++++-------------- 1 file changed, 352 insertions(+), 340 deletions(-) diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile index 5265f37d4..360acc684 100644 --- a/ceph-dev-pipeline/build/Jenkinsfile +++ b/ceph-dev-pipeline/build/Jenkinsfile @@ -141,33 +141,353 @@ def get_ceph_release_repo_text(base_url) { return text.toString() } +def doSourceDistribution() { + if ( ! env.SETUP_BUILD_ID ) { + def setup_build = build( + job: env.SETUP_JOB, + parameters: [ + string(name: "BRANCH", value: env.BRANCH), + // Below are just for ceph-source-dist + string(name: "SHA1", value: env.SHA1), + string(name: "CEPH_REPO", value: env.CEPH_REPO), + string(name: "CEPH_BUILD_BRANCH", value: env.CEPH_BUILD_BRANCH), + // Below are only for actual releases + string(name: 'RELEASE_TYPE', value: env.RELEASE_TYPE ?: ''), + string(name: 'RELEASE_BUILD', value: env.RELEASE_BUILD ?: ''), + string(name: 'VERSION', value: env.VERSION ?: '') + ] + ) + env.SETUP_BUILD_ID = setup_build.getNumber() + } + println "SETUP_BUILD_ID=${env.SETUP_BUILD_ID}" + env.SETUP_BUILD_URL = new URI([env.JENKINS_URL, "job", env.SETUP_JOB, env.SETUP_BUILD_ID].join("/")).normalize() + println "${env.SETUP_BUILD_URL}" +} + +def doCopyArtifactsStage() { + def artifact_filter = "dist/sha1,dist/version,dist/other_envvars,dist/ceph-*.tar.bz2" + def os = get_os_info(env.DIST) + if ( env.CI_COMPILE && os.pkg_type == "deb" ) { + artifact_filter += ",dist/ceph_*.diff.gz,dist/ceph_*.dsc" + } + println artifact_filter + copyArtifacts( + projectName: env.SETUP_JOB, + selector: specific(buildNumber: env.SETUP_BUILD_ID), + filter: artifact_filter, + ) + sh 'sudo journalctl --show-cursor -n 0 --no-pager | tail -n1 | cut -d\" \" -f3 > $WORKSPACE/cursor' + def sha1_trimmed = env.SHA1.trim().toLowerCase() + def sha1_props = readProperties file: "${env.WORKSPACE}/dist/sha1" + def sha1_from_artifact = sha1_props.SHA1.trim().toLowerCase() + if ( env.SHA1 && sha1_from_artifact != sha1_trimmed ) { + error message: "SHA1 from artifact (${sha1_from_artifact}) does not match parameter value (${sha1_trimmed})" + } else if ( ! env.SHA1 ) { + env.SHA1 = sha1_from_artifact + } + println "SHA1=${sha1_trimmed}" + env.VERSION = readFile(file: "${env.WORKSPACE}/dist/version").trim() + // In a release build, dist/other_envvars contains CEPH_REPO, and chacra_url + // as written during ceph-source-dist but we don't to be able to define + // ceph-releases.git or chacra.ceph.com as parameters for ceph-dev-pipeline. + def props = readProperties file: "${env.WORKSPACE}/dist/other_envvars" + for (def key in props.keySet()) { + env."${key}" = props[key] + } + def branch_ui_value = env.BRANCH + def sha1_ui_value = env.SHA1 + if (env.CEPH_REPO?.find(/https?:\/\/github.com\//)) { + // If this is a release build, link to ceph-release.git's $BRANCH-release branch + def suffix = (env.RELEASE_BUILD?.trim() == "true") ? "-release" : "" + + def branch_url = "${env.CEPH_REPO}/tree/${env.BRANCH}${suffix}" + branch_ui_value = "${env.BRANCH}${suffix}" + def commit_url = "${env.CEPH_REPO}/commit/${env.SHA1}" + sha1_ui_value = "${env.SHA1}" + } + def shaman_url = "https://shaman.ceph.com/builds/ceph/${env.BRANCH}/${env.SHA1}" + def build_description = """\ + BRANCH=${branch_ui_value}
+ SHA1=${sha1_ui_value}
+ VERSION=${env.VERSION}
+ DISTROS=${env.DISTROS}
+ ARCHS=${env.ARCHS}
+ FLAVORS=${env.FLAVORS}
+ SETUP_BUILD_ID=${env.SETUP_BUILD_ID}
+ shaman builds for this branch+commit + """.stripIndent() + buildDescription build_description + sh "sha256sum dist/*" + sh "cat dist/sha1 dist/version" + sh '''#!/bin/bash + set -ex + cd dist + mkdir ceph + tar --strip-components=1 -C ceph -xjf ceph-${VERSION}.tar.bz2 ceph-${VERSION}/{container,ceph.spec,ceph.spec.in,debian,Dockerfile.build,do_cmake.sh,install-deps.sh,run-make-check.sh,make-debs.sh,make-dist,make-srpm.sh,src/script} + ''' +} + +def doCheckForBuiltPackagesStage() { + sh './scripts/setup_chacractl.sh' + def chacra_url = sh( + script: '''grep url ~/.chacractl | cut -d'"' -f2''', + returnStdout: true, + ).trim() + def os = get_os_info(env.DIST) + def chacra_endpoint = "ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/${env.ARCH}/flavors/${env.FLAVOR}/" + def chacractl_rc = sh( + script: "${env.HOME}/.local/bin/chacractl exists binaries/${chacra_endpoint}", + returnStatus: true, + ) + if ( chacractl_rc == 0 && env.FORCE != "true" ) { + println("Skipping compilation since chacra already has artifacts. To override, use THROWAWAY=true (to skip this check) or FORCE=true (to re-upload artifacts).") + build_matrix["${env.DIST}_${env.ARCH}"] = false + } + + // Set up Pulp client + if ( env.PULP_UPLOAD == "true" ) { + withCredentials([ + usernamePassword(credentialsId: 'pulp-auth', usernameVariable: 'PULP_USERNAME', passwordVariable: 'PULP_PASSWORD') + ]) { + sh './scripts/setup_pulp.sh' + } + } +} + +def doBuilderContainerStage() { + env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build" + sh '''#!/bin/bash + set -ex + podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION} + podman login -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io + ''' + def ceph_builder_tag_short = "${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}" + def ceph_builder_tag = "${env.SHA1[0..6]}.${ceph_builder_tag_short}" + sh """#!/bin/bash -ex + podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} || \ + podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} || \ + true + """ + def withCrimson = !(env.BRANCH in ['tentacle', 'squid', 'reef']) + sh """#!/bin/bash + set -ex + echo > .env + echo "WITH_CRIMSON=${withCrimson}" >> .env + cd dist/ceph + python3 src/script/build-with-container.py --env-file=${env.WORKSPACE}/.env --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} --image-variant=packages -d ${env.DIST} -e build-container + podman tag ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} + """ + sh """#!/bin/bash -ex + podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} + podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} + """ +} + +def doBuildStage() { + def os = get_os_info(env.DIST) + sh "./scripts/update_shaman.sh started ceph ${os.name} ${os.version_name} ${env.ARCH}" + env.AWS_ACCESS_KEY_ID = env.SCCACHE_BUCKET_CREDS_USR + env.AWS_SECRET_ACCESS_KEY = env.SCCACHE_BUCKET_CREDS_PSW + def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}" + def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${env.DIST} --image-variant=packages --ceph-version ${env.VERSION}" + def bwc_command = bwc_command_base + def bwc_cmd_sccache_flags = "" + if ( env.DWZ == "false" ) { + sh '''#!/bin/bash + echo "DWZ=$DWZ" >> .env + ''' + bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env"; + } + if ( env.SCCACHE == "true" ) { + sh '''#!/bin/bash + echo "SCCACHE=$SCCACHE" >> .env + echo "SCCACHE_CONF=/ceph/sccache.conf" >> .env + echo "SCCACHE_ERROR_LOG=/ceph/sccache_log.txt" >> .env + echo "SCCACHE_LOG=debug" >> .env + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> .env + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> .env + echo "CEPH_BUILD_NORMALIZE_PATHS=true" >> .env + ''' + // TODO: un-hardcode this + writeFile( + file: "dist/ceph/sccache.conf", + text: """\ + [cache.s3] + bucket = "ceph-sccache" + endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud" + use_ssl = true + key_prefix = "" + server_side_encryption = false + no_credentials = false + region = "auto" + """ + ) + bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env"; + } + def ceph_extra_cmake_args = ""; + def deb_build_profiles = ""; + switch (env.FLAVOR) { + case "default": + ceph_extra_cmake_args += " -DALLOCATOR=tcmalloc" + ceph_extra_cmake_args += " -DWITH_SYSTEM_BOOST=OFF -DWITH_BOOST_VALGRIND=ON" + if (os.version_name == "focal") { + ceph_extra_cmake_args += " -DWITH_STATIC_LIBSTDCXX=ON" + } + break + case "debug": + ceph_extra_cmake_args += " -DCMAKE_BUILD_TYPE=Debug" + break + default: + println "FLAVOR=${env.FLAVOR} is invalid" + assert false + } + bwc_command = "${bwc_command} ${bwc_cmd_sccache_flags}" + if ( os.pkg_type == "deb" ) { + def sccache_flag = "-DWITH_SCCACHE=ON" + if ( env.SCCACHE == "true" && ! ceph_extra_cmake_args.contains(sccache_flag) ) { + ceph_extra_cmake_args += " ${sccache_flag}" + } + if ( deb_build_profiles ) { + sh """#!/bin/bash + echo "DEB_BUILD_PROFILES=${deb_build_profiles}" >> .env + """ + } + bwc_command = "${bwc_command} -e debs" + } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) { + def rpmbuild_args = "" + if ( env.SCCACHE == "true" ) rpmbuild_args += " -R--with=sccache" + if ( env.DWZ == "false" ) rpmbuild_args += " -R--without=dwz" + if ( env.FLAVOR == "default" ) rpmbuild_args += " -R--with=tcmalloc" + bwc_command = "${bwc_command}${rpmbuild_args} -e rpm" + } else if ( env.DIST =~ /suse|sles/ ) { + throw new Exception("bwc not implemented for ${env.DIST}") + } else if ( env.DIST =~ /windows/ ) { + throw new Exception("bwc not implemented for ${env.DIST}") + } else { + throw new Exception("DIST '${env.DIST}' is invalid!") + } + sh """#!/bin/bash + echo "CEPH_EXTRA_CMAKE_ARGS=${ceph_extra_cmake_args}" >> .env + """ + sh """#!/bin/bash -ex + cd dist/ceph + ln ../ceph-${env.VERSION}.tar.bz2 . + ${bwc_command} + """ + if ( os.pkg_type == "deb" ) { + sh """#!/bin/bash -ex + cd dist/ceph + ${bwc_command_base} -e custom -- "dpkg-deb --fsys-tarfile /ceph/debs/*/pool/main/c/ceph/cephadm_${env.VERSION}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm" + ln ./cephadm ../../ + """ + } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) { + sh """#!/bin/bash -ex + cd dist/ceph + ${bwc_command_base} -e custom -- "rpm2cpio /ceph/rpmbuild/RPMS/noarch/cephadm-*.rpm | cpio -i --to-stdout *sbin/cephadm > cephadm" + ln ./cephadm ../../ + """ + } +} + +def doBuildPostAlways() { + if (fileExists('dist/ceph/sccache_log.txt')) { + sh """ + if [ -f "${env.WORKSPACE}/dist/ceph/sccache_log.txt" ]; then + ln dist/ceph/sccache_log.txt sccache_log_${env.DIST}_${env.ARCH}_${env.FLAVOR}.txt + fi + """ + // The below is to work around an issue where Jenkins would recurse into + // dist/ceph/qa and get stuck trying to follow the .qa symlinks. This was + // discovered by seeing many messages about "too many levels of symbolic links" + // in the system journal. + sh "find ${env.WORKSPACE}/dist/ceph/ -name .qa -exec rm {} \\;" + archiveArtifacts( + artifacts: 'sccache_log*.txt', + allowEmptyArchive: true, + fingerprint: true, + ) + } +} + +def doUploadPackagesStage() { + def chacra_url = sh( + script: '''grep url ~/.chacractl | cut -d'"' -f2''', + returnStdout: true, + ).trim() + def os = get_os_info(env.DIST) + if ( os.pkg_type == "rpm" ) { + sh """#!/bin/bash + set -ex + cd ./dist/ceph + mkdir -p ./rpmbuild/SRPMS/ + ln ceph-*.src.rpm ./rpmbuild/SRPMS/ + """ + // Push packages to chacra.ceph.com under the 'test' ref if ceph-release-pipeline's TEST=true + env.SHA1 = env.TEST?.toBoolean() ? 'test' : env.SHA1 + def spec_text = get_ceph_release_spec_text("${chacra_url}r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}/") + writeFile( + file: "dist/ceph/rpmbuild/SPECS/ceph-release.spec", + text: spec_text, + ) + def repo_text = get_ceph_release_repo_text("${chacra_url}/r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}") + writeFile( + file: "dist/ceph/rpmbuild/SOURCES/ceph.repo", + text: repo_text, + ) + def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}" + def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${env.DIST} --image-variant=packages --ceph-version ${env.VERSION}" + def bwc_command = "${bwc_command_base} -e custom -- rpmbuild -bb --define \\'_topdir /ceph/rpmbuild\\' /ceph/rpmbuild/SPECS/ceph-release.spec" + sh """#!/bin/bash + set -ex + cd ${env.WORKSPACE}/dist/ceph + ${bwc_command} + """ + } + sh """#!/bin/bash + export CHACRA_URL="${chacra_url}" + export OS_NAME="${os.name}" + export OS_VERSION="${os.version}" + export OS_VERSION_NAME="${os.version_name}" + export OS_PKG_TYPE="${os.pkg_type}" + if [ "${env.THROWAWAY}" != "true" ]; then ./scripts/chacra_upload.sh; fi + """ + + sh """#!/bin/bash + if [ "${env.THROWAWAY}" != "true" ] && [ "${env.PULP_UPLOAD}" == "true" ]; then + export CEPH_VERSION="${env.VERSION}" + export OS_NAME="${os.name}" + export OS_VERSION="${os.version}" + export OS_VERSION_NAME="${os.version_name}" + export OS_PKG_TYPE="${os.pkg_type}" + export SHA1="${env.SHA1}" + export FLAVOR="${env.FLAVOR}" + + ./scripts/pulp_upload.sh + else + echo "Skipping pulp upload because PULP_UPLOAD=${env.PULP_UPLOAD}" + fi + """ +} + +def doContainerStage() { + env.CONTAINER_REPO_USERNAME = env.CONTAINER_REPO_CREDS_USR + env.CONTAINER_REPO_PASSWORD = env.CONTAINER_REPO_CREDS_PSW + def os = get_os_info(env.DIST) + def cephver = env.VERSION.trim() + sh """#!/bin/bash + export DISTRO=${os.name} + export RELEASE=${os.version} + export cephver=${cephver} + ./scripts/build_container + """ +} + pipeline { agent any stages { stage("source distribution") { steps { - script { - if ( ! env.SETUP_BUILD_ID ) { - def setup_build = build( - job: env.SETUP_JOB, - parameters: [ - string(name: "BRANCH", value: env.BRANCH), - // Below are just for ceph-source-dist - string(name: "SHA1", value: env.SHA1), - string(name: "CEPH_REPO", value: env.CEPH_REPO), - string(name: "CEPH_BUILD_BRANCH", value: env.CEPH_BUILD_BRANCH), - // Below are only for actual releases - string(name: 'RELEASE_TYPE', value: env.RELEASE_TYPE ?: ''), - string(name: 'RELEASE_BUILD', value: env.RELEASE_BUILD ?: ''), - string(name: 'VERSION', value: env.VERSION ?: '') - ] - ) - env.SETUP_BUILD_ID = setup_build.getNumber() - } - println "SETUP_BUILD_ID=${env.SETUP_BUILD_ID}" - env.SETUP_BUILD_URL = new URI([env.JENKINS_URL, "job", env.SETUP_JOB, env.SETUP_BUILD_ID].join("/")).normalize() - println "${env.SETUP_BUILD_URL}" - } + script { doSourceDistribution() } } } stage("parallel build") { @@ -252,72 +572,7 @@ pipeline { } stage("copy artifacts") { steps { - script { - def artifact_filter = "dist/sha1,dist/version,dist/other_envvars,dist/ceph-*.tar.bz2" - def os = get_os_info(env.DIST) - if ( env.CI_COMPILE && os.pkg_type == "deb" ) { - artifact_filter += ",dist/ceph_*.diff.gz,dist/ceph_*.dsc" - } - println artifact_filter - copyArtifacts( - projectName: env.SETUP_JOB, - selector: specific(buildNumber: env.SETUP_BUILD_ID), - filter: artifact_filter, - ) - } - sh 'sudo journalctl --show-cursor -n 0 --no-pager | tail -n1 | cut -d\" \" -f3 > $WORKSPACE/cursor' - script { - def sha1_trimmed = env.SHA1.trim().toLowerCase() - def sha1_props = readProperties file: "${WORKSPACE}/dist/sha1" - sha1_from_artifact = sha1_props.SHA1.trim().toLowerCase() - if ( env.SHA1 && sha1_from_artifact != sha1_trimmed ) { - error message: "SHA1 from artifact (${sha1_from_artifact}) does not match parameter value (${sha1_trimmed})" - } else if ( ! env.SHA1 ) { - env.SHA1 = sha1_from_artifact - } - println "SHA1=${sha1_trimmed}" - env.VERSION = readFile(file: "${WORKSPACE}/dist/version").trim() - script { - // In a release build, dist/other_envvars contains CEPH_REPO, and chacra_url - // as written during ceph-source-dist but we don't to be able to define - // ceph-releases.git or chacra.ceph.com as parameters for ceph-dev-pipeline. - def props = readProperties file: "${WORKSPACE}/dist/other_envvars" - for (def key in props.keySet()) { - env."${key}" = props[key] - } - } - def branch_ui_value = env.BRANCH - def sha1_ui_value = env.SHA1 - if (env.CEPH_REPO?.find(/https?:\/\/github.com\//)) { - // If this is a release build, link to ceph-release.git's $BRANCH-release branch - def suffix = (env.RELEASE_BUILD?.trim() == "true") ? "-release" : "" - - def branch_url = "${env.CEPH_REPO}/tree/${env.BRANCH}${suffix}" - branch_ui_value = "${env.BRANCH}${suffix}" - def commit_url = "${env.CEPH_REPO}/commit/${env.SHA1}" - sha1_ui_value = "${env.SHA1}" - } - def shaman_url = "https://shaman.ceph.com/builds/ceph/${env.BRANCH}/${env.SHA1}" - def build_description = """\ - BRANCH=${branch_ui_value}
- SHA1=${sha1_ui_value}
- VERSION=${env.VERSION}
- DISTROS=${env.DISTROS}
- ARCHS=${env.ARCHS}
- FLAVORS=${env.FLAVORS}
- SETUP_BUILD_ID=${env.SETUP_BUILD_ID}
- shaman builds for this branch+commit - """.stripIndent() - buildDescription build_description - } - sh "sha256sum dist/*" - sh "cat dist/sha1 dist/version" - sh '''#!/bin/bash - set -ex - cd dist - mkdir ceph - tar --strip-components=1 -C ceph -xjf ceph-${VERSION}.tar.bz2 ceph-${VERSION}/{container,ceph.spec,ceph.spec.in,debian,Dockerfile.build,do_cmake.sh,install-deps.sh,run-make-check.sh,make-debs.sh,make-dist,make-srpm.sh,src/script} - ''' + script { doCopyArtifactsStage() } } } stage("check for built packages") { @@ -330,32 +585,7 @@ pipeline { SHAMAN_API_KEY = credentials('shaman-api-key') } steps { - script { - sh './scripts/setup_chacractl.sh' - def chacra_url = sh( - script: '''grep url ~/.chacractl | cut -d'"' -f2''', - returnStdout: true, - ).trim() - def os = get_os_info(env.DIST) - def chacra_endpoint = "ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/${env.ARCH}/flavors/${env.FLAVOR}/" - def chacractl_rc = sh( - script: "$HOME/.local/bin/chacractl exists binaries/${chacra_endpoint}", - returnStatus: true, - ) - if ( chacractl_rc == 0 && env.FORCE != "true" ) { - println("Skipping compilation since chacra already has artifacts. To override, use THROWAWAY=true (to skip this check) or FORCE=true (to re-upload artifacts).") - build_matrix["${DIST}_${ARCH}"] = false - } - - // Set up Pulp client - if ( env.PULP_UPLOAD == "true" ) { - withCredentials([ - usernamePassword(credentialsId: 'pulp-auth', usernameVariable: 'PULP_USERNAME', passwordVariable: 'PULP_PASSWORD') - ]) { - sh './scripts/setup_pulp.sh' - } - } - } + script { doCheckForBuiltPackagesStage() } } } stage("builder container") { @@ -367,34 +597,7 @@ pipeline { expression { return build_matrix["${DIST}_${ARCH}"] == true } } steps { - script { - env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build" - sh '''#!/bin/bash - set -ex - podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION} - podman login -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io - ''' - def ceph_builder_tag_short = "${env.BRANCH}.${env.DIST}.${ARCH}.${FLAVOR}" - def ceph_builder_tag = "${env.SHA1[0..6]}.${ceph_builder_tag_short}" - sh """#!/bin/bash -ex - podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} || \ - podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} || \ - true - """ - def withCrimson = !(env.BRANCH in ['tentacle', 'squid', 'reef']) - sh """#!/bin/bash - set -ex - echo > .env - echo "WITH_CRIMSON=${withCrimson}" >> .env - cd dist/ceph - python3 src/script/build-with-container.py --env-file=${env.WORKSPACE}/.env --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} --image-variant=packages -d ${DIST} -e build-container - podman tag ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} - """ - sh """#!/bin/bash -ex - podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} - podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} - """ - } + script { doBuilderContainerStage() } } } stage("build") { @@ -406,138 +609,16 @@ pipeline { expression { return build_matrix["${DIST}_${ARCH}"] == true } } steps { - script { - def os = get_os_info(env.DIST) - sh "./scripts/update_shaman.sh started ceph ${os.name} ${os.version_name} $ARCH" - env.AWS_ACCESS_KEY_ID = env.SCCACHE_BUCKET_CREDS_USR - env.AWS_SECRET_ACCESS_KEY = env.SCCACHE_BUCKET_CREDS_PSW - def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${ARCH}.${FLAVOR}" - def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${DIST} --image-variant=packages --ceph-version ${env.VERSION}" - def bwc_command = bwc_command_base - def bwc_cmd_sccache_flags = "" - if ( env.DWZ == "false" ) { - sh '''#!/bin/bash - echo "DWZ=$DWZ" >> .env - ''' - bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env"; - } - if ( env.SCCACHE == "true" ) { - sh '''#!/bin/bash - echo "SCCACHE=$SCCACHE" >> .env - echo "SCCACHE_CONF=/ceph/sccache.conf" >> .env - echo "SCCACHE_ERROR_LOG=/ceph/sccache_log.txt" >> .env - echo "SCCACHE_LOG=debug" >> .env - echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> .env - echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> .env - echo "CEPH_BUILD_NORMALIZE_PATHS=true" >> .env - ''' - // TODO: un-hardcode this - writeFile( - file: "dist/ceph/sccache.conf", - text: """\ - [cache.s3] - bucket = "ceph-sccache" - endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud" - use_ssl = true - key_prefix = "" - server_side_encryption = false - no_credentials = false - region = "auto" - """ - ) - bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env"; - } - def ceph_extra_cmake_args = ""; - def deb_build_profiles = ""; - switch (env.FLAVOR) { - case "default": - ceph_extra_cmake_args += " -DALLOCATOR=tcmalloc" - ceph_extra_cmake_args += " -DWITH_SYSTEM_BOOST=OFF -DWITH_BOOST_VALGRIND=ON" - if (os.version_name == "focal") { - ceph_extra_cmake_args += " -DWITH_STATIC_LIBSTDCXX=ON" - } - break - case "debug": - ceph_extra_cmake_args += " -DCMAKE_BUILD_TYPE=Debug" - break - default: - println "FLAVOR=${env.FLAVOR} is invalid" - assert false - } - bwc_command = "${bwc_command} ${bwc_cmd_sccache_flags}" - if ( os.pkg_type == "deb" ) { - def sccache_flag = "-DWITH_SCCACHE=ON" - if ( env.SCCACHE == "true" && ! ceph_extra_cmake_args.contains(sccache_flag) ) { - ceph_extra_cmake_args += " ${sccache_flag}" - } - if ( deb_build_profiles ) { - sh """#!/bin/bash - echo "DEB_BUILD_PROFILES=${deb_build_profiles}" >> .env - """ - } - bwc_command = "${bwc_command} -e debs" - } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) { - def rpmbuild_args = "" - if ( env.SCCACHE == "true" ) rpmbuild_args += " -R--with=sccache" - if ( env.DWZ == "false" ) rpmbuild_args += " -R--without=dwz" - if ( env.FLAVOR == "default" ) rpmbuild_args += " -R--with=tcmalloc" - bwc_command = "${bwc_command}${rpmbuild_args} -e rpm" - } else if ( env.DIST =~ /suse|sles/ ) { - throw new Exception("bwc not implemented for ${env.DIST}") - } else if ( env.DIST =~ /windows/ ) { - throw new Exception("bwc not implemented for ${env.DIST}") - } else { - throw new Exception("DIST '${env.DIST}' is invalid!") - } - sh """#!/bin/bash - echo "CEPH_EXTRA_CMAKE_ARGS=${ceph_extra_cmake_args}" >> .env - """ - sh """#!/bin/bash -ex - cd dist/ceph - ln ../ceph-${env.VERSION}.tar.bz2 . - ${bwc_command} - """ - if ( os.pkg_type == "deb" ) { - sh """#!/bin/bash -ex - cd dist/ceph - ${bwc_command_base} -e custom -- "dpkg-deb --fsys-tarfile /ceph/debs/*/pool/main/c/ceph/cephadm_${VERSION}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm" - ln ./cephadm ../../ - """ - } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) { - sh """#!/bin/bash -ex - cd dist/ceph - ${bwc_command_base} -e custom -- "rpm2cpio /ceph/rpmbuild/RPMS/noarch/cephadm-*.rpm | cpio -i --to-stdout *sbin/cephadm > cephadm" - ln ./cephadm ../../ - """ - } - } + script { doBuildStage() } } post { always { - script { - if (fileExists('dist/ceph/sccache_log.txt')) { - sh """ - if [ -f "${env.WORKSPACE}/dist/ceph/sccache_log.txt" ]; then - ln dist/ceph/sccache_log.txt sccache_log_${env.DIST}_${env.ARCH}_${env.FLAVOR}.txt - fi - """ - // The below is to work around an issue where Jenkins would recurse into - // dist/ceph/qa and get stuck trying to follow the .qa symlinks. This was - // discovered by seeing many messages about "too many levels of symbolic links" - // in the system journal. - sh "find ${env.WORKSPACE}/dist/ceph/ -name .qa -exec rm {} \\;" - archiveArtifacts( - artifacts: 'sccache_log*.txt', - allowEmptyArchive: true, - fingerprint: true, - ) - } - } + script { doBuildPostAlways() } } unsuccessful { script { def os = get_os_info(env.DIST) - sh "./scripts/update_shaman.sh failed ceph ${os.name} ${os.version_name} $ARCH" + sh "./scripts/update_shaman.sh failed ceph ${os.name} ${os.version_name} ${env.ARCH}" } } } @@ -551,77 +632,19 @@ pipeline { expression { return build_matrix["${DIST}_${ARCH}"] == true } } steps { - script { - def chacra_url = sh( - script: '''grep url ~/.chacractl | cut -d'"' -f2''', - returnStdout: true, - ).trim() - def os = get_os_info(env.DIST) - if ( os.pkg_type == "rpm" ) { - sh """#!/bin/bash - set -ex - cd ./dist/ceph - mkdir -p ./rpmbuild/SRPMS/ - ln ceph-*.src.rpm ./rpmbuild/SRPMS/ - """ - // Push packages to chacra.ceph.com under the 'test' ref if ceph-release-pipeline's TEST=true - env.SHA1 = env.TEST?.toBoolean() ? 'test' : env.SHA1 - def spec_text = get_ceph_release_spec_text("${chacra_url}r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}/") - writeFile( - file: "dist/ceph/rpmbuild/SPECS/ceph-release.spec", - text: spec_text, - ) - def repo_text = get_ceph_release_repo_text("${chacra_url}/r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}") - writeFile( - file: "dist/ceph/rpmbuild/SOURCES/ceph.repo", - text: repo_text, - ) - def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${ARCH}.${FLAVOR}" - def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${DIST} --image-variant=packages --ceph-version ${env.VERSION}" - bwc_command = "${bwc_command_base} -e custom -- rpmbuild -bb --define \\'_topdir /ceph/rpmbuild\\' /ceph/rpmbuild/SPECS/ceph-release.spec" - sh """#!/bin/bash - set -ex - cd $WORKSPACE/dist/ceph - ${bwc_command} - """ - } - sh """#!/bin/bash - export CHACRA_URL="${chacra_url}" - export OS_NAME="${os.name}" - export OS_VERSION="${os.version}" - export OS_VERSION_NAME="${os.version_name}" - export OS_PKG_TYPE="${os.pkg_type}" - if [ "$THROWAWAY" != "true" ]; then ./scripts/chacra_upload.sh; fi - """ - - sh """#!/bin/bash - if [ "$THROWAWAY" != "true" ] && [ "$PULP_UPLOAD" == "true" ]; then - export CEPH_VERSION="${env.VERSION}" - export OS_NAME="${os.name}" - export OS_VERSION="${os.version}" - export OS_VERSION_NAME="${os.version_name}" - export OS_PKG_TYPE="${os.pkg_type}" - export SHA1="${env.SHA1}" - export FLAVOR="${env.FLAVOR}" - - ./scripts/pulp_upload.sh - else - echo "Skipping pulp upload because PULP_UPLOAD=$PULP_UPLOAD" - fi - """ - } + script { doUploadPackagesStage() } } post { success { script { def os = get_os_info(env.DIST) - sh "./scripts/update_shaman.sh completed ceph ${os.name} ${os.version_name} $ARCH" + sh "./scripts/update_shaman.sh completed ceph ${os.name} ${os.version_name} ${env.ARCH}" } } unsuccessful { script { def os = get_os_info(env.DIST) - sh "./scripts/update_shaman.sh failed ceph ${os.name} ${os.version_name} $ARCH" + sh "./scripts/update_shaman.sh failed ceph ${os.name} ${os.version_name} ${env.ARCH}" } } } @@ -634,18 +657,7 @@ pipeline { CONTAINER_REPO_CREDS = credentials('quay-ceph-io-ceph-ci') } steps { - script { - env.CONTAINER_REPO_USERNAME = env.CONTAINER_REPO_CREDS_USR - env.CONTAINER_REPO_PASSWORD = env.CONTAINER_REPO_CREDS_PSW - def os = get_os_info(env.DIST) - cephver = env.VERSION.trim() - sh """#!/bin/bash - export DISTRO=${os.name} - export RELEASE=${os.version} - export cephver=${cephver} - ./scripts/build_container - """ - } + script { doContainerStage() } } } } From a1c79ffbc9e0cf3b20a5548e6d5a1b95da374f37 Mon Sep 17 00:00:00 2001 From: Shraddha Agrawal Date: Mon, 22 Jun 2026 12:25:49 +0530 Subject: [PATCH 2/2] introduce noble debug builds This commit introduces debug builds for noble which can be used for integration testing. Signed-off-by: Shraddha Agrawal --- ceph-dev-cron/build/Jenkinsfile | 6 +++--- ceph-dev-pipeline/build/Jenkinsfile | 12 ++++++++++-- ceph-trigger-build/build/Jenkinsfile | 4 ++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ceph-dev-cron/build/Jenkinsfile b/ceph-dev-cron/build/Jenkinsfile index 9d52e2e66..281993cb2 100644 --- a/ceph-dev-cron/build/Jenkinsfile +++ b/ceph-dev-cron/build/Jenkinsfile @@ -18,19 +18,19 @@ node('built-in') { tentacle: [ distros: 'noble jammy rocky10 centos9 windows', extras : [ - [distros:'centos9 rocky10', flavors:'debug', archs:'x86_64'] + [distros:'centos9 rocky10 noble', flavors:'debug', archs:'x86_64'] ] ], umbrella: [ distros: 'noble rocky10 centos9 windows', extras : [ - [distros:'centos9 rocky10', flavors:'debug', archs:'x86_64'] + [distros:'centos9 rocky10 noble', flavors:'debug', archs:'x86_64'] ] ], main: [ distros: 'noble jammy rocky10 centos9 windows', extras : [ - [distros:'centos9 rocky10', flavors:'debug', archs:'x86_64'], + [distros:'centos9 rocky10 noble', flavors:'debug', archs:'x86_64'], [distros:'centos9 rocky10', flavors:'default', archs:'arm64'], ] ] diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile index 360acc684..dc02120f1 100644 --- a/ceph-dev-pipeline/build/Jenkinsfile +++ b/ceph-dev-pipeline/build/Jenkinsfile @@ -141,6 +141,7 @@ def get_ceph_release_repo_text(base_url) { return text.toString() } +// Trigger the source-dist setup job (or reuse SETUP_BUILD_ID) and record its number/URL. def doSourceDistribution() { if ( ! env.SETUP_BUILD_ID ) { def setup_build = build( @@ -164,6 +165,7 @@ def doSourceDistribution() { println "${env.SETUP_BUILD_URL}" } +// Copy artifacts from the setup job, verify SHA1, set VERSION/build description, and extract the source tarball. def doCopyArtifactsStage() { def artifact_filter = "dist/sha1,dist/version,dist/other_envvars,dist/ceph-*.tar.bz2" def os = get_os_info(env.DIST) @@ -227,6 +229,7 @@ def doCopyArtifactsStage() { ''' } +// Check chacra for an existing build (and skip if found, unless FORCE) and set up the Pulp client if needed. def doCheckForBuiltPackagesStage() { sh './scripts/setup_chacractl.sh' def chacra_url = sh( @@ -254,6 +257,7 @@ def doCheckForBuiltPackagesStage() { } } +// Log into container registries and pull/build/push the ceph-build container image for this matrix cell. def doBuilderContainerStage() { env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build" sh '''#!/bin/bash @@ -283,6 +287,7 @@ def doBuilderContainerStage() { """ } +// Run the actual package build via build-with-container.py: assemble cmake/sccache/dwz flags per distro+flavor, then extract cephadm. def doBuildStage() { def os = get_os_info(env.DIST) sh "./scripts/update_shaman.sh started ceph ${os.name} ${os.version_name} ${env.ARCH}" @@ -389,6 +394,7 @@ def doBuildStage() { } } +// Post-build always-block: archive the sccache log and remove .qa symlinks Jenkins gets stuck recursing into. def doBuildPostAlways() { if (fileExists('dist/ceph/sccache_log.txt')) { sh """ @@ -409,6 +415,7 @@ def doBuildPostAlways() { } } +// Build the ceph-release rpm (for rpm distros) and upload the packages to chacra and optionally Pulp. def doUploadPackagesStage() { def chacra_url = sh( script: '''grep url ~/.chacractl | cut -d'"' -f2''', @@ -469,6 +476,7 @@ def doUploadPackagesStage() { """ } +// Build the Ceph container image via scripts/build_container. def doContainerStage() { env.CONTAINER_REPO_USERNAME = env.CONTAINER_REPO_CREDS_USR env.CONTAINER_REPO_PASSWORD = env.CONTAINER_REPO_CREDS_PSW @@ -521,7 +529,7 @@ pipeline { values 'default', 'debug' } } - // debug flavor is currently only supported on centos9 and rocky10, x86_64 + // debug flavor is currently only supported on centos9, rocky10 and noble, x86_64 excludes { exclude { axis { @@ -530,7 +538,7 @@ pipeline { } axis { name 'DIST' - notValues 'centos9', 'rocky10' + notValues 'centos9', 'rocky10', 'noble' } } exclude { diff --git a/ceph-trigger-build/build/Jenkinsfile b/ceph-trigger-build/build/Jenkinsfile index 08fb43d39..112866f0b 100644 --- a/ceph-trigger-build/build/Jenkinsfile +++ b/ceph-trigger-build/build/Jenkinsfile @@ -58,7 +58,7 @@ def params_from_branch(initialParams) { if ( !singleSet ) { params << params[0].clone() params[-1]['ARCHS'] = 'x86_64' - params[-1]['DISTROS'] = 'centos9 rocky10' + params[-1]['DISTROS'] = 'centos9 rocky10 noble' params[-1]['FLAVOR'] = 'debug' } else { params[0]['ARCHS'] += ' arm64' @@ -78,7 +78,7 @@ def params_from_branch(initialParams) { if ( !singleSet ) { params << params[0].clone() params[-1]['ARCHS'] = 'x86_64' - params[-1]['DISTROS'] = 'centos9 rocky10' + params[-1]['DISTROS'] = 'centos9 rocky10 noble' params[-1]['FLAVOR'] = 'debug' } else { params[0]['FLAVOR'] += ' debug'