Skip to content

Commit 5af4bf9

Browse files
rajiv-jain-netapprajiv-jain-netapp
authored
Merge pull request #59 from NetApp/feature/CSTACKEX-175
CSTACKEX-175: adding initial changes for CI related workflows
2 parents 446112b + 1b5db38 commit 5af4bf9

22 files changed

Lines changed: 1318 additions & 0 deletions

‎private-cicd/.gitignore‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Local overrides with secrets (templates *.example stay tracked)
19+
config/qa.yaml
20+
marvin/zones/*.cfg
21+
!marvin/zones/*.cfg.example

‎private-cicd/Jenkinsfile‎

Lines changed: 311 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,311 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
/*
21+
* Private downstream Jenkins pipeline for Apache CloudStack.
22+
* Not for inclusion in apache/cloudstack upstream.
23+
*
24+
* Rollout: Phase -1 = preflight (validate private-cicd only). Phase 1 = build-only. Phases 2–3 = marvin / delivery — see private-cicd/docs/IMPLEMENTATION-PHASES.md
25+
*
26+
* Usage:
27+
* - Monorepo: Script Path = private-cicd/Jenkinsfile ; leave CLONE_SEPARATE = false.
28+
* - CI-only repo: set CLONE_SEPARATE = true and point CLOUDSTACK_* at your fork/tag.
29+
*/
30+
31+
pipeline {
32+
agent any
33+
34+
options {
35+
buildDiscarder(logRotator(numToKeepStr: '30'))
36+
timestamps()
37+
timeout(time: 4, unit: 'HOURS')
38+
}
39+
40+
parameters {
41+
choice(
42+
name: 'PIPELINE_PHASE',
43+
choices: ['build-ontap-fast', 'build-only', 'preflight', 'marvin', 'delivery'],
44+
description: 'build-ontap-fast = ONTAP plugin -pl -am test. preflight = Phase -1. build-only = full Maven. marvin / delivery reserved; see private-cicd/docs/IMPLEMENTATION-PHASES.md'
45+
)
46+
string(
47+
name: 'CONFIG_DEFAULTS_FILE',
48+
defaultValue: 'config/defaults.yaml',
49+
description: 'YAML path relative to CICD_ROOT (e.g. config/defaults.yaml or config/team-b.yaml).'
50+
)
51+
string(
52+
name: 'CONFIG_PROFILE',
53+
defaultValue: '',
54+
description: 'Optional: key under profiles in the defaults YAML (e.g. example-lts-branch). Blank = use only top-level cloudstack: from the file.'
55+
)
56+
booleanParam(
57+
name: 'CLONE_SEPARATE',
58+
defaultValue: false,
59+
description: 'If true, clone CloudStack into cloudstack-src/ (use when this job repo is CI-only). If false, expect CloudStack pom.xml at the workspace root (multibranch on your fork). Branch/url below apply only when this is true.'
60+
)
61+
string(
62+
name: 'CLOUDSTACK_GIT_URL',
63+
defaultValue: '',
64+
description: 'Override Git URL for CloudStack. Leave blank to use config/defaults.yaml (after optional CONFIG_PROFILE).'
65+
)
66+
string(
67+
name: 'CLOUDSTACK_GIT_BRANCH',
68+
defaultValue: '',
69+
description: 'Override branch or tag for clone. Leave blank to use config/defaults.yaml (after optional CONFIG_PROFILE). Ignored when CLONE_SEPARATE is false.'
70+
)
71+
booleanParam(
72+
name: 'ENABLE_NOREDIST',
73+
defaultValue: false,
74+
description: 'If true, run third-party non-OSS install script then mvn -Dnoredist (see README / legal for your org).'
75+
)
76+
booleanParam(
77+
name: 'SKIP_TESTS',
78+
defaultValue: true,
79+
description: 'If true, Maven uses -DskipTests=true (faster CI).'
80+
)
81+
booleanParam(
82+
name: 'INSTALL_APT_DEPS',
83+
defaultValue: true,
84+
description: 'If true, run private-cicd/scripts/install-build-deps-ubuntu.sh (requires sudo on agent).'
85+
)
86+
booleanParam(
87+
name: 'SETUP_IPMITOOL_WRAPPER',
88+
defaultValue: false,
89+
description: 'If true, install CloudStack-style ipmitool wrapper (requires sudo).'
90+
)
91+
}
92+
93+
environment {
94+
MAVEN_OPTS = '-Xmx3072m -XX:MaxMetaspaceSize=512m'
95+
}
96+
97+
stages {
98+
stage('Phase gate') {
99+
steps {
100+
script {
101+
switch (params.PIPELINE_PHASE) {
102+
case 'preflight':
103+
case 'build-ontap-fast':
104+
case 'build-only':
105+
break
106+
case 'marvin':
107+
case 'delivery':
108+
error("PIPELINE_PHASE='${params.PIPELINE_PHASE}' is not implemented yet. Use preflight, build-ontap-fast, or build-only. See private-cicd/docs/IMPLEMENTATION-PHASES.md")
109+
default:
110+
error("PIPELINE_PHASE='${params.PIPELINE_PHASE}' is not supported.")
111+
}
112+
}
113+
}
114+
}
115+
116+
stage('Resolve CloudStack directory') {
117+
steps {
118+
script {
119+
if (fileExists("${env.WORKSPACE}/private-cicd/scripts/install-build-deps-ubuntu.sh")) {
120+
env.CICD_SCRIPT_DIR = "${env.WORKSPACE}/private-cicd/scripts"
121+
env.CICD_ROOT = "${env.WORKSPACE}/private-cicd"
122+
} else if (fileExists("${env.WORKSPACE}/scripts/install-build-deps-ubuntu.sh")) {
123+
env.CICD_SCRIPT_DIR = "${env.WORKSPACE}/scripts"
124+
env.CICD_ROOT = env.WORKSPACE
125+
} else {
126+
error('Cannot find install-build-deps-ubuntu.sh (expected private-cicd/scripts or scripts/).')
127+
}
128+
if (params.CLONE_SEPARATE) {
129+
env.CLOUDSTACK_DIR = "${env.WORKSPACE}/cloudstack-src"
130+
} else {
131+
env.CLOUDSTACK_DIR = env.WORKSPACE
132+
}
133+
134+
def rel = params.CONFIG_DEFAULTS_FILE?.trim()
135+
if (!rel) {
136+
rel = 'config/defaults.yaml'
137+
}
138+
rel = rel.replaceAll('^/+', '')
139+
if (rel.contains('..')) {
140+
error('CONFIG_DEFAULTS_FILE must stay under CICD_ROOT (no .. segments).')
141+
}
142+
def cfgPath = "${env.CICD_ROOT}/${rel}"
143+
echo "Loading CI defaults from: ${cfgPath}"
144+
def baseCfg = [cloudstack: [git_url: 'https://github.com/apache/cloudstack.git', branch: 'main']]
145+
if (fileExists(cfgPath)) {
146+
baseCfg = readYaml(file: cfgPath) ?: baseCfg
147+
} else {
148+
echo "No ${cfgPath}; using built-in CloudStack URL/branch defaults."
149+
}
150+
151+
def cs = new LinkedHashMap((baseCfg.cloudstack ?: [:]) as Map)
152+
def profileName = params.CONFIG_PROFILE?.trim()
153+
if (profileName && baseCfg.profiles instanceof Map && baseCfg.profiles[profileName]) {
154+
def overlay = baseCfg.profiles[profileName].cloudstack
155+
if (overlay instanceof Map) {
156+
cs.putAll(overlay as Map)
157+
}
158+
}
159+
160+
def urlOverride = params.CLOUDSTACK_GIT_URL?.trim()
161+
def branchOverride = params.CLOUDSTACK_GIT_BRANCH?.trim()
162+
env.EFFECTIVE_CLOUDSTACK_GIT_URL = urlOverride ?: (cs.git_url as String ?: 'https://github.com/apache/cloudstack.git')
163+
env.EFFECTIVE_CLOUDSTACK_GIT_BRANCH = branchOverride ?: (cs.branch as String ?: 'main')
164+
165+
echo "CICD_ROOT=${env.CICD_ROOT} CICD_SCRIPT_DIR=${env.CICD_SCRIPT_DIR} CLOUDSTACK_DIR=${env.CLOUDSTACK_DIR}"
166+
echo "CONFIG_PROFILE=${profileName ?: '(none)'} EFFECTIVE_CLOUDSTACK_GIT_URL=${env.EFFECTIVE_CLOUDSTACK_GIT_URL} EFFECTIVE_CLOUDSTACK_GIT_BRANCH=${env.EFFECTIVE_CLOUDSTACK_GIT_BRANCH}"
167+
}
168+
}
169+
}
170+
171+
stage('Phase -1: validate private-cicd') {
172+
when {
173+
expression { return params.PIPELINE_PHASE == 'preflight' }
174+
}
175+
steps {
176+
sh """CICD_ROOT='${env.CICD_ROOT}' bash '${env.CICD_SCRIPT_DIR}/validate-local.sh'"""
177+
}
178+
}
179+
180+
stage('Clone CloudStack') {
181+
when {
182+
allOf {
183+
expression { return params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast'] }
184+
expression { return params.CLONE_SEPARATE }
185+
}
186+
}
187+
steps {
188+
sh 'rm -rf cloudstack-src && mkdir cloudstack-src'
189+
dir('cloudstack-src') {
190+
checkout(
191+
[
192+
$class : 'GitSCM',
193+
branches : [[name: "*/${env.EFFECTIVE_CLOUDSTACK_GIT_BRANCH}"]],
194+
doGenerateSubmoduleConfigurations: false,
195+
extensions : [],
196+
submoduleCfg : [],
197+
userRemoteConfigs : [[url: env.EFFECTIVE_CLOUDSTACK_GIT_URL]]
198+
]
199+
)
200+
}
201+
}
202+
}
203+
204+
stage('Validate tree') {
205+
when {
206+
expression { return params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast'] }
207+
}
208+
steps {
209+
script {
210+
def pom = "${env.CLOUDSTACK_DIR}/pom.xml"
211+
if (!fileExists(pom)) {
212+
error("Missing ${pom}. Enable CLONE_SEPARATE or check out CloudStack with Script Path private-cicd/Jenkinsfile at repo root.")
213+
}
214+
}
215+
}
216+
}
217+
218+
stage('Install OS build dependencies') {
219+
when {
220+
allOf {
221+
expression { return params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast'] }
222+
expression { return params.INSTALL_APT_DEPS }
223+
}
224+
}
225+
steps {
226+
sh """bash '${env.CICD_SCRIPT_DIR}/install-build-deps-ubuntu.sh'"""
227+
}
228+
}
229+
230+
stage('Optional ipmitool wrapper') {
231+
when {
232+
allOf {
233+
expression { return params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast'] }
234+
expression { return params.SETUP_IPMITOOL_WRAPPER }
235+
}
236+
}
237+
steps {
238+
sh """bash '${env.CICD_SCRIPT_DIR}/setup-ipmitool-wrapper.sh'"""
239+
}
240+
}
241+
242+
stage('Non-OSS (noredist)') {
243+
when {
244+
allOf {
245+
expression { return params.PIPELINE_PHASE == 'build-only' }
246+
expression { return params.ENABLE_NOREDIST }
247+
}
248+
}
249+
steps {
250+
dir("${env.CLOUDSTACK_DIR}") {
251+
sh '''
252+
set -e
253+
rm -rf nonoss
254+
git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss
255+
cd nonoss && bash -x install-non-oss.sh
256+
cd ..
257+
rm -rf nonoss
258+
'''
259+
}
260+
}
261+
}
262+
263+
stage('Phase 1a: ONTAP fast build') {
264+
when {
265+
expression { return params.PIPELINE_PHASE == 'build-ontap-fast' }
266+
}
267+
steps {
268+
dir("${env.CLOUDSTACK_DIR}") {
269+
script {
270+
def skip = params.SKIP_TESTS ? 'true' : 'false'
271+
sh """SKIP_TESTS='${skip}' CLOUDSTACK_DIR='${env.CLOUDSTACK_DIR}' bash '${env.CICD_SCRIPT_DIR}/mvn-ontap-fast.sh'"""
272+
}
273+
}
274+
}
275+
}
276+
277+
stage('Phase 1: Maven build') {
278+
when {
279+
expression { return params.PIPELINE_PHASE == 'build-only' }
280+
}
281+
steps {
282+
dir("${env.CLOUDSTACK_DIR}") {
283+
script {
284+
def skip = params.SKIP_TESTS ? '-DskipTests=true' : ''
285+
def noredist = params.ENABLE_NOREDIST ? '-Dnoredist' : ''
286+
sh "mvn -B -P developer,systemvm -Dsimulator ${noredist} clean install ${skip} -T\$(nproc)"
287+
}
288+
}
289+
}
290+
}
291+
}
292+
293+
post {
294+
always {
295+
script {
296+
if (params.PIPELINE_PHASE in ['build-only', 'build-ontap-fast']) {
297+
dir("${env.CLOUDSTACK_DIR}") {
298+
def junitGlob = '**/target/surefire-reports/*.xml'
299+
if (params.PIPELINE_PHASE == 'build-ontap-fast') {
300+
junitGlob = 'plugins/storage/volume/ontap/target/surefire-reports/*.xml'
301+
}
302+
junit testResults: junitGlob, allowEmptyResults: true
303+
}
304+
}
305+
}
306+
}
307+
failure {
308+
echo 'Pipeline failed — see console output.'
309+
}
310+
}
311+
}

0 commit comments

Comments
 (0)