From 385406e58714c40ba14c713e78d6da5e868c1ecf Mon Sep 17 00:00:00 2001 From: radnov Date: Wed, 25 Jan 2023 18:28:11 +0200 Subject: [PATCH 01/10] ci: add package develop and commit pipelines --- jenkinsfiles/README.md | 16 +- jenkinsfiles/commit.Jenkinsfile | 422 ++++++++++++++++++ jenkinsfiles/develop.Jenkinsfile | 75 ++++ ...xporter.Jenkinsfile => export.Jenkinsfile} | 5 +- ...ggerer.Jenkinsfile => trigger.Jenkinsfile} | 0 scripts/check-dashboards.sh | 14 +- scripts/check-expressions.sh | 9 +- scripts/make-dashboards-public.sh | 8 + scripts/replace-ou-placeholders.sh | 9 + scripts/run-import-tests.sh | 8 +- 10 files changed, 538 insertions(+), 28 deletions(-) create mode 100644 jenkinsfiles/commit.Jenkinsfile create mode 100644 jenkinsfiles/develop.Jenkinsfile rename jenkinsfiles/{exporter.Jenkinsfile => export.Jenkinsfile} (97%) rename jenkinsfiles/{triggerer.Jenkinsfile => trigger.Jenkinsfile} (100%) create mode 100755 scripts/make-dashboards-public.sh create mode 100755 scripts/replace-ou-placeholders.sh diff --git a/jenkinsfiles/README.md b/jenkinsfiles/README.md index 523461aa..c9129451 100644 --- a/jenkinsfiles/README.md +++ b/jenkinsfiles/README.md @@ -1,11 +1,19 @@ ## Jenkinsfiles -### [Exporter](exporter.Jenkinsfile) +### [Export](export.Jenkinsfile) * Exports metadata packages from a DHIS2 instance and tests them, based on a couple of input parameters (package code/type, dhis2 version and [more](exporter.Jenkinsfile#L8-L17)). * Can export and test a package (based on the provided input parameters) or only test an already exported package by uploading it via the `package_metadata_file` parameter. -* Can be started manually or scheduled in parallel by the [triggerer](##Triggerer) pipeline. +* Can be started manually or scheduled in parallel by the [trigger](##Trigger) pipeline. * If a package is exported from an instance running a "dev snapshot", instead of a stable version - it will only be tested, but not pushed to GitHub. -### [Triggerer](triggerer.Jenkinsfile) -* Triggers the [exporter](##Exporter) pipeline in parallel, based on the enabled packages in the Metadata index spreadsheet (via the [metadata-index-parser](https://github.com/dhis2/dhis2-utils/tree/master/tools/dhis2-metadata-index-parser)) and a list of DHIS2 versions to export from. +### [Trigger](trigger.Jenkinsfile) +* Triggers the [export](##Export) pipeline in parallel, based on the enabled packages in the Metadata index spreadsheet (via the [metadata-index-parser](https://github.com/dhis2/dhis2-utils/tree/master/tools/dhis2-metadata-index-parser)) and a list of DHIS2 versions to export from. * Can be started manually or by a [cron schedule](https://www.jenkins.io/doc/book/pipeline/syntax/#triggers). + +### develop + +### commit + +### export + +### trigger diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile new file mode 100644 index 00000000..92f0fb85 --- /dev/null +++ b/jenkinsfiles/commit.Jenkinsfile @@ -0,0 +1,422 @@ +@Library('pipeline-library') _ + +pipeline { + agent { + label 'ec2-jdk11-large' + } + + parameters { + string(name: 'STAGING_INSTANCE_NAME', defaultValue: 'foobar', description: '[REQUIRED] Full staging instance name will be: "pkg-staging--".') + choice(name: 'DATABASE', choices: ['pkgmaster', 'dev', 'tracker_dev'], description: '[REQUIRED] Master packages database or development database from https://metadata.dev.dhis2.org.') + booleanParam(name: 'REPLACE_DATABASE', defaultValue: true, description: '[OPTIONAL] Replace database if all validations and tests pass.') + string(name: 'DHIS2_VERSION', defaultValue: '2.38.4', description: '[OPTIONAL] DHIS2 version for the instance.') + string(name: 'DEV_INSTANCE_NAME', defaultValue: '', description: '[OPTIONAL] Name of the dev instance to export from.\nOnly needed if you want to export a package specified with PACKAGE_CODE/TYPE') + string(name: 'PACKAGE_CODE', defaultValue: '', description: '[OPTIONAL] Package code to extract with.\nNo need to provide this if you want to uploaded a package file below.') + choice(name: 'PACKAGE_TYPE', choices: ['AGG', 'TRK'], description: '[OPTIONAL] Type of the package to export.\nNo need to provide this if you want to uploaded a package file below.') + string(name: 'PACKAGE_DESC', defaultValue: '', description: '[OPTIONAL] Description of the package.\nNo need to provide this if you want to uploaded a package file below.') + stashedFile(name: 'PACKAGE_FILE_UPLOAD', description: '[OPTIONAL] Custom metadata package file.\nIf you upload one, there\'s no need to provide PACKAGE_CODE/TYPE/DESC.') + } + + options { + ansiColor('xterm') + disableConcurrentBuilds() // Needed in order to sequentially “merge” new packages into the current master database. + } + + environment { + IMAGE_TAG = "${params.DHIS2_VERSION}" + IMAGE_REPOSITORY = 'core' + IM_ENVIRONMENT = 'prod.test.c.dhis2.org' + IM_HOST = "https://api.im.$IM_ENVIRONMENT" + INSTANCE_GROUP_NAME = 'meta-packages' + INSTANCE_NAME_FULL = "pkg-staging-${params.STAGING_INSTANCE_NAME.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER" + INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.im.$IM_ENVIRONMENT" + INSTANCE_URL = "$INSTANCE_HOST/$INSTANCE_NAME_FULL" + HTTP = 'https --check-status' + PKG_IM_CREDENTIALS_ID = 'pkg-im-bot' + PKG_CREDENTIALS = credentials('packages-instance-credentials') + PKG_MASTER_DB_NAME = 'packages-dev.sql.gz' + DATABASE_NAME = "${params.DATABASE == 'pkgmaster' ? params.DATABASE + '.pgc' : params.DATABASE + '.sql.gz'}" + PACKAGE_FILE = 'PACKAGE_FILE_UPLOAD' + PACKAGE_DIFF_FILE = 'package-diff-file' + ALL_METADATA_FILE = 'all-metadata.json' + DHIS2_LOCAL_PORT = 8080 + } + + stages { + stage('Clone git repos') { + steps { + script { + dir('dhis2-utils') { + git url: 'https://github.com/dhis2/dhis2-utils' + } + + dir('dhis2-metadata-checkers') { + git url: 'https://github.com/solid-lines/dhis2-metadata-checkers', branch: 'main' + } + + dir('ALL_METADATA') { + git url: 'https://github.com/dhis2-metadata/ALL_METADATA', credentialsId: 'github-token-as-password' + } + + dir('im-manager') { + gitHelper.sparseCheckout('https://github.com/dhis2-sre/im-manager', 'master', '/scripts') + } + } + } + } + + stage('Create staging instance') { + steps { + script { + withCredentials([usernamePassword(credentialsId: "$PKG_IM_CREDENTIALS_ID", passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager/scripts/databases') { + env.DATABASE_ID = sh( + returnStdout: true, + script: "./list.sh | jq -r '.[] | select(.name == \"$INSTANCE_GROUP_NAME\") .databases[] | select(.name == \"$DATABASE_NAME\") .id'" + ).trim() + + sh '[ -n "$DATABASE_ID" ]' + echo "DATABASE_ID is $DATABASE_ID" + } + + dir('im-manager/scripts/instances') { + echo 'Creating DHIS2 instance ...' + + sh "./deploy-dhis2.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME_FULL" + + timeout(15) { + waitFor.statusOk("$INSTANCE_URL") + } + + NOTIFIER_ENDPOINT = dhis2.generateAnalytics("$INSTANCE_URL", '$PKG_CREDENTIALS') + timeout(15) { + waitFor.analyticsCompleted("${INSTANCE_URL}${NOTIFIER_ENDPOINT}", '$PKG_CREDENTIALS') + } + } + } + } + } + } + + stage('Export package from dev instance') { + when { + expression { + try { + unstash "$PACKAGE_FILE" // Need to unstash the file, otherwise it's always null. + } catch (e) { + echo e.toString() + } + + return params.DEV_INSTANCE_NAME != '' + } + } + + steps { + script { + echo "Exporting package from ${params.DEV_INSTANCE_NAME} ..." + + sh 'echo {\\"dhis\\": {\\"username\\": \\"${PKG_CREDENTIALS_USR}\\", \\"password\\": \\"${PKG_CREDENTIALS_PSW}\\"}} > auth.json' + + PACKAGE_FILE = sh( + returnStdout: true, + script: """#!/bin/bash + set -euxo pipefail + ./scripts/export-package.sh "${params.PACKAGE_CODE}" "${params.PACKAGE_TYPE}" "${params.PACKAGE_DESC}" "${env.INSTANCE_HOST}/${params.DEV_INSTANCE_NAME}" | tail -1 + """ + ).trim() + + PACKAGE_NAME = sh(returnStdout: true, script: "cat $PACKAGE_FILE | jq -r '.package .name'").trim() + } + } + } + + stage('Test package in empty instance') { + when { + anyOf { + expression { params.DEV_INSTANCE_NAME != '' } + + expression { readFile("$PACKAGE_FILE").size() != 0 } + } + + } + + stages { + stage('Validate metadata') { + steps { + script { + catchError(catchInterruptions: false, message: 'Validation errors found!', stageResult: 'FAILURE') { + sh("python3 -u dhis2-utils/tools/dhis2-metadata-package-validator/metadata_package_validator.py -f $WORKSPACE/$PACKAGE_FILE") + } + } + } + } + + stage('Test import') { + steps { + script { + withDockerRegistry([credentialsId: "docker-hub-credentials", url: ""]) { + d2.startCluster("$IMAGE_TAG", "$DHIS2_LOCAL_PORT") + } + + sleep(5) + + dir('test') { + sh "$WORKSPACE/scripts/replace-ou-placeholders.sh $WORKSPACE/$PACKAGE_FILE > ./package.json" + sh "$WORKSPACE/scripts/run-import-tests.sh $DHIS2_LOCAL_PORT" + } + } + } + + post { + always { + script { + sh "d2 cluster compose $IMAGE_TAG logs core > logs_empty_instance.txt" + archiveArtifacts artifacts: 'logs_empty_instance.txt' + } + } + } + } + + stage('Run checks') { + parallel { + stage('Check dashboards') { + steps { + sh './scripts/make-dashboards-public.sh' + sh './scripts/check-dashboards.sh http://localhost:$DHIS2_LOCAL_PORT' + } + } + + stage('Check PR expressions') { + steps { + sh './scripts/check-expressions.sh http://localhost:$DHIS2_LOCAL_PORT' + } + } + } + } + } + } + + stage ('Notify before import') { + when { + anyOf { + expression { params.DEV_INSTANCE_NAME != '' } + + expression { readFile("$PACKAGE_FILE").size() != 0 } + } + + } + + steps { + script { + slackSend( + color: '#00ffff', + channel: '@U01RSD1LPB3', + message: slack.buildUrl() + "\nPausing for review before importing package." + ) + } + } + } + + stage('Pause before import') { + when { + anyOf { + expression { params.DEV_INSTANCE_NAME != '' } + + expression { readFile("$PACKAGE_FILE").size() != 0 } + } + + } + + input { + message "Continue?" + ok "Yes" + parameters { + string(name: 'EXAMPLE_PARAMETER', defaultValue: 'test', description: 'Example description.') + booleanParam(name: 'DELETE_DEV_INSTANCE', defaultValue: false, description: 'Should the dev instance be deleted?') + } + } + + steps { + script { + echo "The value of the EXAMPLE_PARAMETER is ${env.EXAMPLE_PARAMETER}" + // Parameters from the input block are available as env variables only for the current stage, + // so they have to be reassigned to a new environment variable for the next stages. + env.DELETE_DEV_INSTANCE_ENV = env.DELETE_DEV_INSTANCE + } + } + } + + stage('Import package into staging instance') { + when { + anyOf { + expression { params.DEV_INSTANCE_NAME != '' } + + expression { readFile("$PACKAGE_FILE").size() != 0 } + } + + } + + steps { + echo "Importing package into ${params.STAGING_INSTANCE_NAME} ..." + sh "$WORKSPACE/scripts/replace-ou-placeholders.sh $PACKAGE_FILE > updated-$PACKAGE_FILE" + sh "$HTTP --auth \$PKG_CREDENTIALS POST $INSTANCE_URL/api/metadata < updated-$PACKAGE_FILE" + } + } + + stage ('Notify after import') { + steps { + script { + slackSend( + color: '#00ffff', + channel: '@U01RSD1LPB3', + message: slack.buildUrl() + "\nPausing for review after importing package." + ) + } + } + } + + stage('Pause after import') { + input { + message "Continue?" + ok "Yes" + parameters { + string(name: 'EXAMPLE_PARAMETER', defaultValue: 'test', description: 'Example description.') + booleanParam(name: 'DELETE_DEV_INSTANCE', defaultValue: false, description: 'Should the dev instance be deleted?') + } + } + + steps { + script { + echo "The value of the EXAMPLE_PARAMETER is ${env.EXAMPLE_PARAMETER}" + // Parameters from the input block are available as env variables only for the current stage, + // so they have to be reassigned to a new environment variable for the next stages. + env.DELETE_DEV_INSTANCE_ENV = env.DELETE_DEV_INSTANCE + } + } + } + + stage('Test packages in staging instance') { + stages { + stage('Export all metadata') { + steps { + echo "Export metadata from ${params.STAGING_INSTANCE_NAME} ..." + sh "$HTTP --auth \$PKG_CREDENTIALS GET $INSTANCE_URL/api/metadata > $ALL_METADATA_FILE" + } + } + + stage('Validate metadata') { + steps { + script { + catchError(catchInterruptions: false, message: 'Validation errors found!', stageResult: 'FAILURE') { + sh("python3 -u dhis2-utils/tools/dhis2-metadata-package-validator/metadata_package_validator.py -f $WORKSPACE/$ALL_METADATA_FILE") + } + } + } + } + + stage('Run checks') { + parallel { + stage('Check dashboards') { + steps { + catchError(catchInterruptions: false, message: 'Dashboard errors found!', stageResult: 'FAILURE') { + sh 'USER_NAME="$PKG_CREDENTIALS_USR" USER_PASSWORD="$PKG_CREDENTIALS_PSW" ./scripts/check-dashboards.sh $INSTANCE_URL' + } + } + } + + stage('Check PR expressions') { + steps { + catchError(catchInterruptions: false, message: 'PR expression errors found!', stageResult: 'FAILURE') { + sh 'USER_NAME="$PKG_CREDENTIALS_USR" USER_PASSWORD="$PKG_CREDENTIALS_PSW" ./scripts/check-expressions.sh $INSTANCE_URL' + } + } + } + } + } + } + } + + stage('Create diff') { + steps { + dir('dhis2-utils/tools/dhis2-metadatapackagediff') { + sh 'pip3 install -r requirements.txt' + + sh "python3 metadata_package_diff.py $WORKSPACE/ALL_METADATA/$ALL_METADATA_FILE $WORKSPACE/$ALL_METADATA_FILE $WORKSPACE/$PACKAGE_DIFF_FILE" + } + } + + post { + always { + archiveArtifacts artifacts: "${PACKAGE_DIFF_FILE}.xlsx" + } + } + } + + stage('Push metadata to GitHub') { + environment { + GITHUB_CREDS = credentials('github-token-as-password') + GITHUB_EMAIL = 'apps@dhis2.org' + REPOSITORY_NAME = 'ALL_METADATA' + } + + steps { + script { + sh 'git config --global user.email $GITHUB_EMAIL' + sh 'git config --global user.name $GITHUB_CREDS_USR' + + dir("$REPOSITORY_NAME") { + sh 'cp $WORKSPACE/$ALL_METADATA_FILE .' + sh 'git add .' + sh 'git diff-index --quiet HEAD || git commit -m "chore: Upload all metadata"' + sh 'git push https://$GITHUB_CREDS_PSW@github.com/dhis2-metadata/$REPOSITORY_NAME' + } + } + } + } + + stage('Save DB') { + when { + expression { params.REPLACE_DATABASE } + } + + steps { + echo "Saving ${params.DATABASE} DB ..." + script { + withCredentials([usernamePassword(credentialsId: "$PKG_IM_CREDENTIALS_ID", passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager/scripts/databases') { + env.NEW_DATABASE_ID = sh( + returnStdout: true, + script: "./save.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME_FULL | jq -r '.id'" + ).trim() + + timeout(15) { + waitUntil(initialRecurrencePeriod: 5000, quiet: true) { + dbUrl = sh(returnStdout: true, script: "./findById.sh $NEW_DATABASE_ID | jq -r '.url'").trim() + + return (dbUrl != '') + } + } + } + } + } + } + } + } + + post { + success { + script { + withCredentials([usernamePassword(credentialsId: "$PKG_IM_CREDENTIALS_ID", passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager/scripts/instances') { + env.INSTANCES_TO_DELETE = env.INSTANCE_NAME_FULL + if (env.DELETE_DEV_INSTANCE_ENV.toBoolean()) { + env.INSTANCES_TO_DELETE = "${env.INSTANCE_NAME_FULL} ${params.DEV_INSTANCE_NAME}" + } + + sh "./destroy.sh $INSTANCE_GROUP_NAME $INSTANCES_TO_DELETE" + } + } + } + } + } +} diff --git a/jenkinsfiles/develop.Jenkinsfile b/jenkinsfiles/develop.Jenkinsfile new file mode 100644 index 00000000..da2e197a --- /dev/null +++ b/jenkinsfiles/develop.Jenkinsfile @@ -0,0 +1,75 @@ +@Library('pipeline-library') _ + +pipeline { + agent { + label 'ec2-jdk11-large' + } + + parameters { + choice(name: 'DATABASE', choices: ['dev', 'tracker_dev', 'pkgmaster', 'empty'], description: 'Packages development database from https://metadata.dev.dhis2.org, pkgmaster or an empty one.') + string(name: 'INSTANCE_NAME', defaultValue: 'foobar', description: 'Full instance name will be: "pkg-dev--".') + string(name: 'DHIS2_VERSION', defaultValue: '2.38.4', description: 'DHIS2 version for the instance.') + string(name: 'TTL', defaultValue: '', description: 'Time to live for the instance in minutes.') + } + + options { + ansiColor('xterm') + } + + environment { + IMAGE_TAG = "${params.DHIS2_VERSION}" + IMAGE_REPOSITORY = 'core' + IM_ENVIRONMENT = 'prod.test.c.dhis2.org' + IM_HOST = "https://api.im.$IM_ENVIRONMENT" + INSTANCE_GROUP_NAME = 'meta-packages' + INSTANCE_NAME_FULL = "pkg-dev-${params.INSTANCE_NAME.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER" + INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.im.$IM_ENVIRONMENT" + INSTANCE_URL = "$INSTANCE_HOST/$INSTANCE_NAME_FULL" + INSTANCE_TTL = "${params.TTL != '' ? params.TTL.toInteger() * 60 : ''}" + DATABASE_NAME = "${params.DATABASE}.sql.gz" + HTTP = 'https --check-status' + PKG_CREDENTIALS = credentials('packages-instance-credentials') + } + + stages { + stage('Create dev instance') { + steps { + script { + withCredentials([usernamePassword(credentialsId: 'pkg-im-bot', passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager') { + gitHelper.sparseCheckout('https://github.com/dhis2-sre/im-manager', 'master', '/scripts') + + // TODO upload database every time or s3 replication rule? + dir('scripts/databases') { + env.DATABASE_ID = sh( + returnStdout: true, + script: "./list.sh | jq -r '.[] | select(.name == \"$INSTANCE_GROUP_NAME\") .databases[] | select(.name == \"$DATABASE_NAME\") .id'" + ).trim() + sh '[ -n "$DATABASE_ID" ]' + + echo "DATABASE_ID is $DATABASE_ID" + } + + dir('scripts/instances') { + echo 'Creating DHIS2 instance ...' + + sh "./deploy-dhis2.sh ${env.INSTANCE_GROUP_NAME} ${env.INSTANCE_NAME_FULL}" + + timeout(15) { + waitFor.statusOk("$INSTANCE_URL") + } + + if (params.DATABASE != 'empty') { + NOTIFIER_ENDPOINT = dhis2.generateAnalytics("$INSTANCE_URL", '$PKG_CREDENTIALS') + timeout(15) { + waitFor.analyticsCompleted("${INSTANCE_URL}${NOTIFIER_ENDPOINT}", '$PKG_CREDENTIALS') + } + } + } + } + } + } + } + } + } +} diff --git a/jenkinsfiles/exporter.Jenkinsfile b/jenkinsfiles/export.Jenkinsfile similarity index 97% rename from jenkinsfiles/exporter.Jenkinsfile rename to jenkinsfiles/export.Jenkinsfile index d012b9ac..31bc3115 100644 --- a/jenkinsfiles/exporter.Jenkinsfile +++ b/jenkinsfiles/export.Jenkinsfile @@ -7,8 +7,10 @@ pipeline { parameters { stashedFile 'package_metadata_file' + // TODO dynamic list of codes? string(name: 'Package_code', defaultValue: '', description: '[REQUIRED] Package code to extract with.') string(name: 'Package_type', defaultValue: '', description: '[REQUIRED] Type of the package to export.') + // TODO dynamic list of descriptions/names? string(name: 'Package_description', defaultValue: '', description: '[REQUIRED] Description of the package.') string(name: 'Instance_url', defaultValue: 'https://metadata.dev.dhis2.org/dev', description: '[REQUIRED] Instance URL to export package from.') string(name: 'DHIS2_version', defaultValue: '2.37', description: '[OPTIONAL] DHIS2 version to extract the package from. (only major.minor version like 2.37, not 2.37.1)') @@ -160,7 +162,8 @@ pipeline { sleep(5) dir('test') { - sh "$WORKSPACE/scripts/run-import-tests.sh ./package_orig.json $DHIS2_PORT" + sh "$WORKSPACE/scripts/replace-ou-placeholders.sh ./package_orig.json > ./package.json" + sh "$WORKSPACE/scripts/run-import-tests.sh $DHIS2_LOCAL_PORT" } } } diff --git a/jenkinsfiles/triggerer.Jenkinsfile b/jenkinsfiles/trigger.Jenkinsfile similarity index 100% rename from jenkinsfiles/triggerer.Jenkinsfile rename to jenkinsfiles/trigger.Jenkinsfile diff --git a/scripts/check-dashboards.sh b/scripts/check-dashboards.sh index eb5f4965..c8b719f3 100755 --- a/scripts/check-dashboards.sh +++ b/scripts/check-dashboards.sh @@ -2,18 +2,10 @@ set -euxo pipefail -port="$1" -user="${USER_NAME:-admin}" -pass="${USER_PASSWORD:-district}" -ou_root="${OU_ROOT_ID:-GD7TowwI46c}" +instance_url="$1" -db_container=$(docker container ls --filter name=db -q) - -docker exec -i "$db_container" psql -U dhis -d dhis2 -c "UPDATE dashboard SET sharing = jsonb_set(sharing, '{public}', '\"rw------\"');" -docker exec -i "$db_container" psql -U dhis -d dhis2 -c "INSERT INTO usermembership (organisationunitid, userinfoid) VALUES ((SELECT organisationunitid FROM organisationunit WHERE uid = '${ou_root}'), (SELECT userinfoid FROM userinfo WHERE code = '${user}'));" - -echo "{\"dhis\": {\"baseurl\": \"http://localhost:${port}\", \"username\": \"${user}\", \"password\": \"${pass}\"}}" > auth.json +echo "{\"dhis\": {\"username\": \"${USER_NAME:-admin}\", \"password\": \"${USER_PASSWORD:-district}\"}}" > auth.json pip3 install -r dhis2-utils/tools/dhis2-dashboardchecker/requirements.txt -python3 dhis2-utils/tools/dhis2-dashboardchecker/dashboard_checker.py -i=http://localhost:${port} --omit-no_data_warning +python3 dhis2-utils/tools/dhis2-dashboardchecker/dashboard_checker.py -i="$instance_url" --omit-no_data_warning diff --git a/scripts/check-expressions.sh b/scripts/check-expressions.sh index 82e1fe86..1f6dcc0b 100755 --- a/scripts/check-expressions.sh +++ b/scripts/check-expressions.sh @@ -2,10 +2,9 @@ set -euxo pipefail -port="$1" -user="${USER_NAME:-admin}" -pass="${USER_PASSWORD:-district}" +server_url="$1" +default_server_name='default' -echo -e "[localhost]\nserver=http://localhost:${port}/api/\nserver_name=localhost\nuser=${user}\npassword=${pass}\npage_size=500" > credentials.ini +echo -e "[$default_server_name]\nserver=$server_url/api/\nserver_name=$default_server_name\nuser=${USER_NAME:-admin}\npassword=${USER_PASSWORD:-district}\npage_size=500" > credentials.ini -python3 dhis2-metadata-checkers/check_expressions.py --credentials localhost +python3 dhis2-metadata-checkers/check_expressions.py --credentials $default_server_name diff --git a/scripts/make-dashboards-public.sh b/scripts/make-dashboards-public.sh new file mode 100755 index 00000000..c8bd1569 --- /dev/null +++ b/scripts/make-dashboards-public.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +db_container=$(docker container ls --filter name=db -q) + +docker exec -i "$db_container" psql -U dhis -d dhis2 -c "UPDATE dashboard SET sharing = jsonb_set(sharing, '{public}', '\"rw------\"');" +docker exec -i "$db_container" psql -U dhis -d dhis2 -c "INSERT INTO usermembership (organisationunitid, userinfoid) VALUES ((SELECT organisationunitid FROM organisationunit WHERE uid = '${OU_ROOT_ID:-GD7TowwI46c}'), (SELECT userinfoid FROM userinfo WHERE code = '${USER_NAME:-admin}'));" diff --git a/scripts/replace-ou-placeholders.sh b/scripts/replace-ou-placeholders.sh new file mode 100755 index 00000000..569d8017 --- /dev/null +++ b/scripts/replace-ou-placeholders.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +file="$1" + +sed "s//${OU_DISTRICT_UID:-qpXLDdXT3po}/g" "$file" | + sed "s//${OU_FACILITY_UID:-vFr4zVw6Avn}/g" | + sed "s//${OU_ROOT_UID:-GD7TowwI46c}/g" diff --git a/scripts/run-import-tests.sh b/scripts/run-import-tests.sh index 0fb1171f..b862937c 100755 --- a/scripts/run-import-tests.sh +++ b/scripts/run-import-tests.sh @@ -2,17 +2,11 @@ set -euxo pipefail -file="$1" -port="$2" +port="$1" auth="${USER_NAME:-admin}:${USER_PASSWORD:-district}" url="http://localhost:$port" -cat "$file" | - sed "s//${OU_DISTRICT_UID:-qpXLDdXT3po}/g" | - sed "s//${OU_FACILITY_UID:-vFr4zVw6Avn}/g" | - sed "s//${OU_ROOT_UID:-GD7TowwI46c}/g" > ./package.json - ./api-test.sh -v -f ./tests.json -url "$url" -auth "$auth" test ou_import URL="$url" AUTH="$auth" ./run-tests.sh From 578ba2347435d315654b4dff2c6c4ca8fa7354f3 Mon Sep 17 00:00:00 2001 From: radnov Date: Wed, 19 Jul 2023 12:42:10 +0300 Subject: [PATCH 02/10] ci: catch errors for check dashboards and PR expressions in empty instance --- jenkinsfiles/commit.Jenkinsfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index 92f0fb85..69fe9230 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -181,14 +181,18 @@ pipeline { parallel { stage('Check dashboards') { steps { - sh './scripts/make-dashboards-public.sh' - sh './scripts/check-dashboards.sh http://localhost:$DHIS2_LOCAL_PORT' + catchError(catchInterruptions: false, message: 'Dashboard errors found!', stageResult: 'FAILURE') { + sh './scripts/make-dashboards-public.sh' + sh './scripts/check-dashboards.sh http://localhost:$DHIS2_LOCAL_PORT' + } } } stage('Check PR expressions') { steps { - sh './scripts/check-expressions.sh http://localhost:$DHIS2_LOCAL_PORT' + catchError(catchInterruptions: false, message: 'PR expressions errors found!', stageResult: 'FAILURE') { + sh './scripts/check-expressions.sh http://localhost:$DHIS2_LOCAL_PORT' + } } } } From ba6b53a87560c47002bb5bc056c3f7641dd4b8bf Mon Sep 17 00:00:00 2001 From: radnov Date: Wed, 19 Jul 2023 14:37:42 +0300 Subject: [PATCH 03/10] ci: send commit pipeline pause notifications go #pkg-notifications --- jenkinsfiles/commit.Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index 69fe9230..edf43d7f 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -214,7 +214,7 @@ pipeline { script { slackSend( color: '#00ffff', - channel: '@U01RSD1LPB3', + channel: 'pkg-notifications', message: slack.buildUrl() + "\nPausing for review before importing package." ) } @@ -272,7 +272,7 @@ pipeline { script { slackSend( color: '#00ffff', - channel: '@U01RSD1LPB3', + channel: 'pkg-notifications', message: slack.buildUrl() + "\nPausing for review after importing package." ) } From 0852eb95f605697b85dd36a9cdd767b9e3610d26 Mon Sep 17 00:00:00 2001 From: radnov Date: Mon, 24 Jul 2023 15:03:02 +0300 Subject: [PATCH 04/10] ci: change how we save db in the end and check for completion --- jenkinsfiles/commit.Jenkinsfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index edf43d7f..e85323f0 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -388,16 +388,14 @@ pipeline { script { withCredentials([usernamePassword(credentialsId: "$PKG_IM_CREDENTIALS_ID", passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { dir('im-manager/scripts/databases') { - env.NEW_DATABASE_ID = sh( - returnStdout: true, - script: "./save.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME_FULL | jq -r '.id'" - ).trim() + sh "./save.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME_FULL" timeout(15) { waitUntil(initialRecurrencePeriod: 5000, quiet: true) { - dbUrl = sh(returnStdout: true, script: "./findById.sh $NEW_DATABASE_ID | jq -r '.url'").trim() + // TODO change this once we have a better way of knowing the status of a "save" + lock = sh(returnStdout: true, script: "./findById.sh $DATABASE_ID | jq -r '.lock'").trim() - return (dbUrl != '') + return (lock == 'null') } } } From 030e6e329f9a0804dcd1582735063dd87329d4dc Mon Sep 17 00:00:00 2001 From: radnov Date: Tue, 15 Aug 2023 16:11:31 +0300 Subject: [PATCH 05/10] ci: use latest available IM tag instead of master --- jenkinsfiles/commit.Jenkinsfile | 3 ++- jenkinsfiles/develop.Jenkinsfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index e85323f0..f3bfd283 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -25,6 +25,7 @@ pipeline { environment { IMAGE_TAG = "${params.DHIS2_VERSION}" IMAGE_REPOSITORY = 'core' + IM_REPO_URL = "https://github.com/dhis2-sre/im-manager" IM_ENVIRONMENT = 'prod.test.c.dhis2.org' IM_HOST = "https://api.im.$IM_ENVIRONMENT" INSTANCE_GROUP_NAME = 'meta-packages' @@ -59,7 +60,7 @@ pipeline { } dir('im-manager') { - gitHelper.sparseCheckout('https://github.com/dhis2-sre/im-manager', 'master', '/scripts') + gitHelper.sparseCheckout(IM_REPO_URL, "${gitHelper.getLatestTag(IM_REPO_URL)}", '/scripts') } } } diff --git a/jenkinsfiles/develop.Jenkinsfile b/jenkinsfiles/develop.Jenkinsfile index da2e197a..02c6a9e7 100644 --- a/jenkinsfiles/develop.Jenkinsfile +++ b/jenkinsfiles/develop.Jenkinsfile @@ -19,6 +19,7 @@ pipeline { environment { IMAGE_TAG = "${params.DHIS2_VERSION}" IMAGE_REPOSITORY = 'core' + IM_REPO_URL = "https://github.com/dhis2-sre/im-manager" IM_ENVIRONMENT = 'prod.test.c.dhis2.org' IM_HOST = "https://api.im.$IM_ENVIRONMENT" INSTANCE_GROUP_NAME = 'meta-packages' @@ -37,7 +38,7 @@ pipeline { script { withCredentials([usernamePassword(credentialsId: 'pkg-im-bot', passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { dir('im-manager') { - gitHelper.sparseCheckout('https://github.com/dhis2-sre/im-manager', 'master', '/scripts') + gitHelper.sparseCheckout(IM_REPO_URL, "${gitHelper.getLatestTag(IM_REPO_URL)}", '/scripts') // TODO upload database every time or s3 replication rule? dir('scripts/databases') { From b140dcdf00d992551e8a379fcc7bcb6af7e8f3e0 Mon Sep 17 00:00:00 2001 From: radnov Date: Tue, 22 Aug 2023 13:59:36 +0300 Subject: [PATCH 06/10] feat: add image repository parameter --- jenkinsfiles/commit.Jenkinsfile | 6 ++++-- jenkinsfiles/develop.Jenkinsfile | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index f3bfd283..460ebad4 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -9,6 +9,7 @@ pipeline { string(name: 'STAGING_INSTANCE_NAME', defaultValue: 'foobar', description: '[REQUIRED] Full staging instance name will be: "pkg-staging--".') choice(name: 'DATABASE', choices: ['pkgmaster', 'dev', 'tracker_dev'], description: '[REQUIRED] Master packages database or development database from https://metadata.dev.dhis2.org.') booleanParam(name: 'REPLACE_DATABASE', defaultValue: true, description: '[OPTIONAL] Replace database if all validations and tests pass.') + choice(name: 'DHIS2_IMAGE_REPOSITORY', choices: ['core', 'core-dev'], description: 'DHIS2 Docker image repository.') string(name: 'DHIS2_VERSION', defaultValue: '2.38.4', description: '[OPTIONAL] DHIS2 version for the instance.') string(name: 'DEV_INSTANCE_NAME', defaultValue: '', description: '[OPTIONAL] Name of the dev instance to export from.\nOnly needed if you want to export a package specified with PACKAGE_CODE/TYPE') string(name: 'PACKAGE_CODE', defaultValue: '', description: '[OPTIONAL] Package code to extract with.\nNo need to provide this if you want to uploaded a package file below.') @@ -24,7 +25,7 @@ pipeline { environment { IMAGE_TAG = "${params.DHIS2_VERSION}" - IMAGE_REPOSITORY = 'core' + IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}" IM_REPO_URL = "https://github.com/dhis2-sre/im-manager" IM_ENVIRONMENT = 'prod.test.c.dhis2.org' IM_HOST = "https://api.im.$IM_ENVIRONMENT" @@ -156,7 +157,8 @@ pipeline { steps { script { withDockerRegistry([credentialsId: "docker-hub-credentials", url: ""]) { - d2.startCluster("$IMAGE_TAG", "$DHIS2_LOCAL_PORT") + DHIS2_CHANNEL = "${params.DHIS2_IMAGE_REPOSITORY == 'core' ? 'stable' : 'dev'}" + d2.startCluster("$IMAGE_TAG", "$DHIS2_LOCAL_PORT", "$DHIS2_CHANNEL") } sleep(5) diff --git a/jenkinsfiles/develop.Jenkinsfile b/jenkinsfiles/develop.Jenkinsfile index 02c6a9e7..179b976f 100644 --- a/jenkinsfiles/develop.Jenkinsfile +++ b/jenkinsfiles/develop.Jenkinsfile @@ -8,6 +8,7 @@ pipeline { parameters { choice(name: 'DATABASE', choices: ['dev', 'tracker_dev', 'pkgmaster', 'empty'], description: 'Packages development database from https://metadata.dev.dhis2.org, pkgmaster or an empty one.') string(name: 'INSTANCE_NAME', defaultValue: 'foobar', description: 'Full instance name will be: "pkg-dev--".') + choice(name: 'DHIS2_IMAGE_REPOSITORY', choices: ['core', 'core-dev'], description: 'DHIS2 Docker image repository.') string(name: 'DHIS2_VERSION', defaultValue: '2.38.4', description: 'DHIS2 version for the instance.') string(name: 'TTL', defaultValue: '', description: 'Time to live for the instance in minutes.') } @@ -18,7 +19,7 @@ pipeline { environment { IMAGE_TAG = "${params.DHIS2_VERSION}" - IMAGE_REPOSITORY = 'core' + IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}" IM_REPO_URL = "https://github.com/dhis2-sre/im-manager" IM_ENVIRONMENT = 'prod.test.c.dhis2.org' IM_HOST = "https://api.im.$IM_ENVIRONMENT" From 13fa10aa7805165d88664f297c48d469c9e1e0fb Mon Sep 17 00:00:00 2001 From: radnov Date: Tue, 22 Aug 2023 15:54:35 +0300 Subject: [PATCH 07/10] refactor: use docker compose instead of d2 --- hack/docker-compose.yml | 2 +- jenkinsfiles/commit.Jenkinsfile | 27 +++++++++++++++++++++++---- scripts/make-dashboards-public.sh | 4 ++-- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/hack/docker-compose.yml b/hack/docker-compose.yml index 84bd8049..12ffe747 100644 --- a/hack/docker-compose.yml +++ b/hack/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.8" services: - web: + core: image: "${DHIS2_IMAGE:-dhis2/core-dev:local}" ports: - 127.0.0.1:8080:8080 diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index 460ebad4..6f799533 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -157,8 +157,24 @@ pipeline { steps { script { withDockerRegistry([credentialsId: "docker-hub-credentials", url: ""]) { - DHIS2_CHANNEL = "${params.DHIS2_IMAGE_REPOSITORY == 'core' ? 'stable' : 'dev'}" - d2.startCluster("$IMAGE_TAG", "$DHIS2_LOCAL_PORT", "$DHIS2_CHANNEL") +// DHIS2_CHANNEL = "${params.DHIS2_IMAGE_REPOSITORY == 'core' ? 'stable' : 'dev'}" +// d2.startCluster("$IMAGE_TAG", "$DHIS2_LOCAL_PORT", "$DHIS2_CHANNEL") + dir('hack') { + dir('docker') { + sh 'curl "https://raw.githubusercontent.com/dhis2/dhis2-core/master/docker/dhis.conf" -O' + } + + env.DHIS2_HOME = '/opt/dhis2' + env.DHIS2_IMAGE = "dhis2/${params.DHIS2_IMAGE_REPOSITORY}:${IMAGE_TAG}" + + sh 'docker-compose up -d' + + sleep(10) + + timeout(5) { + waitFor.statusOk("http://localhost:${DHIS2_LOCAL_PORT}") + } + } } sleep(5) @@ -173,8 +189,11 @@ pipeline { post { always { script { - sh "d2 cluster compose $IMAGE_TAG logs core > logs_empty_instance.txt" - archiveArtifacts artifacts: 'logs_empty_instance.txt' +// sh "d2 cluster compose $IMAGE_TAG logs core > logs_empty_instance.txt" + dir('hack') { + sh "docker-compose logs core > logs_empty_instance.txt" + archiveArtifacts artifacts: 'logs_empty_instance.txt' + } } } } diff --git a/scripts/make-dashboards-public.sh b/scripts/make-dashboards-public.sh index c8bd1569..79e08b16 100755 --- a/scripts/make-dashboards-public.sh +++ b/scripts/make-dashboards-public.sh @@ -4,5 +4,5 @@ set -euxo pipefail db_container=$(docker container ls --filter name=db -q) -docker exec -i "$db_container" psql -U dhis -d dhis2 -c "UPDATE dashboard SET sharing = jsonb_set(sharing, '{public}', '\"rw------\"');" -docker exec -i "$db_container" psql -U dhis -d dhis2 -c "INSERT INTO usermembership (organisationunitid, userinfoid) VALUES ((SELECT organisationunitid FROM organisationunit WHERE uid = '${OU_ROOT_ID:-GD7TowwI46c}'), (SELECT userinfoid FROM userinfo WHERE code = '${USER_NAME:-admin}'));" +docker exec -i "$db_container" psql -U dhis -d dhis -c "UPDATE dashboard SET sharing = jsonb_set(sharing, '{public}', '\"rw------\"');" +docker exec -i "$db_container" psql -U dhis -d dhis -c "INSERT INTO usermembership (organisationunitid, userinfoid) VALUES ((SELECT organisationunitid FROM organisationunit WHERE uid = '${OU_ROOT_ID:-GD7TowwI46c}'), (SELECT userinfoid FROM userinfo WHERE code = '${USER_NAME:-admin}'));" From 147e48921557981a0a234df37f6ac4ba1ae98e66 Mon Sep 17 00:00:00 2001 From: radnov Date: Thu, 24 Aug 2023 17:45:08 +0300 Subject: [PATCH 08/10] ci: add todo for updating IM hostname references --- jenkinsfiles/commit.Jenkinsfile | 1 + jenkinsfiles/develop.Jenkinsfile | 1 + 2 files changed, 2 insertions(+) diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index 6f799533..e4086100 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -27,6 +27,7 @@ pipeline { IMAGE_TAG = "${params.DHIS2_VERSION}" IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}" IM_REPO_URL = "https://github.com/dhis2-sre/im-manager" + //TODO update prod environment hostname and IM group hostname IM_ENVIRONMENT = 'prod.test.c.dhis2.org' IM_HOST = "https://api.im.$IM_ENVIRONMENT" INSTANCE_GROUP_NAME = 'meta-packages' diff --git a/jenkinsfiles/develop.Jenkinsfile b/jenkinsfiles/develop.Jenkinsfile index 179b976f..3499e585 100644 --- a/jenkinsfiles/develop.Jenkinsfile +++ b/jenkinsfiles/develop.Jenkinsfile @@ -21,6 +21,7 @@ pipeline { IMAGE_TAG = "${params.DHIS2_VERSION}" IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}" IM_REPO_URL = "https://github.com/dhis2-sre/im-manager" + //TODO update prod environment hostname and IM group hostname IM_ENVIRONMENT = 'prod.test.c.dhis2.org' IM_HOST = "https://api.im.$IM_ENVIRONMENT" INSTANCE_GROUP_NAME = 'meta-packages' From 3c6615b2f5f1324698b6b675647d6e4ae71faf55 Mon Sep 17 00:00:00 2001 From: radnov Date: Wed, 27 Sep 2023 13:37:16 +0300 Subject: [PATCH 09/10] refactor: use packages spreadsheet to populate PACKAGE_NAME parameter --- jenkinsfiles/commit.Jenkinsfile | 52 +++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index e4086100..a8ae817d 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -1,21 +1,41 @@ @Library('pipeline-library') _ +node('ec2-jdk8-large-spot') { + dir('dhis2-utils') { + git url: 'https://github.com/dhis2/dhis2-utils', branch: 'use-argparse-insead-of-envvars' + + dir('tools/dhis2-metadata-index-parser') { + sh 'pip3 install -r requirements.txt' + + withCredentials([file(credentialsId: 'metadata-index-parser-service-account', variable: 'GOOGLE_SERVICE_ACCOUNT')]) { + env.SPREADSHEET_ID = '1IIQL2IkGJqiIWLr6Bgg7p9fE78AwQYhHBNGoV-spGOM' + + env.PACKAGES_INDEX_JSON = sh( + returnStdout: true, + script: 'python3 parse-index.py --service-account-file $GOOGLE_SERVICE_ACCOUNT --spreadsheet-id $SPREADSHEET_ID --no-only-ready --no-uncheck-readiness' + ).trim() + + PARAMETER_VALUES = sh(returnStdout:true, script: 'echo $PACKAGES_INDEX_JSON | jq -r \'.[] | .["Component name"]\'').trim() + } + } + } +} + pipeline { agent { label 'ec2-jdk11-large' } parameters { + booleanParam(name: 'REFRESH_PACKAGES', defaultValue: false, description: '[OPTIONAL] Refresh the list of PACKAGE_NAMEs and abort the build.') string(name: 'STAGING_INSTANCE_NAME', defaultValue: 'foobar', description: '[REQUIRED] Full staging instance name will be: "pkg-staging--".') choice(name: 'DATABASE', choices: ['pkgmaster', 'dev', 'tracker_dev'], description: '[REQUIRED] Master packages database or development database from https://metadata.dev.dhis2.org.') booleanParam(name: 'REPLACE_DATABASE', defaultValue: true, description: '[OPTIONAL] Replace database if all validations and tests pass.') choice(name: 'DHIS2_IMAGE_REPOSITORY', choices: ['core', 'core-dev'], description: 'DHIS2 Docker image repository.') string(name: 'DHIS2_VERSION', defaultValue: '2.38.4', description: '[OPTIONAL] DHIS2 version for the instance.') - string(name: 'DEV_INSTANCE_NAME', defaultValue: '', description: '[OPTIONAL] Name of the dev instance to export from.\nOnly needed if you want to export a package specified with PACKAGE_CODE/TYPE') - string(name: 'PACKAGE_CODE', defaultValue: '', description: '[OPTIONAL] Package code to extract with.\nNo need to provide this if you want to uploaded a package file below.') - choice(name: 'PACKAGE_TYPE', choices: ['AGG', 'TRK'], description: '[OPTIONAL] Type of the package to export.\nNo need to provide this if you want to uploaded a package file below.') - string(name: 'PACKAGE_DESC', defaultValue: '', description: '[OPTIONAL] Description of the package.\nNo need to provide this if you want to uploaded a package file below.') - stashedFile(name: 'PACKAGE_FILE_UPLOAD', description: '[OPTIONAL] Custom metadata package file.\nIf you upload one, there\'s no need to provide PACKAGE_CODE/TYPE/DESC.') + string(name: 'DEV_INSTANCE_NAME', defaultValue: '', description: '[OPTIONAL] Name of the dev instance to export from.\nOnly needed if you want to export a package specified with PACKAGE_NAME') + choice(name: 'PACKAGE_NAME', choices: PARAMETER_VALUES, description: '[REQUIRED] Select a package to extract by name.') + stashedFile(name: 'PACKAGE_FILE_UPLOAD', description: '[OPTIONAL] Provide a custom metadata package file instead of exporting it.\nIf a file is uploaded, the "DEV_INSTANCE_NAME" and "PACKAGE_NAME" are obsolete.') } options { @@ -49,6 +69,11 @@ pipeline { stage('Clone git repos') { steps { script { + // Immediately abort pipeline if REFRESH_PACKAGES is enabled. + if (params.REFRESH_PACKAGES.toBoolean()) { + error('This build is only for refreshing the PACKAGE_NAME parameter. Pipeline will be aborted now.') + } + dir('dhis2-utils') { git url: 'https://github.com/dhis2/dhis2-utils' } @@ -71,6 +96,14 @@ pipeline { stage('Create staging instance') { steps { script { + // Get package details based on the selected PACKAGE_NAME parameter value. + env.SELECTED_PACKAGE = sh(returnStdout: true, script: 'echo "$PACKAGES_INDEX_JSON" | jq --arg name "$PACKAGE_NAME" -r \'.[] | select(."Component name" == $name)\'').trim() + env.PACKAGE_CODE = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Extraction code"\'').trim() + env.PACKAGE_TYPE = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Script parameter"\'').trim() + env.PACKAGE_SOURCE_INSTANCE = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Source instance"\'').trim() + env.PACKAGE_HEALTH_AREA_NAME = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Health area"\'').trim() + env.PACKAGE_HEALTH_AREA_CODE = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Health area prefix"\'').trim() + withCredentials([usernamePassword(credentialsId: "$PKG_IM_CREDENTIALS_ID", passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { dir('im-manager/scripts/databases') { env.DATABASE_ID = sh( @@ -122,13 +155,14 @@ pipeline { PACKAGE_FILE = sh( returnStdout: true, - script: """#!/bin/bash + script: '''#!/bin/bash set -euxo pipefail - ./scripts/export-package.sh "${params.PACKAGE_CODE}" "${params.PACKAGE_TYPE}" "${params.PACKAGE_DESC}" "${env.INSTANCE_HOST}/${params.DEV_INSTANCE_NAME}" | tail -1 - """ + ./scripts/export-package.sh "$PACKAGE_CODE" "$PACKAGE_TYPE" "$PACKAGE_NAME" "$PACKAGE_HEALTH_AREA_NAME" "$PACKAGE_HEALTH_AREA_CODE" "$INSTANCE_HOST/$DEV_INSTANCE_NAME" | tail -1 + ''' ).trim() - PACKAGE_NAME = sh(returnStdout: true, script: "cat $PACKAGE_FILE | jq -r '.package .name'").trim() + // TODO obsolete? + PACKAGE_FILE_NAME = sh(returnStdout: true, script: "cat $PACKAGE_FILE | jq -r '.package .name'").trim() } } } From dd8380b9bd5da7294e59e460abb059c7b65f3212 Mon Sep 17 00:00:00 2001 From: radnov Date: Thu, 26 Oct 2023 18:55:43 +0300 Subject: [PATCH 10/10] ci: update IM hostnames --- jenkinsfiles/commit.Jenkinsfile | 9 ++++----- jenkinsfiles/develop.Jenkinsfile | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile index a8ae817d..4d0febb8 100644 --- a/jenkinsfiles/commit.Jenkinsfile +++ b/jenkinsfiles/commit.Jenkinsfile @@ -46,13 +46,12 @@ pipeline { environment { IMAGE_TAG = "${params.DHIS2_VERSION}" IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}" - IM_REPO_URL = "https://github.com/dhis2-sre/im-manager" - //TODO update prod environment hostname and IM group hostname - IM_ENVIRONMENT = 'prod.test.c.dhis2.org' - IM_HOST = "https://api.im.$IM_ENVIRONMENT" + IM_REPO_URL = 'https://github.com/dhis2-sre/im-manager' + IM_ENVIRONMENT = 'im.dhis2.org' + IM_HOST = "https://api.$IM_ENVIRONMENT" INSTANCE_GROUP_NAME = 'meta-packages' INSTANCE_NAME_FULL = "pkg-staging-${params.STAGING_INSTANCE_NAME.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER" - INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.im.$IM_ENVIRONMENT" + INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.$IM_ENVIRONMENT" INSTANCE_URL = "$INSTANCE_HOST/$INSTANCE_NAME_FULL" HTTP = 'https --check-status' PKG_IM_CREDENTIALS_ID = 'pkg-im-bot' diff --git a/jenkinsfiles/develop.Jenkinsfile b/jenkinsfiles/develop.Jenkinsfile index 3499e585..136c68a7 100644 --- a/jenkinsfiles/develop.Jenkinsfile +++ b/jenkinsfiles/develop.Jenkinsfile @@ -20,13 +20,12 @@ pipeline { environment { IMAGE_TAG = "${params.DHIS2_VERSION}" IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}" - IM_REPO_URL = "https://github.com/dhis2-sre/im-manager" - //TODO update prod environment hostname and IM group hostname - IM_ENVIRONMENT = 'prod.test.c.dhis2.org' - IM_HOST = "https://api.im.$IM_ENVIRONMENT" + IM_REPO_URL = 'https://github.com/dhis2-sre/im-manager' + IM_ENVIRONMENT = 'im.dhis2.org' + IM_HOST = "https://api.$IM_ENVIRONMENT" INSTANCE_GROUP_NAME = 'meta-packages' INSTANCE_NAME_FULL = "pkg-dev-${params.INSTANCE_NAME.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER" - INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.im.$IM_ENVIRONMENT" + INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.$IM_ENVIRONMENT" INSTANCE_URL = "$INSTANCE_HOST/$INSTANCE_NAME_FULL" INSTANCE_TTL = "${params.TTL != '' ? params.TTL.toInteger() * 60 : ''}" DATABASE_NAME = "${params.DATABASE}.sql.gz"