diff --git a/hack/docker-compose.yml b/hack/docker-compose.yml index 84bd804..12ffe74 100644 --- a/hack/docker-compose.yml +++ b/hack/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.8" services: - web: + core: image: "${DHIS2_IMAGE:-dhis2/core-dev:local}" ports: - 127.0.0.1:8080:8080 diff --git a/jenkinsfiles/README.md b/jenkinsfiles/README.md index 523461a..c912945 100644 --- a/jenkinsfiles/README.md +++ b/jenkinsfiles/README.md @@ -1,11 +1,19 @@ ## Jenkinsfiles -### [Exporter](exporter.Jenkinsfile) +### [Export](export.Jenkinsfile) * Exports metadata packages from a DHIS2 instance and tests them, based on a couple of input parameters (package code/type, dhis2 version and [more](exporter.Jenkinsfile#L8-L17)). * Can export and test a package (based on the provided input parameters) or only test an already exported package by uploading it via the `package_metadata_file` parameter. -* Can be started manually or scheduled in parallel by the [triggerer](##Triggerer) pipeline. +* Can be started manually or scheduled in parallel by the [trigger](##Trigger) pipeline. * If a package is exported from an instance running a "dev snapshot", instead of a stable version - it will only be tested, but not pushed to GitHub. -### [Triggerer](triggerer.Jenkinsfile) -* Triggers the [exporter](##Exporter) pipeline in parallel, based on the enabled packages in the Metadata index spreadsheet (via the [metadata-index-parser](https://github.com/dhis2/dhis2-utils/tree/master/tools/dhis2-metadata-index-parser)) and a list of DHIS2 versions to export from. +### [Trigger](trigger.Jenkinsfile) +* Triggers the [export](##Export) pipeline in parallel, based on the enabled packages in the Metadata index spreadsheet (via the [metadata-index-parser](https://github.com/dhis2/dhis2-utils/tree/master/tools/dhis2-metadata-index-parser)) and a list of DHIS2 versions to export from. * Can be started manually or by a [cron schedule](https://www.jenkins.io/doc/book/pipeline/syntax/#triggers). + +### develop + +### commit + +### export + +### trigger diff --git a/jenkinsfiles/commit.Jenkinsfile b/jenkinsfiles/commit.Jenkinsfile new file mode 100644 index 0000000..4d0febb --- /dev/null +++ b/jenkinsfiles/commit.Jenkinsfile @@ -0,0 +1,480 @@ +@Library('pipeline-library') _ + +node('ec2-jdk8-large-spot') { + dir('dhis2-utils') { + git url: 'https://github.com/dhis2/dhis2-utils', branch: 'use-argparse-insead-of-envvars' + + dir('tools/dhis2-metadata-index-parser') { + sh 'pip3 install -r requirements.txt' + + withCredentials([file(credentialsId: 'metadata-index-parser-service-account', variable: 'GOOGLE_SERVICE_ACCOUNT')]) { + env.SPREADSHEET_ID = '1IIQL2IkGJqiIWLr6Bgg7p9fE78AwQYhHBNGoV-spGOM' + + env.PACKAGES_INDEX_JSON = sh( + returnStdout: true, + script: 'python3 parse-index.py --service-account-file $GOOGLE_SERVICE_ACCOUNT --spreadsheet-id $SPREADSHEET_ID --no-only-ready --no-uncheck-readiness' + ).trim() + + PARAMETER_VALUES = sh(returnStdout:true, script: 'echo $PACKAGES_INDEX_JSON | jq -r \'.[] | .["Component name"]\'').trim() + } + } + } +} + +pipeline { + agent { + label 'ec2-jdk11-large' + } + + parameters { + booleanParam(name: 'REFRESH_PACKAGES', defaultValue: false, description: '[OPTIONAL] Refresh the list of PACKAGE_NAMEs and abort the build.') + string(name: 'STAGING_INSTANCE_NAME', defaultValue: 'foobar', description: '[REQUIRED] Full staging instance name will be: "pkg-staging--".') + choice(name: 'DATABASE', choices: ['pkgmaster', 'dev', 'tracker_dev'], description: '[REQUIRED] Master packages database or development database from https://metadata.dev.dhis2.org.') + booleanParam(name: 'REPLACE_DATABASE', defaultValue: true, description: '[OPTIONAL] Replace database if all validations and tests pass.') + choice(name: 'DHIS2_IMAGE_REPOSITORY', choices: ['core', 'core-dev'], description: 'DHIS2 Docker image repository.') + string(name: 'DHIS2_VERSION', defaultValue: '2.38.4', description: '[OPTIONAL] DHIS2 version for the instance.') + string(name: 'DEV_INSTANCE_NAME', defaultValue: '', description: '[OPTIONAL] Name of the dev instance to export from.\nOnly needed if you want to export a package specified with PACKAGE_NAME') + choice(name: 'PACKAGE_NAME', choices: PARAMETER_VALUES, description: '[REQUIRED] Select a package to extract by name.') + stashedFile(name: 'PACKAGE_FILE_UPLOAD', description: '[OPTIONAL] Provide a custom metadata package file instead of exporting it.\nIf a file is uploaded, the "DEV_INSTANCE_NAME" and "PACKAGE_NAME" are obsolete.') + } + + options { + ansiColor('xterm') + disableConcurrentBuilds() // Needed in order to sequentially “merge” new packages into the current master database. + } + + environment { + IMAGE_TAG = "${params.DHIS2_VERSION}" + IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}" + IM_REPO_URL = 'https://github.com/dhis2-sre/im-manager' + IM_ENVIRONMENT = 'im.dhis2.org' + IM_HOST = "https://api.$IM_ENVIRONMENT" + INSTANCE_GROUP_NAME = 'meta-packages' + INSTANCE_NAME_FULL = "pkg-staging-${params.STAGING_INSTANCE_NAME.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER" + INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.$IM_ENVIRONMENT" + INSTANCE_URL = "$INSTANCE_HOST/$INSTANCE_NAME_FULL" + HTTP = 'https --check-status' + PKG_IM_CREDENTIALS_ID = 'pkg-im-bot' + PKG_CREDENTIALS = credentials('packages-instance-credentials') + PKG_MASTER_DB_NAME = 'packages-dev.sql.gz' + DATABASE_NAME = "${params.DATABASE == 'pkgmaster' ? params.DATABASE + '.pgc' : params.DATABASE + '.sql.gz'}" + PACKAGE_FILE = 'PACKAGE_FILE_UPLOAD' + PACKAGE_DIFF_FILE = 'package-diff-file' + ALL_METADATA_FILE = 'all-metadata.json' + DHIS2_LOCAL_PORT = 8080 + } + + stages { + stage('Clone git repos') { + steps { + script { + // Immediately abort pipeline if REFRESH_PACKAGES is enabled. + if (params.REFRESH_PACKAGES.toBoolean()) { + error('This build is only for refreshing the PACKAGE_NAME parameter. Pipeline will be aborted now.') + } + + dir('dhis2-utils') { + git url: 'https://github.com/dhis2/dhis2-utils' + } + + dir('dhis2-metadata-checkers') { + git url: 'https://github.com/solid-lines/dhis2-metadata-checkers', branch: 'main' + } + + dir('ALL_METADATA') { + git url: 'https://github.com/dhis2-metadata/ALL_METADATA', credentialsId: 'github-token-as-password' + } + + dir('im-manager') { + gitHelper.sparseCheckout(IM_REPO_URL, "${gitHelper.getLatestTag(IM_REPO_URL)}", '/scripts') + } + } + } + } + + stage('Create staging instance') { + steps { + script { + // Get package details based on the selected PACKAGE_NAME parameter value. + env.SELECTED_PACKAGE = sh(returnStdout: true, script: 'echo "$PACKAGES_INDEX_JSON" | jq --arg name "$PACKAGE_NAME" -r \'.[] | select(."Component name" == $name)\'').trim() + env.PACKAGE_CODE = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Extraction code"\'').trim() + env.PACKAGE_TYPE = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Script parameter"\'').trim() + env.PACKAGE_SOURCE_INSTANCE = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Source instance"\'').trim() + env.PACKAGE_HEALTH_AREA_NAME = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Health area"\'').trim() + env.PACKAGE_HEALTH_AREA_CODE = sh(returnStdout: true, script: 'echo "$SELECTED_PACKAGE" | jq -r \'."Health area prefix"\'').trim() + + withCredentials([usernamePassword(credentialsId: "$PKG_IM_CREDENTIALS_ID", passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager/scripts/databases') { + env.DATABASE_ID = sh( + returnStdout: true, + script: "./list.sh | jq -r '.[] | select(.name == \"$INSTANCE_GROUP_NAME\") .databases[] | select(.name == \"$DATABASE_NAME\") .id'" + ).trim() + + sh '[ -n "$DATABASE_ID" ]' + echo "DATABASE_ID is $DATABASE_ID" + } + + dir('im-manager/scripts/instances') { + echo 'Creating DHIS2 instance ...' + + sh "./deploy-dhis2.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME_FULL" + + timeout(15) { + waitFor.statusOk("$INSTANCE_URL") + } + + NOTIFIER_ENDPOINT = dhis2.generateAnalytics("$INSTANCE_URL", '$PKG_CREDENTIALS') + timeout(15) { + waitFor.analyticsCompleted("${INSTANCE_URL}${NOTIFIER_ENDPOINT}", '$PKG_CREDENTIALS') + } + } + } + } + } + } + + stage('Export package from dev instance') { + when { + expression { + try { + unstash "$PACKAGE_FILE" // Need to unstash the file, otherwise it's always null. + } catch (e) { + echo e.toString() + } + + return params.DEV_INSTANCE_NAME != '' + } + } + + steps { + script { + echo "Exporting package from ${params.DEV_INSTANCE_NAME} ..." + + sh 'echo {\\"dhis\\": {\\"username\\": \\"${PKG_CREDENTIALS_USR}\\", \\"password\\": \\"${PKG_CREDENTIALS_PSW}\\"}} > auth.json' + + PACKAGE_FILE = sh( + returnStdout: true, + script: '''#!/bin/bash + set -euxo pipefail + ./scripts/export-package.sh "$PACKAGE_CODE" "$PACKAGE_TYPE" "$PACKAGE_NAME" "$PACKAGE_HEALTH_AREA_NAME" "$PACKAGE_HEALTH_AREA_CODE" "$INSTANCE_HOST/$DEV_INSTANCE_NAME" | tail -1 + ''' + ).trim() + + // TODO obsolete? + PACKAGE_FILE_NAME = sh(returnStdout: true, script: "cat $PACKAGE_FILE | jq -r '.package .name'").trim() + } + } + } + + stage('Test package in empty instance') { + when { + anyOf { + expression { params.DEV_INSTANCE_NAME != '' } + + expression { readFile("$PACKAGE_FILE").size() != 0 } + } + + } + + stages { + stage('Validate metadata') { + steps { + script { + catchError(catchInterruptions: false, message: 'Validation errors found!', stageResult: 'FAILURE') { + sh("python3 -u dhis2-utils/tools/dhis2-metadata-package-validator/metadata_package_validator.py -f $WORKSPACE/$PACKAGE_FILE") + } + } + } + } + + stage('Test import') { + steps { + script { + withDockerRegistry([credentialsId: "docker-hub-credentials", url: ""]) { +// DHIS2_CHANNEL = "${params.DHIS2_IMAGE_REPOSITORY == 'core' ? 'stable' : 'dev'}" +// d2.startCluster("$IMAGE_TAG", "$DHIS2_LOCAL_PORT", "$DHIS2_CHANNEL") + dir('hack') { + dir('docker') { + sh 'curl "https://raw.githubusercontent.com/dhis2/dhis2-core/master/docker/dhis.conf" -O' + } + + env.DHIS2_HOME = '/opt/dhis2' + env.DHIS2_IMAGE = "dhis2/${params.DHIS2_IMAGE_REPOSITORY}:${IMAGE_TAG}" + + sh 'docker-compose up -d' + + sleep(10) + + timeout(5) { + waitFor.statusOk("http://localhost:${DHIS2_LOCAL_PORT}") + } + } + } + + sleep(5) + + dir('test') { + sh "$WORKSPACE/scripts/replace-ou-placeholders.sh $WORKSPACE/$PACKAGE_FILE > ./package.json" + sh "$WORKSPACE/scripts/run-import-tests.sh $DHIS2_LOCAL_PORT" + } + } + } + + post { + always { + script { +// sh "d2 cluster compose $IMAGE_TAG logs core > logs_empty_instance.txt" + dir('hack') { + sh "docker-compose logs core > logs_empty_instance.txt" + archiveArtifacts artifacts: 'logs_empty_instance.txt' + } + } + } + } + } + + stage('Run checks') { + parallel { + stage('Check dashboards') { + steps { + catchError(catchInterruptions: false, message: 'Dashboard errors found!', stageResult: 'FAILURE') { + sh './scripts/make-dashboards-public.sh' + sh './scripts/check-dashboards.sh http://localhost:$DHIS2_LOCAL_PORT' + } + } + } + + stage('Check PR expressions') { + steps { + catchError(catchInterruptions: false, message: 'PR expressions errors found!', stageResult: 'FAILURE') { + sh './scripts/check-expressions.sh http://localhost:$DHIS2_LOCAL_PORT' + } + } + } + } + } + } + } + + stage ('Notify before import') { + when { + anyOf { + expression { params.DEV_INSTANCE_NAME != '' } + + expression { readFile("$PACKAGE_FILE").size() != 0 } + } + + } + + steps { + script { + slackSend( + color: '#00ffff', + channel: 'pkg-notifications', + message: slack.buildUrl() + "\nPausing for review before importing package." + ) + } + } + } + + stage('Pause before import') { + when { + anyOf { + expression { params.DEV_INSTANCE_NAME != '' } + + expression { readFile("$PACKAGE_FILE").size() != 0 } + } + + } + + input { + message "Continue?" + ok "Yes" + parameters { + string(name: 'EXAMPLE_PARAMETER', defaultValue: 'test', description: 'Example description.') + booleanParam(name: 'DELETE_DEV_INSTANCE', defaultValue: false, description: 'Should the dev instance be deleted?') + } + } + + steps { + script { + echo "The value of the EXAMPLE_PARAMETER is ${env.EXAMPLE_PARAMETER}" + // Parameters from the input block are available as env variables only for the current stage, + // so they have to be reassigned to a new environment variable for the next stages. + env.DELETE_DEV_INSTANCE_ENV = env.DELETE_DEV_INSTANCE + } + } + } + + stage('Import package into staging instance') { + when { + anyOf { + expression { params.DEV_INSTANCE_NAME != '' } + + expression { readFile("$PACKAGE_FILE").size() != 0 } + } + + } + + steps { + echo "Importing package into ${params.STAGING_INSTANCE_NAME} ..." + sh "$WORKSPACE/scripts/replace-ou-placeholders.sh $PACKAGE_FILE > updated-$PACKAGE_FILE" + sh "$HTTP --auth \$PKG_CREDENTIALS POST $INSTANCE_URL/api/metadata < updated-$PACKAGE_FILE" + } + } + + stage ('Notify after import') { + steps { + script { + slackSend( + color: '#00ffff', + channel: 'pkg-notifications', + message: slack.buildUrl() + "\nPausing for review after importing package." + ) + } + } + } + + stage('Pause after import') { + input { + message "Continue?" + ok "Yes" + parameters { + string(name: 'EXAMPLE_PARAMETER', defaultValue: 'test', description: 'Example description.') + booleanParam(name: 'DELETE_DEV_INSTANCE', defaultValue: false, description: 'Should the dev instance be deleted?') + } + } + + steps { + script { + echo "The value of the EXAMPLE_PARAMETER is ${env.EXAMPLE_PARAMETER}" + // Parameters from the input block are available as env variables only for the current stage, + // so they have to be reassigned to a new environment variable for the next stages. + env.DELETE_DEV_INSTANCE_ENV = env.DELETE_DEV_INSTANCE + } + } + } + + stage('Test packages in staging instance') { + stages { + stage('Export all metadata') { + steps { + echo "Export metadata from ${params.STAGING_INSTANCE_NAME} ..." + sh "$HTTP --auth \$PKG_CREDENTIALS GET $INSTANCE_URL/api/metadata > $ALL_METADATA_FILE" + } + } + + stage('Validate metadata') { + steps { + script { + catchError(catchInterruptions: false, message: 'Validation errors found!', stageResult: 'FAILURE') { + sh("python3 -u dhis2-utils/tools/dhis2-metadata-package-validator/metadata_package_validator.py -f $WORKSPACE/$ALL_METADATA_FILE") + } + } + } + } + + stage('Run checks') { + parallel { + stage('Check dashboards') { + steps { + catchError(catchInterruptions: false, message: 'Dashboard errors found!', stageResult: 'FAILURE') { + sh 'USER_NAME="$PKG_CREDENTIALS_USR" USER_PASSWORD="$PKG_CREDENTIALS_PSW" ./scripts/check-dashboards.sh $INSTANCE_URL' + } + } + } + + stage('Check PR expressions') { + steps { + catchError(catchInterruptions: false, message: 'PR expression errors found!', stageResult: 'FAILURE') { + sh 'USER_NAME="$PKG_CREDENTIALS_USR" USER_PASSWORD="$PKG_CREDENTIALS_PSW" ./scripts/check-expressions.sh $INSTANCE_URL' + } + } + } + } + } + } + } + + stage('Create diff') { + steps { + dir('dhis2-utils/tools/dhis2-metadatapackagediff') { + sh 'pip3 install -r requirements.txt' + + sh "python3 metadata_package_diff.py $WORKSPACE/ALL_METADATA/$ALL_METADATA_FILE $WORKSPACE/$ALL_METADATA_FILE $WORKSPACE/$PACKAGE_DIFF_FILE" + } + } + + post { + always { + archiveArtifacts artifacts: "${PACKAGE_DIFF_FILE}.xlsx" + } + } + } + + stage('Push metadata to GitHub') { + environment { + GITHUB_CREDS = credentials('github-token-as-password') + GITHUB_EMAIL = 'apps@dhis2.org' + REPOSITORY_NAME = 'ALL_METADATA' + } + + steps { + script { + sh 'git config --global user.email $GITHUB_EMAIL' + sh 'git config --global user.name $GITHUB_CREDS_USR' + + dir("$REPOSITORY_NAME") { + sh 'cp $WORKSPACE/$ALL_METADATA_FILE .' + sh 'git add .' + sh 'git diff-index --quiet HEAD || git commit -m "chore: Upload all metadata"' + sh 'git push https://$GITHUB_CREDS_PSW@github.com/dhis2-metadata/$REPOSITORY_NAME' + } + } + } + } + + stage('Save DB') { + when { + expression { params.REPLACE_DATABASE } + } + + steps { + echo "Saving ${params.DATABASE} DB ..." + script { + withCredentials([usernamePassword(credentialsId: "$PKG_IM_CREDENTIALS_ID", passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager/scripts/databases') { + sh "./save.sh $INSTANCE_GROUP_NAME $INSTANCE_NAME_FULL" + + timeout(15) { + waitUntil(initialRecurrencePeriod: 5000, quiet: true) { + // TODO change this once we have a better way of knowing the status of a "save" + lock = sh(returnStdout: true, script: "./findById.sh $DATABASE_ID | jq -r '.lock'").trim() + + return (lock == 'null') + } + } + } + } + } + } + } + } + + post { + success { + script { + withCredentials([usernamePassword(credentialsId: "$PKG_IM_CREDENTIALS_ID", passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager/scripts/instances') { + env.INSTANCES_TO_DELETE = env.INSTANCE_NAME_FULL + if (env.DELETE_DEV_INSTANCE_ENV.toBoolean()) { + env.INSTANCES_TO_DELETE = "${env.INSTANCE_NAME_FULL} ${params.DEV_INSTANCE_NAME}" + } + + sh "./destroy.sh $INSTANCE_GROUP_NAME $INSTANCES_TO_DELETE" + } + } + } + } + } +} diff --git a/jenkinsfiles/develop.Jenkinsfile b/jenkinsfiles/develop.Jenkinsfile new file mode 100644 index 0000000..136c68a --- /dev/null +++ b/jenkinsfiles/develop.Jenkinsfile @@ -0,0 +1,77 @@ +@Library('pipeline-library') _ + +pipeline { + agent { + label 'ec2-jdk11-large' + } + + parameters { + choice(name: 'DATABASE', choices: ['dev', 'tracker_dev', 'pkgmaster', 'empty'], description: 'Packages development database from https://metadata.dev.dhis2.org, pkgmaster or an empty one.') + string(name: 'INSTANCE_NAME', defaultValue: 'foobar', description: 'Full instance name will be: "pkg-dev--".') + choice(name: 'DHIS2_IMAGE_REPOSITORY', choices: ['core', 'core-dev'], description: 'DHIS2 Docker image repository.') + string(name: 'DHIS2_VERSION', defaultValue: '2.38.4', description: 'DHIS2 version for the instance.') + string(name: 'TTL', defaultValue: '', description: 'Time to live for the instance in minutes.') + } + + options { + ansiColor('xterm') + } + + environment { + IMAGE_TAG = "${params.DHIS2_VERSION}" + IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}" + IM_REPO_URL = 'https://github.com/dhis2-sre/im-manager' + IM_ENVIRONMENT = 'im.dhis2.org' + IM_HOST = "https://api.$IM_ENVIRONMENT" + INSTANCE_GROUP_NAME = 'meta-packages' + INSTANCE_NAME_FULL = "pkg-dev-${params.INSTANCE_NAME.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER" + INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.$IM_ENVIRONMENT" + INSTANCE_URL = "$INSTANCE_HOST/$INSTANCE_NAME_FULL" + INSTANCE_TTL = "${params.TTL != '' ? params.TTL.toInteger() * 60 : ''}" + DATABASE_NAME = "${params.DATABASE}.sql.gz" + HTTP = 'https --check-status' + PKG_CREDENTIALS = credentials('packages-instance-credentials') + } + + stages { + stage('Create dev instance') { + steps { + script { + withCredentials([usernamePassword(credentialsId: 'pkg-im-bot', passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) { + dir('im-manager') { + gitHelper.sparseCheckout(IM_REPO_URL, "${gitHelper.getLatestTag(IM_REPO_URL)}", '/scripts') + + // TODO upload database every time or s3 replication rule? + dir('scripts/databases') { + env.DATABASE_ID = sh( + returnStdout: true, + script: "./list.sh | jq -r '.[] | select(.name == \"$INSTANCE_GROUP_NAME\") .databases[] | select(.name == \"$DATABASE_NAME\") .id'" + ).trim() + sh '[ -n "$DATABASE_ID" ]' + + echo "DATABASE_ID is $DATABASE_ID" + } + + dir('scripts/instances') { + echo 'Creating DHIS2 instance ...' + + sh "./deploy-dhis2.sh ${env.INSTANCE_GROUP_NAME} ${env.INSTANCE_NAME_FULL}" + + timeout(15) { + waitFor.statusOk("$INSTANCE_URL") + } + + if (params.DATABASE != 'empty') { + NOTIFIER_ENDPOINT = dhis2.generateAnalytics("$INSTANCE_URL", '$PKG_CREDENTIALS') + timeout(15) { + waitFor.analyticsCompleted("${INSTANCE_URL}${NOTIFIER_ENDPOINT}", '$PKG_CREDENTIALS') + } + } + } + } + } + } + } + } + } +} diff --git a/jenkinsfiles/exporter.Jenkinsfile b/jenkinsfiles/export.Jenkinsfile similarity index 97% rename from jenkinsfiles/exporter.Jenkinsfile rename to jenkinsfiles/export.Jenkinsfile index d012b9a..31bc311 100644 --- a/jenkinsfiles/exporter.Jenkinsfile +++ b/jenkinsfiles/export.Jenkinsfile @@ -7,8 +7,10 @@ pipeline { parameters { stashedFile 'package_metadata_file' + // TODO dynamic list of codes? string(name: 'Package_code', defaultValue: '', description: '[REQUIRED] Package code to extract with.') string(name: 'Package_type', defaultValue: '', description: '[REQUIRED] Type of the package to export.') + // TODO dynamic list of descriptions/names? string(name: 'Package_description', defaultValue: '', description: '[REQUIRED] Description of the package.') string(name: 'Instance_url', defaultValue: 'https://metadata.dev.dhis2.org/dev', description: '[REQUIRED] Instance URL to export package from.') string(name: 'DHIS2_version', defaultValue: '2.37', description: '[OPTIONAL] DHIS2 version to extract the package from. (only major.minor version like 2.37, not 2.37.1)') @@ -160,7 +162,8 @@ pipeline { sleep(5) dir('test') { - sh "$WORKSPACE/scripts/run-import-tests.sh ./package_orig.json $DHIS2_PORT" + sh "$WORKSPACE/scripts/replace-ou-placeholders.sh ./package_orig.json > ./package.json" + sh "$WORKSPACE/scripts/run-import-tests.sh $DHIS2_LOCAL_PORT" } } } diff --git a/jenkinsfiles/triggerer.Jenkinsfile b/jenkinsfiles/trigger.Jenkinsfile similarity index 100% rename from jenkinsfiles/triggerer.Jenkinsfile rename to jenkinsfiles/trigger.Jenkinsfile diff --git a/scripts/check-dashboards.sh b/scripts/check-dashboards.sh index eb5f496..c8b719f 100755 --- a/scripts/check-dashboards.sh +++ b/scripts/check-dashboards.sh @@ -2,18 +2,10 @@ set -euxo pipefail -port="$1" -user="${USER_NAME:-admin}" -pass="${USER_PASSWORD:-district}" -ou_root="${OU_ROOT_ID:-GD7TowwI46c}" +instance_url="$1" -db_container=$(docker container ls --filter name=db -q) - -docker exec -i "$db_container" psql -U dhis -d dhis2 -c "UPDATE dashboard SET sharing = jsonb_set(sharing, '{public}', '\"rw------\"');" -docker exec -i "$db_container" psql -U dhis -d dhis2 -c "INSERT INTO usermembership (organisationunitid, userinfoid) VALUES ((SELECT organisationunitid FROM organisationunit WHERE uid = '${ou_root}'), (SELECT userinfoid FROM userinfo WHERE code = '${user}'));" - -echo "{\"dhis\": {\"baseurl\": \"http://localhost:${port}\", \"username\": \"${user}\", \"password\": \"${pass}\"}}" > auth.json +echo "{\"dhis\": {\"username\": \"${USER_NAME:-admin}\", \"password\": \"${USER_PASSWORD:-district}\"}}" > auth.json pip3 install -r dhis2-utils/tools/dhis2-dashboardchecker/requirements.txt -python3 dhis2-utils/tools/dhis2-dashboardchecker/dashboard_checker.py -i=http://localhost:${port} --omit-no_data_warning +python3 dhis2-utils/tools/dhis2-dashboardchecker/dashboard_checker.py -i="$instance_url" --omit-no_data_warning diff --git a/scripts/check-expressions.sh b/scripts/check-expressions.sh index 82e1fe8..1f6dcc0 100755 --- a/scripts/check-expressions.sh +++ b/scripts/check-expressions.sh @@ -2,10 +2,9 @@ set -euxo pipefail -port="$1" -user="${USER_NAME:-admin}" -pass="${USER_PASSWORD:-district}" +server_url="$1" +default_server_name='default' -echo -e "[localhost]\nserver=http://localhost:${port}/api/\nserver_name=localhost\nuser=${user}\npassword=${pass}\npage_size=500" > credentials.ini +echo -e "[$default_server_name]\nserver=$server_url/api/\nserver_name=$default_server_name\nuser=${USER_NAME:-admin}\npassword=${USER_PASSWORD:-district}\npage_size=500" > credentials.ini -python3 dhis2-metadata-checkers/check_expressions.py --credentials localhost +python3 dhis2-metadata-checkers/check_expressions.py --credentials $default_server_name diff --git a/scripts/make-dashboards-public.sh b/scripts/make-dashboards-public.sh new file mode 100755 index 0000000..79e08b1 --- /dev/null +++ b/scripts/make-dashboards-public.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +db_container=$(docker container ls --filter name=db -q) + +docker exec -i "$db_container" psql -U dhis -d dhis -c "UPDATE dashboard SET sharing = jsonb_set(sharing, '{public}', '\"rw------\"');" +docker exec -i "$db_container" psql -U dhis -d dhis -c "INSERT INTO usermembership (organisationunitid, userinfoid) VALUES ((SELECT organisationunitid FROM organisationunit WHERE uid = '${OU_ROOT_ID:-GD7TowwI46c}'), (SELECT userinfoid FROM userinfo WHERE code = '${USER_NAME:-admin}'));" diff --git a/scripts/replace-ou-placeholders.sh b/scripts/replace-ou-placeholders.sh new file mode 100755 index 0000000..569d801 --- /dev/null +++ b/scripts/replace-ou-placeholders.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +file="$1" + +sed "s//${OU_DISTRICT_UID:-qpXLDdXT3po}/g" "$file" | + sed "s//${OU_FACILITY_UID:-vFr4zVw6Avn}/g" | + sed "s//${OU_ROOT_UID:-GD7TowwI46c}/g" diff --git a/scripts/run-import-tests.sh b/scripts/run-import-tests.sh index 0fb1171..b862937 100755 --- a/scripts/run-import-tests.sh +++ b/scripts/run-import-tests.sh @@ -2,17 +2,11 @@ set -euxo pipefail -file="$1" -port="$2" +port="$1" auth="${USER_NAME:-admin}:${USER_PASSWORD:-district}" url="http://localhost:$port" -cat "$file" | - sed "s//${OU_DISTRICT_UID:-qpXLDdXT3po}/g" | - sed "s//${OU_FACILITY_UID:-vFr4zVw6Avn}/g" | - sed "s//${OU_ROOT_UID:-GD7TowwI46c}/g" > ./package.json - ./api-test.sh -v -f ./tests.json -url "$url" -auth "$auth" test ou_import URL="$url" AUTH="$auth" ./run-tests.sh