ci: Add package develop and commit pipelines

hack/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.8"
 
 services:
-  web:
+  core:
     image: "${DHIS2_IMAGE:-dhis2/core-dev:local}"
     ports:
       - 127.0.0.1:8080:8080

jenkinsfiles/README.md

@@ -1,11 +1,19 @@
 ## Jenkinsfiles
 
-### [Exporter](exporter.Jenkinsfile)
+### [Export](export.Jenkinsfile)
 * Exports metadata packages from a DHIS2 instance and tests them, based on a couple of input parameters (package code/type, dhis2 version and [more](exporter.Jenkinsfile#L8-L17)).
 * Can export and test a package (based on the provided input parameters) or only test an already exported package by uploading it via the `package_metadata_file` parameter.
-* Can be started manually or scheduled in parallel by the [triggerer](##Triggerer) pipeline.
+* Can be started manually or scheduled in parallel by the [trigger](##Trigger) pipeline.
 * If a package is exported from an instance running a "dev snapshot", instead of a stable version - it will only be tested, but not pushed to GitHub.
 
-### [Triggerer](triggerer.Jenkinsfile)
-* Triggers the [exporter](##Exporter) pipeline in parallel, based on the enabled packages in the Metadata index spreadsheet (via the [metadata-index-parser](https://github.com/dhis2/dhis2-utils/tree/master/tools/dhis2-metadata-index-parser)) and a list of DHIS2 versions to export from.
+### [Trigger](trigger.Jenkinsfile)
+* Triggers the [export](##Export) pipeline in parallel, based on the enabled packages in the Metadata index spreadsheet (via the [metadata-index-parser](https://github.com/dhis2/dhis2-utils/tree/master/tools/dhis2-metadata-index-parser)) and a list of DHIS2 versions to export from.
 * Can be started manually or by a [cron schedule](https://www.jenkins.io/doc/book/pipeline/syntax/#triggers).
+
+### develop
+
+### commit
+
+### export
+
+### trigger

jenkinsfiles/develop.Jenkinsfile

@@ -0,0 +1,77 @@
+@Library('pipeline-library') _
+
+pipeline {
+    agent {
+        label 'ec2-jdk11-large'
+    }
+
+    parameters {
+        choice(name: 'DATABASE', choices: ['dev', 'tracker_dev', 'pkgmaster', 'empty'], description: 'Packages development database from https://metadata.dev.dhis2.org, pkgmaster or an empty one.')
+        string(name: 'INSTANCE_NAME', defaultValue: 'foobar', description: 'Full instance name will be: "pkg-dev-<INSTANCE_NAME>-<BUILD_NUMBER>".')
+        choice(name: 'DHIS2_IMAGE_REPOSITORY', choices: ['core', 'core-dev'], description: 'DHIS2 Docker image repository.')
+        string(name: 'DHIS2_VERSION', defaultValue: '2.38.4', description: 'DHIS2 version for the instance.')
+        string(name: 'TTL', defaultValue: '', description: 'Time to live for the instance in minutes.')
+    }
+
+    options {
+        ansiColor('xterm')
+    }
+
+    environment {
+        IMAGE_TAG = "${params.DHIS2_VERSION}"
+        IMAGE_REPOSITORY = "${params.DHIS2_IMAGE_REPOSITORY}"
+        IM_REPO_URL = 'https://github.com/dhis2-sre/im-manager'
+        IM_ENVIRONMENT = 'im.dhis2.org'
+        IM_HOST = "https://api.$IM_ENVIRONMENT"
+        INSTANCE_GROUP_NAME = 'meta-packages'
+        INSTANCE_NAME_FULL = "pkg-dev-${params.INSTANCE_NAME.replaceAll("\\P{Alnum}", "").toLowerCase()}-$BUILD_NUMBER"
+        INSTANCE_HOST = "https://${INSTANCE_GROUP_NAME}.$IM_ENVIRONMENT"
+        INSTANCE_URL = "$INSTANCE_HOST/$INSTANCE_NAME_FULL"
+        INSTANCE_TTL = "${params.TTL != '' ? params.TTL.toInteger() * 60 : ''}"
+        DATABASE_NAME = "${params.DATABASE}.sql.gz"
+        HTTP = 'https --check-status'
+        PKG_CREDENTIALS = credentials('packages-instance-credentials')
+    }
+
+    stages {
+        stage('Create dev instance') {
+            steps {
+                script {
+                    withCredentials([usernamePassword(credentialsId: 'pkg-im-bot', passwordVariable: 'PASSWORD', usernameVariable: 'USER_EMAIL')]) {
+                        dir('im-manager') {
+                            gitHelper.sparseCheckout(IM_REPO_URL, "${gitHelper.getLatestTag(IM_REPO_URL)}", '/scripts')
+
+                            // TODO upload database every time or s3 replication rule?
+                            dir('scripts/databases') {
+                                env.DATABASE_ID = sh(
+                                    returnStdout: true,
+                                    script: "./list.sh | jq -r '.[] | select(.name == \"$INSTANCE_GROUP_NAME\") .databases[] | select(.name == \"$DATABASE_NAME\") .id'"
+                                ).trim()
+                                sh '[ -n "$DATABASE_ID" ]'
+
+                                echo "DATABASE_ID is $DATABASE_ID"
+                            }
+
+                            dir('scripts/instances') {
+                                echo 'Creating DHIS2 instance ...'
+
+                                sh "./deploy-dhis2.sh ${env.INSTANCE_GROUP_NAME} ${env.INSTANCE_NAME_FULL}"
+
+                                timeout(15) {
+                                    waitFor.statusOk("$INSTANCE_URL")
+                                }
+
+                                if (params.DATABASE != 'empty') {
+                                    NOTIFIER_ENDPOINT = dhis2.generateAnalytics("$INSTANCE_URL", '$PKG_CREDENTIALS')
+                                    timeout(15) {
+                                        waitFor.analyticsCompleted("${INSTANCE_URL}${NOTIFIER_ENDPOINT}", '$PKG_CREDENTIALS')
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

jenkinsfiles/exporter.Jenkinsfile → jenkinsfiles/export.Jenkinsfile

@@ -7,8 +7,10 @@ pipeline {
 
     parameters {
         stashedFile 'package_metadata_file'
+        // TODO dynamic list of codes?
         string(name: 'Package_code', defaultValue: '', description: '[REQUIRED] Package code to extract with.')
         string(name: 'Package_type', defaultValue: '', description: '[REQUIRED] Type of the package to export.')
+        // TODO dynamic list of descriptions/names?
         string(name: 'Package_description', defaultValue: '', description: '[REQUIRED] Description of the package.')
         string(name: 'Instance_url', defaultValue: 'https://metadata.dev.dhis2.org/dev', description: '[REQUIRED] Instance URL to export package from.')
         string(name: 'DHIS2_version', defaultValue: '2.37', description: '[OPTIONAL] DHIS2 version to extract the package from. (only major.minor version like 2.37, not 2.37.1)')
@@ -160,7 +162,8 @@ pipeline {
                     sleep(5)
 
                     dir('test') {
-                        sh "$WORKSPACE/scripts/run-import-tests.sh ./package_orig.json $DHIS2_PORT"
+                        sh "$WORKSPACE/scripts/replace-ou-placeholders.sh ./package_orig.json > ./package.json"
+                        sh "$WORKSPACE/scripts/run-import-tests.sh $DHIS2_LOCAL_PORT"
                     }
                 }
             }

scripts/check-dashboards.sh

@@ -2,18 +2,10 @@
 
 set -euxo pipefail
 
-port="$1"
-user="${USER_NAME:-admin}"
-pass="${USER_PASSWORD:-district}"
-ou_root="${OU_ROOT_ID:-GD7TowwI46c}"
+instance_url="$1"
 
-db_container=$(docker container ls --filter name=db -q)
-
-docker exec -i "$db_container" psql -U dhis -d dhis2 -c "UPDATE dashboard SET sharing = jsonb_set(sharing, '{public}', '\"rw------\"');"
-docker exec -i "$db_container" psql -U dhis -d dhis2 -c "INSERT INTO usermembership (organisationunitid, userinfoid) VALUES ((SELECT organisationunitid FROM organisationunit WHERE uid = '${ou_root}'), (SELECT userinfoid FROM userinfo WHERE code = '${user}'));"
-
-echo "{\"dhis\": {\"baseurl\": \"http://localhost:${port}\", \"username\": \"${user}\", \"password\": \"${pass}\"}}" > auth.json
+echo "{\"dhis\": {\"username\": \"${USER_NAME:-admin}\", \"password\": \"${USER_PASSWORD:-district}\"}}" > auth.json
 
 pip3 install -r dhis2-utils/tools/dhis2-dashboardchecker/requirements.txt
 
-python3 dhis2-utils/tools/dhis2-dashboardchecker/dashboard_checker.py -i=http://localhost:${port} --omit-no_data_warning
+python3 dhis2-utils/tools/dhis2-dashboardchecker/dashboard_checker.py -i="$instance_url" --omit-no_data_warning

scripts/check-expressions.sh

@@ -2,10 +2,9 @@
 
 set -euxo pipefail
 
-port="$1"
-user="${USER_NAME:-admin}"
-pass="${USER_PASSWORD:-district}"
+server_url="$1"
+default_server_name='default'
 
-echo -e "[localhost]\nserver=http://localhost:${port}/api/\nserver_name=localhost\nuser=${user}\npassword=${pass}\npage_size=500" > credentials.ini
+echo -e "[$default_server_name]\nserver=$server_url/api/\nserver_name=$default_server_name\nuser=${USER_NAME:-admin}\npassword=${USER_PASSWORD:-district}\npage_size=500" > credentials.ini
 
-python3 dhis2-metadata-checkers/check_expressions.py --credentials localhost
+python3 dhis2-metadata-checkers/check_expressions.py --credentials $default_server_name

scripts/make-dashboards-public.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+db_container=$(docker container ls --filter name=db -q)
+
+docker exec -i "$db_container" psql -U dhis -d dhis -c "UPDATE dashboard SET sharing = jsonb_set(sharing, '{public}', '\"rw------\"');"
+docker exec -i "$db_container" psql -U dhis -d dhis -c "INSERT INTO usermembership (organisationunitid, userinfoid) VALUES ((SELECT organisationunitid FROM organisationunit WHERE uid = '${OU_ROOT_ID:-GD7TowwI46c}'), (SELECT userinfoid FROM userinfo WHERE code = '${USER_NAME:-admin}'));"

scripts/replace-ou-placeholders.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+file="$1"
+
+sed "s/<OU_LEVEL_DISTRICT_UID>/${OU_DISTRICT_UID:-qpXLDdXT3po}/g" "$file" |
+  sed "s/<OU_LEVEL_FACILITY_UID>/${OU_FACILITY_UID:-vFr4zVw6Avn}/g" |
+  sed "s/<OU_ROOT_UID>/${OU_ROOT_UID:-GD7TowwI46c}/g"

scripts/run-import-tests.sh

@@ -2,17 +2,11 @@
 
 set -euxo pipefail
 
-file="$1"
-port="$2"
+port="$1"
 
 auth="${USER_NAME:-admin}:${USER_PASSWORD:-district}"
 url="http://localhost:$port"
 
-cat "$file" |
-  sed "s/<OU_LEVEL_DISTRICT_UID>/${OU_DISTRICT_UID:-qpXLDdXT3po}/g" |
-  sed "s/<OU_LEVEL_FACILITY_UID>/${OU_FACILITY_UID:-vFr4zVw6Avn}/g" |
-  sed "s/<OU_ROOT_UID>/${OU_ROOT_UID:-GD7TowwI46c}/g" > ./package.json
-
 ./api-test.sh -v -f ./tests.json -url "$url" -auth "$auth" test ou_import
 
 URL="$url" AUTH="$auth" ./run-tests.sh