improve multicluster support

.env.example

@@ -41,6 +41,7 @@ DASHBOARD_DEV_SERVER=1
 DASHBOARD_SSL=1
 DASHBOARD_URL=
 RGW_MULTISITE=0
+NUMBER_OF_CLUSTERS=1
 
 # Sanity checks' options
 #APPLITOOLS_API_KEY=

docker-compose.yml

@@ -33,6 +33,7 @@ services:
             - PYTHONDONTWRITEBYTECODE=1
             - RGW
             - RGW_MULTISITE=${RGW_MULTISITE:-0}
+            - NUMBER_OF_CLUSTERS=${NUMBER_OF_CLUSTERS:-1}
         cap_add:
             - ALL
         entrypoint: /docker/entrypoint.sh

docker/ceph/ci/sanity-checks.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+source /docker/set-mstart-env.sh
+
 REPO_DIR=/ceph
 [[ "$IS_UPSTREAM" == 1 && "$CEPH_VERSION" -le '14' ]] && PYTHON_VERSION=2 || PYTHON_VERSION=3
 TRANSLATION_FILE=src/pybind/mgr/dashboard/frontend/src/locale/messages.xlf
@@ -244,7 +246,7 @@ run_frontend_e2e_tests() {
     echo 'Running frontend E2E tests...'
 
     cd "$REPO_DIR"/src/pybind/mgr/dashboard/frontend
-    npm ci
+    npm i
     npx --no-install cypress -v && WITH_CYPRESS=1 || WITH_CYPRESS=0
     E2E_CMD="npm run e2e:dev"
     if [[ "${WITH_CYPRESS}" == 1 ]]; then
@@ -264,12 +266,15 @@ run_frontend_e2e_tests() {
 
         DASHBOARD_URL=null
         while [[ "${DASHBOARD_URL}" == 'null' ]]; do
-            export DASHBOARD_URL=$("$CEPH_BIN"/ceph mgr services | jq -r .dashboard)
+            export DASHBOARD_URL=$(CEPH_CLI mgr services | jq -r .dashboard)
+            export DASHBOARD2_URL=$(CEPH2_CLI mgr services | jq -r .dashboard)
             sleep 1
         done
         if [[ "${WITH_CYPRESS}" == 1 ]]; then
             export CYPRESS_BASE_URL="${DASHBOARD_URL}"
+            export CYPRESS_CEPH2_URL="${DASHBOARD2_URL}"
         fi
+
         cd "$REPO_DIR"/src/pybind/mgr/dashboard/frontend
         ANGULAR_VERSION=$(npm run ng version | grep 'Angular: ' | awk '{ print substr($2,1,1) }')
         # In nautilus this flag is required because BASE_URL is not read in protractor config.

docker/ceph/entrypoint.sh

@@ -10,22 +10,23 @@ export_var() {
 export_var IS_CEPH_RPM="$(command -v ceph-mgr 2>/dev/null | wc -l)"
 
 # Env. vars used in vstart.
-export_var CEPH_ASOK_DIR=/ceph/build."${HOSTNAME}"/out
-export_var CEPH_CONF=/ceph/build."${HOSTNAME}"/ceph.conf
-export_var CEPH_CONF_PATH=/ceph/build."${HOSTNAME}"
-export_var CEPH_DEV_DIR=/ceph/build."${HOSTNAME}"/dev
-export_var CEPH_OUT_DIR=/ceph/build."${HOSTNAME}"/out
+export_var CEPH_ASOK_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/asok
+export_var CEPH_CONF=/ceph/build.ceph-dev/run/"${HOSTNAME}"/ceph.conf
+export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${HOSTNAME}"
+export_var CEPH_DEV_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/dev
+export_var CEPH_OUT_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/out
 export_var CEPH_PORT="${CEPH_PORT:-10000}"
 
 rm -rf /ceph/build
+
 mkdir -p "${CEPH_CONF_PATH}"
 
 if [[ "${IS_CEPH_RPM}" == 1 ]]; then
-    export_var CEPH_BIN=/usr/bin
+    export_var CEPH_BIN=/usr/bin/
     export_var CEPH_LIB=/usr/lib64/ceph
     export_var EC_PATH="$CEPH_LIB"/erasure-code
     export_var OBJCLASS_PATH=/usr/lib64/rados-classes
-    
+
     ln -sf "$EC_PATH"/* "$CEPH_LIB"
     ln -sf "$OBJCLASS_PATH"/* "$CEPH_LIB"
 

docker/ceph/set-monitoring.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 
 set -e
+source /docker/set-mstart-env.sh
 
+echo $CEPH1
 # Configure grafana
 set_grafana_api_url() {
     while true; do
         GRAFANA_IP=$(getent ahosts grafana | tail -1 | awk '{print $1}')
         if [[ -n "$GRAFANA_IP" ]]; then
-            "$CEPH_BIN"/ceph dashboard set-grafana-api-url "https://$GRAFANA_IP:$GRAFANA_HOST_PORT"
-
+            CEPH_CLI dashboard set-grafana-api-url "https://$GRAFANA_IP:$GRAFANA_HOST_PORT"
             return
         fi
 
@@ -22,8 +23,7 @@ set_alertmanager_api_host() {
     while true; do
         ALERTMANAGER_IP=$(getent ahosts alertmanager | tail -1 | awk '{print $1}')
         if [[ -n "$ALERTMANAGER_IP" ]]; then
-            "$CEPH_BIN"/ceph dashboard set-alertmanager-api-host "http://$ALERTMANAGER_IP:$ALERTMANAGER_HOST_PORT"
-
+            CEPH_CLI dashboard set-alertmanager-api-host "http://$ALERTMANAGER_IP:$ALERTMANAGER_HOST_PORT"
             return
         fi
 
@@ -37,8 +37,7 @@ set_prometheus_api_host() {
     while true; do
         PROMETHEUS_IP=$(getent ahosts prometheus | tail -1 | awk '{print $1}')
         if [[ -n "$PROMETHEUS_IP" ]]; then
-            "$CEPH_BIN"/ceph dashboard set-prometheus-api-host "http://$PROMETHEUS_IP:$PROMETHEUS_HOST_PORT"
-
+            CEPH_CLI dashboard set-prometheus-api-host "http://$PROMETHEUS_IP:$PROMETHEUS_HOST_PORT"
             return
         fi
 

docker/ceph/set-mstart-env.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+export_var() {
+    export "${1?}"
+    echo "export $1" >> /root/.bashrc
+}
+
+echo $NUMBER_OF_CLUSTERS
+declare -a CEPH_BIN_LIST
+declare -a CEPH_CLUSTERS
+for (( cluster=1; cluster<=$NUMBER_OF_CLUSTERS; cluster++ )); do
+    CEPH_CLUSTERS[cluster]="CEPH${cluster}"
+    CEPH_BIN_LIST[cluster]=${CEPH_CLUSTERS[cluster]}
+    echo ${CEPH_CLUSTERS[cluster]}
+    export_var ${CEPH_BIN_LIST[cluster]}="/ceph/src/mrun ${CEPH_CLUSTERS[cluster]} ceph"
+done
+
+CEPH_CLI() {
+    export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/CEPH1/
+    /ceph/src/mrun CEPH1 ceph "$@"
+}
+
+CEPH2_CLI() {
+    export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/CEPH2/
+    /ceph/src/mrun CEPH2 ceph "$@"
+}
+
+CEPH_CLI_ALL() {
+    for cluster in ${CEPH_BIN_LIST[@]}; do
+        export_var CEPH_BIN=/usr/bin/
+        export_var BUILD_DIR=/ceph/build.ceph-dev/
+        export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${cluster}/"
+
+        cd /ceph/build.ceph-dev
+        ${!cluster} $@
+    done
+}

docker/ceph/set-start-env.sh

@@ -2,6 +2,8 @@
 
 set -e
 
+source /docker/set-mstart-env.sh
+
 [[ -z "$MGR" ]] && export MGR=1
 [[ -z "$MGR_PYTHON_PATH" ]] && export MGR_PYTHON_PATH=/ceph/src/pybind/mgr
 [[ -d "$MGR_PYTHON_PATH"/dashboard/frontend ]] && export IS_UPSTREAM_LUMINOUS=0

docker/ceph/start-ceph.sh

@@ -4,12 +4,13 @@ set -e
 
 source /docker/set-start-env.sh
 
-# Build frontend:
+#Build frontend:
 if [[ "$FRONTEND_BUILD_REQUIRED" == 1 ]]; then
     cd "$MGR_PYTHON_PATH"/dashboard/frontend
 
     # Set dev server proxy:
     TARGET_URL="${HTTP_PROTO}://${HOSTNAME}:${CEPH_MGR_DASHBOARD_PORT}"
+    echo "target" $TARGET_URL
     [[ -n "${DASHBOARD_URL}" ]] && TARGET_URL=${DASHBOARD_URL}
     jq "(.[] | .target)=\""${TARGET_URL}"\"" proxy.conf.json.sample > proxy.conf.json
 
@@ -18,7 +19,7 @@ if [[ "$FRONTEND_BUILD_REQUIRED" == 1 ]]; then
         npm update @angular/cli
     fi
 
-    npm ci
+    npm i
 
     if [[ -z "${DASHBOARD_URL}" ]]; then
         # Required to run dashboard python module.
@@ -40,14 +41,28 @@ fi
 
 rm -rf "$CEPH_CONF_PATH"/*
 
-cd /ceph/build
-../src/vstart.sh ${VSTART_OPTIONS}
+for cluster in ${CEPH_CLUSTERS[@]}; do
+    cd /ceph/build
+    echo ${cluster}
+    # Env. vars used in vstart.
+    export_var CEPH_ASOK_DIR=/ceph/build.ceph-dev/run/"${cluster}"/asok
+    export_var CEPH_CONF=/ceph/build.ceph-dev/run/"${cluster}"/ceph.conf
+    export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${cluster}"
+    export_var CEPH_DEV_DIR=/ceph/build.ceph-dev/run/"${cluster}"/dev
+    export_var CEPH_OUT_DIR=/ceph/build.ceph-dev/run/"${cluster}"/out
+    export VSTART_DEST=/ceph/build.ceph-dev/run/"${cluster}"
+    export CEPH_MGR_DASHBOARD_PORT=$(($CEPH_PORT + 1000))
+
+    ../src/mstart.sh ${cluster} ${VSTART_OPTIONS}
+
+    export CEPH_PORT=$(($CEPH_PORT + 100))
+done
 
 echo 'vstart.sh completed!'
 
 # Create rbd pool:
-"$CEPH_BIN"/ceph osd pool create rbd 8 8 replicated
-"$CEPH_BIN"/ceph osd pool application enable rbd rbd
+CEPH_CLI_ALL osd pool create rbd 8 8 replicated
+CEPH_CLI_ALL osd pool application enable rbd rbd
 
 # Configure Object Gateway:
 if [[ "$RGW" -gt 0  ||  "$RGW_MULTISITE" == 1 ]]; then
@@ -56,7 +71,7 @@ fi
 
 # Enable prometheus module
 if [[ "$IS_FIRST_CLUSTER" == 1 ]]; then
-    "$CEPH_BIN"/ceph mgr module enable prometheus
+    CEPH_CLI mgr module enable prometheus
     echo 'Prometheus mgr module enabled.'
 fi
 
@@ -75,8 +90,8 @@ if [[ "$DASHBOARD_SSL" == 0 && "$VSTART_HAS_SSL_FLAG" == 0 && "$IS_FIRST_CLUSTER
         SSL_OPTIONS=''
     fi
 
-    "$CEPH_BIN"/ceph config set mgr mgr/dashboard/ssl false $SSL_OPTIONS
-    "$CEPH_BIN"/ceph config set mgr mgr/dashboard/x/server_port "$CEPH_MGR_DASHBOARD_PORT" $SSL_OPTIONS
+    CEPH_CLI_ALL config set mgr mgr/dashboard/ssl false $SSL_OPTIONS
+    CEPH_CLI_ALL config set mgr mgr/dashboard/x/server_port "$CEPH_MGR_DASHBOARD_PORT" $SSL_OPTIONS
     /docker/restart-dashboard.sh
 
     echo "SSL disabled."
@@ -90,12 +105,12 @@ fi
 create_test_user() {
     DASHBOARD_TEST_USER_SECRET_FILE="/tmp/dashboard-test-user-secret.txt"
     printf 'test' > "${DASHBOARD_TEST_USER_SECRET_FILE}"
-    "$CEPH_BIN"/ceph dashboard ac-user-create test -i "${DASHBOARD_TEST_USER_SECRET_FILE}" "${DASHBOARD_USER_CREATE_OPTIONS}"
+    CEPH_CLI_ALL dashboard ac-user-create test -i "${DASHBOARD_TEST_USER_SECRET_FILE}" "${DASHBOARD_USER_CREATE_OPTIONS}"
 }
-create_test_user || "$CEPH_BIN"/ceph dashboard ac-user-create test test "${DASHBOARD_USER_CREATE_OPTIONS}"
+create_test_user || CEPH_CLI_ALL dashboard ac-user-create test test "${DASHBOARD_USER_CREATE_OPTIONS}"
 
 # Enable debug mode.
-"$CEPH_BIN"/ceph dashboard debug enable
+CEPH_CLI_ALL dashboard debug enable
 
 # Set monitoring stack:
 /docker/set-monitoring.sh
@@ -104,4 +119,4 @@ create_test_user || "$CEPH_BIN"/ceph dashboard ac-user-create test test "${DASHB
 [[ "$CEPH_VERSION" -le '14' ]] && exit 0
 
 # Set dashboard log level.
-"$CEPH_BIN"/ceph config set mgr mgr/dashboard/log_level debug
+CEPH_CLI_ALL config set mgr mgr/dashboard/log_level debug