diff --git a/.env.example b/.env.example index cbf4a0e504..cdce555529 100644 --- a/.env.example +++ b/.env.example @@ -41,6 +41,7 @@ DASHBOARD_DEV_SERVER=1 DASHBOARD_SSL=1 DASHBOARD_URL= RGW_MULTISITE=0 +NUMBER_OF_CLUSTERS=1 # Sanity checks' options #APPLITOOLS_API_KEY= diff --git a/docker-compose.yml b/docker-compose.yml index 8c6ec1c358..d35bd1ff96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: - PYTHONDONTWRITEBYTECODE=1 - RGW - RGW_MULTISITE=${RGW_MULTISITE:-0} + - NUMBER_OF_CLUSTERS=${NUMBER_OF_CLUSTERS:-1} cap_add: - ALL entrypoint: /docker/entrypoint.sh diff --git a/docker/ceph/ci/sanity-checks.sh b/docker/ceph/ci/sanity-checks.sh index e4422090dd..8fa7620238 100755 --- a/docker/ceph/ci/sanity-checks.sh +++ b/docker/ceph/ci/sanity-checks.sh @@ -2,6 +2,8 @@ set -e +source /docker/set-mstart-env.sh + REPO_DIR=/ceph [[ "$IS_UPSTREAM" == 1 && "$CEPH_VERSION" -le '14' ]] && PYTHON_VERSION=2 || PYTHON_VERSION=3 TRANSLATION_FILE=src/pybind/mgr/dashboard/frontend/src/locale/messages.xlf @@ -244,7 +246,7 @@ run_frontend_e2e_tests() { echo 'Running frontend E2E tests...' cd "$REPO_DIR"/src/pybind/mgr/dashboard/frontend - npm ci + npm i npx --no-install cypress -v && WITH_CYPRESS=1 || WITH_CYPRESS=0 E2E_CMD="npm run e2e:dev" if [[ "${WITH_CYPRESS}" == 1 ]]; then @@ -264,12 +266,15 @@ run_frontend_e2e_tests() { DASHBOARD_URL=null while [[ "${DASHBOARD_URL}" == 'null' ]]; do - export DASHBOARD_URL=$("$CEPH_BIN"/ceph mgr services | jq -r .dashboard) + export DASHBOARD_URL=$(CEPH_CLI mgr services | jq -r .dashboard) + export DASHBOARD2_URL=$(CEPH2_CLI mgr services | jq -r .dashboard) sleep 1 done if [[ "${WITH_CYPRESS}" == 1 ]]; then export CYPRESS_BASE_URL="${DASHBOARD_URL}" + export CYPRESS_CEPH2_URL="${DASHBOARD2_URL}" fi + cd "$REPO_DIR"/src/pybind/mgr/dashboard/frontend ANGULAR_VERSION=$(npm run ng version | grep 'Angular: ' | awk '{ print substr($2,1,1) }') # In nautilus this flag is required because BASE_URL is not read in protractor config. diff --git a/docker/ceph/entrypoint.sh b/docker/ceph/entrypoint.sh index 36d61c6634..dbb36cd9cd 100755 --- a/docker/ceph/entrypoint.sh +++ b/docker/ceph/entrypoint.sh @@ -10,22 +10,23 @@ export_var() { export_var IS_CEPH_RPM="$(command -v ceph-mgr 2>/dev/null | wc -l)" # Env. vars used in vstart. -export_var CEPH_ASOK_DIR=/ceph/build."${HOSTNAME}"/out -export_var CEPH_CONF=/ceph/build."${HOSTNAME}"/ceph.conf -export_var CEPH_CONF_PATH=/ceph/build."${HOSTNAME}" -export_var CEPH_DEV_DIR=/ceph/build."${HOSTNAME}"/dev -export_var CEPH_OUT_DIR=/ceph/build."${HOSTNAME}"/out +export_var CEPH_ASOK_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/asok +export_var CEPH_CONF=/ceph/build.ceph-dev/run/"${HOSTNAME}"/ceph.conf +export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${HOSTNAME}" +export_var CEPH_DEV_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/dev +export_var CEPH_OUT_DIR=/ceph/build.ceph-dev/run/"${HOSTNAME}"/out export_var CEPH_PORT="${CEPH_PORT:-10000}" rm -rf /ceph/build + mkdir -p "${CEPH_CONF_PATH}" if [[ "${IS_CEPH_RPM}" == 1 ]]; then - export_var CEPH_BIN=/usr/bin + export_var CEPH_BIN=/usr/bin/ export_var CEPH_LIB=/usr/lib64/ceph export_var EC_PATH="$CEPH_LIB"/erasure-code export_var OBJCLASS_PATH=/usr/lib64/rados-classes - + ln -sf "$EC_PATH"/* "$CEPH_LIB" ln -sf "$OBJCLASS_PATH"/* "$CEPH_LIB" diff --git a/docker/ceph/set-monitoring.sh b/docker/ceph/set-monitoring.sh index 3015f6bed5..dc72453575 100755 --- a/docker/ceph/set-monitoring.sh +++ b/docker/ceph/set-monitoring.sh @@ -1,14 +1,15 @@ #!/bin/bash set -e +source /docker/set-mstart-env.sh +echo $CEPH1 # Configure grafana set_grafana_api_url() { while true; do GRAFANA_IP=$(getent ahosts grafana | tail -1 | awk '{print $1}') if [[ -n "$GRAFANA_IP" ]]; then - "$CEPH_BIN"/ceph dashboard set-grafana-api-url "https://$GRAFANA_IP:$GRAFANA_HOST_PORT" - + CEPH_CLI dashboard set-grafana-api-url "https://$GRAFANA_IP:$GRAFANA_HOST_PORT" return fi @@ -22,8 +23,7 @@ set_alertmanager_api_host() { while true; do ALERTMANAGER_IP=$(getent ahosts alertmanager | tail -1 | awk '{print $1}') if [[ -n "$ALERTMANAGER_IP" ]]; then - "$CEPH_BIN"/ceph dashboard set-alertmanager-api-host "http://$ALERTMANAGER_IP:$ALERTMANAGER_HOST_PORT" - + CEPH_CLI dashboard set-alertmanager-api-host "http://$ALERTMANAGER_IP:$ALERTMANAGER_HOST_PORT" return fi @@ -37,8 +37,7 @@ set_prometheus_api_host() { while true; do PROMETHEUS_IP=$(getent ahosts prometheus | tail -1 | awk '{print $1}') if [[ -n "$PROMETHEUS_IP" ]]; then - "$CEPH_BIN"/ceph dashboard set-prometheus-api-host "http://$PROMETHEUS_IP:$PROMETHEUS_HOST_PORT" - + CEPH_CLI dashboard set-prometheus-api-host "http://$PROMETHEUS_IP:$PROMETHEUS_HOST_PORT" return fi diff --git a/docker/ceph/set-mstart-env.sh b/docker/ceph/set-mstart-env.sh new file mode 100755 index 0000000000..b062770661 --- /dev/null +++ b/docker/ceph/set-mstart-env.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +export_var() { + export "${1?}" + echo "export $1" >> /root/.bashrc +} + +echo $NUMBER_OF_CLUSTERS +declare -a CEPH_BIN_LIST +declare -a CEPH_CLUSTERS +for (( cluster=1; cluster<=$NUMBER_OF_CLUSTERS; cluster++ )); do + CEPH_CLUSTERS[cluster]="CEPH${cluster}" + CEPH_BIN_LIST[cluster]=${CEPH_CLUSTERS[cluster]} + echo ${CEPH_CLUSTERS[cluster]} + export_var ${CEPH_BIN_LIST[cluster]}="/ceph/src/mrun ${CEPH_CLUSTERS[cluster]} ceph" +done + +CEPH_CLI() { + export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/CEPH1/ + /ceph/src/mrun CEPH1 ceph "$@" +} + +CEPH2_CLI() { + export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/CEPH2/ + /ceph/src/mrun CEPH2 ceph "$@" +} + +CEPH_CLI_ALL() { + for cluster in ${CEPH_BIN_LIST[@]}; do + export_var CEPH_BIN=/usr/bin/ + export_var BUILD_DIR=/ceph/build.ceph-dev/ + export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${cluster}/" + + cd /ceph/build.ceph-dev + ${!cluster} $@ + done +} diff --git a/docker/ceph/set-start-env.sh b/docker/ceph/set-start-env.sh index c940e37e2a..03623b818e 100755 --- a/docker/ceph/set-start-env.sh +++ b/docker/ceph/set-start-env.sh @@ -2,6 +2,8 @@ set -e +source /docker/set-mstart-env.sh + [[ -z "$MGR" ]] && export MGR=1 [[ -z "$MGR_PYTHON_PATH" ]] && export MGR_PYTHON_PATH=/ceph/src/pybind/mgr [[ -d "$MGR_PYTHON_PATH"/dashboard/frontend ]] && export IS_UPSTREAM_LUMINOUS=0 diff --git a/docker/ceph/start-ceph.sh b/docker/ceph/start-ceph.sh index ee5d304a4f..af53b2a2ef 100755 --- a/docker/ceph/start-ceph.sh +++ b/docker/ceph/start-ceph.sh @@ -4,12 +4,13 @@ set -e source /docker/set-start-env.sh -# Build frontend: +#Build frontend: if [[ "$FRONTEND_BUILD_REQUIRED" == 1 ]]; then cd "$MGR_PYTHON_PATH"/dashboard/frontend # Set dev server proxy: TARGET_URL="${HTTP_PROTO}://${HOSTNAME}:${CEPH_MGR_DASHBOARD_PORT}" + echo "target" $TARGET_URL [[ -n "${DASHBOARD_URL}" ]] && TARGET_URL=${DASHBOARD_URL} jq "(.[] | .target)=\""${TARGET_URL}"\"" proxy.conf.json.sample > proxy.conf.json @@ -18,7 +19,7 @@ if [[ "$FRONTEND_BUILD_REQUIRED" == 1 ]]; then npm update @angular/cli fi - npm ci + npm i if [[ -z "${DASHBOARD_URL}" ]]; then # Required to run dashboard python module. @@ -40,14 +41,28 @@ fi rm -rf "$CEPH_CONF_PATH"/* -cd /ceph/build -../src/vstart.sh ${VSTART_OPTIONS} +for cluster in ${CEPH_CLUSTERS[@]}; do + cd /ceph/build + echo ${cluster} + # Env. vars used in vstart. + export_var CEPH_ASOK_DIR=/ceph/build.ceph-dev/run/"${cluster}"/asok + export_var CEPH_CONF=/ceph/build.ceph-dev/run/"${cluster}"/ceph.conf + export_var CEPH_CONF_PATH=/ceph/build.ceph-dev/run/"${cluster}" + export_var CEPH_DEV_DIR=/ceph/build.ceph-dev/run/"${cluster}"/dev + export_var CEPH_OUT_DIR=/ceph/build.ceph-dev/run/"${cluster}"/out + export VSTART_DEST=/ceph/build.ceph-dev/run/"${cluster}" + export CEPH_MGR_DASHBOARD_PORT=$(($CEPH_PORT + 1000)) + + ../src/mstart.sh ${cluster} ${VSTART_OPTIONS} + + export CEPH_PORT=$(($CEPH_PORT + 100)) +done echo 'vstart.sh completed!' # Create rbd pool: -"$CEPH_BIN"/ceph osd pool create rbd 8 8 replicated -"$CEPH_BIN"/ceph osd pool application enable rbd rbd +CEPH_CLI_ALL osd pool create rbd 8 8 replicated +CEPH_CLI_ALL osd pool application enable rbd rbd # Configure Object Gateway: if [[ "$RGW" -gt 0 || "$RGW_MULTISITE" == 1 ]]; then @@ -56,7 +71,7 @@ fi # Enable prometheus module if [[ "$IS_FIRST_CLUSTER" == 1 ]]; then - "$CEPH_BIN"/ceph mgr module enable prometheus + CEPH_CLI mgr module enable prometheus echo 'Prometheus mgr module enabled.' fi @@ -75,8 +90,8 @@ if [[ "$DASHBOARD_SSL" == 0 && "$VSTART_HAS_SSL_FLAG" == 0 && "$IS_FIRST_CLUSTER SSL_OPTIONS='' fi - "$CEPH_BIN"/ceph config set mgr mgr/dashboard/ssl false $SSL_OPTIONS - "$CEPH_BIN"/ceph config set mgr mgr/dashboard/x/server_port "$CEPH_MGR_DASHBOARD_PORT" $SSL_OPTIONS + CEPH_CLI_ALL config set mgr mgr/dashboard/ssl false $SSL_OPTIONS + CEPH_CLI_ALL config set mgr mgr/dashboard/x/server_port "$CEPH_MGR_DASHBOARD_PORT" $SSL_OPTIONS /docker/restart-dashboard.sh echo "SSL disabled." @@ -90,12 +105,12 @@ fi create_test_user() { DASHBOARD_TEST_USER_SECRET_FILE="/tmp/dashboard-test-user-secret.txt" printf 'test' > "${DASHBOARD_TEST_USER_SECRET_FILE}" - "$CEPH_BIN"/ceph dashboard ac-user-create test -i "${DASHBOARD_TEST_USER_SECRET_FILE}" "${DASHBOARD_USER_CREATE_OPTIONS}" + CEPH_CLI_ALL dashboard ac-user-create test -i "${DASHBOARD_TEST_USER_SECRET_FILE}" "${DASHBOARD_USER_CREATE_OPTIONS}" } -create_test_user || "$CEPH_BIN"/ceph dashboard ac-user-create test test "${DASHBOARD_USER_CREATE_OPTIONS}" +create_test_user || CEPH_CLI_ALL dashboard ac-user-create test test "${DASHBOARD_USER_CREATE_OPTIONS}" # Enable debug mode. -"$CEPH_BIN"/ceph dashboard debug enable +CEPH_CLI_ALL dashboard debug enable # Set monitoring stack: /docker/set-monitoring.sh @@ -104,4 +119,4 @@ create_test_user || "$CEPH_BIN"/ceph dashboard ac-user-create test test "${DASHB [[ "$CEPH_VERSION" -le '14' ]] && exit 0 # Set dashboard log level. -"$CEPH_BIN"/ceph config set mgr mgr/dashboard/log_level debug +CEPH_CLI_ALL config set mgr mgr/dashboard/log_level debug