From 2f0a0c6d382d670d50e651be4e489233dd11063e Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Thu, 29 Jan 2026 14:07:34 -0500 Subject: [PATCH 1/6] feat(clone): this allows you to use the dotcms image as an init container and clone an another dotCMS env ref: #34442 --- docker/dev-env/Dockerfile | 3 +- .../docker/original/ROOT/srv/10-import-env.sh | 232 ++++++++++++++++++ .../ROOT/srv/40-custom-starter-zip.sh | 58 ++++- .../original/ROOT/srv/50-load-dump-sql.sh | 5 +- .../docker/original/ROOT/srv/entrypoint.sh | 11 + .../docker/original/ROOT/srv/test-import.sh | 24 ++ 6 files changed, 317 insertions(+), 16 deletions(-) create mode 100755 dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh create mode 100755 dotCMS/src/main/docker/original/ROOT/srv/test-import.sh diff --git a/docker/dev-env/Dockerfile b/docker/dev-env/Dockerfile index 8faa4050551e..ccfd09a2d504 100644 --- a/docker/dev-env/Dockerfile +++ b/docker/dev-env/Dockerfile @@ -36,8 +36,7 @@ RUN apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends bash zip unzip wget libtcnative-1\ tzdata tini ca-certificates openssl libapr1 libpq-dev curl gnupg\ - vim libarchive-tools postgresql-common libmimalloc2.0 - + vim libarchive-tools postgresql-common libmimalloc2.0 libarchive-tools RUN /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y diff --git a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh new file mode 100755 index 000000000000..da4501aaf3c2 --- /dev/null +++ b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh @@ -0,0 +1,232 @@ +#!/bin/bash -e + + +## Drops the dotCMS database in preperation for a new import +drop_db_tables () { + + echo "- DOT_IMPORT_DROP_DB - attempting to drop db schema" + # Extract hostname and database name from JDBC URL (jdbc:postgresql://host/dbname) + DB_HOST="${DB_BASE_URL#jdbc:postgresql://}" # Remove prefix -> host/dbname + DB_HOST="${DB_HOST%%/*}" # Remove /dbname -> host + DB_NAME="${DB_BASE_URL##*/}" # Remove everything before last / -> dbname + + # Export password for psql (avoids password prompt) + export PGPASSWORD="${DB_PASSWORD}" + + psql -h "${DB_HOST}" -d "${DB_NAME}" -U "${DB_USERNAME}" -c "DROP SCHEMA public CASCADE;CREATE SCHEMA public;GRANT ALL ON SCHEMA public TO public;" + # Clear the password from environment + unset PGPASSWORD + +} + + + +## Imports the dotcms_db.sql.gz file into the postgres database specified by the DB_BASE_URL environment variable. +import_postgres () { + + if [ -s $DB_BACKUP_FILE ]; then + + if [ -z $DB_BASE_URL ]; then + echo "DB_BASE_URL environment variable not set, cannont continue without importing database" + return 0 + fi + + # Extract hostname and database name from JDBC URL (jdbc:postgresql://host/dbname) + DB_HOST="${DB_BASE_URL#jdbc:postgresql://}" # Remove prefix -> host/dbname + DB_HOST="${DB_HOST%%/*}" # Remove /dbname -> host + DB_NAME="${DB_BASE_URL##*/}" # Remove everything before last / -> dbname + + # Export password for psql (avoids password prompt) + export PGPASSWORD="${DB_PASSWORD}" + + # Check if database already has data (inode table exists with records) + INODE_COUNT=$(psql -h "${DB_HOST}" -d "${DB_NAME}" -U "${DB_USERNAME}" -qtAX -c \ + "SELECT CASE WHEN EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'inode') + THEN (SELECT count(*) FROM inode) ELSE 0 END" 2>/dev/null | tr -d '[:space:]') + + if [ -n "$INODE_COUNT" ] && [ "$INODE_COUNT" -gt 0 ]; then + echo "- Database already contains data ($INODE_COUNT inodes), skipping import." + return 0 + fi + + + # Run the query using psql + cat $DB_BACKUP_FILE | gzip -d | psql -h "${DB_HOST}" -d "${DB_NAME}" -U "${DB_USERNAME}" + + # Clear the password from environment + unset PGPASSWORD + + fi + +} + + +## Downloads the assets.zip and dotcms_db.sql.gz files from the specified environment. +download_dotcms_db_assets () { + + # If these are 0 length files, delete them + if [ ! -s $ASSETS_BACKUP_FILE ] ; then + rm -rf $ASSETS_BACKUP_FILE + fi + + if [ ! -s $DB_BACKUP_FILE ] ; then + rm -rf $DB_BACKUP_FILE + fi + + echo "- Pulling Environment from $DOT_IMPORT_ENVIRONMENT" + + if [ -n "$DOT_IMPORT_API_TOKEN" ]; then + echo "- Using Authorization: Bearer" + export AUTH_HEADER="Authorization: Bearer $DOT_IMPORT_API_TOKEN" + elif [ -n "$DOT_IMPORT_USERNAME_PASSWORD" ]; then + echo "- Using Authorization: Basic" + export AUTH_HEADER="Authorization: Basic $(echo -n $DOT_IMPORT_USERNAME_PASSWORD | base64)" + fi + + + mkdir -p $SHARED_DATA_DIR/assets + chown -R dotcms:dotcms $SHARED_DATA_DIR || echo "cannot chown" + + if [ -s "$ASSETS_BACKUP_FILE" ] && [ -s $DB_BACKUP_FILE ]; then + + echo "- DB and Assets backups exist, skipping" + echo "- Delete $ASSETS_BACKUP_FILE and $DB_BACKUP_FILE to force a re-download" + return + fi + + if [ ! -s "$ASSETS_BACKUP_FILE" ]; then + rm -rf $ASSETS_BACKUP_FILE.tmp + echo "- Downloading ASSETS" + echo "AUTH_HEADER: $AUTH_HEADER" + echo "wget --no-check-certificate --header=\"$AUTH_HEADER\" -t 1 -O ${ASSETS_BACKUP_FILE}.tmp ${DOT_IMPORT_ENVIRONMENT}/api/v1/maintenance/_downloadAssets\?oldAssets=${DOT_IMPORT_ALL_ASSETS}\&maxSize=${DOT_IMPORT_MAX_ASSET_SIZE}" + + wget --no-check-certificate --header="$AUTH_HEADER" -t 1 -O ${ASSETS_BACKUP_FILE}.tmp ${DOT_IMPORT_ENVIRONMENT}/api/v1/maintenance/_downloadAssets\?oldAssets=${DOT_IMPORT_ALL_ASSETS}\&maxSize=${DOT_IMPORT_MAX_ASSET_SIZE} + if [ -s ${ASSETS_BACKUP_FILE}.tmp ]; then + mv ${ASSETS_BACKUP_FILE}.tmp $ASSETS_BACKUP_FILE + else + rm -rf ${ASSETS_BACKUP_FILE}.tmp + echo "asset download failed, please check your credentials and try again" + exit 1 + fi + fi + + if [ ! -f "$DB_BACKUP_FILE" ]; then + echo "- Downloading database" + rm -rf ${DB_BACKUP_FILE}.tmp + wget --no-check-certificate --header="$AUTH_HEADER" -t 1 -O ${DB_BACKUP_FILE}.tmp "${DOT_IMPORT_ENVIRONMENT}/api/v1/maintenance/_downloadDb" || exit 1 + if [ -s ${DB_BACKUP_FILE}.tmp ]; then + mv ${DB_BACKUP_FILE}.tmp $DB_BACKUP_FILE + else + rm -rf ${DB_BACKUP_FILE}.tmp + echo "database download failed, please check your credentials and try again" + exit 1 + fi + fi + + unset AUTH_HEADER + +} + +## Unpacks the assets.zip file if it exists and has not been unpacked +unpack_assets(){ + if [ ! -s "$ASSETS_BACKUP_FILE" ]; then + return 0 + fi + + echo "- Extracting assets.zip" + local tar_lang="${DOT_IMPORT_TAR_LANG:-C.UTF-8}" + local DOT_IMPORT_IGNORE_ASSET_ERRORS=${DOT_IMPORT_IGNORE_ASSET_ERRORS:-"true"} + + + if ! LANG="$tar_lang" bsdtar -xf "$ASSETS_BACKUP_FILE" -C "$SHARED_DATA_DIR"; then + if [ "${DOT_IMPORT_IGNORE_ASSET_ERRORS}" = "true" ]; then + echo "WARNING: assets extraction reported errors; continuing due to DOT_IMPORT_IGNORE_ASSET_ERRORS=true" + return 0 + fi + return 1 + fi +} + + + +#### Script main() + +export DOT_IMPORT_DROP_DB=${DOT_IMPORT_DROP_DB:-"false"} +export DOT_IMPORT_NON_LIVE_ASSETS=${DOT_IMPORT_NON_LIVE_ASSETS:-"false"} +export DOT_IMPORT_MAX_ASSET_SIZE=${DOT_IMPORT_MAX_ASSET_SIZE:-"100mb"} +export SHARED_DATA_DIR=${SHARED_DATA_DIR:-"/data/shared"} +export IMPORT_DATA_DIR=${IMPORT_DATA_DIR:-"$SHARED_DATA_DIR/import"} +export IMPORT_IN_PROCESS=$IMPORT_DATA_DIR/lock.txt +export IMPORT_COMPLETE=$IMPORT_DATA_DIR/import_complete.txt + +if [ -z "$DOT_IMPORT_ENVIRONMENT" ]; then + echo "- No dotCMS env to import, starting normally" + exit 0 +fi + +if [ -z "$DOT_IMPORT_API_TOKEN" -a -z "$DOT_IMPORT_USERNAME_PASSWORD" ]; then + echo "- Set DOT_IMPORT_ENVIRONMENT, DOT_IMPORT_USERNAME_PASSWORD and/or DOT_IMPORT_API_TOKEN to import from another environment on first run" + echo 0 +fi + +export DOT_IMPORT_HOST="${DOT_IMPORT_ENVIRONMENT#http://}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST#https://}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST%%/*}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST%%:*}" + + + +if [ -f "$IMPORT_COMPLETE" ]; then + echo "- Cloning of $DOT_IMPORT_HOST completed. Delete ${IMPORT_COMPLETE} to try again" + echo 0 +fi + +if [ -f "$IMPORT_IN_PROCESS" ]; then + # Get lock file age in minutes (portable for Linux and macOS) + if stat -c %Y "$IMPORT_IN_PROCESS" >/dev/null 2>&1; then + FILE_MTIME=$(stat -c %Y "$IMPORT_IN_PROCESS") # Linux + else + FILE_MTIME=$(stat -f %m "$IMPORT_IN_PROCESS") # macOS + fi + CURRENT_TIME=$(date +%s) + LOCK_AGE_MINUTES=$(( (CURRENT_TIME - FILE_MTIME) / 60 )) + + # Check if import process file is older than 30 minutes (stale lock) + if [ "$LOCK_AGE_MINUTES" -ge 30 ]; then + echo "ERROR: Import process appears stale (lock file is ${LOCK_AGE_MINUTES} minutes old). Removing lock file." + rm -f "$IMPORT_IN_PROCESS" + exit 1 + fi + echo "ERROR: Lock file found: ${IMPORT_IN_PROCESS} (${LOCK_AGE_MINUTES} minutes old)." + echo " Delete lock file or wait until it's 30 minutes old and try again" + echo " sleeping for 3m" + sleep 180 + exit 1 +fi +mkdir -p $IMPORT_DATA_DIR +touch $IMPORT_IN_PROCESS + + +HASHED_ENV=$(echo -n "$DOT_IMPORT_ENVIRONMENT" | md5sum | cut -d ' ' -f 1) + + +export ASSETS_BACKUP_FILE="${IMPORT_DATA_DIR}/${HASHED_ENV}_assets.zip" +export DB_BACKUP_FILE="${IMPORT_DATA_DIR}/${HASHED_ENV}_dotcms_db.sql.gz" + +# Step 1. download files if needed +download_dotcms_db_assets || { echo "Unable to download dotcms backup"; rm $IMPORT_IN_PROCESS; exit 1; } + +# Step 2. wipe out database if needed +if [ "$DOT_IMPORT_DROP_DB" = "true" ]; then + drop_db_tables || { echo "unable to drop the dotcms db schema"; rm $IMPORT_IN_PROCESS; exit 1; } +fi + +# Step 3. import postgres +import_postgres || { echo "Unable to import postgres backup"; rm $IMPORT_IN_PROCESS; exit 1; } + +# Step 4. unpack assets +unpack_assets || { echo "Unable to unzip assets"; rm $IMPORT_IN_PROCESS; exit 1; } + +if rm -f "$IMPORT_IN_PROCESS" && touch "$IMPORT_COMPLETE"; then + echo "dotCMS Environment $DOT_IMPORT_HOST Imported, exiting." + exit 13 +fi +echo "Unable complete import" +exit 1 diff --git a/dotCMS/src/main/docker/original/ROOT/srv/40-custom-starter-zip.sh b/dotCMS/src/main/docker/original/ROOT/srv/40-custom-starter-zip.sh index 817e79acf1e6..addde6271dd8 100755 --- a/dotCMS/src/main/docker/original/ROOT/srv/40-custom-starter-zip.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/40-custom-starter-zip.sh @@ -3,26 +3,60 @@ set -e +### URL to download the custom starter from +CUSTOM_STARTER_URL=${CUSTOM_STARTER_URL:-""} -CUSTOM_STARTER=custom_starter.zip +### dotCMS API Token +CUSTOM_STARTER_URL_AUTH_TOKEN=${CUSTOM_STARTER_URL_AUTH_TOKEN:-""} -## if we have a custom starter +### Or basic auth - in the form of username:password +CUSTOM_STARTER_URL_BASIC_AUTH=${CUSTOM_STARTER_URL_BASIC_AUTH:-""} + +### Folder where the custom starter will be downloaded +CUSTOM_STARTER_DATA_FOLDER=${CUSTOM_STARTER_DATA_FOLDER:-"/data/shared"} + + +## if we dont have a custom starter if [ -z ${CUSTOM_STARTER_URL} ]; then echo "Using default starter"; -else - if [[ ! -f /data/shared/$CUSTOM_STARTER ]]; then - touch /data/shared/$CUSTOM_STARTER - echo "Downloading Custom Starter:" $CUSTOM_STARTER_URL - mkdir -p /data/shared - curl -s -L -o /data/shared/$CUSTOM_STARTER $CUSTOM_STARTER_URL - if [[ -s /data/shared/$CUSTOM_STARTER ]] ; then - export DOT_STARTER_DATA_LOAD=/data/shared/$CUSTOM_STARTER +else + + HASHED_URL=$(echo -n "$CUSTOM_STARTER_URL" | md5sum | cut -d ' ' -f 1) + CUSTOM_STARTER=dotcms-starter-$HASHED_URL.zip + + if [[ ! -f $CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER ]]; then + + echo "CUSTOM_STARTER_DATA_FOLDER: $CUSTOM_STARTER_DATA_FOLDER" + echo "CUSTOM_STARTER_URL: $CUSTOM_STARTER_URL" + echo "HASHED_URL: $HASHED_URL" + echo "CUSTOM_STARTER file: $CUSTOM_STARTER" + if [[ -n $CUSTOM_STARTER_URL_AUTH_TOKEN ]]; then + echo "CUSTOM_STARTER_URL_AUTH_TOKEN: XXXXXX" + fi + if [[ -n $CUSTOM_STARTER_URL_AUTH_TOKEN ]]; then + echo "CUSTOM_STARTER_URL_BASIC_AUTH: XXXXXX:XXXXXX" + fi + mkdir -p $CUSTOM_STARTER_DATA_FOLDER + if [[ -n $CUSTOM_STARTER_URL_AUTH_TOKEN ]]; then + echo "Downloading Custom Starter with Auth Token:" $CUSTOM_STARTER_URL + echo "curl -s -L -o $CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER -HAuthorization: Bearer $CUSTOM_STARTER_URL_AUTH_TOKEN $CUSTOM_STARTER_URL" + curl -k -s -L -o $CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER -H"Authorization: Bearer $CUSTOM_STARTER_URL_AUTH_TOKEN" "$CUSTOM_STARTER_URL" || echo "Failed to download starter with auth token" + elif [[ -n $CUSTOM_STARTER_URL_BASIC_AUTH ]]; then + echo "Downloading Custom Starter with Basic Auth:" $CUSTOM_STARTER_URL + curl -k -s -L -o $CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER -u"$CUSTOM_STARTER_URL_BASIC_AUTH" "$CUSTOM_STARTER_URL" || echo "Failed to download starter with basic auth" + else + echo "Downloading Custom Starter:" $CUSTOM_STARTER_URL + curl -k -s -L -o $CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER "$CUSTOM_STARTER_URL" || echo "Failed to download starter" + fi + + if [[ -s $CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER ]] ; then + export DOT_STARTER_DATA_LOAD=$CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER else - rm /data/shared/$CUSTOM_STARTER + rm -f $CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER echo "No starter downloaded, skipping" fi else echo "custom starter already downloaded" - echo "if you need to redownload a new starter, delete the existing custom starter file found here: /data/shared/$CUSTOM_STARTER" + echo "if you need to redownload a new starter, delete the existing custom starter file found here: $CUSTOM_STARTER_DATA_FOLDER/$CUSTOM_STARTER" fi fi diff --git a/dotCMS/src/main/docker/original/ROOT/srv/50-load-dump-sql.sh b/dotCMS/src/main/docker/original/ROOT/srv/50-load-dump-sql.sh index d3ade85147b9..b7a3e32cd1d3 100644 --- a/dotCMS/src/main/docker/original/ROOT/srv/50-load-dump-sql.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/50-load-dump-sql.sh @@ -1,16 +1,17 @@ #!/bin/bash set -e + # Check if the DB_LOAD_DUMP environment variable is set and the file exists if [[ -n "${DB_LOAD_DUMP_SQL}" && -f "${DB_LOAD_DUMP_SQL}" && -z ${CUSTOM_STARTER_URL} ]]; then echo "Importing database dump from ${DB_LOAD_DUMP_SQL}..." sleep 10 export PGPASSWORD=${DB_PASSWORD} /usr/bin/psql -h "${DB_HOST}" -U "${DB_USERNAME}" -d "${DB_NAME}" -f "${DB_LOAD_DUMP_SQL}" - + unset PGPASSWORD echo "Dump successfully imported." elif [[ -n ${DOT_STARTER_DATA_LOAD} ]]; then echo "Importing data from starter ${CUSTOM_STARTER_URL}..." else echo "Dump file not found [${DB_LOAD_DUMP_SQL}]" -fi \ No newline at end of file +fi diff --git a/dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh b/dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh index 99ef7015ba02..70a27f5599b4 100644 --- a/dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh @@ -6,6 +6,17 @@ umask 007 export TOMCAT_HOME=/srv/dotserver/tomcat +/srv/10-import-env.sh +exit_status=$? +# Check the value +if [ $exit_status -eq 13 ]; then + echo "Import completed, init signing off" + exit 0; +fi + +exit 0 + + source /srv/20-copy-overriden-files.sh source /srv/25-generate-dev-ssl-cert.sh source /srv/30-override-config-props.sh diff --git a/dotCMS/src/main/docker/original/ROOT/srv/test-import.sh b/dotCMS/src/main/docker/original/ROOT/srv/test-import.sh new file mode 100755 index 000000000000..19472408249c --- /dev/null +++ b/dotCMS/src/main/docker/original/ROOT/srv/test-import.sh @@ -0,0 +1,24 @@ +#!/bin/bash -e + + + +cat > app.env << 'EOF' +#SHARED_DATA_DIR=/Users/will/git/dotcms/data +DOT_IMPORT_ENVIRONMENT=https://demo.dotcms.com +DOT_IMPORT_API_TOKEN= +DOT_IMPORT_USERNAME_PASSWORD=admin@dotcms.com:admin +DOT_IMPORT_DROP_DB=true +DB_BASE_URL=jdbc:postgresql://db.dotcms.site/dotcms +DB_DRIVER=org.postgresql.Driver +DB_PASSWORD=password +DB_USERNAME=dotcmsdbuser +DOT_ENABLE_SCRIPTING=true +DOT_ES_AUTH_BASIC_PASSWORD=admin +DOT_ES_AUTH_BASIC_USER=admin +DOT_ES_AUTH_TYPE=BASIC +DOT_ES_ENDPOINTS=https://es.dotcms.site:9200 + +EOF + +docker run --env-file app.env --rm -p8080:8082 -p8433:8433 dotcms/dotcms-test:1.0.0-SNAPSHOT + From 86f151f1fbb361b900d45bff4ec7dedc900fa127 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Thu, 29 Jan 2026 14:17:03 -0500 Subject: [PATCH 2/6] feat(clone): run entrypoint setup after import success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the unconditional `exit 0` so the entrypoint continues to source startup scripts, clarify the import script’s exit-13 success path, and install `libarchive-tools` to support asset unpacking during imports. ref: #34442 --- dotCMS/src/main/docker/original/Dockerfile | 1 + dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh | 3 +++ dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh | 3 --- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dotCMS/src/main/docker/original/Dockerfile b/dotCMS/src/main/docker/original/Dockerfile index 955cdbbe06cf..3dfb79e4c0b9 100644 --- a/dotCMS/src/main/docker/original/Dockerfile +++ b/dotCMS/src/main/docker/original/Dockerfile @@ -55,6 +55,7 @@ RUN apt update && \ libmimalloc2.0 \ openssl \ libapr1 \ + libarchive-tools \ libpq-dev && \ rm -rf /var/lib/apt/lists/* diff --git a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh index da4501aaf3c2..d7387a05d5ce 100755 --- a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh @@ -224,9 +224,12 @@ import_postgres || { echo "Unable to import postgres backup"; rm $IMPORT_IN_PROC # Step 4. unpack assets unpack_assets || { echo "Unable to unzip assets"; rm $IMPORT_IN_PROCESS; exit 1; } +# Step 5: exit 13 if the clone worked if rm -f "$IMPORT_IN_PROCESS" && touch "$IMPORT_COMPLETE"; then echo "dotCMS Environment $DOT_IMPORT_HOST Imported, exiting." exit 13 fi + +# Otherwise, die echo "Unable complete import" exit 1 diff --git a/dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh b/dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh index 70a27f5599b4..966811bcabd3 100644 --- a/dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/entrypoint.sh @@ -14,9 +14,6 @@ if [ $exit_status -eq 13 ]; then exit 0; fi -exit 0 - - source /srv/20-copy-overriden-files.sh source /srv/25-generate-dev-ssl-cert.sh source /srv/30-override-config-props.sh From 02ec090062dec3ba7f1457662f41f45b975af26a Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Thu, 29 Jan 2026 14:45:05 -0500 Subject: [PATCH 3/6] feat(clone): run entrypoint setup after import success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the unconditional `exit 0` so the entrypoint continues to source startup scripts, clarify the import script’s exit-13 success path, and install `libarchive-tools` to support asset unpacking during imports. ref: #34442 --- .../docker/original/ROOT/srv/10-import-env.sh | 31 +++++++++---------- .../docker/original/ROOT/srv/test-import.sh | 12 +++++-- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh index d7387a05d5ce..dcfcd317330e 100755 --- a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh @@ -1,7 +1,7 @@ #!/bin/bash -e -## Drops the dotCMS database in preperation for a new import +## Drops the contents of the dotCMS database in preperation for a new import (only if requested) drop_db_tables () { echo "- DOT_IMPORT_DROP_DB - attempting to drop db schema" @@ -160,24 +160,23 @@ export IMPORT_IN_PROCESS=$IMPORT_DATA_DIR/lock.txt export IMPORT_COMPLETE=$IMPORT_DATA_DIR/import_complete.txt if [ -z "$DOT_IMPORT_ENVIRONMENT" ]; then - echo "- No dotCMS env to import, starting normally" exit 0 fi if [ -z "$DOT_IMPORT_API_TOKEN" -a -z "$DOT_IMPORT_USERNAME_PASSWORD" ]; then echo "- Set DOT_IMPORT_ENVIRONMENT, DOT_IMPORT_USERNAME_PASSWORD and/or DOT_IMPORT_API_TOKEN to import from another environment on first run" - echo 0 + exit 0 fi export DOT_IMPORT_HOST="${DOT_IMPORT_ENVIRONMENT#http://}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST#https://}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST%%/*}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST%%:*}" - - +# Exit normally if already cloned if [ -f "$IMPORT_COMPLETE" ]; then - echo "- Cloning of $DOT_IMPORT_HOST completed. Delete ${IMPORT_COMPLETE} to try again" - echo 0 + echo "- Import of $DOT_IMPORT_HOST completed. Delete ${IMPORT_COMPLETE} to try again." + exit 0 fi +# lock other pods out if importing if [ -f "$IMPORT_IN_PROCESS" ]; then # Get lock file age in minutes (portable for Linux and macOS) if stat -c %Y "$IMPORT_IN_PROCESS" >/dev/null 2>&1; then @@ -200,36 +199,34 @@ if [ -f "$IMPORT_IN_PROCESS" ]; then sleep 180 exit 1 fi -mkdir -p $IMPORT_DATA_DIR -touch $IMPORT_IN_PROCESS +mkdir -p $IMPORT_DATA_DIR && touch $IMPORT_IN_PROCESS HASHED_ENV=$(echo -n "$DOT_IMPORT_ENVIRONMENT" | md5sum | cut -d ' ' -f 1) - export ASSETS_BACKUP_FILE="${IMPORT_DATA_DIR}/${HASHED_ENV}_assets.zip" export DB_BACKUP_FILE="${IMPORT_DATA_DIR}/${HASHED_ENV}_dotcms_db.sql.gz" -# Step 1. download files if needed +# Step 1. download db and assets (if needed) download_dotcms_db_assets || { echo "Unable to download dotcms backup"; rm $IMPORT_IN_PROCESS; exit 1; } -# Step 2. wipe out database if needed +# Step 2. wipe database clean (if requested) if [ "$DOT_IMPORT_DROP_DB" = "true" ]; then drop_db_tables || { echo "unable to drop the dotcms db schema"; rm $IMPORT_IN_PROCESS; exit 1; } fi -# Step 3. import postgres +# Step 3. import postgres db import_postgres || { echo "Unable to import postgres backup"; rm $IMPORT_IN_PROCESS; exit 1; } -# Step 4. unpack assets +# Step 4. unpack assets.zip unpack_assets || { echo "Unable to unzip assets"; rm $IMPORT_IN_PROCESS; exit 1; } -# Step 5: exit 13 if the clone worked +# Step 5: exit sig 13 if the clone worked if rm -f "$IMPORT_IN_PROCESS" && touch "$IMPORT_COMPLETE"; then - echo "dotCMS Environment $DOT_IMPORT_HOST Imported, exiting." + echo "dotCMS Environment $DOT_IMPORT_HOST Imported, happily exiting." exit 13 fi -# Otherwise, die +# Otherwise, die ugly echo "Unable complete import" exit 1 diff --git a/dotCMS/src/main/docker/original/ROOT/srv/test-import.sh b/dotCMS/src/main/docker/original/ROOT/srv/test-import.sh index 19472408249c..416e698602b7 100755 --- a/dotCMS/src/main/docker/original/ROOT/srv/test-import.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/test-import.sh @@ -1,13 +1,21 @@ #!/bin/bash -e +# This script can be used to test the importing functionality of dotCMS. +# Use the environmental variables below to spin up a dotcms docker image +# and clone different environments. If you are running locally, run a +# +# `just build-quicker` +# +# from the project root to quickly build a testable docker image cat > app.env << 'EOF' -#SHARED_DATA_DIR=/Users/will/git/dotcms/data DOT_IMPORT_ENVIRONMENT=https://demo.dotcms.com DOT_IMPORT_API_TOKEN= DOT_IMPORT_USERNAME_PASSWORD=admin@dotcms.com:admin DOT_IMPORT_DROP_DB=true + + DB_BASE_URL=jdbc:postgresql://db.dotcms.site/dotcms DB_DRIVER=org.postgresql.Driver DB_PASSWORD=password @@ -20,5 +28,5 @@ DOT_ES_ENDPOINTS=https://es.dotcms.site:9200 EOF -docker run --env-file app.env --rm -p8080:8082 -p8433:8433 dotcms/dotcms-test:1.0.0-SNAPSHOT +docker run --env-file app.env -v $PWD/data:/data --rm -p8080:8082 -p8443:8443 dotcms/dotcms-test:1.0.0-SNAPSHOT From 338da8f4a6cd623c809245876931074992aa2763 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 30 Jan 2026 09:23:47 -0500 Subject: [PATCH 4/6] feat(clone): run entrypoint setup after import success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the unconditional `exit 0` so the entrypoint continues to source startup scripts, clarify the import script’s exit-13 success path, and install `libarchive-tools` to support asset unpacking during imports. ref: #34442 --- dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh index dcfcd317330e..f47743c369cb 100755 --- a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh @@ -172,7 +172,7 @@ export DOT_IMPORT_HOST="${DOT_IMPORT_ENVIRONMENT#http://}"; DOT_IMPORT_HOST="${D # Exit normally if already cloned if [ -f "$IMPORT_COMPLETE" ]; then - echo "- Import of $DOT_IMPORT_HOST completed. Delete ${IMPORT_COMPLETE} to try again." + echo "dotCMS environment already inited. Delete ${IMPORT_COMPLETE} to import again." exit 0 fi From fba967533a9f0214fed1939d1e0c1e3e50033650 Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Fri, 30 Jan 2026 18:42:31 -0500 Subject: [PATCH 5/6] feat(clone): run entrypoint setup after import success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the unconditional so the entrypoint continues to source startup scripts, clarify the import script’s exit-13 success path, and install to support asset unpacking during imports. ref: #34442 --- .../docker/original/ROOT/srv/10-import-env.sh | 103 +++++++++--------- 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh index f47743c369cb..9fcb973cd528 100755 --- a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh @@ -3,42 +3,33 @@ ## Drops the contents of the dotCMS database in preperation for a new import (only if requested) drop_db_tables () { - - echo "- DOT_IMPORT_DROP_DB - attempting to drop db schema" - # Extract hostname and database name from JDBC URL (jdbc:postgresql://host/dbname) - DB_HOST="${DB_BASE_URL#jdbc:postgresql://}" # Remove prefix -> host/dbname - DB_HOST="${DB_HOST%%/*}" # Remove /dbname -> host - DB_NAME="${DB_BASE_URL##*/}" # Remove everything before last / -> dbname - - # Export password for psql (avoids password prompt) - export PGPASSWORD="${DB_PASSWORD}" - + echo "- DOT_IMPORT_DROP_DB - attempting to drop db schema" psql -h "${DB_HOST}" -d "${DB_NAME}" -U "${DB_USERNAME}" -c "DROP SCHEMA public CASCADE;CREATE SCHEMA public;GRANT ALL ON SCHEMA public TO public;" - # Clear the password from environment - unset PGPASSWORD - } - +## This checks active connections to the dotCMS database - we can only proceed if there are no connections +check_active_connections() { + # Export password for psql (avoids password prompt) + + ACTIVE=$(psql -h "$DB_HOST" -d "$DB_NAME" -U "$DB_USERNAME" -qtAX -c \ + "SELECT count(*) FROM pg_stat_activity + WHERE datname = '$DB_NAME' + AND pid != pg_backend_pid() + AND state != 'idle'" 2>/dev/null || echo "0") + + if [ "$ACTIVE" -gt 0 ]; then + echo "ERROR: Database has $ACTIVE active connections" + echo "Cannot import while database is in use" + echo "This script is designed for initial deployment only" + echo "Stop all pods before performing refresh" + exit 1 + fi +} ## Imports the dotcms_db.sql.gz file into the postgres database specified by the DB_BASE_URL environment variable. import_postgres () { if [ -s $DB_BACKUP_FILE ]; then - - if [ -z $DB_BASE_URL ]; then - echo "DB_BASE_URL environment variable not set, cannont continue without importing database" - return 0 - fi - - # Extract hostname and database name from JDBC URL (jdbc:postgresql://host/dbname) - DB_HOST="${DB_BASE_URL#jdbc:postgresql://}" # Remove prefix -> host/dbname - DB_HOST="${DB_HOST%%/*}" # Remove /dbname -> host - DB_NAME="${DB_BASE_URL##*/}" # Remove everything before last / -> dbname - - # Export password for psql (avoids password prompt) - export PGPASSWORD="${DB_PASSWORD}" - # Check if database already has data (inode table exists with records) INODE_COUNT=$(psql -h "${DB_HOST}" -d "${DB_NAME}" -U "${DB_USERNAME}" -qtAX -c \ "SELECT CASE WHEN EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'inode') @@ -49,13 +40,9 @@ import_postgres () { return 0 fi - # Run the query using psql cat $DB_BACKUP_FILE | gzip -d | psql -h "${DB_HOST}" -d "${DB_NAME}" -U "${DB_USERNAME}" - # Clear the password from environment - unset PGPASSWORD - fi } @@ -96,10 +83,8 @@ download_dotcms_db_assets () { if [ ! -s "$ASSETS_BACKUP_FILE" ]; then rm -rf $ASSETS_BACKUP_FILE.tmp - echo "- Downloading ASSETS" - echo "AUTH_HEADER: $AUTH_HEADER" - echo "wget --no-check-certificate --header=\"$AUTH_HEADER\" -t 1 -O ${ASSETS_BACKUP_FILE}.tmp ${DOT_IMPORT_ENVIRONMENT}/api/v1/maintenance/_downloadAssets\?oldAssets=${DOT_IMPORT_ALL_ASSETS}\&maxSize=${DOT_IMPORT_MAX_ASSET_SIZE}" - + echo "- Downloading ASSETS from ${DOT_IMPORT_ENVIRONMENT}" + wget --no-check-certificate --header="$AUTH_HEADER" -t 1 -O ${ASSETS_BACKUP_FILE}.tmp ${DOT_IMPORT_ENVIRONMENT}/api/v1/maintenance/_downloadAssets\?oldAssets=${DOT_IMPORT_ALL_ASSETS}\&maxSize=${DOT_IMPORT_MAX_ASSET_SIZE} if [ -s ${ASSETS_BACKUP_FILE}.tmp ]; then mv ${ASSETS_BACKUP_FILE}.tmp $ASSETS_BACKUP_FILE @@ -156,8 +141,15 @@ export DOT_IMPORT_NON_LIVE_ASSETS=${DOT_IMPORT_NON_LIVE_ASSETS:-"false"} export DOT_IMPORT_MAX_ASSET_SIZE=${DOT_IMPORT_MAX_ASSET_SIZE:-"100mb"} export SHARED_DATA_DIR=${SHARED_DATA_DIR:-"/data/shared"} export IMPORT_DATA_DIR=${IMPORT_DATA_DIR:-"$SHARED_DATA_DIR/import"} -export IMPORT_IN_PROCESS=$IMPORT_DATA_DIR/lock.txt export IMPORT_COMPLETE=$IMPORT_DATA_DIR/import_complete.txt +# Extract hostname and database name from JDBC URL (jdbc:postgresql://host/dbname) +export DB_HOST="${DB_BASE_URL#jdbc:postgresql://}" # Remove prefix -> host/dbname +export DB_HOST="${DB_HOST%%/*}" # Remove /dbname -> host +export DB_NAME="${DB_BASE_URL##*/}" # Remove everything before last / -> dbname +export PGPASSWORD="${DB_PASSWORD}" + +# Clear the password from environment on exit +trap "unset PGPASSWORD" EXIT if [ -z "$DOT_IMPORT_ENVIRONMENT" ]; then exit 0 @@ -168,6 +160,13 @@ if [ -z "$DOT_IMPORT_API_TOKEN" -a -z "$DOT_IMPORT_USERNAME_PASSWORD" ]; then exit 0 fi +if [ -z $DB_BASE_URL ]; then + echo "DB_BASE_URL environment variable not set, cannot continue without importing database" + exit 0 +fi + +mkdir -p $IMPORT_DATA_DIR + export DOT_IMPORT_HOST="${DOT_IMPORT_ENVIRONMENT#http://}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST#https://}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST%%/*}"; DOT_IMPORT_HOST="${DOT_IMPORT_HOST%%:*}" # Exit normally if already cloned @@ -176,13 +175,15 @@ if [ -f "$IMPORT_COMPLETE" ]; then exit 0 fi -# lock other pods out if importing -if [ -f "$IMPORT_IN_PROCESS" ]; then +LOCK_DIR="$IMPORT_DATA_DIR/.lock" +if mkdir "$LOCK_DIR" 2>/dev/null; then + trap "rm -rf '$LOCK_DIR' 2>/dev/null; unset PGPASSWORD" EXIT +else # Get lock file age in minutes (portable for Linux and macOS) - if stat -c %Y "$IMPORT_IN_PROCESS" >/dev/null 2>&1; then - FILE_MTIME=$(stat -c %Y "$IMPORT_IN_PROCESS") # Linux + if stat -c %Y "$LOCK_DIR" >/dev/null 2>&1; then + FILE_MTIME=$(stat -c %Y "$LOCK_DIR") # Linux else - FILE_MTIME=$(stat -f %m "$IMPORT_IN_PROCESS") # macOS + FILE_MTIME=$(stat -f %m "$LOCK_DIR") # macOS fi CURRENT_TIME=$(date +%s) LOCK_AGE_MINUTES=$(( (CURRENT_TIME - FILE_MTIME) / 60 )) @@ -190,39 +191,39 @@ if [ -f "$IMPORT_IN_PROCESS" ]; then # Check if import process file is older than 30 minutes (stale lock) if [ "$LOCK_AGE_MINUTES" -ge 30 ]; then echo "ERROR: Import process appears stale (lock file is ${LOCK_AGE_MINUTES} minutes old). Removing lock file." - rm -f "$IMPORT_IN_PROCESS" + rm -rf "$LOCK_DIR" exit 1 fi - echo "ERROR: Lock file found: ${IMPORT_IN_PROCESS} (${LOCK_AGE_MINUTES} minutes old)." + echo "ERROR: Lock found: ${LOCK_DIR} (${LOCK_AGE_MINUTES} minutes old)." echo " Delete lock file or wait until it's 30 minutes old and try again" echo " sleeping for 3m" sleep 180 exit 1 fi -mkdir -p $IMPORT_DATA_DIR && touch $IMPORT_IN_PROCESS -HASHED_ENV=$(echo -n "$DOT_IMPORT_ENVIRONMENT" | md5sum | cut -d ' ' -f 1) +HASHED_ENV=$(echo -n "$DOT_IMPORT_ENVIRONMENT" | tr -cs 'a-zA-Z0-9' '_') + export ASSETS_BACKUP_FILE="${IMPORT_DATA_DIR}/${HASHED_ENV}_assets.zip" export DB_BACKUP_FILE="${IMPORT_DATA_DIR}/${HASHED_ENV}_dotcms_db.sql.gz" # Step 1. download db and assets (if needed) -download_dotcms_db_assets || { echo "Unable to download dotcms backup"; rm $IMPORT_IN_PROCESS; exit 1; } +download_dotcms_db_assets || { echo "Unable to download dotcms backup"; exit 1; } # Step 2. wipe database clean (if requested) if [ "$DOT_IMPORT_DROP_DB" = "true" ]; then - drop_db_tables || { echo "unable to drop the dotcms db schema"; rm $IMPORT_IN_PROCESS; exit 1; } + drop_db_tables || { echo "unable to drop the dotcms db schema"; exit 1; } fi # Step 3. import postgres db -import_postgres || { echo "Unable to import postgres backup"; rm $IMPORT_IN_PROCESS; exit 1; } +import_postgres || { echo "Unable to import postgres backup"; exit 1; } # Step 4. unpack assets.zip -unpack_assets || { echo "Unable to unzip assets"; rm $IMPORT_IN_PROCESS; exit 1; } +unpack_assets || { echo "Unable to unzip assets"; exit 1; } # Step 5: exit sig 13 if the clone worked -if rm -f "$IMPORT_IN_PROCESS" && touch "$IMPORT_COMPLETE"; then +if rm -rf "$LOCK_DIR" && touch "$IMPORT_COMPLETE"; then echo "dotCMS Environment $DOT_IMPORT_HOST Imported, happily exiting." exit 13 fi From 43957c6f4d09fef999c0c59f200dd745a55676ec Mon Sep 17 00:00:00 2001 From: Will Ezell Date: Tue, 3 Feb 2026 12:30:03 -0500 Subject: [PATCH 6/6] feat(import): checking for active connections before proceeding ref: #34442 --- .../docker/original/ROOT/srv/10-import-env.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh index 9fcb973cd528..bde36ebda615 100755 --- a/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh +++ b/dotCMS/src/main/docker/original/ROOT/srv/10-import-env.sh @@ -9,7 +9,6 @@ drop_db_tables () { ## This checks active connections to the dotCMS database - we can only proceed if there are no connections check_active_connections() { - # Export password for psql (avoids password prompt) ACTIVE=$(psql -h "$DB_HOST" -d "$DB_NAME" -U "$DB_USERNAME" -qtAX -c \ "SELECT count(*) FROM pg_stat_activity @@ -142,6 +141,9 @@ export DOT_IMPORT_MAX_ASSET_SIZE=${DOT_IMPORT_MAX_ASSET_SIZE:-"100mb"} export SHARED_DATA_DIR=${SHARED_DATA_DIR:-"/data/shared"} export IMPORT_DATA_DIR=${IMPORT_DATA_DIR:-"$SHARED_DATA_DIR/import"} export IMPORT_COMPLETE=$IMPORT_DATA_DIR/import_complete.txt +export DOT_IMPORT_ALL_ASSETS=${DOT_IMPORT_ALL_ASSETS:-"false"} + + # Extract hostname and database name from JDBC URL (jdbc:postgresql://host/dbname) export DB_HOST="${DB_BASE_URL#jdbc:postgresql://}" # Remove prefix -> host/dbname export DB_HOST="${DB_HOST%%/*}" # Remove /dbname -> host @@ -211,18 +213,21 @@ export DB_BACKUP_FILE="${IMPORT_DATA_DIR}/${HASHED_ENV}_dotcms_db.sql.gz" # Step 1. download db and assets (if needed) download_dotcms_db_assets || { echo "Unable to download dotcms backup"; exit 1; } -# Step 2. wipe database clean (if requested) +# Step 2. check no active connections +check_active_connections || { echo "There are active connections to the db, cannot proceed."; exit 1; } + +# Step 3. wipe database clean (if requested) if [ "$DOT_IMPORT_DROP_DB" = "true" ]; then drop_db_tables || { echo "unable to drop the dotcms db schema"; exit 1; } fi -# Step 3. import postgres db +# Step 4. import postgres db import_postgres || { echo "Unable to import postgres backup"; exit 1; } -# Step 4. unpack assets.zip +# Step 5. unpack assets.zip unpack_assets || { echo "Unable to unzip assets"; exit 1; } -# Step 5: exit sig 13 if the clone worked +# Step 6: exit sig 13 if the clone worked if rm -rf "$LOCK_DIR" && touch "$IMPORT_COMPLETE"; then echo "dotCMS Environment $DOT_IMPORT_HOST Imported, happily exiting." exit 13