diff --git a/backup_tests/inc_backup_load_tests.sh b/backup_tests/inc_backup_load_tests.sh index 37e8e2ed..bf5f650c 100755 --- a/backup_tests/inc_backup_load_tests.sh +++ b/backup_tests/inc_backup_load_tests.sh @@ -23,16 +23,21 @@ export PATH="$PATH:$xtrabackup_dir" export qascripts="$HOME/percona-qa" export logdir="$HOME/backuplogs" export mysql_start_timeout=60 -declare -A KMIP_CONFIGS=( - # PyKMIP Docker Configuration - ["pykmip"]="addr=127.0.0.1,image=mohitpercona/kmip:latest,port=5696,name=kmip_pykmip" - +declare -gA KMIP_CONFIGS=( # Hashicorp Docker Setup Configuration - # ["hashicorp"]="addr=127.0.0.1,port=5696,name=kmip_hashicorp,setup_script=hashicorp-kmip-setup.sh" + ["hashicorp"]="addr=127.0.0.1,port=5696,name=kmip_hashicorp,setup_script=hashicorp-kmip-setup.sh" + + # Fortanix Setup Configuration + ["fortanix"]="addr=216.180.120.88,port=5696,name=kmip_fortanix,setup_script=fortanix_kmip_setup.py" + + # PyKMIP Docker Configuration + #["pykmip"]="addr=127.0.0.1,image=satyapercona/kmip:latest,port=5696,name=kmip_pykmip" # API Configuration # ["ciphertrust"]="addr=127.0.0.1,port=5696,name=kmip_ciphertrust,setup_script=setup_kmip_api.py" ) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEFAULT_LICENSE="${SCRIPT_DIR}/vault.hclic" # Set tool variables load_tool="pstress" # Set value as pstress/sysbench @@ -976,24 +981,26 @@ cleanup() { rm -rf $mysqldir/lib/plugin/component_keyring_file echo "..Deleted" fi - echo "Checking for previously started containers..." - if [ -z "${KMIP_CONTAINER_NAMES+x}" ] || [ ${#KMIP_CONTAINER_NAMES[@]} -eq 0 ]; then - get_kmip_container_names - fi - containers_found=false + if declare -p KMIP_CONFIGS >/dev/null 2>&1 && declare -f get_kmip_container_names >/dev/null 2>&1; then + echo "Checking for previously started containers..." + if [ -z "${KMIP_CONTAINER_NAMES+x}" ] || [ ${#KMIP_CONTAINER_NAMES[@]} -eq 0 ]; then + get_kmip_container_names + fi + containers_found=false - for name in "${KMIP_CONTAINER_NAMES[@]}"; do - if docker ps -aq --filter "name=$name" | grep -q .; then - containers_found=true - break - fi - done + for name in "${KMIP_CONTAINER_NAMES[@]}"; do + if docker ps -aq --filter "name=$name" | grep -q .; then + containers_found=true + break + fi + done - if [[ "$containers_found" == true ]]; then - echo "Killing previously started containers if any..." - for name in "${KMIP_CONTAINER_NAMES[@]}"; do - cleanup_existing_container "$name" - done + if [[ "$containers_found" == true ]]; then + echo "Killing previously started containers if any..." + for name in "${KMIP_CONTAINER_NAMES[@]}"; do + cleanup_existing_container "$name" + done + fi fi # Only cleanup vault directory if it exists diff --git a/backup_tests/kmip_helper.sh b/backup_tests/kmip_helper.sh index ab7fb18c..58e8c1ae 100644 --- a/backup_tests/kmip_helper.sh +++ b/backup_tests/kmip_helper.sh @@ -10,24 +10,28 @@ # Global variables declare -ga KMIP_CONTAINER_NAMES -declare -gA KMIP_CONFIGS 2>/dev/null || true # 1. Safely declare the global array (no error if already exists) +declare -gA KMIP_CONFIGS_DEFAULTS=( + #[pykmip]="addr=127.0.0.1,image=satyapercona/kmip:latest,port=5696,name=kmip_pykmip" + [hashicorp]="addr=127.0.0.1,port=5696,name=kmip_hashicorp,setup_script=hashicorp-kmip-setup.sh" + [fortanix]="addr=216.180.120.88,port=5696,name=kmip_fortanix,setup_script=fortanix_kmip_setup.py" + #[ciphertrust]="addr=127.0.0.1,port=5696,name=kmip_ciphertrust,setup_script=setup_kmip_api.py" +) # Initialize default configurations if not already set init_kmip_configs() { - # Check if array is empty without triggering nounset errors + # If KMIP_CONFIGS not set in main script, initialize with defaults if [[ -z "${KMIP_CONFIGS[*]-}" ]]; then - KMIP_CONFIGS=( - # PyKMIP Docker Configuration - ["pykmip"]="addr=127.0.0.1,image=mohitpercona/kmip:latest,port=5696,name=kmip_pykmip" + declare -gA KMIP_CONFIGS=() + fi - # Hashicorp Docker Setup Configuration - ["hashicorp"]="addr=127.0.0.1,port=5696,name=kmip_hashicorp,setup_script=hashicorp-kmip-setup.sh" + # Apply defaults for all keys defined in main script if not set + for key in "${!KMIP_CONFIGS[@]}"; do + if [[ -z "${KMIP_CONFIGS[$key]}" ]]; then + KMIP_CONFIGS[$key]="${KMIP_CONFIGS_DEFAULTS[$key]}" + fi + done - # API Configuration - # ["ciphertrust"]="addr=127.0.0.1,port=5696,name=kmip_ciphertrust,setup_script=setup_kmip_api.py" - ) - echo "Initialized default KMIP configurations" >&2 - fi + echo "KMIP configurations initialized from Defaults" >&2 } # Cleanup existing Docker container @@ -275,10 +279,10 @@ setup_hashicorp() { fi fi - echo "Starting Docker KMIP server in (script method): $setup_script" # Download first, then execute the hashicorp setup - script=$(wget -qO- https://raw.githubusercontent.com/Percona-QA/percona-qa/refs/heads/master/"$setup_script") - wget_exit_code=$? + # ToDo Remove before Merge + # script=$(wget -qO- https://raw.githubusercontent.com/Percona-QA/percona-qa/refs/heads/master/"$setup_script") + script=$(wget -qO- https://raw.githubusercontent.com/Percona-QA/percona-qa/cad02909729f1347fa01079247c0ca03f2e3acab/"$setup_script") if [ $wget_exit_code -ne 0 ]; then echo "Failed to download script (wget exit code: $wget_exit_code)" @@ -299,21 +303,86 @@ setup_hashicorp() { echo "ERROR: Failed to create certificate directory: $cert_dir" >&2 return 1 } + + # Check if license file exists + if [[ ! -f "$DEFAULT_LICENSE" ]]; then + echo "ERROR: License file not found at: $DEFAULT_LICENSE" >&2 + exit 1 + fi + + echo "Executing script: $script" # Execute the script - echo "$script" | bash -s -- --cert-dir="$cert_dir" + echo "$script" | bash -s -- --cert-dir="$cert_dir" --license="$DEFAULT_LICENSE" exit_code=$? if [ $exit_code -ne 0 ]; then - echo "Failed to execute script $setup_script, (exit code: $exit_code)" - exit 1 + echo "Failed to execute script $setup_script, (exit code: $exit_code)" >&2 + return 1 fi generate_kmip_config "$type" "$addr" "$port" "$cert_dir" || { - echo "Failed to generate KMIP config"; exit 1; } + echo "Failed to generate KMIP config" >&2; return 1; } echo "Hashicorp server started successfully on address $addr and port $port" return 0 } +setup_fortanix() { + local type="fortanix" + local container_name="${kmip_config[name]}" + local addr="${kmip_config[addr]}" + local port="${kmip_config[port]}" + local email="${kmip_config[email]}" + local password="${kmip_config[password]}" + local setup_script="${kmip_config[setup_script]}" + local cert_dir="${HOME}/${kmip_config[cert_dir]}" + + # Check if both variables are set and not empty + if [[ -z "$email" || -z "$password" ]]; then + echo "Error: Both email and password must be set in Config or Script for Fortanix KMIP Provider!!" >&2 + exit 1 + fi + + echo "Checking port availability... " + if validate_port_available "$port"; then + echo "Available" + else + echo "Unavailable" + echo "Port $port is in use by:" + lsof -i :"$port" + return 1 + fi + + echo "Starting Fortanix KMIP server in (script method): $setup_script" + # Download first, then execute the fortanix setup script + script=$(wget -qO- https://raw.githubusercontent.com/Percona-QA/percona-qa/8ab34a4da257070518825fcdf8ae547f99705597/"$setup_script") + + # To-Do Remove B4 Merge + # script=$(wget -qO- https://raw.githubusercontent.com/Percona-QA/percona-qa/refs/heads/master/"$setup_script") + wget_exit_code=$? + + if [ $wget_exit_code -ne 0 ]; then + echo "Failed to download script (wget exit code: $wget_exit_code)" + exit 1 + fi + + if [ -z "$script" ]; then + echo "Downloaded script is empty" + exit 1 + fi + + mkdir -p "$cert_dir" || true + + # Execute the Python script from a variable + echo "$script" | python3 - --cert-dir="$cert_dir" --email="$email" --password="$password" + exit_code=$? + + generate_kmip_config "$type" "$addr" "$port" "$cert_dir" || { + echo "Failed to generate KMIP config"; exit 1; } + + echo "Fortanix server started successfully on address $addr and port $port" + return 0 +} + # Placeholder for CipherTrust setup setup_cipher_api() { echo "CipherTrust setup not implemented yet" @@ -330,6 +399,7 @@ start_kmip_server() { case "$type" in pykmip) setup_pykmip ;; hashicorp) setup_hashicorp ;; + fortanix) setup_fortanix ;; ciphertrust) setup_cipher_api ;; *) echo "Unsupported KMIP Type: $type"; return 1 ;; esac diff --git a/backup_tests/xbstream_fifo_test.sh b/backup_tests/xbstream_fifo_test.sh index 3675e092..3c010a00 100755 --- a/backup_tests/xbstream_fifo_test.sh +++ b/backup_tests/xbstream_fifo_test.sh @@ -8,13 +8,16 @@ SOCKET=/tmp/mysql_22000.sock BACKUP_DIR=/tmp/backup PSTRESS_BIN=$HOME/pstress/src ENCRYPTION=0; COMPRESS=0; ENCRYPT=""; DECRYPT=""; ENCRYPT_KEY="" -declare -A KMIP_CONFIGS=( +declare -gA KMIP_CONFIGS=( # PyKMIP Docker Configuration ["pykmip"]="addr=127.0.0.1,image=mohitpercona/kmip:latest,port=5696,name=kmip_pykmip" # Hashicorp Docker Setup Configuration # ["hashicorp"]="addr=127.0.0.1,port=5696,name=kmip_hashicorp,setup_script=hashicorp-kmip-setup.sh" + # Fortanix Setup Configuration + # ["fortanix"]="addr=216.180.120.88,port=5696,name=kmip_fortanix,setup_script=fortanix_kmip_setup.py" + # API Configuration # ["ciphertrust"]="addr=127.0.0.1,port=5696,name=kmip_ciphertrust,setup_script=setup_kmip_api.py" ) @@ -160,12 +163,7 @@ init_datadir() { "components": "file://component_keyring_kmip" }' > "$PS_DIR/bin/mysqld.my" - if ! source ./kmip_helper.sh; then - echo "ERROR: Failed to load KMIP helper library" - exit 1 - fi - init_kmip_configs - start_kmip_server "$kmip_type" + start_kmip_server $kmip_type [ -f "${HOME}/${kmip_config[cert_dir]}/component_keyring_kmip.cnf" ] && cp "${HOME}/${kmip_config[cert_dir]}/component_keyring_kmip.cnf" "$PS_DIR/lib/plugin/" elif [ "$keyring_type" = "keyring_file" ]; then @@ -608,9 +606,9 @@ echo "Copy the backup in datadir" $XTRABACKUP_DIR/bin/xtrabackup --no-defaults --copy-back --target_dir=$BACKUP_DIR/full --datadir=$DATADIR --core-file > $LOGDIR/copy_back5.log 2>&1 start_server -echo "##############################################################################" -echo "# 6. Test FIFO xbstream: Test with encrypted tables w/ keyring kmip - pykmip #" -echo "##############################################################################" +echo "###############################################################################" +echo "# 6. Test FIFO xbstream: Test with encrypted tables w/ component keyring kmip #" +echo "###############################################################################" LOGDIR=$HOME/6 if [ -d $LOGDIR ]; then @@ -625,7 +623,17 @@ echo "..Cleanup completed" ENCRYPTION=1 stop_server rm -rf $DATADIR -init_datadir "keyring_kmip" "pykmip" + +if ! source ./kmip_helper.sh; then + echo "ERROR: Failed to load KMIP helper library" + exit 1 +fi +init_kmip_configs +for vault_type in "${!KMIP_CONFIGS[@]}"; do + echo "Testing Encryption with $vault_type..." + init_datadir "keyring_kmip" $vault_type +done + start_server echo "=>Run pstress load" pstress_run_load @@ -648,46 +656,6 @@ echo "Copy the backup in datadir" $XTRABACKUP_DIR/bin/xtrabackup --no-defaults --copy-back --target_dir=$BACKUP_DIR/full --datadir=$DATADIR --core-file > $LOGDIR/copy_back6.log 2>&1 start_server -echo "#####################################################################################" -echo "# 6.5 Test FIFO xbstream: Test with encrypted tables w/ keyring kmip - hashicorp ####" -echo "#####################################################################################" - -LOGDIR=$HOME/6.5 -if [ -d $LOGDIR ]; then - rm -rf $LOGDIR/* -else - mkdir $LOGDIR -fi -echo "=>Cleanup in progress" -cleanup -echo "..Cleanup completed" - -ENCRYPTION=1 -stop_server -rm -rf $DATADIR -init_datadir "keyring_kmip" "hashicorp" -start_server -echo "=>Run pstress load" -pstress_run_load - -incremental_backup_and_restore "keyring_kmip" -echo "=>Shutting down MySQL server" -stop_server -echo "..Successful" - -echo "=>Taking backup of original datadir" -if [ ! -d ${DATADIR}_bk6.5 ]; then - mv $DATADIR ${DATADIR}_bk6 -else - rm -rf ${DATADIR}_bk6.5 - mv $DATADIR ${DATADIR}_bk6.5 -fi -echo "..Successful" - -echo "Copy the backup in datadir" -$XTRABACKUP_DIR/bin/xtrabackup --no-defaults --copy-back --target_dir=$BACKUP_DIR/full --datadir=$DATADIR --core-file > $LOGDIR/copy_back6.5.log 2>&1 -start_server - echo "#######################################################" echo "# 7. Test FIFO xbstream: Test with encrypted backup #" echo "#######################################################"