From b2b53245fe58ec965d7d6e35b39788bac6722863 Mon Sep 17 00:00:00 2001 From: Alex Williams-Ferreira Date: Mon, 29 Jun 2026 17:09:45 -0700 Subject: [PATCH] Fix Sysbench MySQL TPCC restart-on-crash failure; bump package to rev6 QoS MySQL Sysbench TPCC goals fail 100% on VirtualClient restart-on-crash. The shipped sysbench package's cleanup-database.py has no TPCC branch (Cleanup is a no-op) and populate-database.py's 'prepare' is not idempotent. When a crash leaves the TPCC tables populated, every restart re-runs 'prepare' against non-empty tables -> MySQL error 1062 (Duplicate entry itemN.PRIMARY) / 1061 -> 6 crashes -> System.AggregateException -> experiment runs to full timeout. - populate-database.py: drop existing TPCC tables before 'prepare' so populate is idempotent regardless of prior (crash-interrupted) state. This also re-syncs the repo copy with the deployed package lineage (the repo had drifted to a stale rev3-era version using --host + a truncate loop; the shipped rev5 uses --hostIpAddress + a run_command FATAL-detector and no truncate loop). - cleanup-database.py: add the missing MySQL TPCC branch so Cleanup drops tables. - Bump all 4 SYSBENCH profiles (MYSQL/POSTGRESQL x OLTP/TPCC) rev5 -> rev6. NOTE: the fixed scripts ship inside sysbench-1.0.20.rev6.zip in the 'packages' blob store (not the build), so that package must be uploaded for this to take effect. Reproduced and verified on an Ubuntu 24.04 VM with VC-equivalent sysbench 1.1.0. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- VERSION | 2 +- .../Sysbench/cleanup-database.py | 6 ++ .../Sysbench/populate-database.py | 97 ++++++------------- .../profiles/PERF-MYSQL-SYSBENCH-OLTP.json | 2 +- .../profiles/PERF-MYSQL-SYSBENCH-TPCC.json | 2 +- .../PERF-POSTGRESQL-SYSBENCH-OLTP.json | 2 +- .../PERF-POSTGRESQL-SYSBENCH-TPCC.json | 2 +- 7 files changed, 40 insertions(+), 73 deletions(-) diff --git a/VERSION b/VERSION index fbb5333616..c141006c22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.14 \ No newline at end of file +3.3.15 \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/cleanup-database.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/cleanup-database.py index e43175272a..04fa0a6333 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/cleanup-database.py +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/cleanup-database.py @@ -18,5 +18,11 @@ if databaseSystem == "MySQL": if benchmark == "OLTP": subprocess.run(f'sudo src/sysbench oltp_common --tables={tableCount} --mysql-db={dbName} --mysql-host={hostIp} cleanup',shell=True, check=True) + elif benchmark == "TPCC": + # Drop the TPCC tables so the database is restored to a clean state. + # Without a TPCC branch here the Cleanup action is a no-op, leaving the + # tables populated; a subsequent 'prepare' then fails with MySQL error + # 1062 (Duplicate entry) / 1061 (Duplicate key name). + subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale=1 --mysql-db={dbName} --mysql-host={hostIp} --use_fk=0 cleanup',shell=True, check=True) else: parser.error("You are running on a database type that has not been onboarded to Virtual Client. Available options are: MySQL") \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py b/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py index 21dc5dbbbc..34e695a88e 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py @@ -1,4 +1,4 @@ -import subprocess, argparse, os +import subprocess, argparse, os, sys parser = argparse.ArgumentParser() @@ -10,7 +10,7 @@ parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False) parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True) parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False) -parser.add_argument('--host', type=str, help="Database Server Host IP Address", required=False) +parser.add_argument('--hostIpAddress', type=str, help="Database Server Host IP Address", required=False) args = parser.parse_args() @@ -22,80 +22,41 @@ recordCount = args.recordCount threadCount = args.threadCount password = args.password -host = args.host +host = args.hostIpAddress -def add_host_if_needed(command_base, host, benchmark): - if host and benchmark != "TPCC": - if "sysbench" not in command_base: - return command_base.replace('-u', f'-h {host} -u') - else: - return f"{command_base} --mysql-host={host}" - else: - return command_base +def run_command(command): + """Run a shell command, check exit code AND scan for FATAL errors in output.""" + result = subprocess.run(command, shell=True, capture_output=True, text=True) + combined_output = result.stdout + result.stderr + + if result.stdout: + print(result.stdout, end='') + if result.stderr: + print(result.stderr, end='', file=sys.stderr) + + if result.returncode != 0: + sys.exit(result.returncode) + + # sysbench may exit 0 even on FATAL connection errors + if "FATAL:" in combined_output: + print("ERROR: sysbench reported FATAL errors but exited with code 0. Failing the population step.", file=sys.stderr) + sys.exit(1) if databaseSystem == "MySQL": if benchmark == "TPCC": - subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare', shell=True, check=True) - if int(warehouses) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - # drop idxs - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_customer{i} ON customer{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_orders{i} ON orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_stock_2{i} ON stock{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_order_line_2{i} ON order_line{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_1{i} ON history{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_2{i} ON history{i};"', shell=True, check=True) - - # truncate, to make distributing faster - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE district{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE customer{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE history{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE order_line{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE stock{i};"', shell=True, check=True) - subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE item{i};"', shell=True, check=True) + # Drop any tables left from a prior (possibly crash-interrupted) run so + # 'prepare' is idempotent across VirtualClient restart-on-crash. Otherwise + # re-running 'prepare' against populated tables fails with MySQL error + # 1062 (Duplicate entry) / 1061 (Duplicate key name). + subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --mysql-host={host} --use_fk=0 cleanup', shell=True) + run_command(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --mysql-host={host} --use_fk=0 prepare') else: - command_base = f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare' - command = add_host_if_needed(command_base, host, benchmark) - subprocess.run(command, shell=True, check=True) - if int(recordCount) == 1: - for i in range(1, int(tableCount) + 1): - drop_index_command = f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX k_{i} ON sbtest{i};"' - drop_index_command = add_host_if_needed(drop_index_command, host, benchmark) - subprocess.run(drop_index_command, shell=True, check=True) + run_command(f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} --mysql-host={host} prepare') elif databaseSystem == "PostgreSQL": os.environ['PGPASSWORD'] = password if benchmark == "TPCC": - subprocess.run(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare', shell=True, check=True) - if int(warehouses) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - # drop idxs - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_customer{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_stock_2{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_order_line_2{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_1{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_2{i};"', shell=True, check=True) - - # truncate, to make distributing faster - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE warehouse{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE district{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE customer{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE history{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE new_orders{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE order_line{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE stock{i};"', shell=True, check=True) - subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE item{i};"', shell=True, check=True) + run_command(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare') else: - subprocess.run(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare', shell=True, check=True) - if int(recordCount) == 1: - for i in range(1,int(tableCount)+1): - table = str(i) - subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"', shell=True, check=True) + run_command(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare') else: parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL") \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json index 27cd86e1df..cbae2cb6d4 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json @@ -193,7 +193,7 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev5.zip", + "BlobName": "sysbench-1.0.20.rev6.zip", "PackageName": "sysbench", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json index b23b59b725..bbb777b196 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json @@ -81,7 +81,7 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev5.zip", + "BlobName": "sysbench-1.0.20.rev6.zip", "PackageName": "sysbench", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json index 8fd94e04e5..7da17aac6f 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json @@ -194,7 +194,7 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev5.zip", + "BlobName": "sysbench-1.0.20.rev6.zip", "PackageName": "sysbench", "Extract": true } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json index 8b7ac7a628..55f074bada 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json @@ -82,7 +82,7 @@ "Parameters": { "Scenario": "DownloadSysbenchPackage", "BlobContainer": "packages", - "BlobName": "sysbench-1.0.20.rev5.zip", + "BlobName": "sysbench-1.0.20.rev6.zip", "PackageName": "sysbench", "Extract": true }