Skip to content

Fix Sysbench MySQL TPCC restart-on-crash failure; bump sysbench package to rev6 #738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AlexWFMS wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix Sysbench MySQL TPCC restart-on-crash failure; bump sysbench package to rev6 #738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.3.14
3.3.15
6 changes: 6 additions & 0 deletions src/VirtualClient/VirtualClient.Actions/Sysbench/cleanup-database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,11 @@
if databaseSystem == "MySQL":
if benchmark == "OLTP":
subprocess.run(f'sudo src/sysbench oltp_common --tables={tableCount} --mysql-db={dbName} --mysql-host={hostIp} cleanup',shell=True, check=True)
elif benchmark == "TPCC":
# Drop the TPCC tables so the database is restored to a clean state.
# Without a TPCC branch here the Cleanup action is a no-op, leaving the
# tables populated; a subsequent 'prepare' then fails with MySQL error
# 1062 (Duplicate entry) / 1061 (Duplicate key name).
subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale=1 --mysql-db={dbName} --mysql-host={hostIp} --use_fk=0 cleanup',shell=True, check=True)
else:
parser.error("You are running on a database type that has not been onboarded to Virtual Client. Available options are: MySQL")
97 changes: 29 additions & 68 deletions src/VirtualClient/VirtualClient.Actions/Sysbench/populate-database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import subprocess, argparse, os
import subprocess, argparse, os, sys

parser = argparse.ArgumentParser()

Expand All @@ -10,7 +10,7 @@
parser.add_argument('-u', '--warehouses', type=str, help='Warehouse Count', required=False)
parser.add_argument('-e', '--threadCount', type=str, help="Number of Threads", required=True)
parser.add_argument('--password', type=str, help="PostgreSQL Password", required=False)
parser.add_argument('--host', type=str, help="Database Server Host IP Address", required=False)
parser.add_argument('--hostIpAddress', type=str, help="Database Server Host IP Address", required=False)


args = parser.parse_args()
Expand All @@ -22,80 +22,41 @@
recordCount = args.recordCount
threadCount = args.threadCount
password = args.password
host = args.host
host = args.hostIpAddress

def add_host_if_needed(command_base, host, benchmark):
if host and benchmark != "TPCC":
if "sysbench" not in command_base:
return command_base.replace('-u', f'-h {host} -u')
else:
return f"{command_base} --mysql-host={host}"
else:
return command_base
def run_command(command):
"""Run a shell command, check exit code AND scan for FATAL errors in output."""
result = subprocess.run(command, shell=True, capture_output=True, text=True)
combined_output = result.stdout + result.stderr

if result.stdout:
print(result.stdout, end='')
if result.stderr:
print(result.stderr, end='', file=sys.stderr)

if result.returncode != 0:
sys.exit(result.returncode)

# sysbench may exit 0 even on FATAL connection errors
if "FATAL:" in combined_output:
print("ERROR: sysbench reported FATAL errors but exited with code 0. Failing the population step.", file=sys.stderr)
sys.exit(1)

if databaseSystem == "MySQL":
if benchmark == "TPCC":
subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --use_fk=0 prepare', shell=True, check=True)
if int(warehouses) == 1:
for i in range(1,int(tableCount)+1):
table = str(i)
# drop idxs
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_customer{i} ON customer{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX idx_orders{i} ON orders{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_stock_2{i} ON stock{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_order_line_2{i} ON order_line{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_1{i} ON history{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX fkey_history_2{i} ON history{i};"', shell=True, check=True)

# truncate, to make distributing faster
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE warehouse{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE district{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE customer{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE history{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE orders{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE new_orders{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE order_line{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE stock{i};"', shell=True, check=True)
subprocess.run(f'sudo mysql -u {dbName} {dbName} -e "TRUNCATE TABLE item{i};"', shell=True, check=True)
# Drop any tables left from a prior (possibly crash-interrupted) run so
# 'prepare' is idempotent across VirtualClient restart-on-crash. Otherwise
# re-running 'prepare' against populated tables fails with MySQL error
# 1062 (Duplicate entry) / 1061 (Duplicate key name).
subprocess.run(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --mysql-host={host} --use_fk=0 cleanup', shell=True)
run_command(f'sudo src/sysbench tpcc --tables={tableCount} --scale={warehouses} --threads={threadCount} --mysql-db={dbName} --mysql-host={host} --use_fk=0 prepare')
else:
command_base = f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} prepare'
command = add_host_if_needed(command_base, host, benchmark)
subprocess.run(command, shell=True, check=True)
if int(recordCount) == 1:
for i in range(1, int(tableCount) + 1):
drop_index_command = f'sudo mysql -u {dbName} {dbName} -e "DROP INDEX k_{i} ON sbtest{i};"'
drop_index_command = add_host_if_needed(drop_index_command, host, benchmark)
subprocess.run(drop_index_command, shell=True, check=True)
run_command(f'sudo src/sysbench oltp_common --tables={tableCount} --table-size={recordCount} --threads={threadCount} --mysql-db={dbName} --mysql-host={host} prepare')
elif databaseSystem == "PostgreSQL":
os.environ['PGPASSWORD'] = password
if benchmark == "TPCC":
subprocess.run(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare', shell=True, check=True)
if int(warehouses) == 1:
for i in range(1,int(tableCount)+1):
table = str(i)
# drop idxs
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_customer{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS idx_orders{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_stock_2{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_order_line_2{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_1{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS fkey_history_2{i};"', shell=True, check=True)

# truncate, to make distributing faster
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE warehouse{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE district{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE customer{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE history{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE orders{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE new_orders{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE order_line{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE stock{i};"', shell=True, check=True)
subprocess.run(f'psql -U postgres -d {dbName} -c "TRUNCATE TABLE item{i};"', shell=True, check=True)
run_command(f'sudo src/sysbench tpcc --db-driver=pgsql --tables={tableCount} --scale={warehouses} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} --use_fk=0 prepare')
else:
subprocess.run(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare', shell=True, check=True)
if int(recordCount) == 1:
for i in range(1,int(tableCount)+1):
table = str(i)
subprocess.run(f'psql -U postgres -d {dbName} -c "DROP INDEX IF EXISTS k_{i};"', shell=True, check=True)
run_command(f'sudo src/sysbench oltp_common --db-driver=pgsql --tables={tableCount} --table-size={recordCount} --threads={threadCount} --pgsql-user=postgres --pgsql-password={password} --pgsql-db={dbName} prepare')
else:
parser.error("You are running on a database system type that has not been onboarded to Virtual Client. Available options are: MySQL, PostgreSQL")
2 changes: 1 addition & 1 deletion src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@
"Parameters": {
"Scenario": "DownloadSysbenchPackage",
"BlobContainer": "packages",
"BlobName": "sysbench-1.0.20.rev5.zip",
"BlobName": "sysbench-1.0.20.rev6.zip",
"PackageName": "sysbench",
"Extract": true
}
Expand Down
2 changes: 1 addition & 1 deletion src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@
"Parameters": {
"Scenario": "DownloadSysbenchPackage",
"BlobContainer": "packages",
"BlobName": "sysbench-1.0.20.rev5.zip",
"BlobName": "sysbench-1.0.20.rev6.zip",
"PackageName": "sysbench",
"Extract": true
}
Expand Down
2 changes: 1 addition & 1 deletion src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@
"Parameters": {
"Scenario": "DownloadSysbenchPackage",
"BlobContainer": "packages",
"BlobName": "sysbench-1.0.20.rev5.zip",
"BlobName": "sysbench-1.0.20.rev6.zip",
"PackageName": "sysbench",
"Extract": true
}
Expand Down
2 changes: 1 addition & 1 deletion src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
"Parameters": {
"Scenario": "DownloadSysbenchPackage",
"BlobContainer": "packages",
"BlobName": "sysbench-1.0.20.rev5.zip",
"BlobName": "sysbench-1.0.20.rev6.zip",
"PackageName": "sysbench",
"Extract": true
}
Expand Down
Loading