From 5bd3d384a898503106a5fae0b62df5b35ba8b1a8 Mon Sep 17 00:00:00 2001 From: Fariha Shaikh Date: Mon, 2 Mar 2026 19:13:29 +0000 Subject: [PATCH] MDEV-38936 Proactive handling of InnoDB tablespace full condition InnoDB write failures occur when tablespace files exceed filesystem size limits. Current behavior logs errors but continues accepting transactions, causing repeated failures and potential data integrity issues. Add proactive monitoring by emitting warnings when InnoDB tablespaces approach a configurable size threshold. Warnings use a tiered frequency approach to balance early notification with log spam prevention. Key features: - Three new system variables: * innodb_tablespace_size_warning_threshold (default 16TB): Maximum tablespace size in bytes before warnings begin * innodb_tablespace_size_warning_pct (default 70%): Percentage of threshold at which to start emitting warnings * innodb_tablespace_size_warning_enabled (default ON): Andon cord to enable/disable the warning feature - Tiered warning frequency: * Below warning_pct: No warnings * Between warning_pct and 90%: At most twice per 10% interval with minimum 5% gap (e.g., 70%, 77%, 81%, 89%) * Above 90%: Every 1% increase (90%, 91%, 92%, etc.) - Per-tablespace tracking with automatic reset on TRUNCATE/DROP or threshold changes - Zero overhead when disabled - Progressive warnings capped at 100% Implementation hooks into fsp_try_extend_data_file() for O(1) size checking during tablespace extension. Adds 11 bytes per tablespace (m_last_size_warning_pct, m_last_warning_threshold, m_warning_count_in_decade) to fil_space_t structure. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. --- .../innodb/r/tablespace_size_warning.result | 64 ++++++++ .../innodb/t/tablespace_size_warning.test | 149 ++++++++++++++++++ .../suite/sys_vars/r/sysvars_innodb.result | 36 +++++ storage/innobase/fsp/fsp0fsp.cc | 81 ++++++++++ storage/innobase/handler/ha_innodb.cc | 27 ++++ storage/innobase/include/fil0fil.h | 10 ++ storage/innobase/include/srv0srv.h | 9 ++ storage/innobase/srv/srv0srv.cc | 9 ++ 8 files changed, 385 insertions(+) create mode 100644 mysql-test/suite/innodb/r/tablespace_size_warning.result create mode 100644 mysql-test/suite/innodb/t/tablespace_size_warning.test diff --git a/mysql-test/suite/innodb/r/tablespace_size_warning.result b/mysql-test/suite/innodb/r/tablespace_size_warning.result new file mode 100644 index 0000000000000..7ba286797aa69 --- /dev/null +++ b/mysql-test/suite/innodb/r/tablespace_size_warning.result @@ -0,0 +1,64 @@ +# +# MDEV-38936: Proactive handling of InnoDB tablespace full condition +# +SET @old_threshold = @@global.innodb_tablespace_size_warning_threshold; +SET @old_enabled = @@global.innodb_tablespace_size_warning_enabled; +SET @old_pct = @@global.innodb_tablespace_size_warning_pct; +# Test system variables +SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_threshold'; +Variable_name Value +innodb_tablespace_size_warning_threshold 17592186044416 +SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_enabled'; +Variable_name Value +innodb_tablespace_size_warning_enabled ON +SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_pct'; +Variable_name Value +innodb_tablespace_size_warning_pct 70 +# Test basic warning emission +SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760; +SET GLOBAL innodb_tablespace_size_warning_pct = 70; +CREATE TABLE t1 ( +id INT AUTO_INCREMENT PRIMARY KEY, +data LONGBLOB +) ENGINE=InnoDB; +FOUND 1 /Tablespace 'test/t1' size .* bytes .* exceeds warning threshold/ in mysqld.1.err +DROP TABLE t1; +# Test configurable warning percentage +SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760; +SET GLOBAL innodb_tablespace_size_warning_pct = 80; +CREATE TABLE t1b ( +id INT AUTO_INCREMENT PRIMARY KEY, +data LONGBLOB +) ENGINE=InnoDB; +FOUND 1 /Tablespace 'test/t1b' size .* bytes .* exceeds warning threshold/ in mysqld.1.err +DROP TABLE t1b; +# Test threshold set to zero disables warnings +SET GLOBAL innodb_tablespace_size_warning_threshold = 0; +CREATE TABLE t2 ( +id INT AUTO_INCREMENT PRIMARY KEY, +data LONGBLOB +) ENGINE=InnoDB; +DROP TABLE t2; +# Test andon cord disable/enable +SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760; +SET GLOBAL innodb_tablespace_size_warning_pct = 70; +SET GLOBAL innodb_tablespace_size_warning_enabled = FALSE; +CREATE TABLE t3 ( +id INT AUTO_INCREMENT PRIMARY KEY, +data LONGBLOB +) ENGINE=InnoDB; +SET GLOBAL innodb_tablespace_size_warning_enabled = TRUE; +FOUND 1 /Tablespace 'test/t3' size .* bytes .* exceeds warning threshold/ in mysqld.1.err +DROP TABLE t3; +# Test TRUNCATE TABLE resets warning state +CREATE TABLE t4 ( +id INT AUTO_INCREMENT PRIMARY KEY, +data LONGBLOB +) ENGINE=InnoDB; +FOUND 1 /Tablespace 'test/t4' size .* bytes .* exceeds warning threshold/ in mysqld.1.err +TRUNCATE TABLE t4; +FOUND 1 /Tablespace 'test/t4' size .* bytes .* exceeds warning threshold/ in mysqld.1.err +DROP TABLE t4; +SET GLOBAL innodb_tablespace_size_warning_threshold = @old_threshold; +SET GLOBAL innodb_tablespace_size_warning_enabled = @old_enabled; +SET GLOBAL innodb_tablespace_size_warning_pct = @old_pct; diff --git a/mysql-test/suite/innodb/t/tablespace_size_warning.test b/mysql-test/suite/innodb/t/tablespace_size_warning.test new file mode 100644 index 0000000000000..25193ae783331 --- /dev/null +++ b/mysql-test/suite/innodb/t/tablespace_size_warning.test @@ -0,0 +1,149 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc + +--disable_query_log +call mtr.add_suppression("Tablespace .* size .* bytes .* exceeds warning threshold"); +--enable_query_log + +--echo # +--echo # MDEV-38936: Proactive handling of InnoDB tablespace full condition +--echo # + +# Save original values +SET @old_threshold = @@global.innodb_tablespace_size_warning_threshold; +SET @old_enabled = @@global.innodb_tablespace_size_warning_enabled; +SET @old_pct = @@global.innodb_tablespace_size_warning_pct; + +--echo # Test system variables +SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_threshold'; +SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_enabled'; +SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_pct'; + +--echo # Test basic warning emission +SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760; +SET GLOBAL innodb_tablespace_size_warning_pct = 70; + +CREATE TABLE t1 ( + id INT AUTO_INCREMENT PRIMARY KEY, + data LONGBLOB +) ENGINE=InnoDB; + +--disable_query_log +let $i = 10; +while ($i) { + eval INSERT INTO t1 (data) VALUES (REPEAT('a', 1024*1024)); + dec $i; +} +--enable_query_log + +let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=Tablespace 'test/t1' size .* bytes .* exceeds warning threshold; +--source include/search_pattern_in_file.inc + +DROP TABLE t1; + +--echo # Test configurable warning percentage +SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760; +SET GLOBAL innodb_tablespace_size_warning_pct = 80; + +CREATE TABLE t1b ( + id INT AUTO_INCREMENT PRIMARY KEY, + data LONGBLOB +) ENGINE=InnoDB; + +--disable_query_log +let $i = 8; +while ($i) { + eval INSERT INTO t1b (data) VALUES (REPEAT('x', 1024*1024)); + dec $i; +} +--enable_query_log + +let SEARCH_PATTERN=Tablespace 'test/t1b' size .* bytes .* exceeds warning threshold; +--source include/search_pattern_in_file.inc + +DROP TABLE t1b; + +--echo # Test threshold set to zero disables warnings +SET GLOBAL innodb_tablespace_size_warning_threshold = 0; + +CREATE TABLE t2 ( + id INT AUTO_INCREMENT PRIMARY KEY, + data LONGBLOB +) ENGINE=InnoDB; + +--disable_query_log +let $i = 5; +while ($i) { + eval INSERT INTO t2 (data) VALUES (REPEAT('b', 1024*1024)); + dec $i; +} +--enable_query_log + +DROP TABLE t2; + +--echo # Test andon cord disable/enable +SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760; +SET GLOBAL innodb_tablespace_size_warning_pct = 70; +SET GLOBAL innodb_tablespace_size_warning_enabled = FALSE; + +CREATE TABLE t3 ( + id INT AUTO_INCREMENT PRIMARY KEY, + data LONGBLOB +) ENGINE=InnoDB; + +--disable_query_log +let $i = 10; +while ($i) { + eval INSERT INTO t3 (data) VALUES (REPEAT('c', 1024*1024)); + dec $i; +} +--enable_query_log + +SET GLOBAL innodb_tablespace_size_warning_enabled = TRUE; + +--disable_query_log +INSERT INTO t3 (data) VALUES (REPEAT('d', 1024*1024)); +--enable_query_log + +let SEARCH_PATTERN=Tablespace 'test/t3' size .* bytes .* exceeds warning threshold; +--source include/search_pattern_in_file.inc + +DROP TABLE t3; + +--echo # Test TRUNCATE TABLE resets warning state +CREATE TABLE t4 ( + id INT AUTO_INCREMENT PRIMARY KEY, + data LONGBLOB +) ENGINE=InnoDB; + +--disable_query_log +let $i = 10; +while ($i) { + eval INSERT INTO t4 (data) VALUES (REPEAT('e', 1024*1024)); + dec $i; +} +--enable_query_log + +let SEARCH_PATTERN=Tablespace 'test/t4' size .* bytes .* exceeds warning threshold; +--source include/search_pattern_in_file.inc + +TRUNCATE TABLE t4; + +--disable_query_log +let $i = 10; +while ($i) { + eval INSERT INTO t4 (data) VALUES (REPEAT('f', 1024*1024)); + dec $i; +} +--enable_query_log + +let SEARCH_PATTERN=Tablespace 'test/t4' size .* bytes .* exceeds warning threshold; +--source include/search_pattern_in_file.inc + +DROP TABLE t4; + +# Restore original values +SET GLOBAL innodb_tablespace_size_warning_threshold = @old_threshold; +SET GLOBAL innodb_tablespace_size_warning_enabled = @old_enabled; +SET GLOBAL innodb_tablespace_size_warning_pct = @old_pct; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 36bd5ca6894ec..d2b02de5f5faa 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1616,6 +1616,42 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME INNODB_TABLESPACE_SIZE_WARNING_ENABLED +SESSION_VALUE NULL +DEFAULT_VALUE ON +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Enable/disable tablespace size warning feature +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME INNODB_TABLESPACE_SIZE_WARNING_PCT +SESSION_VALUE NULL +DEFAULT_VALUE 70 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE INT UNSIGNED +VARIABLE_COMMENT Percentage at which to start emitting tablespace size warnings +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 100 +NUMERIC_BLOCK_SIZE 0 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME INNODB_TABLESPACE_SIZE_WARNING_THRESHOLD +SESSION_VALUE NULL +DEFAULT_VALUE 17592186044416 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT Threshold in bytes for tablespace size warnings (0 = disabled) +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 0 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_TABLE_LOCKS SESSION_VALUE ON DEFAULT_VALUE ON diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 2179496f5088a..80e436e6cf403 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -658,6 +658,84 @@ static uint32_t fsp_get_pages_to_extend_ibd(unsigned physical_size, return extent_size; } +/** Check if tablespace size exceeds warning threshold. +@param[in] space Tablespace +@param[in] new_size New size in pages +@return true if warning was emitted */ +static bool fsp_check_size_warning(fil_space_t *space, uint32_t new_size) noexcept +{ + /* Named constant for high-resolution warning threshold */ + constexpr uint8_t high_resolution_pct = 90; + + if (!srv_tablespace_size_warning_enabled) + return false; + + if (srv_tablespace_size_warning_threshold == 0) + return false; + + /* Reset state if threshold changed */ + if (space->m_last_warning_threshold != srv_tablespace_size_warning_threshold) { + space->m_last_size_warning_pct= 0; + space->m_last_warning_threshold= srv_tablespace_size_warning_threshold; + space->m_warning_count_in_decade= 0; + } + + uint64_t current_bytes= + static_cast(new_size) * space->physical_size(); + uint64_t current_pct= + (current_bytes * 100) / srv_tablespace_size_warning_threshold; + uint64_t display_pct= std::min(current_pct, static_cast(100)); + + bool should_warn= false; + + if (display_pct < srv_tablespace_size_warning_pct) + return false; + + if (display_pct >= high_resolution_pct) { + /* Above high_resolution_pct: print on every 1% increase */ + should_warn= (display_pct > space->m_last_size_warning_pct); + } else { + /* Between tablespace_size_warning_pct and high_resolution_pct: + print at most twice per 10% (e.g., 70%, 77%, 81%, 89%) */ + uint8_t current_decade= static_cast(display_pct / 10); + uint8_t last_decade= space->m_last_size_warning_pct / 10; + + /* If we've moved to a new decade, reset the counter */ + if (current_decade > last_decade) { + space->m_warning_count_in_decade= 0; + } + + /* Warn if we haven't warned twice yet in this decade, percentage increased, + and there's at least a 5% gap since last warning (or it's the first warning) */ + if (space->m_warning_count_in_decade < 2 && + display_pct > space->m_last_size_warning_pct && + (space->m_warning_count_in_decade == 0 || + display_pct >= static_cast(space->m_last_size_warning_pct + 5))) { + should_warn= true; + } + } + + if (should_warn) { + const auto name= space->name(); + ib::warn() << "Tablespace '" << std::string_view(name.data(), name.size()) + << "' size " << current_bytes + << " bytes (" << display_pct << "%)" + << " exceeds warning threshold of " + << srv_tablespace_size_warning_threshold << " bytes"; + + space->m_last_size_warning_pct= static_cast(display_pct); + + /* Increment counter only for the tiered warning range (below high_resolution_pct) */ + if (display_pct < high_resolution_pct) { + space->m_warning_count_in_decade++; + } + + return true; + } + + return false; +} + /** Try to extend the last data file of a tablespace if it is auto-extending. @param[in,out] space tablespace @param[in,out] header tablespace header @@ -758,6 +836,9 @@ fsp_try_extend_data_file(fil_space_t *space, buf_block_t *header, mtr_t *mtr) + header->page.frame, space->size_in_header); + /* Check if tablespace size exceeds warning threshold */ + fsp_check_size_warning(space, space->size_in_header); + return(size_increase); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4f8fa2c5a6ee2..9bdfcef71770c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -19138,6 +19138,30 @@ static MYSQL_SYSVAR_ULONG(purge_batch_size, srv_purge_batch_size, 1, /* Minimum value */ innodb_purge_batch_size_MAX, 0); +static MYSQL_SYSVAR_ULONGLONG(tablespace_size_warning_threshold, + srv_tablespace_size_warning_threshold, + PLUGIN_VAR_RQCMDARG, + "Threshold in bytes for tablespace size warnings (0 = disabled)", + NULL, NULL, + 17592186044416ULL, /* Default setting */ + 0, /* Minimum value */ + ULLONG_MAX, 0); /* Maximum value */ + +static MYSQL_SYSVAR_UINT(tablespace_size_warning_pct, + srv_tablespace_size_warning_pct, + PLUGIN_VAR_RQCMDARG, + "Percentage at which to start emitting tablespace size warnings", + NULL, NULL, + 70, /* Default setting */ + 0, /* Minimum value */ + 100, 0); /* Maximum value */ + +static MYSQL_SYSVAR_BOOL(tablespace_size_warning_enabled, + srv_tablespace_size_warning_enabled, + PLUGIN_VAR_OPCMDARG, + "Enable/disable tablespace size warning feature", + NULL, NULL, TRUE); + extern void srv_update_purge_thread_count(uint n); static @@ -20102,6 +20126,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(monitor_reset_all), MYSQL_SYSVAR(purge_threads), MYSQL_SYSVAR(purge_batch_size), + MYSQL_SYSVAR(tablespace_size_warning_threshold), + MYSQL_SYSVAR(tablespace_size_warning_pct), + MYSQL_SYSVAR(tablespace_size_warning_enabled), MYSQL_SYSVAR(log_checkpoint_now), #ifdef UNIV_DEBUG MYSQL_SYSVAR(buf_flush_list_now), diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 48697750c7dea..caaf3835d386b 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -420,7 +420,17 @@ struct fil_space_t final /** LSN of undo tablespace creation or 0; protected by latch */ lsn_t create_lsn= 0; + public: + /** Last percentage at which we emitted a size warning (0-100) */ + uint8_t m_last_size_warning_pct{0}; + + /** Threshold value used for the last warning */ + ulonglong m_last_warning_threshold{0}; + + /** Count of warnings emitted in the current decade (0-2) */ + uint8_t m_warning_count_in_decade{0}; + /** @return whether this is the temporary tablespace */ bool is_temporary() const noexcept { return UNIV_UNLIKELY(id == SRV_TMP_SPACE_ID); } diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 1eb429a94e16a..e9c1199ed9f71 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -238,6 +238,15 @@ extern ulong srv_buf_pool_load_pages_abort; /** Lock table size in bytes */ extern ulint srv_lock_table_size; +/** Threshold in bytes for tablespace size warnings (0 = disabled) */ +extern ulonglong srv_tablespace_size_warning_threshold; + +/** Percentage at which to start emitting tablespace size warnings */ +extern uint srv_tablespace_size_warning_pct; + +/** Enable/disable tablespace size warning feature */ +extern my_bool srv_tablespace_size_warning_enabled; + /** the value of innodb_checksum_algorithm */ extern ulong srv_checksum_algorithm; extern my_bool srv_random_read_ahead; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index f9fffde59bb2e..0d5fd1918bfe5 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -242,6 +242,15 @@ NULL value when collecting statistics. By default, it is set to SRV_STATS_NULLS_EQUAL(0), ie. all NULL value are treated equal */ ulong srv_innodb_stats_method; +/** Threshold in bytes for tablespace size warnings (0 = disabled) */ +ulonglong srv_tablespace_size_warning_threshold = 17592186044416ULL; + +/** Percentage at which to start emitting tablespace size warnings */ +uint srv_tablespace_size_warning_pct = 70; + +/** Enable/disable tablespace size warning feature */ +my_bool srv_tablespace_size_warning_enabled = TRUE; + srv_stats_t srv_stats; /* structure to pass status variables to MySQL */