Skip to content

Commit 5bd3d38

Browse files
FarihaISFarihaIS
committed
MDEV-38936 Proactive handling of InnoDB tablespace full condition
InnoDB write failures occur when tablespace files exceed filesystem size limits. Current behavior logs errors but continues accepting transactions, causing repeated failures and potential data integrity issues. Add proactive monitoring by emitting warnings when InnoDB tablespaces approach a configurable size threshold. Warnings use a tiered frequency approach to balance early notification with log spam prevention. Key features: - Three new system variables: * innodb_tablespace_size_warning_threshold (default 16TB): Maximum tablespace size in bytes before warnings begin * innodb_tablespace_size_warning_pct (default 70%): Percentage of threshold at which to start emitting warnings * innodb_tablespace_size_warning_enabled (default ON): Andon cord to enable/disable the warning feature - Tiered warning frequency: * Below warning_pct: No warnings * Between warning_pct and 90%: At most twice per 10% interval with minimum 5% gap (e.g., 70%, 77%, 81%, 89%) * Above 90%: Every 1% increase (90%, 91%, 92%, etc.) - Per-tablespace tracking with automatic reset on TRUNCATE/DROP or threshold changes - Zero overhead when disabled - Progressive warnings capped at 100% Implementation hooks into fsp_try_extend_data_file() for O(1) size checking during tablespace extension. Adds 11 bytes per tablespace (m_last_size_warning_pct, m_last_warning_threshold, m_warning_count_in_decade) to fil_space_t structure. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc.
1 parent d755574 commit 5bd3d38

8 files changed

Lines changed: 385 additions & 0 deletions

File tree

‎mysql-test/suite/innodb/r/tablespace_size_warning.result‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#
2+
# MDEV-38936: Proactive handling of InnoDB tablespace full condition
3+
#
4+
SET @old_threshold = @@global.innodb_tablespace_size_warning_threshold;
5+
SET @old_enabled = @@global.innodb_tablespace_size_warning_enabled;
6+
SET @old_pct = @@global.innodb_tablespace_size_warning_pct;
7+
# Test system variables
8+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_threshold';
9+
Variable_name Value
10+
innodb_tablespace_size_warning_threshold 17592186044416
11+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_enabled';
12+
Variable_name Value
13+
innodb_tablespace_size_warning_enabled ON
14+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_pct';
15+
Variable_name Value
16+
innodb_tablespace_size_warning_pct 70
17+
# Test basic warning emission
18+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
19+
SET GLOBAL innodb_tablespace_size_warning_pct = 70;
20+
CREATE TABLE t1 (
21+
id INT AUTO_INCREMENT PRIMARY KEY,
22+
data LONGBLOB
23+
) ENGINE=InnoDB;
24+
FOUND 1 /Tablespace 'test/t1' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
25+
DROP TABLE t1;
26+
# Test configurable warning percentage
27+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
28+
SET GLOBAL innodb_tablespace_size_warning_pct = 80;
29+
CREATE TABLE t1b (
30+
id INT AUTO_INCREMENT PRIMARY KEY,
31+
data LONGBLOB
32+
) ENGINE=InnoDB;
33+
FOUND 1 /Tablespace 'test/t1b' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
34+
DROP TABLE t1b;
35+
# Test threshold set to zero disables warnings
36+
SET GLOBAL innodb_tablespace_size_warning_threshold = 0;
37+
CREATE TABLE t2 (
38+
id INT AUTO_INCREMENT PRIMARY KEY,
39+
data LONGBLOB
40+
) ENGINE=InnoDB;
41+
DROP TABLE t2;
42+
# Test andon cord disable/enable
43+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
44+
SET GLOBAL innodb_tablespace_size_warning_pct = 70;
45+
SET GLOBAL innodb_tablespace_size_warning_enabled = FALSE;
46+
CREATE TABLE t3 (
47+
id INT AUTO_INCREMENT PRIMARY KEY,
48+
data LONGBLOB
49+
) ENGINE=InnoDB;
50+
SET GLOBAL innodb_tablespace_size_warning_enabled = TRUE;
51+
FOUND 1 /Tablespace 'test/t3' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
52+
DROP TABLE t3;
53+
# Test TRUNCATE TABLE resets warning state
54+
CREATE TABLE t4 (
55+
id INT AUTO_INCREMENT PRIMARY KEY,
56+
data LONGBLOB
57+
) ENGINE=InnoDB;
58+
FOUND 1 /Tablespace 'test/t4' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
59+
TRUNCATE TABLE t4;
60+
FOUND 1 /Tablespace 'test/t4' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
61+
DROP TABLE t4;
62+
SET GLOBAL innodb_tablespace_size_warning_threshold = @old_threshold;
63+
SET GLOBAL innodb_tablespace_size_warning_enabled = @old_enabled;
64+
SET GLOBAL innodb_tablespace_size_warning_pct = @old_pct;

‎mysql-test/suite/innodb/t/tablespace_size_warning.test‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
--source include/have_innodb.inc
2+
--source include/not_embedded.inc
3+
4+
--disable_query_log
5+
call mtr.add_suppression("Tablespace .* size .* bytes .* exceeds warning threshold");
6+
--enable_query_log
7+
8+
--echo #
9+
--echo # MDEV-38936: Proactive handling of InnoDB tablespace full condition
10+
--echo #
11+
12+
# Save original values
13+
SET @old_threshold = @@global.innodb_tablespace_size_warning_threshold;
14+
SET @old_enabled = @@global.innodb_tablespace_size_warning_enabled;
15+
SET @old_pct = @@global.innodb_tablespace_size_warning_pct;
16+
17+
--echo # Test system variables
18+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_threshold';
19+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_enabled';
20+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_pct';
21+
22+
--echo # Test basic warning emission
23+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
24+
SET GLOBAL innodb_tablespace_size_warning_pct = 70;
25+
26+
CREATE TABLE t1 (
27+
id INT AUTO_INCREMENT PRIMARY KEY,
28+
data LONGBLOB
29+
) ENGINE=InnoDB;
30+
31+
--disable_query_log
32+
let $i = 10;
33+
while ($i) {
34+
eval INSERT INTO t1 (data) VALUES (REPEAT('a', 1024*1024));
35+
dec $i;
36+
}
37+
--enable_query_log
38+
39+
let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err;
40+
let SEARCH_PATTERN=Tablespace 'test/t1' size .* bytes .* exceeds warning threshold;
41+
--source include/search_pattern_in_file.inc
42+
43+
DROP TABLE t1;
44+
45+
--echo # Test configurable warning percentage
46+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
47+
SET GLOBAL innodb_tablespace_size_warning_pct = 80;
48+
49+
CREATE TABLE t1b (
50+
id INT AUTO_INCREMENT PRIMARY KEY,
51+
data LONGBLOB
52+
) ENGINE=InnoDB;
53+
54+
--disable_query_log
55+
let $i = 8;
56+
while ($i) {
57+
eval INSERT INTO t1b (data) VALUES (REPEAT('x', 1024*1024));
58+
dec $i;
59+
}
60+
--enable_query_log
61+
62+
let SEARCH_PATTERN=Tablespace 'test/t1b' size .* bytes .* exceeds warning threshold;
63+
--source include/search_pattern_in_file.inc
64+
65+
DROP TABLE t1b;
66+
67+
--echo # Test threshold set to zero disables warnings
68+
SET GLOBAL innodb_tablespace_size_warning_threshold = 0;
69+
70+
CREATE TABLE t2 (
71+
id INT AUTO_INCREMENT PRIMARY KEY,
72+
data LONGBLOB
73+
) ENGINE=InnoDB;
74+
75+
--disable_query_log
76+
let $i = 5;
77+
while ($i) {
78+
eval INSERT INTO t2 (data) VALUES (REPEAT('b', 1024*1024));
79+
dec $i;
80+
}
81+
--enable_query_log
82+
83+
DROP TABLE t2;
84+
85+
--echo # Test andon cord disable/enable
86+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
87+
SET GLOBAL innodb_tablespace_size_warning_pct = 70;
88+
SET GLOBAL innodb_tablespace_size_warning_enabled = FALSE;
89+
90+
CREATE TABLE t3 (
91+
id INT AUTO_INCREMENT PRIMARY KEY,
92+
data LONGBLOB
93+
) ENGINE=InnoDB;
94+
95+
--disable_query_log
96+
let $i = 10;
97+
while ($i) {
98+
eval INSERT INTO t3 (data) VALUES (REPEAT('c', 1024*1024));
99+
dec $i;
100+
}
101+
--enable_query_log
102+
103+
SET GLOBAL innodb_tablespace_size_warning_enabled = TRUE;
104+
105+
--disable_query_log
106+
INSERT INTO t3 (data) VALUES (REPEAT('d', 1024*1024));
107+
--enable_query_log
108+
109+
let SEARCH_PATTERN=Tablespace 'test/t3' size .* bytes .* exceeds warning threshold;
110+
--source include/search_pattern_in_file.inc
111+
112+
DROP TABLE t3;
113+
114+
--echo # Test TRUNCATE TABLE resets warning state
115+
CREATE TABLE t4 (
116+
id INT AUTO_INCREMENT PRIMARY KEY,
117+
data LONGBLOB
118+
) ENGINE=InnoDB;
119+
120+
--disable_query_log
121+
let $i = 10;
122+
while ($i) {
123+
eval INSERT INTO t4 (data) VALUES (REPEAT('e', 1024*1024));
124+
dec $i;
125+
}
126+
--enable_query_log
127+
128+
let SEARCH_PATTERN=Tablespace 'test/t4' size .* bytes .* exceeds warning threshold;
129+
--source include/search_pattern_in_file.inc
130+
131+
TRUNCATE TABLE t4;
132+
133+
--disable_query_log
134+
let $i = 10;
135+
while ($i) {
136+
eval INSERT INTO t4 (data) VALUES (REPEAT('f', 1024*1024));
137+
dec $i;
138+
}
139+
--enable_query_log
140+
141+
let SEARCH_PATTERN=Tablespace 'test/t4' size .* bytes .* exceeds warning threshold;
142+
--source include/search_pattern_in_file.inc
143+
144+
DROP TABLE t4;
145+
146+
# Restore original values
147+
SET GLOBAL innodb_tablespace_size_warning_threshold = @old_threshold;
148+
SET GLOBAL innodb_tablespace_size_warning_enabled = @old_enabled;
149+
SET GLOBAL innodb_tablespace_size_warning_pct = @old_pct;

‎mysql-test/suite/sys_vars/r/sysvars_innodb.result‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1616,6 +1616,42 @@ NUMERIC_BLOCK_SIZE 0
16161616
ENUM_VALUE_LIST NULL
16171617
READ_ONLY NO
16181618
COMMAND_LINE_ARGUMENT REQUIRED
1619+
VARIABLE_NAME INNODB_TABLESPACE_SIZE_WARNING_ENABLED
1620+
SESSION_VALUE NULL
1621+
DEFAULT_VALUE ON
1622+
VARIABLE_SCOPE GLOBAL
1623+
VARIABLE_TYPE BOOLEAN
1624+
VARIABLE_COMMENT Enable/disable tablespace size warning feature
1625+
NUMERIC_MIN_VALUE NULL
1626+
NUMERIC_MAX_VALUE NULL
1627+
NUMERIC_BLOCK_SIZE NULL
1628+
ENUM_VALUE_LIST OFF,ON
1629+
READ_ONLY NO
1630+
COMMAND_LINE_ARGUMENT OPTIONAL
1631+
VARIABLE_NAME INNODB_TABLESPACE_SIZE_WARNING_PCT
1632+
SESSION_VALUE NULL
1633+
DEFAULT_VALUE 70
1634+
VARIABLE_SCOPE GLOBAL
1635+
VARIABLE_TYPE INT UNSIGNED
1636+
VARIABLE_COMMENT Percentage at which to start emitting tablespace size warnings
1637+
NUMERIC_MIN_VALUE 0
1638+
NUMERIC_MAX_VALUE 100
1639+
NUMERIC_BLOCK_SIZE 0
1640+
ENUM_VALUE_LIST NULL
1641+
READ_ONLY NO
1642+
COMMAND_LINE_ARGUMENT REQUIRED
1643+
VARIABLE_NAME INNODB_TABLESPACE_SIZE_WARNING_THRESHOLD
1644+
SESSION_VALUE NULL
1645+
DEFAULT_VALUE 17592186044416
1646+
VARIABLE_SCOPE GLOBAL
1647+
VARIABLE_TYPE BIGINT UNSIGNED
1648+
VARIABLE_COMMENT Threshold in bytes for tablespace size warnings (0 = disabled)
1649+
NUMERIC_MIN_VALUE 0
1650+
NUMERIC_MAX_VALUE 18446744073709551615
1651+
NUMERIC_BLOCK_SIZE 0
1652+
ENUM_VALUE_LIST NULL
1653+
READ_ONLY NO
1654+
COMMAND_LINE_ARGUMENT REQUIRED
16191655
VARIABLE_NAME INNODB_TABLE_LOCKS
16201656
SESSION_VALUE ON
16211657
DEFAULT_VALUE ON

‎storage/innobase/fsp/fsp0fsp.cc‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,6 +658,84 @@ static uint32_t fsp_get_pages_to_extend_ibd(unsigned physical_size,
658658
return extent_size;
659659
}
660660

661+
/** Check if tablespace size exceeds warning threshold.
662+
@param[in] space Tablespace
663+
@param[in] new_size New size in pages
664+
@return true if warning was emitted */
665+
static bool fsp_check_size_warning(fil_space_t *space, uint32_t new_size) noexcept
666+
{
667+
/* Named constant for high-resolution warning threshold */
668+
constexpr uint8_t high_resolution_pct = 90;
669+
670+
if (!srv_tablespace_size_warning_enabled)
671+
return false;
672+
673+
if (srv_tablespace_size_warning_threshold == 0)
674+
return false;
675+
676+
/* Reset state if threshold changed */
677+
if (space->m_last_warning_threshold != srv_tablespace_size_warning_threshold) {
678+
space->m_last_size_warning_pct= 0;
679+
space->m_last_warning_threshold= srv_tablespace_size_warning_threshold;
680+
space->m_warning_count_in_decade= 0;
681+
}
682+
683+
uint64_t current_bytes=
684+
static_cast<uint64_t>(new_size) * space->physical_size();
685+
uint64_t current_pct=
686+
(current_bytes * 100) / srv_tablespace_size_warning_threshold;
687+
uint64_t display_pct= std::min(current_pct, static_cast<uint64_t>(100));
688+
689+
bool should_warn= false;
690+
691+
if (display_pct < srv_tablespace_size_warning_pct)
692+
return false;
693+
694+
if (display_pct >= high_resolution_pct) {
695+
/* Above high_resolution_pct: print on every 1% increase */
696+
should_warn= (display_pct > space->m_last_size_warning_pct);
697+
} else {
698+
/* Between tablespace_size_warning_pct and high_resolution_pct:
699+
print at most twice per 10% (e.g., 70%, 77%, 81%, 89%) */
700+
uint8_t current_decade= static_cast<uint8_t>(display_pct / 10);
701+
uint8_t last_decade= space->m_last_size_warning_pct / 10;
702+
703+
/* If we've moved to a new decade, reset the counter */
704+
if (current_decade > last_decade) {
705+
space->m_warning_count_in_decade= 0;
706+
}
707+
708+
/* Warn if we haven't warned twice yet in this decade, percentage increased,
709+
and there's at least a 5% gap since last warning (or it's the first warning) */
710+
if (space->m_warning_count_in_decade < 2 &&
711+
display_pct > space->m_last_size_warning_pct &&
712+
(space->m_warning_count_in_decade == 0 ||
713+
display_pct >= static_cast<uint64_t>(space->m_last_size_warning_pct + 5))) {
714+
should_warn= true;
715+
}
716+
}
717+
718+
if (should_warn) {
719+
const auto name= space->name();
720+
ib::warn() << "Tablespace '" << std::string_view(name.data(), name.size())
721+
<< "' size " << current_bytes
722+
<< " bytes (" << display_pct << "%)"
723+
<< " exceeds warning threshold of "
724+
<< srv_tablespace_size_warning_threshold << " bytes";
725+
726+
space->m_last_size_warning_pct= static_cast<uint8_t>(display_pct);
727+
728+
/* Increment counter only for the tiered warning range (below high_resolution_pct) */
729+
if (display_pct < high_resolution_pct) {
730+
space->m_warning_count_in_decade++;
731+
}
732+
733+
return true;
734+
}
735+
736+
return false;
737+
}
738+
661739
/** Try to extend the last data file of a tablespace if it is auto-extending.
662740
@param[in,out] space tablespace
663741
@param[in,out] header tablespace header
@@ -758,6 +836,9 @@ fsp_try_extend_data_file(fil_space_t *space, buf_block_t *header, mtr_t *mtr)
758836
+ header->page.frame,
759837
space->size_in_header);
760838

839+
/* Check if tablespace size exceeds warning threshold */
840+
fsp_check_size_warning(space, space->size_in_header);
841+
761842
return(size_increase);
762843
}
763844

‎storage/innobase/handler/ha_innodb.cc‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19138,6 +19138,30 @@ static MYSQL_SYSVAR_ULONG(purge_batch_size, srv_purge_batch_size,
1913819138
1, /* Minimum value */
1913919139
innodb_purge_batch_size_MAX, 0);
1914019140

19141+
static MYSQL_SYSVAR_ULONGLONG(tablespace_size_warning_threshold,
19142+
srv_tablespace_size_warning_threshold,
19143+
PLUGIN_VAR_RQCMDARG,
19144+
"Threshold in bytes for tablespace size warnings (0 = disabled)",
19145+
NULL, NULL,
19146+
17592186044416ULL, /* Default setting */
19147+
0, /* Minimum value */
19148+
ULLONG_MAX, 0); /* Maximum value */
19149+
19150+
static MYSQL_SYSVAR_UINT(tablespace_size_warning_pct,
19151+
srv_tablespace_size_warning_pct,
19152+
PLUGIN_VAR_RQCMDARG,
19153+
"Percentage at which to start emitting tablespace size warnings",
19154+
NULL, NULL,
19155+
70, /* Default setting */
19156+
0, /* Minimum value */
19157+
100, 0); /* Maximum value */
19158+
19159+
static MYSQL_SYSVAR_BOOL(tablespace_size_warning_enabled,
19160+
srv_tablespace_size_warning_enabled,
19161+
PLUGIN_VAR_OPCMDARG,
19162+
"Enable/disable tablespace size warning feature",
19163+
NULL, NULL, TRUE);
19164+
1914119165
extern void srv_update_purge_thread_count(uint n);
1914219166

1914319167
static
@@ -20102,6 +20126,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
2010220126
MYSQL_SYSVAR(monitor_reset_all),
2010320127
MYSQL_SYSVAR(purge_threads),
2010420128
MYSQL_SYSVAR(purge_batch_size),
20129+
MYSQL_SYSVAR(tablespace_size_warning_threshold),
20130+
MYSQL_SYSVAR(tablespace_size_warning_pct),
20131+
MYSQL_SYSVAR(tablespace_size_warning_enabled),
2010520132
MYSQL_SYSVAR(log_checkpoint_now),
2010620133
#ifdef UNIV_DEBUG
2010720134
MYSQL_SYSVAR(buf_flush_list_now),

0 commit comments

Comments
 (0)