Skip to content

Commit 1dca122

Browse files
FarihaISFarihaIS
committed
MDEV-38936 Proactive handling of InnoDB tablespace full condition
InnoDB write failures occur when tablespace files exceed filesystem size limits. Current behavior logs errors but continues accepting transactions, causing repeated failures and potential data integrity issues. Add proactive monitoring by emitting warnings when InnoDB tablespaces exceed 90% of a configurable threshold. Warnings appear at each percentage point (90%, 91%, etc.) to prevent log flooding. Key features: - Two new system variables: innodb_tablespace_size_warning_threshold (default 16TB) and innodb_tablespace_size_warning_enabled (andon cord) - Per-tablespace tracking with automatic reset on TRUNCATE/DROP - Zero overhead when disabled - Progressive warnings capped at 100% Implementation hooks into fsp_try_extend_data_file() for O(1) size checking during tablespace extension. Adds 9 bytes per tablespace (m_last_size_warning_pct, m_last_warning_threshold) to fil_space_t structure. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc.
1 parent d755574 commit 1dca122

7 files changed

Lines changed: 252 additions & 0 deletions

File tree

‎mysql-test/suite/innodb/r/tablespace_size_warning.result‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#
2+
# MDEV-38936: Proactive handling of InnoDB tablespace full condition
3+
#
4+
SET @old_threshold = @@global.innodb_tablespace_size_warning_threshold;
5+
SET @old_enabled = @@global.innodb_tablespace_size_warning_enabled;
6+
# Test system variables
7+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_threshold';
8+
Variable_name Value
9+
innodb_tablespace_size_warning_threshold 17592186044416
10+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_enabled';
11+
Variable_name Value
12+
innodb_tablespace_size_warning_enabled ON
13+
# Test basic warning emission
14+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
15+
CREATE TABLE t1 (
16+
id INT AUTO_INCREMENT PRIMARY KEY,
17+
data LONGBLOB
18+
) ENGINE=InnoDB;
19+
FOUND 1 /Tablespace 'test/t1' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
20+
DROP TABLE t1;
21+
# Test threshold set to zero disables warnings
22+
SET GLOBAL innodb_tablespace_size_warning_threshold = 0;
23+
CREATE TABLE t2 (
24+
id INT AUTO_INCREMENT PRIMARY KEY,
25+
data LONGBLOB
26+
) ENGINE=InnoDB;
27+
DROP TABLE t2;
28+
# Test andon cord disable/enable
29+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
30+
SET GLOBAL innodb_tablespace_size_warning_enabled = FALSE;
31+
CREATE TABLE t3 (
32+
id INT AUTO_INCREMENT PRIMARY KEY,
33+
data LONGBLOB
34+
) ENGINE=InnoDB;
35+
SET GLOBAL innodb_tablespace_size_warning_enabled = TRUE;
36+
FOUND 1 /Tablespace 'test/t3' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
37+
DROP TABLE t3;
38+
# Test TRUNCATE TABLE resets warning state
39+
CREATE TABLE t4 (
40+
id INT AUTO_INCREMENT PRIMARY KEY,
41+
data LONGBLOB
42+
) ENGINE=InnoDB;
43+
FOUND 1 /Tablespace 'test/t4' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
44+
TRUNCATE TABLE t4;
45+
FOUND 1 /Tablespace 'test/t4' size .* bytes .* exceeds warning threshold/ in mysqld.1.err
46+
DROP TABLE t4;
47+
SET GLOBAL innodb_tablespace_size_warning_threshold = @old_threshold;
48+
SET GLOBAL innodb_tablespace_size_warning_enabled = @old_enabled;

‎mysql-test/suite/innodb/t/tablespace_size_warning.test‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
--source include/have_innodb.inc
2+
3+
--disable_query_log
4+
call mtr.add_suppression("Tablespace .* size .* bytes .* exceeds warning threshold");
5+
--enable_query_log
6+
7+
--echo #
8+
--echo # MDEV-38936: Proactive handling of InnoDB tablespace full condition
9+
--echo #
10+
11+
# Save original values
12+
SET @old_threshold = @@global.innodb_tablespace_size_warning_threshold;
13+
SET @old_enabled = @@global.innodb_tablespace_size_warning_enabled;
14+
15+
--echo # Test system variables
16+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_threshold';
17+
SHOW VARIABLES LIKE 'innodb_tablespace_size_warning_enabled';
18+
19+
--echo # Test basic warning emission
20+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
21+
22+
CREATE TABLE t1 (
23+
id INT AUTO_INCREMENT PRIMARY KEY,
24+
data LONGBLOB
25+
) ENGINE=InnoDB;
26+
27+
--disable_query_log
28+
let $i = 10;
29+
while ($i) {
30+
eval INSERT INTO t1 (data) VALUES (REPEAT('a', 1024*1024));
31+
dec $i;
32+
}
33+
--enable_query_log
34+
35+
let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err;
36+
let SEARCH_PATTERN=Tablespace 'test/t1' size .* bytes .* exceeds warning threshold;
37+
--source include/search_pattern_in_file.inc
38+
39+
DROP TABLE t1;
40+
41+
--echo # Test threshold set to zero disables warnings
42+
SET GLOBAL innodb_tablespace_size_warning_threshold = 0;
43+
44+
CREATE TABLE t2 (
45+
id INT AUTO_INCREMENT PRIMARY KEY,
46+
data LONGBLOB
47+
) ENGINE=InnoDB;
48+
49+
--disable_query_log
50+
let $i = 5;
51+
while ($i) {
52+
eval INSERT INTO t2 (data) VALUES (REPEAT('b', 1024*1024));
53+
dec $i;
54+
}
55+
--enable_query_log
56+
57+
DROP TABLE t2;
58+
59+
--echo # Test andon cord disable/enable
60+
SET GLOBAL innodb_tablespace_size_warning_threshold = 10485760;
61+
SET GLOBAL innodb_tablespace_size_warning_enabled = FALSE;
62+
63+
CREATE TABLE t3 (
64+
id INT AUTO_INCREMENT PRIMARY KEY,
65+
data LONGBLOB
66+
) ENGINE=InnoDB;
67+
68+
--disable_query_log
69+
let $i = 10;
70+
while ($i) {
71+
eval INSERT INTO t3 (data) VALUES (REPEAT('c', 1024*1024));
72+
dec $i;
73+
}
74+
--enable_query_log
75+
76+
SET GLOBAL innodb_tablespace_size_warning_enabled = TRUE;
77+
78+
--disable_query_log
79+
INSERT INTO t3 (data) VALUES (REPEAT('d', 1024*1024));
80+
--enable_query_log
81+
82+
let SEARCH_PATTERN=Tablespace 'test/t3' size .* bytes .* exceeds warning threshold;
83+
--source include/search_pattern_in_file.inc
84+
85+
DROP TABLE t3;
86+
87+
--echo # Test TRUNCATE TABLE resets warning state
88+
CREATE TABLE t4 (
89+
id INT AUTO_INCREMENT PRIMARY KEY,
90+
data LONGBLOB
91+
) ENGINE=InnoDB;
92+
93+
--disable_query_log
94+
let $i = 10;
95+
while ($i) {
96+
eval INSERT INTO t4 (data) VALUES (REPEAT('e', 1024*1024));
97+
dec $i;
98+
}
99+
--enable_query_log
100+
101+
let SEARCH_PATTERN=Tablespace 'test/t4' size .* bytes .* exceeds warning threshold;
102+
--source include/search_pattern_in_file.inc
103+
104+
TRUNCATE TABLE t4;
105+
106+
--disable_query_log
107+
let $i = 10;
108+
while ($i) {
109+
eval INSERT INTO t4 (data) VALUES (REPEAT('f', 1024*1024));
110+
dec $i;
111+
}
112+
--enable_query_log
113+
114+
let SEARCH_PATTERN=Tablespace 'test/t4' size .* bytes .* exceeds warning threshold;
115+
--source include/search_pattern_in_file.inc
116+
117+
DROP TABLE t4;
118+
119+
# Restore original values
120+
SET GLOBAL innodb_tablespace_size_warning_threshold = @old_threshold;
121+
SET GLOBAL innodb_tablespace_size_warning_enabled = @old_enabled;

‎storage/innobase/fsp/fsp0fsp.cc‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,6 +658,50 @@ static uint32_t fsp_get_pages_to_extend_ibd(unsigned physical_size,
658658
return extent_size;
659659
}
660660

661+
/** Check if tablespace size exceeds warning threshold.
662+
@param[in] space Tablespace
663+
@param[in] new_size New size in pages
664+
@return true if warning was emitted */
665+
static bool fsp_check_size_warning(fil_space_t *space, uint32_t new_size) noexcept
666+
{
667+
if (!srv_tablespace_size_warning_enabled)
668+
return false;
669+
670+
if (srv_tablespace_size_warning_threshold == 0)
671+
return false;
672+
673+
/* Reset state if threshold changed */
674+
if (space->m_last_warning_threshold != srv_tablespace_size_warning_threshold)
675+
{
676+
space->m_last_size_warning_pct= 0;
677+
space->m_last_warning_threshold= srv_tablespace_size_warning_threshold;
678+
}
679+
680+
uint64_t current_bytes=
681+
static_cast<uint64_t>(new_size) * space->physical_size();
682+
uint64_t current_pct=
683+
(current_bytes * 100) / srv_tablespace_size_warning_threshold;
684+
uint64_t display_pct= std::min(current_pct, static_cast<uint64_t>(100));
685+
686+
if (display_pct >= 90 && display_pct > space->m_last_size_warning_pct)
687+
{
688+
const auto name= space->name();
689+
ib::warn() << "Tablespace '" << std::string_view(name.data(), name.size())
690+
<< "' size " << current_bytes
691+
<< " bytes (" << display_pct << "%)"
692+
<< " exceeds warning threshold of "
693+
<< srv_tablespace_size_warning_threshold << " bytes";
694+
695+
space->m_last_size_warning_pct= static_cast<uint8_t>(display_pct);
696+
return true;
697+
}
698+
699+
if (display_pct < 90 && space->m_last_size_warning_pct > 0)
700+
space->m_last_size_warning_pct= 0;
701+
702+
return false;
703+
}
704+
661705
/** Try to extend the last data file of a tablespace if it is auto-extending.
662706
@param[in,out] space tablespace
663707
@param[in,out] header tablespace header
@@ -758,6 +802,9 @@ fsp_try_extend_data_file(fil_space_t *space, buf_block_t *header, mtr_t *mtr)
758802
+ header->page.frame,
759803
space->size_in_header);
760804

805+
/* Check if tablespace size exceeds warning threshold */
806+
fsp_check_size_warning(space, space->size_in_header);
807+
761808
return(size_increase);
762809
}
763810

‎storage/innobase/handler/ha_innodb.cc‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19138,6 +19138,21 @@ static MYSQL_SYSVAR_ULONG(purge_batch_size, srv_purge_batch_size,
1913819138
1, /* Minimum value */
1913919139
innodb_purge_batch_size_MAX, 0);
1914019140

19141+
static MYSQL_SYSVAR_ULONGLONG(tablespace_size_warning_threshold,
19142+
srv_tablespace_size_warning_threshold,
19143+
PLUGIN_VAR_RQCMDARG,
19144+
"Threshold in bytes for tablespace size warnings (0 = disabled)",
19145+
NULL, NULL,
19146+
17592186044416ULL, /* Default setting */
19147+
0, /* Minimum value */
19148+
ULLONG_MAX, 0); /* Maximum value */
19149+
19150+
static MYSQL_SYSVAR_BOOL(tablespace_size_warning_enabled,
19151+
srv_tablespace_size_warning_enabled,
19152+
PLUGIN_VAR_OPCMDARG,
19153+
"Enable/disable tablespace size warning feature",
19154+
NULL, NULL, TRUE);
19155+
1914119156
extern void srv_update_purge_thread_count(uint n);
1914219157

1914319158
static
@@ -20102,6 +20117,8 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
2010220117
MYSQL_SYSVAR(monitor_reset_all),
2010320118
MYSQL_SYSVAR(purge_threads),
2010420119
MYSQL_SYSVAR(purge_batch_size),
20120+
MYSQL_SYSVAR(tablespace_size_warning_threshold),
20121+
MYSQL_SYSVAR(tablespace_size_warning_enabled),
2010520122
MYSQL_SYSVAR(log_checkpoint_now),
2010620123
#ifdef UNIV_DEBUG
2010720124
MYSQL_SYSVAR(buf_flush_list_now),

‎storage/innobase/include/fil0fil.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,7 +420,14 @@ struct fil_space_t final
420420

421421
/** LSN of undo tablespace creation or 0; protected by latch */
422422
lsn_t create_lsn= 0;
423+
423424
public:
425+
/** Last percentage at which we emitted a size warning (0-100) */
426+
uint8_t m_last_size_warning_pct{0};
427+
428+
/** Threshold value used for the last warning */
429+
ulonglong m_last_warning_threshold{0};
430+
424431
/** @return whether this is the temporary tablespace */
425432
bool is_temporary() const noexcept
426433
{ return UNIV_UNLIKELY(id == SRV_TMP_SPACE_ID); }

‎storage/innobase/include/srv0srv.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,12 @@ extern ulong srv_buf_pool_load_pages_abort;
238238
/** Lock table size in bytes */
239239
extern ulint srv_lock_table_size;
240240

241+
/** Threshold in bytes for tablespace size warnings (0 = disabled) */
242+
extern ulonglong srv_tablespace_size_warning_threshold;
243+
244+
/** Enable/disable tablespace size warning feature */
245+
extern my_bool srv_tablespace_size_warning_enabled;
246+
241247
/** the value of innodb_checksum_algorithm */
242248
extern ulong srv_checksum_algorithm;
243249
extern my_bool srv_random_read_ahead;

‎storage/innobase/srv/srv0srv.cc‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,12 @@ NULL value when collecting statistics. By default, it is set to
242242
SRV_STATS_NULLS_EQUAL(0), ie. all NULL value are treated equal */
243243
ulong srv_innodb_stats_method;
244244

245+
/** Threshold in bytes for tablespace size warnings (0 = disabled) */
246+
ulonglong srv_tablespace_size_warning_threshold = 17592186044416ULL;
247+
248+
/** Enable/disable tablespace size warning feature */
249+
my_bool srv_tablespace_size_warning_enabled = TRUE;
250+
245251
srv_stats_t srv_stats;
246252

247253
/* structure to pass status variables to MySQL */

0 commit comments

Comments
 (0)