MDEV-38947 Reimplement SET GLOBAL innodb_buffer_pool_size

extra/mariabackup/innobackupex.cc

@@ -44,8 +44,8 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA
 #include <string.h>
 #include <mysql.h>
 #include <my_dir.h>
-#include <ut0mem.h>
 #include <os0file.h>
+#include "buf0buf.h"
 #include <srv0start.h>
 #include <algorithm>
 #include <mysqld.h>
@@ -608,8 +608,9 @@ static struct my_option ibx_long_options[] =
 	 "--apply-log.",
 	 (uchar*) &ibx_xtrabackup_use_memory,
 	 (uchar*) &ibx_xtrabackup_use_memory,
-	 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
-	 1024*1024L, 0},
+	 0, GET_LL, REQUIRED_ARG, 96 << 20,
+	 innodb_buffer_pool_extent_size, SIZE_T_MAX, 0,
+	 innodb_buffer_pool_extent_size, 0},
 
 	{"innodb-force-recovery", OPT_INNODB_FORCE_RECOVERY,
 	 "This option starts up the embedded InnoDB instance in crash "

extra/mariabackup/xtrabackup.cc

@@ -1284,8 +1284,9 @@ struct my_option xb_client_options[]= {
      "The value is used in place of innodb_buffer_pool_size. "
      "This option is only relevant when the --prepare option is specified.",
      (G_PTR *) &xtrabackup_use_memory, (G_PTR *) &xtrabackup_use_memory, 0,
-     GET_LL, REQUIRED_ARG, 100 * 1024 * 1024L, 1024 * 1024L, LONGLONG_MAX, 0,
-     1024 * 1024L, 0},
+     GET_ULL, REQUIRED_ARG, 96 << 20, innodb_buffer_pool_extent_size,
+     size_t(-ssize_t(innodb_buffer_pool_extent_size)),
+     0, innodb_buffer_pool_extent_size, 0},
     {"throttle", OPT_XTRA_THROTTLE,
      "limit count of IO operations (pairs of read&write) per second to IOS "
      "values (for '--backup')",
@@ -2327,7 +2328,7 @@ static bool innodb_init_param()
 	}
 
 	srv_sys_space.normalize_size();
-	srv_lock_table_size = 5 * (srv_buf_pool_size >> srv_page_size_shift);
+	srv_lock_table_size = 5 * buf_pool.curr_size();
 
 	/* -------------- Log files ---------------------------*/
 
@@ -2349,11 +2350,8 @@ static bool innodb_init_param()
 
 	srv_adaptive_flushing = FALSE;
 
-        /* We set srv_pool_size here in units of 1 kB. InnoDB internally
-        changes the value so that it becomes the number of database pages. */
-
-	srv_buf_pool_size = (ulint) xtrabackup_use_memory;
-	srv_buf_pool_chunk_unit = (ulong)srv_buf_pool_size;
+	buf_pool.size_in_bytes_max = size_t(xtrabackup_use_memory);
+	buf_pool.size_in_bytes_requested = buf_pool.size_in_bytes_max;
 
 	srv_n_read_io_threads = (uint) innobase_read_io_threads;
 	srv_n_write_io_threads = (uint) innobase_write_io_threads;

include/my_sys.h

@@ -172,9 +172,15 @@ extern void my_free(void *ptr);
 extern void *my_memdup(PSI_memory_key key, const void *from,size_t length,myf MyFlags);
 extern char *my_strdup(PSI_memory_key key, const char *from,myf MyFlags);
 extern char *my_strndup(PSI_memory_key key, const char *from, size_t length, myf MyFlags);
+extern my_bool my_use_large_pages;
 
-int my_init_large_pages(my_bool super_large_pages);
+int my_init_large_pages(void);
 uchar *my_large_malloc(size_t *size, myf my_flags);
+#ifdef _WIN32
+/* On Windows, use my_virtual_mem_reserve() and my_virtual_mem_commit(). */
+#else
+char *my_large_virtual_alloc(size_t *size);
+#endif
 void my_large_free(void *ptr, size_t size);
 
 #ifdef _WIN32

include/my_virtual_mem.h

@@ -0,0 +1,37 @@
+/* Copyright (c) 2025, MariaDB
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA */
+
+#pragma once
+/*
+  Functionality for handling virtual memory
+  (reserve, commit, decommit, release)
+*/
+#include <stddef.h> /*size_t*/
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _WIN32
+char *my_virtual_mem_reserve(size_t *size);
+#endif
+char *my_virtual_mem_commit(char *ptr, size_t size);
+void my_virtual_mem_decommit(char *ptr, size_t size);
+void my_virtual_mem_release(char *ptr, size_t size);
+
+#ifdef __cplusplus
+}
+#endif
+

mysql-test/main/large_pages.opt

@@ -1 +1 @@
---large-pages
+--large-pages --loose-innodb-buffer-pool-size-max=16m

mysql-test/main/large_pages.result

@@ -1,4 +1,5 @@
 call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*");
+call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*");
 create table t1 (
 a int not null auto_increment,
 b char(16) not null,

mysql-test/main/large_pages.test

@@ -1,11 +1,9 @@
 # Test of large pages (or at least the fallback to conventional allocation)
 
-# Windows needs SeLockMemoryPrivilege
---source include/not_windows.inc
 --source include/have_innodb.inc
 
 call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*");
-
+call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*");
 create table t1 (
   a int not null auto_increment,
   b char(16) not null,

mysql-test/suite/encryption/t/innodb_encrypt_temporary_tables.opt

@@ -1,2 +1,2 @@
---innodb_buffer_pool_size=5M
+--innodb_buffer_pool_size=6M
 --innodb_encrypt_temporary_tables=1

mysql-test/suite/galera/r/galera_sst_mariabackup_use_memory.result

@@ -8,6 +8,6 @@ connection node_1;
 Cleaning var directory ...
 connection node_2;
 Starting server ...
-include/assert_grep.inc [mariabackup: Using 128974848 bytes for buffer pool \(set by --use-memory parameter\)]
+include/assert_grep.inc [mariabackup: Using 125829120 bytes for buffer pool \(set by --use-memory parameter\)]
 disconnect node_2;
 disconnect node_1;

mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.cnf

@@ -5,4 +5,4 @@ wsrep_sst_method=mariabackup
 wsrep_sst_auth="root:"
 
 [mariabackup]
-use_memory=123m
+use_memory=121m

mysql-test/suite/galera/t/galera_sst_mariabackup_use_memory.test

@@ -40,8 +40,8 @@ let $restart_noprint=2;
 --source include/wait_condition.inc
 
 # Confirm that IST did not take place
---let $assert_text = mariabackup: Using 128974848 bytes for buffer pool \(set by --use-memory parameter\)
---let $assert_select = mariabackup: Using 128974848 bytes for buffer pool \(set by --use-memory parameter\)
+--let $assert_text = mariabackup: Using 125829120 bytes for buffer pool \(set by --use-memory parameter\)
+--let $assert_select = mariabackup: Using 125829120 bytes for buffer pool \(set by --use-memory parameter\)
 --let $assert_count = 1
 --let $assert_file = $MYSQLTEST_VARDIR/mysqld.2/data/mariabackup.prepare.log
 --let $assert_only_after = Starting InnoDB instance for recovery

mysql-test/suite/innodb/r/innodb_buffer_pool_fail.result

@@ -1,9 +1,9 @@
-call mtr.add_suppression("InnoDB: Cannot allocate memory for the buffer pool");
+call mtr.add_suppression("InnoDB: Cannot map innodb_buffer_pool_size_max=");
 call mtr.add_suppression("InnoDB: Plugin initialization aborted at srv0start.cc.*");
 call mtr.add_suppression("Plugin 'InnoDB' init function returned error.");
 call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed.");
 #
 # MDEV-25019 memory allocation failures during startup cause server failure in different, confusing ways
 #
 # restart: --debug_dbug=+d,ib_buf_chunk_init_fails
-FOUND 1 /\[ERROR\] InnoDB: Cannot allocate memory for the buffer pool/ in mysqld.1.err
+FOUND 1 /\[ERROR\] InnoDB: Cannot map innodb_buffer_pool_size_max=16m/ in mysqld.1.err

mysql-test/suite/innodb/r/innodb_buffer_pool_resize.result

@@ -1,34 +1,60 @@
+#
+# MDEV-29445: Reorganize buffer pool (and remove chunks)
+#
 set global innodb_adaptive_hash_index=ON;
 select @@innodb_buffer_pool_size;
 @@innodb_buffer_pool_size
 8388608
+set global innodb_buffer_pool_size = 9437184;
 set global innodb_buffer_pool_size = 10485760;
-Warnings:
-Warning	1292	Truncated incorrect innodb_buffer_pool_size value: '10485760'
 select @@innodb_buffer_pool_size;
 @@innodb_buffer_pool_size
-16777216
-create table t1 (id int not null, val int not null default '0', primary key (id)) ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
-create or replace view view0 as select 1 union all select 1;
-set @`v_id` := 0;
-set @`v_val` := 0;
-replace into t1 select (@`v_id` := (@`v_id` + 4) mod 4294967296) as id, (@`v_val` := (@`v_val` + 4) mod 4294967296) as val from view0 v0, view0 v1, view0 v2, view0 v3, view0 v4, view0 v5, view0 v6, view0 v7, view0 v8, view0 v9, view0 v10, view0 v11, view0 v12, view0 v13, view0 v14, view0 v15, view0 v16, view0 v17;
+10485760
+create table t1 (id int primary key, val int not null)
+ENGINE=InnoDB ROW_FORMAT=COMPRESSED;
+create table t2 (id int primary key, val int not null)
+ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=$kbs;
+SET STATEMENT foreign_key_checks=0, unique_checks=0 FOR
+INSERT INTO t1 SELECT seq*4,seq*4 FROM seq_1_to_262144;
+SET STATEMENT foreign_key_checks=0, unique_checks=0 FOR
+INSERT INTO t2 SELECT seq*4,seq*4 FROM seq_1_to_16384;
+SELECT @@GLOBAL.innodb_adaptive_hash_index;
+@@GLOBAL.innodb_adaptive_hash_index
+1
+SET STATEMENT max_statement_time=1e-9 FOR
+SET GLOBAL innodb_buffer_pool_size = 7340032;
+SELECT @@GLOBAL.innodb_adaptive_hash_index;
+@@GLOBAL.innodb_adaptive_hash_index
+1
+FOUND 1 /innodb_buffer_pool_size=7m.*resized from|innodb_buffer_pool_size change aborted/ in mysqld.1.err
 set global innodb_buffer_pool_size = 7340032;
-Warnings:
-Warning	1292	Truncated incorrect innodb_buffer_pool_size value: '7340032'
-select @@innodb_buffer_pool_size;
-@@innodb_buffer_pool_size
-8388608
 select count(val) from t1;
 count(val)
 262144
+select count(val) from t2;
+count(val)
+16384
 set global innodb_adaptive_hash_index=OFF;
-set global innodb_buffer_pool_size = 25165824;
+set global innodb_buffer_pool_size = 24117248;
+set global innodb_buffer_pool_size = 26214400;
+Warnings:
+Warning	1292	Truncated incorrect innodb_buffer_pool_size value: '26214400'
 select @@innodb_buffer_pool_size;
 @@innodb_buffer_pool_size
 25165824
 select count(val) from t1;
 count(val)
 262144
-drop table t1;
-drop view view0;
+select count(val) from t2;
+count(val)
+16384
+drop table t1,t2;
+SET GLOBAL innodb_max_purge_lag_wait = 0;
+SET @save_pct= @@GLOBAL.innodb_max_dirty_pages_pct;
+SET @save_pct_lwm= @@GLOBAL.innodb_max_dirty_pages_pct_lwm;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm = 0.0;
+SET GLOBAL innodb_max_dirty_pages_pct = 0.0;
+SET GLOBAL innodb_buffer_pool_size = @old_innodb_buffer_pool_size;
+SET GLOBAL innodb_adaptive_hash_index = @old_innodb_adaptive_hash_index;
+SET GLOBAL innodb_max_dirty_pages_pct = @save_pct;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm = @save_pct_lwm;

mysql-test/suite/innodb/r/innodb_buffer_pool_resize_temporary.result

@@ -2,9 +2,44 @@ SET @save_limit=@@GLOBAL.innodb_limit_optimistic_insert_debug;
 SET @save_size=@@GLOBAL.innodb_buffer_pool_size;
 SET GLOBAL innodb_limit_optimistic_insert_debug=2;
 SET GLOBAL innodb_buffer_pool_size=16777216;
+call mtr.add_suppression("innodb_buffer_pool_size change aborted");
+SET @old_innodb_adaptive_hash_index = @@innodb_adaptive_hash_index;
+SET GLOBAL innodb_adaptive_hash_index = ON;
+SET STATEMENT debug_dbug='+d,buf_shrink_fail' FOR
+SET GLOBAL innodb_buffer_pool_size=8388608;
+ERROR HY000: innodb_buffer_pool_size change aborted
+SELECT @@GLOBAL.innodb_adaptive_hash_index,@@GLOBAL.innodb_buffer_pool_size;
+@@GLOBAL.innodb_adaptive_hash_index	@@GLOBAL.innodb_buffer_pool_size
+1	16777216
+SET GLOBAL innodb_adaptive_hash_index = @old_innodb_adaptive_hash_index;
 CREATE TEMPORARY TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 SELECT seq FROM seq_1_to_200;
+SET GLOBAL innodb_max_purge_lag_wait=0;
+SET @save_pct= @@GLOBAL.innodb_max_dirty_pages_pct;
+SET @save_pct_lwm= @@GLOBAL.innodb_max_dirty_pages_pct_lwm;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm = 0.0;
+SET GLOBAL innodb_max_dirty_pages_pct = 0.0;
+SHOW STATUS LIKE 'innodb_buffer_pool_resize_status';
+Variable_name	Value
+Innodb_buffer_pool_resize_status	
+connect con1,localhost,root;
+SET DEBUG_SYNC='buf_pool_shrink_before_wakeup SIGNAL blocked WAIT_FOR go';
 SET GLOBAL innodb_buffer_pool_size=8388608;
+connection default;
+SET DEBUG_SYNC='now WAIT_FOR blocked';
+SHOW STATUS LIKE 'innodb_buffer_pool_resize_status';
+Variable_name	Value
+Innodb_buffer_pool_resize_status	Withdrawing blocks. (505/505).
+SET DEBUG_SYNC='now SIGNAL go';
+connection con1;
+disconnect con1;
+connection default;
+SHOW STATUS LIKE 'innodb_buffer_pool_resize_status';
+Variable_name	Value
+Innodb_buffer_pool_resize_status	
+SET DEBUG_SYNC=RESET;
+SET GLOBAL innodb_max_dirty_pages_pct = @save_pct;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm = @save_pct_lwm;
 SELECT COUNT(*),MIN(a),MAX(a) FROM t1;
 COUNT(*)	MIN(a)	MAX(a)
 200	1	200

mysql-test/suite/innodb/r/innodb_buffer_pool_shrink.result

@@ -0,0 +1,11 @@
+call mtr.add_suppression("innodb_buffer_pool_size change aborted");
+CREATE TABLE t (c INT) ENGINE=InnoDB PARTITION BY HASH(c) PARTITIONS 512;
+BEGIN;
+SELECT * FROM t LOCK IN SHARE MODE;
+c
+SET @save_size = @@GLOBAL.innodb_buffer_pool_size;
+SET GLOBAL innodb_buffer_pool_size=6291456;
+COMMIT;
+SET GLOBAL innodb_buffer_pool_size=6291456;
+SET GLOBAL innodb_buffer_pool_size = @save_size;
+DROP TABLE t;

mysql-test/suite/innodb/r/innodb_buffer_pool_shrink_temporary.result

@@ -0,0 +1,17 @@
+call mtr.add_suppression("innodb_buffer_pool_size change aborted");
+SET @b=REPEAT('0',1048576);
+CREATE TEMPORARY TABLE t (c MEDIUMTEXT) ENGINE=InnoDB;
+INSERT INTO t VALUES
+(@b),(@b),(@b),(@b),(@b),(@b),(@b),(@b),(@b),(@b),(@b);
+SET STATEMENT max_statement_time=0.000001 FOR
+SET GLOBAL innodb_buffer_pool_size=6291456;
+SET STATEMENT max_statement_time=0.000001 FOR
+SET GLOBAL innodb_buffer_pool_size=6291456;
+SET STATEMENT max_statement_time=0.000001 FOR
+SET GLOBAL innodb_buffer_pool_size=6291456;
+SET GLOBAL innodb_buffer_pool_size=6291456;
+SET GLOBAL innodb_buffer_pool_size=16777216;
+CHECKSUM TABLE t;
+Table	Checksum
+test.t	4050893687
+DROP TEMPORARY TABLE t;

mysql-test/suite/innodb/r/lock_memory_debug.result

@@ -5,7 +5,7 @@ call mtr.add_suppression("\\[Warning\\] InnoDB: Over 67 percent of the buffer po
 CREATE TABLE t1 (col1 INT) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1),(2),(3),(4),(5);
 SET STATEMENT debug_dbug='+d,innodb_skip_lock_bitmap' FOR
-INSERT INTO t1 SELECT a.* FROM t1 a, t1 b, t1 c, t1 d, t1 e, t1 f, t1 g LIMIT 45000;
+INSERT INTO t1 SELECT a.* FROM t1 a, t1 b, t1 c, t1 d, t1 e, t1 f, t1 g;
 ERROR HY000: The total number of locks exceeds the lock table size
 SELECT COUNT(*) FROM t1;
 COUNT(*)

mysql-test/suite/innodb/r/log_upgrade_101_flags.result

@@ -1,7 +1,7 @@
 call mtr.add_suppression("InnoDB: The change buffer is corrupted");
 call mtr.add_suppression("InnoDB: Tablespace size stored in header is 768 pages, but the sum of data file sizes is 384 pages");
 call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of file");
-# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-force-recovery=5 --innodb-log-file-size=4m --innodb_page_size=32k --innodb_buffer_pool_size=10M
+# restart: --innodb-data-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-log-group-home-dir=MYSQLTEST_VARDIR/tmp/log_upgrade --innodb-force-recovery=5 --innodb-log-file-size=4m --innodb_page_size=32k --innodb_buffer_pool_size=11M
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');