Skip to content

Commit 2383896

Browse files
committed
MDEV-38975: HEAP engine BLOB/TEXT column support
Allow BLOB/TEXT columns in MEMORY (HEAP) engine tables by storing blob data in variable-length continuation record chains within the existing `HP_BLOCK` structure. **Continuation runs**: blob data is split across contiguous sequences of `recbuffer`-sized records. Each run stores a 10-byte header (`next_cont` pointer + `run_rec_count`) in the first record; subsequent records carry pure payload. Runs are linked via `next_cont` pointers. Individual runs are capped at 65,535 records (`uint16` format limit); larger blobs are automatically split into multiple runs. **Free list scavenging**: on insert, the free list is walked read-only (peek) tracking contiguous groups in descending address order (LIFO). Qualifying groups (>= `min_run_records`) are unlinked and used. The first non-qualifying group terminates the scan — remaining data is allocated from the block tail. The free list is never disturbed when no qualifying group is found. **Record counting**: new `HP_SHARE::total_records` tracks all physical records (primary + continuation). `HP_SHARE::records` remains logical (primary-only) to preserve linear hash bucket mapping correctness. **Reassembly**: per-handle `HP_INFO::blob_buff` reassembles blob data on read, following the same pattern as InnoDB's `blob_heap`. Freed on `heap_reset()` / `HA_EXTRA_NORMAL` / `heap_close()`. **Hash functions**: `hp_rec_hashnr()`, `hp_rec_key_cmp()`, `hp_key_cmp()`, `hp_make_key()` updated to handle `HA_BLOB_PART` key segments — reading actual blob data via pointer dereference or chain materialization. **SQL layer**: `choose_engine()` no longer rejects HEAP for blob tables (replaced `blob_fields` check with `reclength > HA_MAX_REC_LENGTH`). `remove_duplicates()` routes HEAP+blob to `remove_dup_with_compare()`. `ha_heap::remember_rnd_pos()` / `restart_rnd_next()` implemented for DISTINCT deduplication support.
1 parent 8e3c93b commit 2383896

37 files changed

Lines changed: 2283 additions & 92 deletions

include/heap.h

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,21 +131,27 @@ typedef struct st_hp_keydef /* Key definition with open */
131131
uint (*get_key_length)(struct st_hp_keydef *keydef, const uchar *key);
132132
} HP_KEYDEF;
133133

134+
typedef struct st_hp_blob_desc
135+
{
136+
uint offset; /* Byte offset of blob descriptor within record buffer */
137+
uint packlength; /* 1, 2, 3, or 4: length prefix size */
138+
} HP_BLOB_DESC;
139+
134140
typedef struct st_heap_share
135141
{
136142
HP_BLOCK block;
137143
HP_KEYDEF *keydef;
138144
ulonglong data_length,index_length,max_table_size;
139145
ulonglong auto_increment;
140146
ulong min_records,max_records; /* Params to open */
141-
ulong records; /* records */
147+
ulong records; /* Logical (primary) record count */
142148
ulong blength; /* records rounded up to 2^n */
143149
ulong deleted; /* Deleted records in database */
144150
uint key_stat_version; /* version to indicate insert/delete */
145151
uint key_version; /* Updated on key change */
146152
uint file_version; /* Update on clear */
147153
uint reclength; /* Length of one record */
148-
uint visible; /* Offset to the visible/deleted mark */
154+
uint visible; /* Offset to the flags byte (active/deleted/continuation) */
149155
uint changed;
150156
uint keys,max_key_length;
151157
uint currently_disabled_keys; /* saved value from "keys" when disabled */
@@ -156,6 +162,9 @@ typedef struct st_heap_share
156162
THR_LOCK lock;
157163
my_bool delete_on_close;
158164
my_bool internal; /* Internal temporary table */
165+
HP_BLOB_DESC *blob_descs; /* Array of blob column descriptors */
166+
uint blob_count; /* Number of blob columns */
167+
ulong total_records; /* All active records (primary + blob continuation) */
159168
LIST open_list;
160169
uint auto_key;
161170
uint auto_key_type; /* real type of the auto key segment */
@@ -181,6 +190,8 @@ typedef struct st_heap_info
181190
uint file_version; /* Version at scan */
182191
uint lastkey_len;
183192
my_bool implicit_emptied;
193+
uchar *blob_buff; /* Reassembly buffer for blob reads */
194+
uint32 blob_buff_len; /* Current allocated size of blob_buff */
184195
THR_LOCK_DATA lock;
185196
LIST open_list;
186197
} HP_INFO;
@@ -204,6 +215,8 @@ typedef struct st_heap_create_info
204215
open_count to 1. Is only looked at if not internal_table.
205216
*/
206217
my_bool pin_share;
218+
HP_BLOB_DESC *blob_descs;
219+
uint blob_count;
207220
} HP_CREATE_INFO;
208221

209222
/* Prototypes for heap-functions */

mysql-test/main/cte_recursive.test

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3212,6 +3212,8 @@ show create table t2;
32123212
--eval insert ignore into t2 $query;
32133213
drop table t2;
32143214
set @@sql_mode="";
3215+
# Rows with identical (level, mid) due to overflow have non-deterministic order
3216+
--sorted_result
32153217
--eval $query
32163218
--eval create table t2 as $query;
32173219
show create table t2;

mysql-test/main/distinct.result

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1189,7 +1189,7 @@ insert into t1 values (1, 'Aa123456', 'abc'), (2, 'Bb7897777', 'def'),
11891189
(3, 'Cc01287', 'xyz'), (5, 'd12345', 'efg');
11901190
select distinct if(sum(a), b, 0) from t1 group by value(c) with rollup;
11911191
if(sum(a), b, 0)
1192-
Aa123456
1192+
SOME_B_VALUE
11931193
drop table t1;
11941194
#
11951195
# end of 10.5 tests

mysql-test/main/distinct.test

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,9 @@ create table t1 (a int, b longtext, c varchar(18));
915915
insert into t1 values (1, 'Aa123456', 'abc'), (2, 'Bb7897777', 'def'),
916916
(3, 'Cc01287', 'xyz'), (5, 'd12345', 'efg');
917917

918+
# ROLLUP row's b value is indeterminate (depends on last group processed),
919+
# which varies by temp table engine (HEAP vs Aria). Mask the value.
920+
--replace_regex /(Aa123456|Bb7897777|Cc01287|d12345)/SOME_B_VALUE/
918921
select distinct if(sum(a), b, 0) from t1 group by value(c) with rollup;
919922
drop table t1;
920923

mysql-test/main/group_by.result

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2510,10 +2510,10 @@ SELECT f3, MIN(f2) FROM t1 GROUP BY f1 LIMIT 1;
25102510
f3 MIN(f2)
25112511
blob NULL
25122512
DROP TABLE t1;
2513-
the value below *must* be 1
2513+
the value below *must* be 0 (HEAP supports blobs)
25142514
show status like 'Created_tmp_disk_tables';
25152515
Variable_name Value
2516-
Created_tmp_disk_tables 1
2516+
Created_tmp_disk_tables 0
25172517
#
25182518
# Bug #1002146: Unneeded filesort if usage of join buffer is not allowed
25192519
# (bug mdev-645)

mysql-test/main/group_by.test

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1671,14 +1671,14 @@ DROP TABLE t1, t2;
16711671
--disable_ps2_protocol
16721672
--disable_view_protocol
16731673
--disable_cursor_protocol
1674-
FLUSH STATUS; # this test case *must* use Aria temp tables
1674+
FLUSH STATUS;
16751675

16761676
CREATE TABLE t1 (f1 INT, f2 decimal(20,1), f3 blob);
16771677
INSERT INTO t1 values(11,NULL,'blob'),(11,NULL,'blob');
16781678
SELECT f3, MIN(f2) FROM t1 GROUP BY f1 LIMIT 1;
16791679
DROP TABLE t1;
16801680

1681-
--echo the value below *must* be 1
1681+
--echo the value below *must* be 0 (HEAP supports blobs)
16821682
show status like 'Created_tmp_disk_tables';
16831683
--enable_cursor_protocol
16841684
--enable_view_protocol

mysql-test/main/select.result

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -576,18 +576,18 @@ bedlam 1
576576
bedpost 1
577577
boasted 1
578578
set tmp_memory_table_size=default;
579-
select distinct fld3,repeat("a",length(fld3)),count(*) from t2 group by companynr,fld3 limit 100,10;
579+
select distinct fld3,repeat("a",length(fld3)),count(*) from t2 group by companynr,fld3 order by fld3 limit 100,10;
580580
fld3 repeat("a",length(fld3)) count(*)
581-
circus aaaaaa 1
582-
cited aaaaa 1
583-
Colombo aaaaaaa 1
584-
congresswoman aaaaaaaaaaaaa 1
585-
contrition aaaaaaaaaa 1
586-
corny aaaaa 1
587-
cultivation aaaaaaaaaaa 1
588-
definiteness aaaaaaaaaaaa 1
589-
demultiplex aaaaaaaaaaa 1
590-
disappointing aaaaaaaaaaaaa 1
581+
Baird aaaaa 1
582+
balled aaaaaa 1
583+
ballgown aaaaaaaa 1
584+
Baltimorean aaaaaaaaaaa 1
585+
bankruptcies aaaaaaaaaaaa 1
586+
Barry aaaaa 1
587+
batting aaaaaaa 1
588+
beaner aaaaaa 1
589+
beasts aaaaaa 1
590+
beaters aaaaaaa 1
591591
select distinct companynr,rtrim(space(512+companynr)) from t3 order by 1,2;
592592
companynr rtrim(space(512+companynr))
593593
37

mysql-test/main/select.test

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1442,7 +1442,8 @@ select distinct fld3,count(*) from t2 group by companynr,fld3 limit 10;
14421442
set tmp_memory_table_size=0; # force on-disk tmp table
14431443
select distinct fld3,count(*) from t2 group by companynr,fld3 limit 10;
14441444
set tmp_memory_table_size=default;
1445-
select distinct fld3,repeat("a",length(fld3)),count(*) from t2 group by companynr,fld3 limit 100,10;
1445+
# ORDER BY fld3 ensures deterministic LIMIT window regardless of temp table engine
1446+
select distinct fld3,repeat("a",length(fld3)),count(*) from t2 group by companynr,fld3 order by fld3 limit 100,10;
14461447

14471448
#
14481449
# A big order by that should trigger a merge in filesort
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
CREATE TABLE t1 (a mediumtext) ENGINE=HEAP;
2+
INSERT INTO t1 VALUES ('abc'),('def');
3+
SELECT DISTINCT a FROM t1;
4+
a
5+
abc
6+
def
7+
DROP TABLE t1;
8+
CREATE TABLE t1 (a mediumtext);
9+
CREATE TABLE t2 (b varchar(20));
10+
INSERT INTO t1 VALUES ('a'),('b');
11+
SELECT left(a,100000000) FROM t1 UNION SELECT b FROM t2;
12+
left(a,100000000)
13+
a
14+
b
15+
DROP TABLE t1, t2;
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
CREATE TABLE t1 (a mediumtext) ENGINE=HEAP;
2+
INSERT INTO t1 VALUES ('abc'),('def');
3+
SELECT DISTINCT a FROM t1;
4+
DROP TABLE t1;
5+
6+
CREATE TABLE t1 (a mediumtext);
7+
CREATE TABLE t2 (b varchar(20));
8+
INSERT INTO t1 VALUES ('a'),('b');
9+
SELECT left(a,100000000) FROM t1 UNION SELECT b FROM t2;
10+
DROP TABLE t1, t2;

0 commit comments

Comments
 (0)