MDEV-21181 Add Support for Generated Invisible Primary Keys

jaeheonshim · jaeheonshim · commit 829e2a2c231e · 2026-03-02T22:38:21.000-06:00
This PR implements a new session/global option
`sql_generate_invisible_primary_key`. When this option is on, MariaDB
will automatically generate an invisible primary key for tables created
without an explicit primary key. The type of the auto-generated primary
key is `BIGINT UNSIGNED AUTO_INCREMENT`. If an ALTER TABLE ADD PRIMARY
KEY statement is executed on a table with an generated primary key, the
generated primary key will be silently dropped.

Notes:
- Internally, the generated primary key has the INVISIBLE_FULL
  attribute. This means that the key and column effectively do not
  exist to the user and will not appear in SHOW CREATE TABLE and SHOW
  COLUMNS. Furthermore, the user cannot explicitly drop the
  generated primary key.
- The column name of the invisible generated primary key is _inv_PK.
  If the user creates their own column named _inv_PK, the generated
  primary key column's name will be suffixed with a number.
- If the storage engine does not support auto increment, no generated
  primary key will be added.
diff --git a/mysql-test/main/generate_invisible_primary_key.result b/mysql-test/main/generate_invisible_primary_key.result
@@ -0,0 +1,51 @@
+SET SESSION debug_dbug="+d,test_invisible_index,test_completely_invisible";
+CREATE TABLE t0 (c1 INT, c2 INT);
+SHOW INDEX FROM t0;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
+t0	1	invisible	1	invisible	A	NULL	NULL	NULL	YES	BTREE			NO
+SET sql_generate_invisible_primary_key=ON;
+CREATE TABLE t1 (c1 INT, c2 INT);
+SHOW INDEX FROM t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
+t1	0	PRIMARY	1	_inv_PK	A	0	NULL	NULL		BTREE			NO
+t1	1	invisible	1	invisible	A	NULL	NULL	NULL	YES	BTREE			NO
+INSERT INTO t1 VALUES (3, 0);
+INSERT INTO t1 VALUES (4, 0);
+SELECT _rowid, t1.* FROM t1;
+_rowid	c1	c2
+1	3	0
+2	4	0
+ALTER TABLE t1 ADD c3 INT;
+SHOW INDEX FROM t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
+t1	1	invisible	1	invisible	A	NULL	NULL	NULL	YES	BTREE			NO
+t1	0	PRIMARY	1	_inv_PK	A	2	NULL	NULL		BTREE			NO
+ALTER TABLE t1 ADD _inv_PK INT;
+ALTER TABLE t1 ADD _inv_PK1 INT;
+SHOW INDEX FROM t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
+t1	1	invisible	1	invisible	A	NULL	NULL	NULL	YES	BTREE			NO
+t1	0	PRIMARY	1	_inv_PK2	A	2	NULL	NULL		BTREE			NO
+ALTER TABLE t1 DROP PRIMARY KEY;
+ERROR 42000: Can't DROP INDEX `PRIMARY`; check that it exists
+ALTER TABLE t1 ADD PRIMARY KEY (c1);
+SHOW INDEX FROM t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
+t1	0	PRIMARY	1	c1	A	2	NULL	NULL		BTREE			NO
+t1	1	invisible	1	invisible	A	NULL	NULL	NULL	YES	BTREE			NO
+SELECT _rowid, t1.* FROM t1;
+_rowid	c1	c2	c3	_inv_PK	_inv_PK1
+3	3	0	NULL	NULL	NULL
+4	4	0	NULL	NULL	NULL
+ALTER TABLE t1 DROP PRIMARY KEY;
+CREATE TABLE t2 (c1 INT, c2 INT, _inv_PK INT);
+SHOW INDEX FROM t2;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
+t2	0	PRIMARY	1	_inv_PK1	A	0	NULL	NULL		BTREE			NO
+t2	1	invisible	1	invisible	A	NULL	NULL	NULL	YES	BTREE			NO
+SET SESSION debug_dbug="";
+CREATE TABLE t3 (c1 INT, c2 INT);
+SHOW INDEX FROM t3;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment	Ignored
+DROP TABLE t0, t1, t2, t3;
+SET sql_generate_invisible_primary_key=OFF;
diff --git a/mysql-test/main/generate_invisible_primary_key.test b/mysql-test/main/generate_invisible_primary_key.test
@@ -0,0 +1,72 @@
+#
+# Test of sql_generate_invisible_primary_key option (MDEV-21181)
+#
+--source include/have_debug.inc
+
+SET SESSION debug_dbug="+d,test_invisible_index,test_completely_invisible";
+
+#
+# When option is off, invisible PK is not generated
+#
+CREATE TABLE t0 (c1 INT, c2 INT);
+SHOW INDEX FROM t0;
+
+SET sql_generate_invisible_primary_key=ON;
+
+#
+# Check that PK index gets created when not specified in CREATE TABLE
+#
+CREATE TABLE t1 (c1 INT, c2 INT);
+SHOW INDEX FROM t1;
+
+INSERT INTO t1 VALUES (3, 0);
+INSERT INTO t1 VALUES (4, 0);
+SELECT _rowid, t1.* FROM t1;
+
+#
+# Check that generated PK persists on ALTER TABLE without new PK
+#
+ALTER TABLE t1 ADD c3 INT;
+SHOW INDEX FROM t1;
+
+#
+# Check that adding column with name of generated PK succeeds
+#
+ALTER TABLE t1 ADD _inv_PK INT;
+ALTER TABLE t1 ADD _inv_PK1 INT;
+SHOW INDEX FROM t1;
+
+#
+# Trying to drop autogenerated PK should fail
+#
+--error 1091
+ALTER TABLE t1 DROP PRIMARY KEY;
+
+#
+# Check that generated PK is dropped on ALTER TABLE with new PK
+#
+ALTER TABLE t1 ADD PRIMARY KEY (c1);
+SHOW INDEX FROM t1;
+SELECT _rowid, t1.* FROM t1;
+
+#
+# Check that user PK that replaced generated PK can be dropped
+#
+ALTER TABLE t1 DROP PRIMARY KEY;
+
+#
+# Check that creating table with column named _inv_PK succeeds
+#
+CREATE TABLE t2 (c1 INT, c2 INT, _inv_PK INT);
+SHOW INDEX FROM t2;
+
+SET SESSION debug_dbug="";
+
+#
+# Generated PK should be fully invisible to user
+#
+CREATE TABLE t3 (c1 INT, c2 INT);
+SHOW INDEX FROM t3;
+
+DROP TABLE t0, t1, t2, t3;
+SET sql_generate_invisible_primary_key=OFF;
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
@@ -3672,6 +3672,16 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	OFF,ON
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	NULL
+VARIABLE_NAME	SQL_GENERATE_INVISIBLE_PRIMARY_KEY
+VARIABLE_SCOPE	SESSION
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	Automatically generate an invisible auto-increment primary key for tables created without a primary key
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	SQL_IF_EXISTS
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BOOLEAN
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
@@ -4472,6 +4472,16 @@ NUMERIC_BLOCK_SIZE	NULL
 ENUM_VALUE_LIST	OFF,ON
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	NULL
+VARIABLE_NAME	SQL_GENERATE_INVISIBLE_PRIMARY_KEY
+VARIABLE_SCOPE	SESSION
+VARIABLE_TYPE	BOOLEAN
+VARIABLE_COMMENT	Automatically generate an invisible auto-increment primary key for tables created without a primary key
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	OFF,ON
+READ_ONLY	NO
+COMMAND_LINE_ARGUMENT	OPTIONAL
 VARIABLE_NAME	SQL_IF_EXISTS
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BOOLEAN
diff --git a/sql/sql_class.h b/sql/sql_class.h
@@ -928,6 +928,7 @@ typedef struct system_variables
 #endif // USER_VAR_TRACKING
   my_bool tcp_nodelay;
   my_bool optimizer_record_context;
+  my_bool generate_invisible_primary_key;
   plugin_ref table_plugin;
   plugin_ref tmp_table_plugin;
   plugin_ref enforced_table_plugin;
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
@@ -2751,6 +2751,16 @@ int mysql_add_invisible_field(THD *thd, List<Create_field> * field_list,
   return 0;
 }
 
+static bool is_auto_pk_field(Field *field)
+{
+  return field && field->invisible == INVISIBLE_FULL &&
+         (field->flags & (AUTO_INCREMENT_FLAG | NOT_NULL_FLAG | UNSIGNED_FLAG |
+                          UNIQUE_KEY_FLAG)) ==
+             (AUTO_INCREMENT_FLAG | NOT_NULL_FLAG | UNSIGNED_FLAG |
+              UNIQUE_KEY_FLAG) &&
+         field->type_handler() == &type_handler_ulonglong;
+}
+
 #define INTERNAL_FIELD_NAME_LENGTH  30
 
 static Lex_ident_column make_internal_field_name(THD *thd, const char *prefix,
@@ -2784,18 +2794,50 @@ static Create_field *add_internal_field(THD *thd, Type_handler *type_handler,
   return cf;
 }
 
-Key *
-mysql_add_invisible_index(THD *thd, List<Key> *key_list,
-        LEX_CSTRING* field_name, enum Key::Keytype type)
+Key *mysql_add_invisible_index(THD *thd, List<Key> *key_list,
+                               LEX_CSTRING *field_name, enum Key::Keytype type,
+                               bool generated_arg= false)
 {
-  Key *key= new (thd->mem_root) Key(type, &null_clex_str, HA_KEY_ALG_UNDEF,
-                                    false, DDL_options(DDL_options::OPT_NONE));
+  Key *key= new (thd->mem_root)
+      Key(type, &null_clex_str, HA_KEY_ALG_UNDEF, generated_arg,
+          DDL_options(DDL_options::OPT_NONE));
   key->columns.push_back(new(thd->mem_root) Key_part_spec(field_name, 0, true),
           thd->mem_root);
   key_list->push_back(key, thd->mem_root);
   return key;
 }
 
+static int add_generated_invisible_pk(THD *thd, List<Create_field> *field_list,
+                                      Alter_info *alter_info)
+{
+  class Column_derived_attributes dat(&my_charset_bin);
+  Create_field *fld= new (thd->mem_root) Create_field();
+
+  Lex_ident_column automatic_pk_name;
+  if ((automatic_pk_name= make_unique_invisible_field_name(
+           thd, "_inv_PK"_Lex_ident_column, &alter_info->create_list))
+          .str)
+    fld->field_name= automatic_pk_name;
+  else
+    return 1;
+
+  fld->set_handler(&type_handler_ulonglong);
+  fld->prepare_stage1(thd, thd->mem_root, COLUMN_DEFINITION_TABLE_FIELD, &dat);
+  fld->invisible= INVISIBLE_FULL;
+  fld->flags|=
+      AUTO_INCREMENT_FLAG | NOT_NULL_FLAG | UNSIGNED_FLAG | UNIQUE_KEY_FLAG;
+
+  if (fld->check(thd))
+    return 1;
+
+  field_list->push_front(fld, thd->mem_root);
+
+  Key *key= mysql_add_invisible_index(thd, &(alter_info->key_list),
+                                      &(fld->field_name), Key::PRIMARY, true);
+  key->invisible= true;
+
+  return 0;
+}
 
 bool Type_handler_string::Key_part_spec_init_ft(Key_part_spec *part,
                                                 const Column_definition &def)
@@ -3300,6 +3342,29 @@ mysql_prepare_create_table_finalize(THD *thd, HA_CREATE_INFO *create_info,
       DBUG_RETURN(TRUE);
   }
 
+  /*
+    Check if primary_key exists. If a primary key does not exist, and
+    the generate_invisible_primary_key option is set, and the handler
+    supports auto increment, add an INVISIBLE_FULL primary key (MDEV-21181)
+  */
+  if (!create_info->sequence &&
+      thd->variables.generate_invisible_primary_key &&
+      !(file->ha_table_flags() & HA_NO_AUTO_INCREMENT))
+  {
+    List_iterator<Key> auto_pk_key_iterator(alter_info->key_list);
+    bool has_primary_key= 0;
+    Key *auto_pk_key;
+    while (!has_primary_key && (auto_pk_key= auto_pk_key_iterator++))
+    {
+      if (auto_pk_key->type == Key::PRIMARY)
+        has_primary_key= 1;
+    }
+
+    if (!has_primary_key)
+      if (add_generated_invisible_pk(thd, &alter_info->create_list,
+                                     alter_info))
+        DBUG_RETURN(TRUE);
+  }
 
   for (field_no=0; (sql_field=it++) ; field_no++)
   {
@@ -8710,14 +8775,59 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
 
   table->file->get_foreign_key_list(thd, &fk_list);
 
+  /*
+    If there is an existing primary key that is defined on a single field
+    and the field is an autogenerated pk field, check if the user added a
+    primary key of their own.
+  */
+  bool should_replace_generated_pk= false;
+  Field *generated_pk_field= nullptr;
+  if (table->s->primary_key != MAX_KEY)
+  {
+    KEY *primary_key_info= table->key_info + table->s->primary_key;
+    if (primary_key_info->user_defined_key_parts == 1 &&
+        is_auto_pk_field(
+            table->field[primary_key_info->key_part[0].fieldnr - 1]))
+    {
+      generated_pk_field=
+          table->field[primary_key_info->key_part[0].fieldnr - 1];
+
+      List_iterator<Key> add_key_it(alter_info->key_list);
+      for (Key *key; (key= add_key_it++);)
+      {
+        if (key->type == Key::PRIMARY && !key->generated)
+        {
+          should_replace_generated_pk= true;
+          break;
+        }
+      }
+    }
+  }
+
   /*
     First collect all fields from table which isn't in drop_list
   */
   bitmap_clear_all(&table->tmp_set);
   for (f_ptr=table->field ; (field= *f_ptr) ; f_ptr++)
   {
     if (field->invisible == INVISIBLE_FULL)
+    {
+      /*
+        If the user added a pk of their own, and this was a generated pk field,
+        we drop the generated pk field. Other wise, we want to carry it over to
+        the new table.
+      */
+      if (should_replace_generated_pk && generated_pk_field == field)
+      {
         continue;
+      }
+      if (generated_pk_field == field)
+      {
+        def= new (root) Create_field(thd, field, field);
+        new_create_list.push_back(def, root);
+      }
+      continue;
+    }
     Alter_drop *drop;
     if (field->type() == MYSQL_TYPE_VARCHAR)
       create_info->varchar= TRUE;
@@ -9118,7 +9228,13 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
   for (uint i= 0; i < table->s->total_keys; i++, key_info++)
   {
     bool long_hash_key= false;
-    if (key_info->flags & HA_INVISIBLE_KEY)
+    /*
+      If this is the generated PK and the user is not adding their own PK,
+      keep it
+    */
+    if ((key_info->flags & HA_INVISIBLE_KEY) &&
+        !(generated_pk_field != nullptr && i == table->s->primary_key &&
+          !should_replace_generated_pk))
       continue;
     Lex_ident_column key_name(key_info->name);
     const bool primary_key= table->s->primary_key == i;
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
@@ -7737,3 +7737,9 @@ static Sys_var_path Sys_path(
         "path", "Comma-separated list of schema names that defines the search "
         "order for stored routines",
         SESSION_VAR(path), NO_CMD_LINE, NOT_IN_BINLOG);
+
+static Sys_var_mybool Sys_generate_invisible_primary_key(
+    "sql_generate_invisible_primary_key",
+    "Automatically generate an invisible auto-increment primary key for "
+    "tables created without a primary key",
+    SESSION_VAR(generate_invisible_primary_key), CMD_LINE(OPT_ARG), DEFAULT(FALSE));
