MDEV-40417 Fix default value for compressed columns

mysql-test/main/column_compression.result

@@ -1323,7 +1323,7 @@ LENGTH(a)
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` blob /*M!100301 COMPRESSED*/ NOT NULL DEFAULT ''
+  `a` blob /*M!100301 COMPRESSED*/ NOT NULL
 ) ENGINE=CSV DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 DROP TABLE t1;
@@ -3051,3 +3051,125 @@ REPLACE INTO t VALUES ('abcdefghijklm');
 UPDATE t SET c=MID(c,2,4);
 DROP TABLE t;
 # End of 10.6 tests
+#
+# MDEV-40417 A column with compression enabled automatically adds DEFAULT ''
+#
+CREATE TABLE t (
+c1 LONGTEXT COMPRESSED NOT NULL,
+nc1 LONGTEXT NOT NULL,
+c2 LONGTEXT COMPRESSED NOT NULL DEFAULT '',
+nc2 LONGTEXT NOT NULL DEFAULT '',
+c3 VARCHAR(100) COMPRESSED NOT NULL,
+nc3 VARCHAR(100) NOT NULL
+) ENGINE=InnoDB;
+SHOW CREATE TABLE t;
+Table	Create Table
+t	CREATE TABLE `t` (
+  `c1` longtext /*M!100301 COMPRESSED*/ NOT NULL,
+  `nc1` longtext NOT NULL,
+  `c2` longtext /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `nc2` longtext NOT NULL DEFAULT '',
+  `c3` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL,
+  `nc3` varchar(100) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+SELECT COLUMN_NAME, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t' ORDER BY ORDINAL_POSITION;
+COLUMN_NAME	COLUMN_DEFAULT
+c1	NULL
+nc1	NULL
+c2	''
+nc2	''
+c3	NULL
+nc3	NULL
+# A compressed column must be as mandatory as an uncompressed one
+INSERT INTO t (nc1) VALUES ('x');
+ERROR HY000: Field 'c1' doesn't have a default value
+INSERT INTO t (c1, nc1, nc3) VALUES ('x', 'x', 'x');
+ERROR HY000: Field 'c3' doesn't have a default value
+INSERT INTO t (c1, nc1, c3, nc3) VALUES ('x', 'x', 'x', 'x');
+SELECT * FROM t;
+c1	nc1	c2	nc2	c3	nc3
+x	x			x	x
+DROP TABLE t;
+# ALTER TABLE must not add an implicit default either
+CREATE TABLE t (c LONGTEXT COMPRESSED NOT NULL DEFAULT 'x') ENGINE=InnoDB;
+ALTER TABLE t MODIFY c LONGTEXT COMPRESSED NOT NULL;
+ALTER TABLE t ADD c2 VARCHAR(100) COMPRESSED NOT NULL;
+SHOW CREATE TABLE t;
+Table	Create Table
+t	CREATE TABLE `t` (
+  `c` longtext /*M!100301 COMPRESSED*/ NOT NULL,
+  `c2` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+DROP TABLE t;
+#
+# MDEV-40417 upgrade: a table created before the fix is repaired when
+# its FRM is read.  Only blobs can be repaired - for VARCHAR the FRM
+# does not tell an explicit DEFAULT '' from the wrong implicit one,
+# so those columns are deliberately left alone.
+#
+# MDEV-40417.frm was created by 10.11.19 as:
+#   CREATE TABLE mdev40417 (
+#     blob_nodef  LONGTEXT COMPRESSED NOT NULL,
+#     blob_def    LONGTEXT COMPRESSED NOT NULL DEFAULT 'x',
+#     vc_nodef    VARCHAR(100) COMPRESSED NOT NULL,
+#     vc_def      VARCHAR(100) COMPRESSED NOT NULL DEFAULT '',
+#     plain_nodef LONGTEXT NOT NULL,
+#     pad         INT) ENGINE=MyISAM CHARSET=latin1;
+#   INSERT INTO mdev40417 (plain_nodef, pad) VALUES ('p', 1);
+#
+SHOW CREATE TABLE mdev40417;
+Table	Create Table
+mdev40417	CREATE TABLE `mdev40417` (
+  `blob_nodef` longtext /*M!100301 COMPRESSED*/ NOT NULL,
+  `blob_def` longtext /*M!100301 COMPRESSED*/ NOT NULL DEFAULT 'x',
+  `vc_nodef` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `vc_def` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `plain_nodef` longtext NOT NULL,
+  `pad` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+SELECT COLUMN_NAME, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA='test' AND TABLE_NAME='mdev40417' ORDER BY ORDINAL_POSITION;
+COLUMN_NAME	COLUMN_DEFAULT
+blob_nodef	NULL
+blob_def	'x'
+vc_nodef	''
+vc_def	''
+plain_nodef	NULL
+pad	NULL
+# The row that the old server allowed to be inserted is still readable
+SELECT * FROM mdev40417;
+blob_nodef	blob_def	vc_nodef	vc_def	plain_nodef	pad
+	x			p	1
+# blob_nodef is mandatory again
+INSERT INTO mdev40417 (blob_def, vc_nodef, vc_def, plain_nodef)
+VALUES ('a', 'b', 'c', 'd');
+ERROR HY000: Field 'blob_nodef' doesn't have a default value
+# plain_nodef was never affected by the bug
+INSERT INTO mdev40417 (blob_nodef, blob_def, vc_nodef, vc_def)
+VALUES ('a', 'b', 'c', 'd');
+ERROR HY000: Field 'plain_nodef' doesn't have a default value
+# blob_def keeps its explicit DEFAULT
+INSERT INTO mdev40417 (blob_nodef, vc_nodef, vc_def, plain_nodef)
+VALUES ('a', 'b', 'c', 'd');
+# vc_nodef is not touched, it still has the old implicit DEFAULT ''
+INSERT INTO mdev40417 (blob_nodef, blob_def, plain_nodef) VALUES ('a', 'b', 'd');
+SELECT blob_nodef, blob_def, vc_nodef, vc_def, plain_nodef FROM mdev40417;
+blob_nodef	blob_def	vc_nodef	vc_def	plain_nodef
+	x			p
+a	b			d
+a	x	b	c	d
+# ALTER TABLE ... FORCE writes the repaired flag back into the FRM
+ALTER TABLE mdev40417 FORCE;
+SHOW CREATE TABLE mdev40417;
+Table	Create Table
+mdev40417	CREATE TABLE `mdev40417` (
+  `blob_nodef` longtext /*M!100301 COMPRESSED*/ NOT NULL,
+  `blob_def` longtext /*M!100301 COMPRESSED*/ NOT NULL DEFAULT 'x',
+  `vc_nodef` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `vc_def` varchar(100) /*M!100301 COMPRESSED*/ NOT NULL DEFAULT '',
+  `plain_nodef` longtext NOT NULL,
+  `pad` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+DROP TABLE mdev40417;
+# End of 10.11 tests

mysql-test/main/column_compression.test

@@ -594,3 +594,85 @@ UPDATE t SET c=MID(c,2,4);
 DROP TABLE t;
 
 --echo # End of 10.6 tests
+
+--echo #
+--echo # MDEV-40417 A column with compression enabled automatically adds DEFAULT ''
+--echo #
+
+CREATE TABLE t (
+  c1 LONGTEXT COMPRESSED NOT NULL,
+  nc1 LONGTEXT NOT NULL,
+  c2 LONGTEXT COMPRESSED NOT NULL DEFAULT '',
+  nc2 LONGTEXT NOT NULL DEFAULT '',
+  c3 VARCHAR(100) COMPRESSED NOT NULL,
+  nc3 VARCHAR(100) NOT NULL
+) ENGINE=InnoDB;
+SHOW CREATE TABLE t;
+SELECT COLUMN_NAME, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t' ORDER BY ORDINAL_POSITION;
+
+--echo # A compressed column must be as mandatory as an uncompressed one
+--error ER_NO_DEFAULT_FOR_FIELD
+INSERT INTO t (nc1) VALUES ('x');
+--error ER_NO_DEFAULT_FOR_FIELD
+INSERT INTO t (c1, nc1, nc3) VALUES ('x', 'x', 'x');
+INSERT INTO t (c1, nc1, c3, nc3) VALUES ('x', 'x', 'x', 'x');
+SELECT * FROM t;
+DROP TABLE t;
+
+--echo # ALTER TABLE must not add an implicit default either
+CREATE TABLE t (c LONGTEXT COMPRESSED NOT NULL DEFAULT 'x') ENGINE=InnoDB;
+ALTER TABLE t MODIFY c LONGTEXT COMPRESSED NOT NULL;
+ALTER TABLE t ADD c2 VARCHAR(100) COMPRESSED NOT NULL;
+SHOW CREATE TABLE t;
+DROP TABLE t;
+
+--echo #
+--echo # MDEV-40417 upgrade: a table created before the fix is repaired when
+--echo # its FRM is read.  Only blobs can be repaired - for VARCHAR the FRM
+--echo # does not tell an explicit DEFAULT '' from the wrong implicit one,
+--echo # so those columns are deliberately left alone.
+--echo #
+--echo # MDEV-40417.frm was created by 10.11.19 as:
+--echo #   CREATE TABLE mdev40417 (
+--echo #     blob_nodef  LONGTEXT COMPRESSED NOT NULL,
+--echo #     blob_def    LONGTEXT COMPRESSED NOT NULL DEFAULT 'x',
+--echo #     vc_nodef    VARCHAR(100) COMPRESSED NOT NULL,
+--echo #     vc_def      VARCHAR(100) COMPRESSED NOT NULL DEFAULT '',
+--echo #     plain_nodef LONGTEXT NOT NULL,
+--echo #     pad         INT) ENGINE=MyISAM CHARSET=latin1;
+--echo #   INSERT INTO mdev40417 (plain_nodef, pad) VALUES ('p', 1);
+--echo #
+
+--copy_file std_data/MDEV-40417.frm $MYSQLD_DATADIR/test/mdev40417.frm
+--copy_file std_data/MDEV-40417.MYD $MYSQLD_DATADIR/test/mdev40417.MYD
+--copy_file std_data/MDEV-40417.MYI $MYSQLD_DATADIR/test/mdev40417.MYI
+
+SHOW CREATE TABLE mdev40417;
+SELECT COLUMN_NAME, COLUMN_DEFAULT FROM INFORMATION_SCHEMA.COLUMNS
+WHERE TABLE_SCHEMA='test' AND TABLE_NAME='mdev40417' ORDER BY ORDINAL_POSITION;
+--echo # The row that the old server allowed to be inserted is still readable
+SELECT * FROM mdev40417;
+
+--echo # blob_nodef is mandatory again
+--error ER_NO_DEFAULT_FOR_FIELD
+INSERT INTO mdev40417 (blob_def, vc_nodef, vc_def, plain_nodef)
+VALUES ('a', 'b', 'c', 'd');
+--echo # plain_nodef was never affected by the bug
+--error ER_NO_DEFAULT_FOR_FIELD
+INSERT INTO mdev40417 (blob_nodef, blob_def, vc_nodef, vc_def)
+VALUES ('a', 'b', 'c', 'd');
+--echo # blob_def keeps its explicit DEFAULT
+INSERT INTO mdev40417 (blob_nodef, vc_nodef, vc_def, plain_nodef)
+VALUES ('a', 'b', 'c', 'd');
+--echo # vc_nodef is not touched, it still has the old implicit DEFAULT ''
+INSERT INTO mdev40417 (blob_nodef, blob_def, plain_nodef) VALUES ('a', 'b', 'd');
+--sorted_result
+SELECT blob_nodef, blob_def, vc_nodef, vc_def, plain_nodef FROM mdev40417;
+
+--echo # ALTER TABLE ... FORCE writes the repaired flag back into the FRM
+ALTER TABLE mdev40417 FORCE;
+SHOW CREATE TABLE mdev40417;
+DROP TABLE mdev40417;
+
+--echo # End of 10.11 tests

sql/field.cc

@@ -10937,7 +10937,7 @@ bool Column_definition::check(THD *thd)
     We need to do this check here and in mysql_create_prepare_table() as
     sp_head::fill_field_definition() calls this function.
   */
-  if (!default_value && unireg_check == Field::NONE && (flags & NOT_NULL_FLAG))
+  if (!default_value && !has_default_function() && (flags & NOT_NULL_FLAG))
   {
     /*
       TIMESTAMP columns get implicit DEFAULT value when

sql/field.h

@@ -5486,7 +5486,12 @@ class Column_definition: public Sql_alloc,
 
   bool has_default_function() const
   {
-    return unireg_check != Field::NONE;
+    /*
+      TMYSQL_COMPRESSED is not a default function, it is stored in
+      unireg_check only to remember that the column is COMPRESSED.
+    */
+    return unireg_check != Field::NONE &&
+           unireg_check != Field::TMYSQL_COMPRESSED;
   }
 
   Field *make_field(TABLE_SHARE *share, MEM_ROOT *mem_root,

sql/table.cc

@@ -3315,6 +3315,32 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
               (uint) (share->table_check_constraints -
                       share->field_check_constraints));
 
+  /*
+    Tables created before MDEV-40417 was fixed have no FIELDFLAG_NO_DEFAULT
+    in the FRM for a COMPRESSED NOT NULL column without an explicit DEFAULT
+    clause, so such a column wrongly looks like it has DEFAULT ''.
+
+    This can be repaired for blobs: an explicit DEFAULT of a blob column is
+    always stored in the FRM as an expression, see
+    Column_definition::has_default_expression(), hence a blob that has no
+    default_value provably had no DEFAULT clause.  Note that the check is
+    also correct for FRMs written after the fix, where the flag is set
+    already.
+
+    VARCHAR and VARBINARY cannot be repaired here.  A constant DEFAULT of a
+    non-blob column is stored in the default record, exactly like the wrong
+    implicit default, which makes the two cases indistinguishable.
+  */
+  for (field_ptr= share->field; *field_ptr; field_ptr++)
+  {
+    reg_field= *field_ptr;
+    if (reg_field->compression_method() && !reg_field->default_value &&
+        !reg_field->vcol_info &&
+        (reg_field->flags & (NOT_NULL_FLAG | BLOB_FLAG)) ==
+          (NOT_NULL_FLAG | BLOB_FLAG))
+      reg_field->flags|= NO_DEFAULT_VALUE_FLAG;
+  }
+
   if (options.str)
   {
     DBUG_ASSERT(options.length);