Binlog Commit Optimization for Large Transactions

mysql-test/r/all_persisted_variables.result

@@ -48,17 +48,17 @@ include/assert.inc [Expect 500+ variables in the table. Due to open Bugs, we are
 
 # Test SET PERSIST
 
-include/assert.inc [Expect 451 persisted variables in the table.]
+include/assert.inc [Expect 452 persisted variables in the table.]
 
 ************************************************************
 * 3. Restart server, it must preserve the persisted variable
 *    settings. Verify persisted configuration.
 ************************************************************
 # restart
 
-include/assert.inc [Expect 451 persisted variables in persisted_variables table.]
-include/assert.inc [Expect 451 persisted variables shown as PERSISTED in variables_info table.]
-include/assert.inc [Expect 451 persisted variables with matching peristed and global values.]
+include/assert.inc [Expect 452 persisted variables in persisted_variables table.]
+include/assert.inc [Expect 452 persisted variables shown as PERSISTED in variables_info table.]
+include/assert.inc [Expect 452 persisted variables with matching peristed and global values.]
 
 ************************************************************
 * 4. Test RESET PERSIST IF EXISTS. Verify persisted variable

mysql-test/r/mysqld--help-notwin.result

@@ -187,6 +187,15 @@ The following options may be given as the first argument:
  --binlog-ignore-db=name 
  Exclude updates to the specified database when writing
  the binary log.
+ --binlog-large-commit-threshold=# 
+ Increases transaction concurrency for large transactions
+ (i.e. those with sizes larger than this value) by
+ renaming the large transaction's binlog cache temporary
+ file to a new binary log file at commit time, instead of
+ copying the transaction cache data to the end of the
+ active binary log file while holding a lock that prevents
+ other transactions from binlogging. 0 disables the
+ feature.
  --binlog-max-flush-queue-time=# 
  The maximum time that the binary log group commit will
  keep reading transactions before it flush the
@@ -1696,6 +1705,7 @@ binlog-format ROW
 binlog-group-commit-sync-delay 0
 binlog-group-commit-sync-no-delay-count 0
 binlog-gtid-simple-recovery TRUE
+binlog-large-commit-threshold 134217728
 binlog-max-flush-queue-time 0
 binlog-order-commits TRUE
 binlog-rotate-encryption-master-key-at-startup FALSE

mysql-test/suite/binlog/r/binlog_commit_by_rotate_atomic.result

@@ -0,0 +1,66 @@
+RESET BINARY LOGS AND GTIDS;
+#
+# A binlog cache file is created in the #binlog_cache_files directory
+# and it is deleted at disconnect.
+#
+CREATE TABLE t1 (c1 LONGTEXT) ENGINE = InnoDB;
+# list #binlog_cache_files/ (empty)
+INSERT INTO t1 values(repeat("1", 5242880));
+INSERT INTO t1 values(repeat("1", 5242880));
+FLUSH BINARY LOGS;
+# list #binlog_cache_files/ (one cache file)
+ML_BINLOG_CACHE_FILE
+# The binlog cache file is deleted at disconnection.
+# list #binlog_cache_files/ (empty)
+#
+# Reserved space is not big enough for the header events. The rename is
+# not done, but the rotation already created a fresh binary log file, so
+# the transaction falls back to a normal commit into that new file.
+#
+SET GLOBAL binlog_large_commit_threshold = 10 * 1024 * 1024;
+SET SESSION debug = "+d,simulate_reserve_size_not_enough";
+UPDATE t1 SET c1 = repeat('2', 5242880);
+SET SESSION debug = "-d,simulate_reserve_size_not_enough";
+include/rpl/assert_binlog_events.inc [!Gtid_or_anon]
+#
+# Crash happens before renaming the file.
+#
+SET GLOBAL binlog_large_commit_threshold = 10 * 1024 * 1024;
+SET SESSION debug = "+d,binlog_commit_by_rotate_crash_before_rename";
+UPDATE t1 SET c1 = repeat('4', 5242880);
+ERROR HY000: Lost connection to MySQL server during query
+# One cache file left after crash.
+# list #binlog_cache_files/
+ML_BINLOG_CACHE_FILE
+non_binlog_cache
+# restart
+# The cache files are deleted at startup (only the foreign file remains).
+# list #binlog_cache_files/
+non_binlog_cache
+include/assert_grep.inc [warning: non_binlog_cache is not a binlog cache file]
+SELECT * FROM t1 WHERE c1 = 4;
+c1
+include/rpl/assert_binlog_events.inc [()]
+#
+# Crash happens just after the rotation renamed the cache to the new
+# binary log file (the whole transaction, including its Xid event, is
+# already in that file) but before the engine commit finished. On restart
+# crash recovery commits the transaction, and no cache file is left.
+#
+SET GLOBAL binlog_large_commit_threshold = 10 * 1024 * 1024;
+BEGIN;
+UPDATE t1 SET c1 = repeat('5', 5242880);
+SAVEPOINT s1;
+UPDATE t1 SET c1 = repeat('6', 5242880);
+UPDATE t1 SET c1 = repeat('7', 5242880);
+ROLLBACK TO SAVEPOINT s1;
+INSERT INTO t1 VALUES('a');
+SET SESSION debug = "+d,binlog_commit_by_rotate_crash_after_rotate";
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+# No cache file left after crash.
+# list #binlog_cache_files/
+# restart
+include/rpl/assert_binlog_events.inc [Ignorable]
+call mtr.add_suppression(".*not a binlog cache file.*");
+DROP TABLE t1;

mysql-test/suite/binlog/t/binlog_commit_by_rotate_atomic-master.opt

@@ -0,0 +1 @@
+--log-bin=master-bin --binlog-large-commit-threshold=10485760

mysql-test/suite/binlog/t/binlog_commit_by_rotate_atomic.test

@@ -0,0 +1,135 @@
+################################################################################
+# Rename binlog cache to binlog file (ported to MySQL trunk)
+#
+# It verifies that the rename logic is handled correctly if an error or a crash
+# happens.
+################################################################################
+--source include/have_binlog_format_row.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+RESET BINARY LOGS AND GTIDS;
+
+--echo #
+--echo # A binlog cache file is created in the #binlog_cache_files directory
+--echo # and it is deleted at disconnect.
+--echo #
+--connect(con1,localhost,root,,)
+CREATE TABLE t1 (c1 LONGTEXT) ENGINE = InnoDB;
+
+--echo # list #binlog_cache_files/ (empty)
+--let $datadir= `SELECT @@datadir`
+--list_files $datadir/#binlog_cache_files
+
+INSERT INTO t1 values(repeat("1", 5242880));
+INSERT INTO t1 values(repeat("1", 5242880));
+FLUSH BINARY LOGS;
+
+--echo # list #binlog_cache_files/ (one cache file)
+--replace_regex /ML_[0-9]+/ML_BINLOG_CACHE_FILE/
+--list_files $datadir/#binlog_cache_files
+
+--disconnect con1
+--connection default
+# Wait until the cache file of the disconnected session has been deleted.
+--let $wait_condition= SELECT COUNT(*) = 0 FROM performance_schema.threads WHERE PROCESSLIST_USER = 'root' AND PROCESSLIST_ID != CONNECTION_ID()
+--source include/wait_condition.inc
+
+--echo # The binlog cache file is deleted at disconnection.
+--echo # list #binlog_cache_files/ (empty)
+--list_files $datadir/#binlog_cache_files
+
+--echo #
+--echo # Reserved space is not big enough for the header events. The rename is
+--echo # not done, but the rotation already created a fresh binary log file, so
+--echo # the transaction falls back to a normal commit into that new file.
+--echo #
+SET GLOBAL binlog_large_commit_threshold = 10 * 1024 * 1024;
+--let $before_file= query_get_value(SHOW BINARY LOG STATUS, File, 1)
+SET SESSION debug = "+d,simulate_reserve_size_not_enough";
+UPDATE t1 SET c1 = repeat('2', 5242880);
+SET SESSION debug = "-d,simulate_reserve_size_not_enough";
+
+--let $event_sequence= !Gtid_or_anon
+--let $limit= 3
+--let $binlog_file= master-bin.000003
+--let $binlog_position= 4
+--source include/rpl/assert_binlog_events.inc
+
+--echo #
+--echo # Crash happens before renaming the file.
+--echo #
+SET GLOBAL binlog_large_commit_threshold = 10 * 1024 * 1024;
+--let $datadir= `SELECT @@datadir`
+
+SET SESSION debug = "+d,binlog_commit_by_rotate_crash_before_rename";
+--source include/expect_crash.inc
+--error CR_SERVER_LOST
+UPDATE t1 SET c1 = repeat('4', 5242880);
+
+--write_file $datadir/#binlog_cache_files/non_binlog_cache
+It is not a binlog cache file
+EOF
+
+--echo # One cache file left after crash.
+--echo # list #binlog_cache_files/
+--replace_regex /ML_[0-9]+/ML_BINLOG_CACHE_FILE/
+--list_files $datadir/#binlog_cache_files
+
+--source include/start_mysqld.inc
+
+--echo # The cache files are deleted at startup (only the foreign file remains).
+--echo # list #binlog_cache_files/
+--list_files $datadir/#binlog_cache_files
+
+--let $assert_text= warning: non_binlog_cache is not a binlog cache file
+--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err
+--let $assert_select= not a binlog cache file
+--let $assert_count= 1
+--let $assert_only_after= CURRENT_TEST: binlog.binlog_commit_by_rotate_atomic
+--source include/assert_grep.inc
+
+--remove_file $datadir/#binlog_cache_files/non_binlog_cache
+
+SELECT * FROM t1 WHERE c1 = 4;
+
+--let $event_sequence= ()
+--let $binlog_file= master-bin.000004
+--let $binlog_position= 4
+--source include/rpl/assert_binlog_events.inc
+
+--echo #
+--echo # Crash happens just after the rotation renamed the cache to the new
+--echo # binary log file (the whole transaction, including its Xid event, is
+--echo # already in that file) but before the engine commit finished. On restart
+--echo # crash recovery commits the transaction, and no cache file is left.
+--echo #
+SET GLOBAL binlog_large_commit_threshold = 10 * 1024 * 1024;
+
+BEGIN;
+UPDATE t1 SET c1 = repeat('5', 5242880);
+SAVEPOINT s1;
+UPDATE t1 SET c1 = repeat('6', 5242880);
+UPDATE t1 SET c1 = repeat('7', 5242880);
+ROLLBACK TO SAVEPOINT s1;
+INSERT INTO t1 VALUES('a');
+
+SET SESSION debug = "+d,binlog_commit_by_rotate_crash_after_rotate";
+--source include/expect_crash.inc
+--error CR_SERVER_LOST
+COMMIT;
+
+--echo # No cache file left after crash.
+--echo # list #binlog_cache_files/
+--replace_regex /ML_[0-9]+/ML_BINLOG_CACHE_FILE/
+--list_files $datadir/#binlog_cache_files
+
+--source include/start_mysqld.inc
+
+--let $event_sequence= Ignorable
+--let $limit= 3
+--let $binlog_file= master-bin.000005
+--let $binlog_position= 4
+--source include/rpl/assert_binlog_events.inc
+
+call mtr.add_suppression(".*not a binlog cache file.*");
+DROP TABLE t1;

mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result

@@ -47,7 +47,7 @@ INSERT INTO aliases(name) VALUES
 ('slave_parallel_workers'), ('slave_pending_jobs_size_max'),
 ('pseudo_slave_mode'), ('skip_slave_start');
 
-include/assert.inc [Expect 110 variables in the table.]
+include/assert.inc [Expect 111 variables in the table.]
 
 # Test SET PERSIST_ONLY
 SET PERSIST_ONLY binlog_cache_size = @@GLOBAL.binlog_cache_size;
@@ -64,6 +64,7 @@ Warning	1287	'@@binlog_format' is deprecated and will be removed in a future rel
 SET PERSIST_ONLY binlog_group_commit_sync_delay = @@GLOBAL.binlog_group_commit_sync_delay;
 SET PERSIST_ONLY binlog_group_commit_sync_no_delay_count = @@GLOBAL.binlog_group_commit_sync_no_delay_count;
 SET PERSIST_ONLY binlog_gtid_simple_recovery = @@GLOBAL.binlog_gtid_simple_recovery;
+SET PERSIST_ONLY binlog_large_commit_threshold = @@GLOBAL.binlog_large_commit_threshold;
 SET PERSIST_ONLY binlog_max_flush_queue_time = @@GLOBAL.binlog_max_flush_queue_time;
 Warnings:
 Warning	1287	'@@binlog_max_flush_queue_time' is deprecated and will be removed in a future release.
@@ -257,16 +258,16 @@ Warning	1287	'@@sync_relay_log_info' is deprecated and will be removed in a futu
 Warning	1287	'@@sync_relay_log_info' is deprecated and will be removed in a future release.
 SET PERSIST_ONLY sync_source_info = @@GLOBAL.sync_source_info;
 
-include/assert.inc [Expect 99 persisted variables in persisted_variables table.]
+include/assert.inc [Expect 100 persisted variables in persisted_variables table.]
 
 ############################################################
 # 2. Restart server, it must preserve the persisted variable
 #    settings. Verify persisted configuration.
 # restart
 
-include/assert.inc [Expect 99 persisted variables in persisted_variables table.]
-include/assert.inc [Expect 99 persisted variables shown as PERSISTED in variables_info table.]
-include/assert.inc [Expect 99 persisted variables with matching persisted and global values.]
+include/assert.inc [Expect 100 persisted variables in persisted_variables table.]
+include/assert.inc [Expect 100 persisted variables shown as PERSISTED in variables_info table.]
+include/assert.inc [Expect 100 persisted variables with matching persisted and global values.]
 
 ############################################################
 # 3. Test RESET PERSIST. Verify persisted variable settings
@@ -282,6 +283,7 @@ RESET PERSIST binlog_format;
 RESET PERSIST binlog_group_commit_sync_delay;
 RESET PERSIST binlog_group_commit_sync_no_delay_count;
 RESET PERSIST binlog_gtid_simple_recovery;
+RESET PERSIST binlog_large_commit_threshold;
 RESET PERSIST binlog_max_flush_queue_time;
 RESET PERSIST binlog_order_commits;
 RESET PERSIST binlog_rotate_encryption_master_key_at_startup;

mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test

@@ -84,7 +84,7 @@ INSERT INTO aliases(name) VALUES
 # If this count differs, it means a variable has been added or removed.
 # In that case, this testcase needs to be updated accordingly.
 --echo
---let $expected = 110
+--let $expected = 111
 --let $assert_text = Expect $expected variables in the table.
 --let $assert_cond = [SELECT COUNT(*) as count FROM rplvars, count, 1] = $expected
 --source include/assert.inc
@@ -116,7 +116,7 @@ while ( $varid <= $countvars )
 }
 
 --echo
---let $expected = 99
+--let $expected = 100
 --let $assert_text = Expect $expected persisted variables in persisted_variables table.
 --let $assert_cond = [SELECT COUNT(*) as count FROM performance_schema.persisted_variables, count, 1] = $expected
 --source include/assert.inc