[MDEV-34009] Introduce server-initiated instant failover mechanism (and TLS fixes)

.gitlab-ci.yml

@@ -298,6 +298,7 @@ centos7:
  main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
  main.mysql_upgrade_noengine : upgrade output order does not match the expected
  main.func_math : MDEV-20966 - Wrong error code
+ main.ssl_autoverify : See comments by dlenski@ on https://jira.mariadb.org/browse/MDEV-31855; almost everything about the "feature" AND the test are wrong
  " > skiplist
  - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $RESTART_POLICY
 

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "libmariadb"]
  path = libmariadb
- url = https://github.com/MariaDB/mariadb-connector-c.git
+ url = ../mariadb-connector-c
 [submodule "storage/rocksdb/rocksdb"]
  path = storage/rocksdb/rocksdb
  url = https://github.com/facebook/rocksdb.git

client/mysql.cc

@@ -248,7 +248,8 @@ static my_bool ignore_errors=0,wait_flag=0,quick=0,
  default_pager_set= 0, opt_sigint_ignore= 0,
  auto_vertical_output= 0, show_query_cost= 0,
  show_warnings= 0, executing_query= 0,
- ignore_spaces= 0, opt_binhex= 0, opt_progress_reports;
+ ignore_spaces= 0, opt_binhex= 0, opt_progress_reports,
+ opt_follow_instant_failovers= 1;
 static my_bool debug_info_flag, debug_check_flag, batch_abort_on_error;
 static my_bool column_types_flag;
 static my_bool preserve_comments= 0;
@@ -1503,6 +1504,8 @@ static bool do_connect(MYSQL *mysql, const char *host, const char *user,
  if (opt_secure_auth)
  mysql_options(mysql, MYSQL_SECURE_AUTH, (char *) &opt_secure_auth);
  SET_SSL_OPTS_WITH_CHECK(mysql);
+ mysql_options(mysql,MARIADB_OPT_FOLLOW_INSTANT_FAILOVERS,
+ (char*)&opt_follow_instant_failovers);
  if (opt_protocol)
  mysql_options(mysql,MYSQL_OPT_PROTOCOL,(char*)&opt_protocol);
  if (opt_plugin_dir && *opt_plugin_dir)
@@ -1805,6 +1808,11 @@ static struct my_option my_long_options[] =
  &opt_mysql_unix_port, &opt_mysql_unix_port, 0, GET_STR_ALLOC,
  REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 #include "sslopt-longopts.h"
+ {"follow-instant-failovers", 0,
+ "Follow instant failovers. Disable with "
+ "--disable-follow-instant-failovers. This option is enabled by default.",
+ &opt_follow_instant_failovers, &opt_follow_instant_failovers,
+ 0, GET_BOOL, OPT_ARG, 1, 0, 0, 0, 0, 0},
  {"table", 't', "Output in table format.", &output_tables,
  &output_tables, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
  {"tee", OPT_TEE,

client/mysqltest.cc

@@ -6015,8 +6015,8 @@ int connect_n_handle_errors(struct st_command *command,
 enum use_ssl
 {
  USE_SSL_FORBIDDEN = -1,
- USE_SSL_IF_POSSIBLE,
- USE_SSL_REQUIRED
+ USE_SSL_IF_POSSIBLE = 0,
+ USE_SSL_REQUIRED = 1
 };
 
 void do_connect(struct st_command *command)

include/sslopt-vars.h

@@ -36,9 +36,21 @@ SSL_STATIC char *opt_ssl_fp = 0;
 SSL_STATIC char *opt_ssl_fplist = 0;
 SSL_STATIC my_bool opt_ssl_verify_server_cert= 2;
 
+/* FIXME: there's a major TLS-related hole here. This macro treats
+ * USE_SSL_FORBIDDEN (-1) identically to USE_SSL_REQUIRED (1). See
+ * 'enum use_ssl' in mysqltest.cc for their definitions.
+ *
+ * This means that, among other things, when MTR tests THINK they are
+ * connecting WITHOUT SSL, they may actually be connecting with
+ * obligatory SSL.
+ *
+ * Tests that use 'NOSSL' are NOT TESTING WHAT THEY INTEND TO TEST.
+ */
+
 #define SET_SSL_OPTS(M) \
  do { \
- if (opt_use_ssl) \
+ /* if (opt_use_ssl == -1) {} else */ /* SSL forbidden */ \
+ if (opt_use_ssl) /* SSL required */ \
  { \
  mysql_ssl_set((M), opt_ssl_key, opt_ssl_cert, opt_ssl_ca, \
  opt_ssl_capath, opt_ssl_cipher); \
@@ -48,7 +60,7 @@ SSL_STATIC my_bool opt_ssl_verify_server_cert= 2;
  mysql_options((M), MARIADB_OPT_TLS_PEER_FP, opt_ssl_fp); \
  mysql_options((M), MARIADB_OPT_TLS_PEER_FP_LIST, opt_ssl_fplist); \
  } \
- else   \
+ else /* SSL if available */ \
  opt_ssl_verify_server_cert= 0; \
  mysql_options((M),MYSQL_OPT_SSL_VERIFY_SERVER_CERT, \
  &opt_ssl_verify_server_cert); \

mysql-test/main/instant_failover.cnf

@@ -0,0 +1,7 @@
+!include include/default_my.cnf
+
+[mysqld.1]
+[email protected]_EXTRA_PORT
+
+[ENV]
+MASTER_EXTRA_PORT= @OPT.port

mysql-test/main/instant_failover.result

@@ -0,0 +1,40 @@
+call mtr.add_suppression("\\[Warning\\] Redirecting connection \\d+ via \\S+ to INSTANT_FAILOVER_TARGET=\\S+ \\(INSTANT_FAILOVER_MODE=\\w+\\)");
+Verify that a TCP connection works with --disable-follow-instant-failovers (before enabling instant failover)
+default_port_tcp_works
+1
+Verify that connections via local socket and extra port SUCCEED
+connection default;
+connect local_sock,localhost,root,,test;
+disconnect local_sock;
+connect tcp_sock,127.0.0.1,root,,test,$MASTER_EXTRA_PORT;
+disconnect tcp_sock;
+Enable instant failover in its default mode (ON)
+connection default;
+set global instant_failover_mode=ON;
+set global instant_failover_target="127.0.0.1:$MASTER_EXTRA_PORT"
+With --disable-follow-instant-failovers, connecting to the default port should now fail
+With --follow-instant-failovers (the client library default), this should redirect to the extra port
+connect OKAY,127.0.0.1,root,,test,$MASTER_MYPORT;
+disconnect OKAY;
+Setup a redirect loop, and verify that connections fail due to the loop
+connection default;
+set global instant_failover_target="127.0.0.1:$MASTER_MYPORT"
+connect(127.0.0.1,root,,test,MASTER_MYPORT,MASTER_MYSOCK);
+connect fail_con,127.0.0.1,root,,test,$MASTER_MYPORT;
+ERROR HY000: Too many instant failovers (>= 8)
+Change instant failover mode to ALL, and verify that even connections via local socket and extra port now FAIL due to the loop
+connection default;
+set global instant_failover_mode=ALL;
+connect(localhost,root,,test,MASTER_MYPORT,MASTER_MYSOCK);
+connect fail_con,localhost,root,,test;
+ERROR HY000: Too many instant failovers (>= 8)
+connect(127.0.0.1,root,,test,MASTER_EXTRA_PORT,MASTER_MYSOCK);
+connect fail_con,127.0.0.1,root,,test,$MASTER_EXTRA_PORT;
+ERROR HY000: Too many instant failovers (>= 8)
+Turn instant failover back off
+connection default;
+set global instant_failover_mode=OFF;
+set global instant_failover_target=DEFAULT;
+Connections should now succeed again, even with --disable-follow-instant-failovers
+default_port_tcp_works
+1

mysql-test/main/instant_failover.test

@@ -0,0 +1,64 @@
+# We need to ignore the redirection warnings in the server logs, e.g.
+# "Redirecting connection N via SSL/TLS to INSTANT_FAILOVER_TARGET=127.0.0.1:$MASTER_EXTRA_PORT (INSTANT_FAILOVER_MODE=ON)"
+call mtr.add_suppression("\\[Warning\\] Redirecting connection \\d+ via \\S+ to INSTANT_FAILOVER_TARGET=\\S+ \\(INSTANT_FAILOVER_MODE=\\w+\\)");
+
+##########
+--echo Verify that a TCP connection works with --disable-follow-instant-failovers (before enabling instant failover)
+--exec $MYSQL --disable-follow-instant-failovers --host=127.0.0.1 --port=$MASTER_MYPORT test -e "select 1 as default_port_tcp_works"
+
+--echo Verify that connections via local socket and extra port SUCCEED
+--connection default
+--connect local_sock,localhost,root,,test
+--disconnect local_sock
+--connect tcp_sock,127.0.0.1,root,,test,$MASTER_EXTRA_PORT
+--disconnect tcp_sock
+
+##########
+--echo Enable instant failover in its default mode (ON)
+--connection default
+set global instant_failover_mode=ON;
+--echo set global instant_failover_target="127.0.0.1:\$MASTER_EXTRA_PORT"
+--disable_query_log
+--eval set global instant_failover_target="127.0.0.1:$MASTER_EXTRA_PORT"
+--enable_query_log
+
+##########
+--echo With --disable-follow-instant-failovers, connecting to the default port should now fail
+--error 1
+--exec $MYSQL --disable-follow-instant-failovers --host=127.0.0.1 --port=$MASTER_MYPORT test -e "select 1"
+
+--echo With --follow-instant-failovers (the client library default), this should redirect to the extra port
+--connect OKAY,127.0.0.1,root,,test,$MASTER_MYPORT
+--disconnect OKAY
+
+##########
+--echo Setup a redirect loop, and verify that connections fail due to the loop
+--connection default
+--echo set global instant_failover_target="127.0.0.1:\$MASTER_MYPORT"
+--disable_query_log
+--eval set global instant_failover_target="127.0.0.1:$MASTER_MYPORT"
+--enable_query_log
+--replace_result $MASTER_MYSOCK MASTER_MYSOCK $MASTER_MYPORT MASTER_MYPORT
+--error ER_INSTANT_FAILOVER
+--connect fail_con,127.0.0.1,root,,test,$MASTER_MYPORT
+
+##########
+--echo Change instant failover mode to ALL, and verify that even connections via local socket and extra port now FAIL due to the loop
+--connection default
+set global instant_failover_mode=ALL;
+--replace_result $MASTER_MYSOCK MASTER_MYSOCK $MASTER_MYPORT MASTER_MYPORT
+--error ER_INSTANT_FAILOVER
+--connect fail_con,localhost,root,,test
+--replace_result $MASTER_MYSOCK MASTER_MYSOCK $MASTER_EXTRA_PORT MASTER_EXTRA_PORT
+--error ER_INSTANT_FAILOVER
+--connect fail_con,127.0.0.1,root,,test,$MASTER_EXTRA_PORT
+
+##########
+--echo Turn instant failover back off
+--connection default
+set global instant_failover_mode=OFF;
+set global instant_failover_target=DEFAULT;
+
+--echo Connections should now succeed again, even with --disable-follow-instant-failovers
+--error 0
+--exec $MYSQL --disable-follow-instant-failovers --host=127.0.0.1 --port=$MASTER_MYPORT test -e "select 1 as default_port_tcp_works"

mysql-test/main/mysql.result

@@ -649,7 +649,7 @@ MYSQL --disable-ssl-verify-server-cert -e "\s"
 SSL: Cipher in use is XXX, cert is UNKNOWN
 
 
-MYSQL --ssl-verify-server-cert -e "\s"
+MYSQL --ssl-verify-server-cert --ssl-ca=cacert.pem -e "\s"
 
 SSL: Cipher in use is XXX, cert is OK
 

mysql-test/main/mysql.test

@@ -733,9 +733,9 @@ create user ser@localhost identified by "ass";
 --echo MYSQL --disable-ssl-verify-server-cert -e "\\s"
 --replace_regex /^.[^S].*// /\b[-A-Z_0-9]+,/XXX,/
 --exec $MYSQL -user -pass --disable-ssl-verify-server-cert -e "\\s"
---echo MYSQL --ssl-verify-server-cert -e "\\s"
+--echo MYSQL --ssl-verify-server-cert --ssl-ca=cacert.pem -e "\\s"
 --replace_regex /^.[^S].*// /\b[-A-Z_0-9]+,/XXX,/
---exec $MYSQL -user -pass --ssl-verify-server-cert -e "\\s"
+--exec $MYSQL -user -pass --ssl-verify-server-cert --ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem -e "\\s"
 drop user ser@localhost;
 
 --echo #

mysql-test/main/mysqld--help.result

@@ -433,6 +433,20 @@ The following specify which files/extra groups are read (specified before remain
  Set the replication role. One of: MASTER, SLAVE
  --init-slave=name Command(s) that are executed by a slave server each time
  the SQL thread starts
+ --instant-failover-mode=name 
+ Instant failover mode. Possible modes are: OFF - No
+ instant failover, ON: Unconditionally redirect new
+ clients connecting over the network via the standard
+ server port to INSTANT_FAILOVER_TARGET (no redirection of
+ local socket-based connections, nor of connections to the
+ EXTRA_PORT), ALL: Unconditionally redirect all new
+ clients to INSTANT_FAILOVER_TARGET (even via local
+ socket-based connections and the EXTRA_PORT).
+ --instant-failover-target=name 
+ Instant failover target. This should be a hostname, an IP
+ address, or a hostname or IP address followed by ':PORT'.
+ Instant failover will not be activated unless
+ INSTANT_FAILOVER_MODE is also set.
  --interactive-timeout=# 
  The number of seconds the server waits for activity on an
  interactive connection before closing it
@@ -1706,6 +1720,8 @@ init-connect
 init-file (No default value)
 init-rpl-role MASTER
 init-slave 
+instant-failover-mode OFF
+instant-failover-target (No default value)
 interactive-timeout 28800
 join-buffer-size 262144
 join-buffer-space-limit 2097152

mysql-test/main/ssl_7937,nossl.result

@@ -3,15 +3,13 @@ select if(variable_value > '','yes','no') as 'have_ssl'
  from information_schema.session_status
 where variable_name='ssl_cipher';
 mysql --ssl-ca=cacert.pem -e "call test.have_ssl()"
-have_ssl
-no
+ERROR 2026 (HY000): Client requires TLS/SSL, but the server does not support it
 mysql --ssl -e "call test.have_ssl()"
-have_ssl
-no
+ERROR 2026 (HY000): Client requires TLS/SSL, but the server does not support it
 mysql --ssl-ca=cacert.pem --ssl-verify-server-cert -e "call test.have_ssl()"
-ERROR 2026 (HY000): TLS/SSL error: SSL is required, but the server does not support it
+ERROR 2026 (HY000): Client requires TLS/SSL, but the server does not support it
 mysql --ssl --ssl-verify-server-cert -e "call test.have_ssl()"
-ERROR 2026 (HY000): TLS/SSL error: SSL is required, but the server does not support it
+ERROR 2026 (HY000): Client requires TLS/SSL, but the server does not support it
 #
 # MDEV-27105 --ssl option set as default for mariadb CLI
 #

mysql-test/main/ssl_7937.test

@@ -26,6 +26,14 @@ if($is_win)
 {
  let $host=--host=127.0.0.2;
 }
+
+# The replace_regex below replaces
+# "self signed certificate in certificate chain" with "Failed to verify the server certificate"
+#
+# This replacement was added in 6484288cd260cc9ad34d93a35502e66c034f01a7, and it
+# is intended to paper over a difference between various versions of OpenSSL (and its derivatives)
+# in terms of exactly what error message is printed in case of a TLS error caused by a
+# self-signed certificate.
 --echo mysql --ssl --ssl-verify-server-cert -e "call test.have_ssl()"
 --replace_regex /TLS\/SSL error.*certificate[^\n]*/TLS\/SSL error: Failed to verify the server certificate/
 --exec $MYSQL --protocol tcp $host --ssl --ssl-verify-server-cert -e "call test.have_ssl()" 2>&1

mysql-test/main/ssl_crl.result

@@ -2,4 +2,4 @@
 Variable_name Value
 Ssl_version TLS_VERSION
 # try logging in with a certificate in the server's --ssl-crl : should fail
-ERROR 2026 (HY000): TLS/SSL error: sslv3 alert certificate revoked
+ERROR 2026 (HY000): TLS/SSL error: ssl/tls alert certificate revoked

mysql-test/main/ssl_crl.test

@@ -8,6 +8,7 @@
 
 --echo # try logging in with a certificate in the server's --ssl-crl : should fail
 # OpenSSL 1.1.1a correctly rejects the certificate, but the error message is different
---replace_regex /ERROR 2013 \(HY000\): Lost connection to server at '.*', system error: [0-9]+/ERROR 2026 (HY000): TLS\/SSL error: sslv3 alert certificate revoked/
+# In OpenSSL 3.2.0, the error message has changed again ("sslv3 alert" changed to "ssl/tls alert" in https://github.com/openssl/openssl/commit/81b741f68984b2620166d0d6271fbd946bab9e7f)
+--replace_regex /ERROR 2013 \(HY000\): Lost connection to server at '.*', system error: [0-9]+/ERROR 2026 (HY000): TLS\/SSL error: ssl\/tls alert certificate revoked/ /sslv3 alert/ssl\/tls alert/
 --error 1
 --exec $MYSQL --ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem --ssl-key=$MYSQL_TEST_DIR/std_data/client-key.pem --ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem test -e "SHOW STATUS LIKE 'Ssl_version'" 2>&1

mysql-test/main/ssl_fp.result

@@ -4,7 +4,7 @@ return (select if(variable_value > '','yes','no') as 'have_ssl'
 where variable_name='ssl_cipher');
 # mysql --protocol tcp -uroot --ssl-verify-server-cert -e "select test.have_ssl()"
 ERROR 2026 (HY000): TLS/SSL error: Failed to verify the server certificate
-# mysql --protocol tcp -uroot --ssl-fp=F1:D0:08:AF:A1:D2:F4:15:79:B4:39:06:41:F4:20:96:F1:90:A9:65 --ssl-verify-server-cert -e "select test.have_ssl()"
+# mysql --protocol tcp -uroot --ssl-fp=F1:D0:08:AF:A1:D2:F4:15:79:B4:39:06:41:F4:20:96:F1:90:A9:65 -e "select test.have_ssl()"
 test.have_ssl()
 yes
 # mysql --protocol tcp -uroot --ssl-fp=00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33 --disable-ssl-verify-server-cert -e "select test.have_ssl()"

mysql-test/main/ssl_fp.test

@@ -21,8 +21,8 @@ if($is_win)
 #
 # fingerprint based cert verification:
 #
---echo # mysql --protocol tcp -uroot --ssl-fp=F1:D0:08:AF:A1:D2:F4:15:79:B4:39:06:41:F4:20:96:F1:90:A9:65 --ssl-verify-server-cert -e "select test.have_ssl()"
---exec $MYSQL --protocol tcp $host -uroot --ssl-fp=F1:D0:08:AF:A1:D2:F4:15:79:B4:39:06:41:F4:20:96:F1:90:A9:65 --ssl-verify-server-cert -e "select test.have_ssl()" 2>&1
+--echo # mysql --protocol tcp -uroot --ssl-fp=F1:D0:08:AF:A1:D2:F4:15:79:B4:39:06:41:F4:20:96:F1:90:A9:65 -e "select test.have_ssl()"
+--exec $MYSQL --protocol tcp $host -uroot --ssl-fp=F1:D0:08:AF:A1:D2:F4:15:79:B4:39:06:41:F4:20:96:F1:90:A9:65 -e "select test.have_ssl()" 2>&1
 #
 # wrong fingerprint fails even with --disable-ssl-verify-server-cert
 #

mysql-test/mariadb-test-run.pl

@@ -3111,7 +3111,7 @@ sub mysql_install_db {
  mtr_add_arg($args, "--core-file");
  mtr_add_arg($args, "--console");
  mtr_add_arg($args, "--character-set-server=latin1");
- mtr_add_arg($args, "--disable-performance-schema");
+ mtr_add_arg($args, "--loose-disable-performance-schema");
 
  if ( $opt_debug )
  {

sql/mysqld.cc

@@ -1526,6 +1526,12 @@ static Atomic_counter<uint> extra_connection_count;
 
 my_bool opt_gtid_strict_mode= FALSE;
 
+/**
+ Instant failover
+ */
+
+const char *instant_failover_target = NullS;
+ulong instant_failover_mode= INSTANT_FAILOVER_MODE_OFF;
 
 /* Function declarations */
 

sql/mysqld.h

@@ -1002,6 +1002,14 @@ extern ulong opt_binlog_dbug_fsync_sleep;
 extern uint volatile global_disable_checkpoint;
 extern my_bool opt_help;
 
+extern const char *instant_failover_target;
+enum enum_instant_failover_mode {
+ INSTANT_FAILOVER_MODE_OFF = 0,
+ INSTANT_FAILOVER_MODE_ON = 1,
+ INSTANT_FAILOVER_MODE_ALL = 2
+};
+extern ulong instant_failover_mode;
+
 extern int mysqld_main(int argc, char **argv);
 
 #ifdef _WIN32

sql/share/errmsg-utf8.txt

@@ -12280,3 +12280,7 @@ ER_SEQUENCE_TABLE_CANNOT_HAVE_ANY_CONSTRAINTS
  eng "Sequence tables cannot have any constraints"
 ER_SEQUENCE_TABLE_ORDER_BY
  eng "ORDER BY"
+ER_INSTANT_FAILOVER
+ eng "|Server is directing clients to the alternative server '%1$s'|%1$s"
+ fra "|Ce serveur dirige ses clients vers le serveur alternatif '%1$s'|%1$s"
+ spa "|Este servidor está dirigiendo sus clientes al servidor alternativo '%1$s'|%1$s"