Merge 10.6 into 10.7

.gitlab-ci.yml

@@ -118,8 +118,10 @@ fedora-ninja:
  - mkdir builddir; cd builddir
  - cmake -DRPM=generic $CMAKE_FLAGS -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -G Ninja .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
  - ninja -t graph > ../dependencies.dot && dot -Tpng -o ../dependencies.png ../dependencies.dot
- - eatmydata ninja package --verbose 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
- # Ninja builds are not affected by bug https://jira.mariadb.org/browse/MDEV-25968
+ - eatmydata ninja package -j 2 --verbose 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ # @TODO: Unlike other builds, the Ninja builds using Gitlab.com runners don't get stuck, but they do get
+ # stuck on runners with more processors, see https://jira.mariadb.org/browse/MDEV-25968.
+ # Thus, use the same limitation on Ninja builds as well to ensure it never gets stuck due to this bug.
  - ninja test
  - *rpm_listfiles
  - mkdir ../rpm; mv *.rpm ../rpm

appveyor.yml

@@ -25,6 +25,6 @@ test_script:
  - set PATH=C:\Strawberry\perl\bin;%PATH%;C:\Program Files (x86)\Windows Kits\10\Debuggers\x64
  - cd %APPVEYOR_BUILD_FOLDER%\_build\mysql-test
  - set /A parallel=4*%NUMBER_OF_PROCESSORS%
- - perl mysql-test-run.pl --force --max-test-fail=10 --retry=2 -parallel=%parallel% --testcase-timeout=4 --suite=main --skip-test-list=%APPVEYOR_BUILD_FOLDER%\win\appveyor_skip_tests.txt --mysqld=--loose-innodb-flush-log-at-trx-commit=2
+ - perl mysql-test-run.pl --force --max-test-fail=10 --retry=2 --parallel=%parallel% --testcase-timeout=4 --suite=main --skip-test-list=%APPVEYOR_BUILD_FOLDER%\win\appveyor_skip_tests.txt --mysqld=--loose-innodb-flush-log-at-trx-commit=2
 
 image: Visual Studio 2022

client/mysql.cc

@@ -42,7 +42,7 @@
 #include <violite.h>
 #include <my_sys.h>
 #include <source_revision.h>
-#if defined(USE_LIBEDIT_INTERFACE) && defined(HAVE_LOCALE_H)
+#if defined(HAVE_LOCALE_H)
 #include <locale.h>
 #endif
 
@@ -2751,6 +2751,9 @@ static void initialize_readline ()
  /* Allow conditional parsing of the ~/.inputrc file. */
  rl_readline_name= (char *) "mysql";
  rl_terminal_name= getenv("TERM");
+#ifdef HAVE_SETLOCALE
+ setlocale(LC_ALL,"");
+#endif
 
  /* Tell the completer that we want a crack first. */
 #if defined(USE_NEW_READLINE_INTERFACE)
@@ -2759,9 +2762,6 @@ static void initialize_readline ()
 
  rl_add_defun("magic-space", (rl_command_func_t *)&fake_magic_space, -1);
 #elif defined(USE_LIBEDIT_INTERFACE)
-#ifdef HAVE_LOCALE_H
- setlocale(LC_ALL,""); /* so as libedit use isprint */
-#endif
  rl_attempted_completion_function= (CPPFunction*)&new_mysql_completion;
  rl_completion_entry_function= &no_completion;
  rl_add_defun("magic-space", (Function*)&fake_magic_space, -1);

cmake/libutils.cmake

@@ -154,7 +154,11 @@ MACRO(MERGE_STATIC_LIBS TARGET OUTPUT_NAME LIBS_TO_MERGE)
  # (can be a static or shared lib)
  IF(LIB_TYPE STREQUAL "STATIC_LIBRARY")
  SET(STATIC_TGTS ${STATIC_TGTS} ${LIB})
- SET(STATIC_LIBS ${STATIC_LIBS} $<TARGET_FILE:${LIB}>)
+ IF(MSVC)
+ TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ${LIB})
+ ELSE()
+ SET(STATIC_LIBS ${STATIC_LIBS} $<TARGET_FILE:${LIB}>)
+ ENDIF()
  ADD_DEPENDENCIES(${TARGET} ${LIB})
  # Extract dependent OS libraries
  GET_DEPENDEND_OS_LIBS(${LIB} LIB_OSLIBS)

cmake/os/FreeBSD.cmake

@@ -28,7 +28,3 @@ SET(EXECINFO_ROOT /usr/local CACHE INTERNAL "Where to find execinfo library and
 INCLUDE_DIRECTORIES(${EXECINFO_ROOT}/include)
 SET(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${EXECINFO_ROOT}/include)
 SET(ENV{LIB} "$ENV{LIB}:${EXECINFO_ROOT}/lib")
-FIND_LIBRARY(EXECINFO NAMES execinfo)
-IF(EXECINFO)
- SET(LIBEXECINFO ${EXECINFO})
-ENDIF()

cmake/os/WindowsCache.cmake

@@ -296,6 +296,11 @@ SET(HAVE_SOCKPEERCRED CACHE INTERNAL "")
 SET(HAVE_ABI_CXA_DEMANGLE CACHE INTERNAL "")
 SET(HAVE_GCC_C11_ATOMICS CACHE INTERNAL "")
 SET(HAVE_VISIBILITY_HIDDEN CACHE INTERNAL "")
+SET(HAVE_GCC_C11_ATOMICS_WITHOUT_LIBATOMIC CACHE INTERNAL "")
+SET(HAVE_GCC_C11_ATOMICS_WITH_LIBATOMIC CACHE INTERNAL "")
+SET(HAVE_MPROTECT CACHE INTERNAL "")
+SET(HAVE_C__Werror CACHE INTERNAL "")
+SET(HAVE_NANOSLEEP CACHE INTERNAL "")
 SET(HAVE_ACCEPT4 CACHE INTERNAL "")
 SET(HAVE_LINK_H CACHE INTERNAL "")
 SET(HAVE_DLADDR CACHE INTERNAL "")

configure.cmake

@@ -134,6 +134,8 @@ IF(UNIX)
  IF(NOT LIBRT)
  MY_SEARCH_LIBS(clock_gettime rt LIBRT)
  ENDIF()
+ MY_SEARCH_LIBS(backtrace_symbols_fd execinfo LIBEXECINFO)
+
  set(THREADS_PREFER_PTHREAD_FLAG ON)
  FIND_PACKAGE(Threads)
 

extra/mariabackup/ds_compress.cc

@@ -35,6 +35,7 @@ typedef struct {
  uint num;
  pthread_mutex_t data_mutex;
  pthread_cond_t data_cond;
+ pthread_cond_t done_cond;
  my_bool data_avail;
  my_bool cancelled;
  const char *from;
@@ -229,25 +230,24 @@ compress_write(ds_file_t *file, const uchar *buf, size_t len)
  thd = threads + i;
 
  pthread_mutex_lock(&thd->data_mutex);
- while (thd->data_avail == TRUE) {
- pthread_cond_wait(&thd->data_cond,
+ while (!thd->to_len) {
+ pthread_cond_wait(&thd->done_cond,
  &thd->data_mutex);
  }
 
- xb_a(threads[i].to_len > 0);
-
  bool fail = ds_write(dest_file, "NEWBNEWB", 8) ||
  write_uint64_le(dest_file,
  comp_file->bytes_processed);
- comp_file->bytes_processed += threads[i].from_len;
+ comp_file->bytes_processed += thd->from_len;
 
  if (!fail) {
- fail = write_uint32_le(dest_file, threads[i].adler) ||
- ds_write(dest_file, threads[i].to,
- threads[i].to_len);
+ fail = write_uint32_le(dest_file, thd->adler) ||
+ ds_write(dest_file, thd->to,
+ thd->to_len);
  }
 
- pthread_mutex_unlock(&threads[i].data_mutex);
+ thd->to_len = 0;
+ pthread_mutex_unlock(&thd->data_mutex);
 
  if (fail) {
  msg("compress: write to the destination stream "
@@ -335,6 +335,7 @@ destroy_worker_thread(comp_thread_ctxt_t *thd)
  pthread_join(thd->id, NULL);
 
  pthread_cond_destroy(&thd->data_cond);
+ pthread_cond_destroy(&thd->done_cond);
  pthread_mutex_destroy(&thd->data_mutex);
 
  my_free(thd->to);
@@ -347,22 +348,24 @@ create_worker_threads(uint n)
  comp_thread_ctxt_t *threads;
  uint i;
 
- threads = (comp_thread_ctxt_t *) my_malloc(PSI_NOT_INSTRUMENTED,
- sizeof(comp_thread_ctxt_t) * n, MYF(MY_FAE));
+ threads = static_cast<comp_thread_ctxt_t*>
+ (my_malloc(PSI_NOT_INSTRUMENTED, n * sizeof *threads,
+ MYF(MY_ZEROFILL|MY_FAE)));
 
  for (i = 0; i < n; i++) {
  comp_thread_ctxt_t *thd = threads + i;
 
  thd->num = i + 1;
- thd->cancelled = FALSE;
- thd->data_avail = FALSE;
-
- thd->to = (char *) my_malloc(PSI_NOT_INSTRUMENTED,
-  COMPRESS_CHUNK_SIZE + MY_QLZ_COMPRESS_OVERHEAD, MYF(MY_FAE));
+ thd->to = static_cast<char*>
+  (my_malloc(PSI_NOT_INSTRUMENTED,
+ COMPRESS_CHUNK_SIZE +
+  MY_QLZ_COMPRESS_OVERHEAD,
+  MYF(MY_FAE)));
 
  /* Initialize and data mutex and condition var */
  if (pthread_mutex_init(&thd->data_mutex, NULL) ||
- pthread_cond_init(&thd->data_cond, NULL)) {
+ pthread_cond_init(&thd->data_cond, NULL) ||
+ pthread_cond_init(&thd->done_cond, NULL)) {
  goto err;
  }
 
@@ -413,7 +416,7 @@ compress_worker_thread_func(void *arg)
 
  if (thd->cancelled)
  break;
-
+ thd->data_avail = FALSE;
  thd->to_len = qlz_compress(thd->from, thd->to, thd->from_len,
  &thd->state);
 
@@ -428,6 +431,7 @@ compress_worker_thread_func(void *arg)
 
  thd->adler = adler32(0x00000001, (uchar *) thd->to,
  (uInt)thd->to_len);
+ pthread_cond_signal(&thd->done_cond);
  }
 
  pthread_mutex_unlock(&thd->data_mutex);

extra/mariabackup/xtrabackup.cc

@@ -846,7 +846,6 @@ static void backup_file_op(uint32_t space_id, int type,
  case FILE_MODIFY:
  ddl_tracker.insert_defer_id(
  space_id, filename_to_spacename(name, len));
- msg("DDL tracking : modify %u \"%.*s\"", space_id, int(len), name);
  break;
  case FILE_RENAME:
  {
@@ -896,7 +895,6 @@ static void backup_file_op_fail(uint32_t space_id, int type,
  filename_to_spacename(name, len).c_str());
  break;
  case FILE_MODIFY:
- msg("DDL tracking : modify %u \"%.*s\"", space_id, int(len), name);
  break;
  case FILE_RENAME:
  msg("DDL tracking : rename %u \"%.*s\",\"%.*s\"",

libmysqld/CMakeLists.txt

@@ -1,5 +1,5 @@
 # Copyright (c) 2006, 2011, Oracle and/or its affiliates.
-# Copyright (c) 2009, 2020, MariaDB Corporation
+# Copyright (c) 2009, 2022, MariaDB Corporation
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -152,6 +152,7 @@ ENDIF()
 IF(TARGET libfmt)
  ADD_DEPENDENCIES(sql_embedded libfmt)
 ENDIF()
+TARGET_LINK_LIBRARIES(sql_embedded LINK_PRIVATE tpool ${CRC32_LIBRARY})
 
 # On Windows, static embedded server library is called mysqlserver.lib
 # On Unix, it is libmysqld.a

libmysqld/lib_sql.cc

@@ -496,7 +496,7 @@ MYSQL_METHODS embedded_methods=
 
 char **copy_arguments(int argc, char **argv)
 {
- uint length= 0;
+ size_t length= 0;
  char **from, **res, **end= argv+argc;
 
  for (from=argv ; from != end ; from++)
@@ -1111,11 +1111,11 @@ bool Protocol_text::store_field_metadata(const THD * thd,
  client_field->flags= (uint16) server_field.flags;
  client_field->decimals= server_field.decimals;
 
- client_field->db_length= strlen(client_field->db);
- client_field->table_length= strlen(client_field->table);
- client_field->name_length= strlen(client_field->name);
- client_field->org_name_length= strlen(client_field->org_name);
- client_field->org_table_length= strlen(client_field->org_table);
+ client_field->db_length= (uint)strlen(client_field->db);
+ client_field->table_length= (uint)strlen(client_field->table);
+ client_field->name_length= (uint)strlen(client_field->name);
+ client_field->org_name_length= (uint)strlen(client_field->org_name);
+ client_field->org_table_length= (uint)strlen(client_field->org_table);
 
  client_field->catalog= dup_str_aux(field_alloc, "def", 3, cs, thd_cs);
  client_field->catalog_length= 3;
@@ -1389,12 +1389,12 @@ bool Protocol::net_store_data(const uchar *from, size_t length)
 
  if (!(field_buf= (char*) alloc_root(alloc, length + sizeof(uint) + 1)))
  return TRUE;
- *(uint *)field_buf= length;
+ *(uint *)field_buf= (uint)length;
  *next_field= field_buf + sizeof(uint);
  memcpy((uchar*) *next_field, from, length);
  (*next_field)[length]= 0;
  if (next_mysql_field->max_length < length)
- next_mysql_field->max_length=length;
+ next_mysql_field->max_length=(ulong)length;
  ++next_field;
  ++next_mysql_field;
  return FALSE;
@@ -1404,7 +1404,7 @@ bool Protocol::net_store_data(const uchar *from, size_t length)
 bool Protocol::net_store_data_cs(const uchar *from, size_t length,
  CHARSET_INFO *from_cs, CHARSET_INFO *to_cs)
 {
- uint conv_length= to_cs->mbmaxlen * length / from_cs->mbminlen;
+ size_t conv_length= length * to_cs->mbmaxlen / from_cs->mbminlen;
  uint dummy_error;
  char *field_buf;
  if (!thd->mysql) // bootstrap file handling
@@ -1415,10 +1415,10 @@ bool Protocol::net_store_data_cs(const uchar *from, size_t length,
  *next_field= field_buf + sizeof(uint);
  length= copy_and_convert(*next_field, conv_length, to_cs,
  (const char*) from, length, from_cs, &dummy_error);
- *(uint *) field_buf= length;
+ *(uint *) field_buf= (uint)length;
  (*next_field)[length]= 0;
  if (next_mysql_field->max_length < length)
- next_mysql_field->max_length= length;
+ next_mysql_field->max_length= (ulong)length;
  ++next_field;
  ++next_mysql_field;
  return false;

mysql-test/include/ensure_binlog_row_event_columns.inc

@@ -0,0 +1,72 @@
+#
+# Helper file to ensure that a binary log file properly writes the expected
+# fields based on the binlog_row_image value.
+#
+# ==== Usage ====
+#
+# --let $expected_columns= (COLUMN_IDS)
+# --let $binlog_filename= FILENAME
+# --source include/count_binlog_row_event_columns.inc
+#
+# Parameters:
+# expected_columns (list<uint>, in) : A list of positive integers which
+# correspond to the column numbers that should be output in a binary
+# log's write_rows event
+# binlog_filename (string, in) : Name of the binary log file to analyze
+#
+
+if (!$expected_columns)
+{
+ --die expected_columns parameter is required but was not set
+}
+
+if (!$binlog_filename)
+{
+ --die binlog_filename parameter is required but was not set
+}
+
+--let $include_filename= ensure_binlog_row_event_columns.inc [$expected_columns]
+--source include/begin_include_file.inc
+
+--let $assert_file=$MYSQLTEST_VARDIR/tmp/binlog_decoded.out
+--let mysqld_datadir=`select @@datadir`
+
+--echo # MYSQL_BINLOG mysqld_datadir/binlog_filename -vv > assert_file
+--exec $MYSQL_BINLOG $mysqld_datadir/$binlog_filename -vv > $assert_file
+
+--echo # Verifying all expected column ids appear in binlog event output..
+--let num_cols_found=0
+--let last_expected_col= `SELECT GREATEST $expected_columns`
+--let i= 1
+while($i <= $last_expected_col)
+{
+ # By default, assume the column is not expected to be in the binary log.
+ # If the column id is set in expected_columns, then override assertion
+ # parameters.
+ --let assert_count= 0
+ --let assert_text= Column @$i should not be in binary log
+
+ if (`SELECT $i IN $expected_columns`)
+ {
+ --let assert_count= 1
+ --let assert_text= Column @$i should be in binary log
+
+ --inc $num_cols_found
+ }
+
+ --let assert_select= @$i
+ --source include/assert_grep.inc
+
+ --inc $i
+}
+--echo # ..success
+
+--echo # Verifying only expected column ids appear in binlog event output..
+--let assert_count= $num_cols_found
+--let assert_text= The binlog event should only have $num_cols_found columns
+--let assert_select= @[\d]+
+--source include/assert_grep.inc
+--echo # ..success
+
+--let $include_filename= ensure_binlog_row_event_columns.inc [$expected_columns]
+--source include/end_include_file.inc

mysql-test/include/mtr_warnings.sql

@@ -157,6 +157,10 @@ INSERT INTO global_suppressions VALUES
  ("InnoDB: Error: table `test`.`t[123]` .*does not exist in the InnoDB internal"),
  ("InnoDB: Warning: semaphore wait:"),
 
+ /* MDEV-28976: Tests that kill the server do not ensure that the
+ old process has terminated before starting a new one */
+ ("InnoDB: Unable to lock"),
+
  /*
  BUG#32080 - Excessive warnings on Solaris: setrlimit could not
  change the size of core files