Merge 10.9 into 10.10

.gitlab-ci.yml

@@ -172,6 +172,39 @@ fedora-clang:
  - dependencies.dot
  - dependencies.png
 
+fedora-sanitizer:
+ stage: build
+ variables:
+ GIT_STRATEGY: fetch
+ GIT_SUBMODULE_STRATEGY: normal
+ script:
+ - yum install -y yum-utils rpm-build openssl-devel clang
+ - yum install -y /usr/lib64/libasan.so.6.0.0 /usr/lib64/libtsan.so.0.0.0 /usr/lib64/libubsan.so.1.0.0
+ # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+ - yum-builddep -y mariadb-server
+ - mkdir builddir; cd builddir
+ - export CXX=${CXX:-clang++}
+ - export CC=${CC:-clang}
+ - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+ - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+ - export CFLAGS='-Wno-unused-command-line-argument'
+ - export CXXFLAGS='-Wno-unused-command-line-argument'
+ - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS $SANITIZER .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ # @TODO: the build will fail consistently at 24% when trying to make using eatmydata
+ - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - *rpm_listfiles
+ - mkdir ../rpm; mv *.rpm ../rpm
+ artifacts:
+ when: always # Must be able to see logs
+ paths:
+ - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+ - rpm
+ - builddir/_CPack_Packages/Linux/RPM/SPECS/
+ parallel:
+ matrix:
+ - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES, -DWITH_MSAN=YES]
+
 centos8:
  stage: build
  image: quay.io/centos/centos:stream8 # CentOS 8 is deprecated, use this Stream8 instead
@@ -246,10 +279,8 @@ centos7:
  - rpm
  - builddir/_CPack_Packages/Linux/RPM/SPECS/
 
-mysql-test-run:
+.mysql-test-run: &mysql-test-run-def
  stage: test
- dependencies:
- - fedora
  script:
  # Install packages so tests and the dependencies install
  # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now
@@ -265,7 +296,76 @@ mysql-test-run:
  main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
  main.mysql_upgrade_noengine : upgrade output order does not match the expected
  " > skiplist
- - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist
+ - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $RESTART_POLICY
+
+mysql-test-run:
+ stage: test
+ dependencies:
+ - fedora
+ <<: *mysql-test-run-def
+ artifacts:
+ when: always # Also show results when tests fail
+ reports:
+ junit:
+ - junit.xml
+
+# Duplicate of the above jobs, except we use sanitizer build jobs as a dependency. This is so we can keep
+# sanitizer errors separate from functional test failures. Currently, there is no way to run the same
+# job for different dependencies.
+#
+# Additionally, for each sanitizer MTR job, we enable --force-restart so that 
+# sanitizer errors can be traced to individual tests. The difference in test 
+# suite runtime as a result of this flag is negligable (~30s for the entire test suite).
+# (see https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_MYSQL_TEST_RUN_PL.html)
+mysql-test-run-asan:
+ stage: test
+ variables:
+ RESTART_POLICY: "--force-restart"
+ dependencies:
+ - "fedora-sanitizer: [-DWITH_ASAN=YES]"
+ <<: *mysql-test-run-def
+ artifacts:
+ when: always # Also show results when tests fail
+ reports:
+ junit:
+ - junit.xml
+
+mysql-test-run-tsan:
+ stage: test
+ variables:
+ RESTART_POLICY: "--force-restart"
+ dependencies:
+ - "fedora-sanitizer: [-DWITH_TSAN=YES]"
+ <<: *mysql-test-run-def
+ allow_failure: true
+ artifacts:
+ when: always # Also show results when tests fail
+ reports:
+ junit:
+ - junit.xml
+
+mysql-test-run-ubsan:
+ stage: test
+ variables:
+ RESTART_POLICY: "--force-restart"
+ dependencies:
+ - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
+ <<: *mysql-test-run-def
+ allow_failure: true
+ artifacts:
+ when: always # Also show results when tests fail
+ reports:
+ junit:
+ - junit.xml
+
+mysql-test-run-msan:
+ stage: test
+ variables:
+ RESTART_POLICY: "--force-restart"
+ dependencies:
+ - "fedora-sanitizer: [-DWITH_MSAN=YES]"
+ <<: *mysql-test-run-def
+ allow_failure: true
  artifacts:
  when: always # Also show results when tests fail
  reports:

CMakeLists.txt

@@ -541,7 +541,10 @@ INSTALL_DOCUMENTATION(README.md CREDITS COPYING THIRDPARTY COMPONENT Readme)
 # ${CMAKE_BINARY_DIR}/Docs/INFO_BIN)
 
 IF(UNIX)
- INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY Docs/README-wsrep COMPONENT Readme)
+ INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY COMPONENT Readme)
+ IF(WITH_WSREP)
+ INSTALL_DOCUMENTATION(Docs/README-wsrep COMPONENT Readme)
+ ENDIF()
 ENDIF()
 
 INCLUDE(build_depends)

cmake/install_macros.cmake

@@ -265,6 +265,11 @@ SET(DEBUGBUILDDIR "${BINARY_PARENTDIR}/debug" CACHE INTERNAL "Directory of debug
 
 FUNCTION(INSTALL_MYSQL_TEST from to)
  IF(INSTALL_MYSQLTESTDIR)
+ IF(NOT WITH_WSREP)
+ SET(EXCL_GALERA "(suite/(galera|wsrep|sys_vars/[rt]/(sysvars_)?wsrep).*|include/((w.*)?wsrep.*|.*galera.*)\\.inc|std_data/(galera|wsrep).*)")
+ ELSE()
+ SET(EXCL_GALERA "^DOES_NOT_EXIST$")
+ ENDIF()
  INSTALL(
  DIRECTORY ${from}
  DESTINATION "${INSTALL_MYSQLTESTDIR}/${to}"
@@ -286,6 +291,7 @@ FUNCTION(INSTALL_MYSQL_TEST from to)
  PATTERN "*.vcxproj.user" EXCLUDE
  PATTERN "CTest*" EXCLUDE
  PATTERN "*~" EXCLUDE
+ REGEX "${EXCL_GALERA}" EXCLUDE
  )
  ENDIF()
 ENDFUNCTION()

cmake/systemd.cmake

@@ -45,7 +45,10 @@ MACRO(CHECK_SYSTEMD)
  IF(HAVE_SYSTEMD_SD_DAEMON_H AND HAVE_SYSTEMD_SD_LISTEN_FDS
  AND HAVE_SYSTEMD_SD_NOTIFY AND HAVE_SYSTEMD_SD_NOTIFYF)
  SET(HAVE_SYSTEMD TRUE)
- SET(SYSTEMD_SCRIPTS mariadb-service-convert galera_new_cluster galera_recovery)
+ SET(SYSTEMD_SCRIPTS mariadb-service-convert)
+ IF(WITH_WSREP)
+ SET(SYSTEMD_SCRIPTS ${SYSTEMD_SCRIPTS} galera_new_cluster galera_recovery)
+ ENDIF()
  IF(DEB)
  SET(SYSTEMD_EXECSTARTPRE "ExecStartPre=/usr/bin/install -m 755 -o mysql -g root -d /var/run/mysqld")
  SET(SYSTEMD_EXECSTARTPOST "ExecStartPost=/etc/mysql/debian-start")

extra/mariabackup/backup_copy.cc

@@ -561,8 +561,8 @@ datafile_read(datafile_cur_t *cursor)
  }
 
  if (os_file_read(IORequestRead,
-  cursor->file, cursor->buf, cursor->buf_offset,
-  to_read) != DB_SUCCESS) {
+ cursor->file, cursor->buf, cursor->buf_offset,
+ to_read, nullptr) != DB_SUCCESS) {
  return(XB_FIL_CUR_ERROR);
  }
 

extra/mariabackup/changed_page_bitmap.cc

@@ -188,18 +188,15 @@ log_online_read_bitmap_page(
 {
  ulint checksum;
  ulint actual_checksum;
- ibool success;
 
  ut_a(bitmap_file->size >= MODIFIED_PAGE_BLOCK_SIZE);
  ut_a(bitmap_file->offset
  <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE);
  ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0);
- success = os_file_read(IORequestRead,
- bitmap_file->file, page, bitmap_file->offset,
- MODIFIED_PAGE_BLOCK_SIZE) == DB_SUCCESS;
-
- if (UNIV_UNLIKELY(!success)) {
-
+ if (DB_SUCCESS !=
+ os_file_read(IORequestRead, bitmap_file->file, page,
+ bitmap_file->offset, MODIFIED_PAGE_BLOCK_SIZE,
+ nullptr)) {
  /* The following call prints an error message */
  os_file_get_last_error(TRUE);
  msg("InnoDB: Warning: failed reading changed page bitmap "

extra/mariabackup/fil_cur.cc

@@ -223,7 +223,7 @@ xb_fil_cur_open(
  if (!node->space->crypt_data
  && os_file_read(IORequestRead,
  node->handle, cursor->buf, 0,
- cursor->page_size) == DB_SUCCESS) {
+ cursor->page_size, nullptr) == DB_SUCCESS) {
  mysql_mutex_lock(&fil_system.mutex);
  if (!node->space->crypt_data) {
  node->space->crypt_data = fil_space_read_crypt_data(
@@ -415,12 +415,12 @@ xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t* cursor,
  cursor->buf_page_no = static_cast<unsigned>(offset / page_size);
 
  if (os_file_read(IORequestRead, cursor->file, cursor->buf, offset,
-  (ulint) to_read) != DB_SUCCESS) {
+ (ulint) to_read, nullptr) != DB_SUCCESS) {
  ret = XB_FIL_CUR_ERROR;
  goto func_exit;
  }
 
- defer = space->is_deferred();
+ defer = UT_LIST_GET_FIRST(space->chain)->deferred;
  /* check pages for corruption and re-read if necessary. i.e. in case of
  partially written pages */
  for (page = cursor->buf, i = 0; i < npages;

extra/mariabackup/xtrabackup.cc

@@ -3857,7 +3857,7 @@ static dberr_t xb_assign_undo_space_start()
  byte* page = static_cast<byte*>
  (aligned_malloc(srv_page_size, srv_page_size));
 
- if (os_file_read(IORequestRead, file, page, 0, srv_page_size)
+ if (os_file_read(IORequestRead, file, page, 0, srv_page_size, nullptr)
  != DB_SUCCESS) {
  msg("Reading first page failed.\n");
  error = DB_ERROR;
@@ -3869,7 +3869,7 @@ static dberr_t xb_assign_undo_space_start()
 retry:
  if (os_file_read(IORequestRead, file, page,
  TRX_SYS_PAGE_NO << srv_page_size_shift,
- srv_page_size) != DB_SUCCESS) {
+ srv_page_size, nullptr) != DB_SUCCESS) {
  msg("Reading TRX_SYS page failed.");
  error = DB_ERROR;
  goto func_exit;
@@ -5315,7 +5315,8 @@ xtrabackup_apply_delta(
  offset = ((incremental_buffers * (page_size / 4))
  << page_size_shift);
  if (os_file_read(IORequestRead, src_file,
- incremental_buffer, offset, page_size)
+ incremental_buffer, offset, page_size,
+ nullptr)
  != DB_SUCCESS) {
  goto error;
  }
@@ -5348,7 +5349,7 @@ xtrabackup_apply_delta(
  /* read whole of the cluster */
  if (os_file_read(IORequestRead, src_file,
  incremental_buffer,
- offset, page_in_buffer * page_size)
+ offset, page_in_buffer * page_size, nullptr)
  != DB_SUCCESS) {
  goto error;
  }

include/CMakeLists.txt

@@ -74,7 +74,15 @@ FOREACH(f ${HEADERS_GEN_CONFIGURE})
  INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${f}
  DESTINATION ${INSTALL_INCLUDEDIR}/server COMPONENT Development)
 ENDFOREACH(f)
-INSTALL(DIRECTORY mysql/ DESTINATION ${INSTALL_INCLUDEDIR}/server/mysql COMPONENT Development FILES_MATCHING PATTERN "*.h")
+IF(NOT WITH_WSREP)
+ SET(EXCL_SERVICE_WSREP "service_wsrep.h")
+ SET(EXCL_WSREP "wsrep.h")
+ENDIF()
+INSTALL(DIRECTORY mysql/
+ DESTINATION ${INSTALL_INCLUDEDIR}/server/mysql COMPONENT Development
+ FILES_MATCHING PATTERN "*.h"
+ PATTERN "${EXCL_SERVICE_WSREP}" EXCLUDE
+)
 
 STRING(REPLACE "." "\\." EXCL_RE "${HEADERS};${HEADERS_GEN_CONFIGURE}")
 STRING(REPLACE ";" "|" EXCL_RE "${EXCL_RE}")
@@ -85,6 +93,7 @@ MACRO(INSTALL_PRIVATE DIR)
  FILES_MATCHING PATTERN "*.h"
  PATTERN CMakeFiles EXCLUDE
  PATTERN mysql EXCLUDE
+ PATTERN "${EXCL_WSREP}" EXCLUDE
  REGEX "\\./(${EXCL_RE}$)" EXCLUDE)
 ENDMACRO()
 

man/CMakeLists.txt

@@ -12,8 +12,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA
+
 SET(MAN1_WSREP wsrep_sst_rsync.1 wsrep_sst_common.1 wsrep_sst_mariabackup.1
- wsrep_sst_rsync_wan.1)
+ wsrep_sst_mysqldump.1 wsrep_sst_rsync_wan.1 galera_recovery.1 galera_new_cluster.1)
 SET(MAN1_SERVER innochecksum.1 myisam_ftdump.1 myisamchk.1
  aria_chk.1 aria_dump_log.1 aria_ftdump.1 aria_pack.1 aria_read_log.1
  aria_s3_copy.1
@@ -24,10 +25,11 @@ SET(MAN1_SERVER innochecksum.1 myisam_ftdump.1 myisamchk.1
  mysqld_multi.1 mysqld_safe.1
  resolveip.1 mariadb-service-convert.1
  mysqld_safe_helper.1
- wsrep_sst_mysqldump.1
- galera_recovery.1 galera_new_cluster.1
  mysql_ldb.1 myrocks_hotbackup.1
- mbstream.1 mariabackup.1 ${MAN1_WSREP})
+ mbstream.1 mariabackup.1)
+IF(WITH_WSREP)
+ SET(MAN1_SERVER ${MAN1_SERVER} ${MAN1_WSREP})
+ENDIF()
 SET(MAN8_SERVER mysqld.8)
 SET(MAN1_CLIENT msql2mysql.1 mysql.1 mysql_find_rows.1 mysql_waitpid.1
  mysqldumpslow.1

mysql-test/suite/encryption/r/encrypt_and_grep,undo3.rdiff

@@ -0,0 +1,32 @@
+--- encrypt_and_grep.result 2022-09-02 22:36:21.669650278 +0530
++++ encrypt_and_grep.reject 2022-11-29 19:01:22.080027528 +0530
+@@ -14,6 +14,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry
+@@ -35,6 +38,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry
+@@ -62,6 +68,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry

mysql-test/suite/encryption/r/innodb-remove-encryption,undo3.rdiff

@@ -0,0 +1,32 @@
+--- innodb-remove-encryption.result 2022-09-02 20:44:59.960430396 +0530
++++ innodb-remove-encryption,undo3.reject 2022-11-29 19:02:24.813094277 +0530
+@@ -13,6 +13,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry
+@@ -24,6 +27,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry
+@@ -36,6 +42,9 @@
+ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0;
+ NAME
+ innodb_system
++innodb_undo001
++innodb_undo002
++innodb_undo003
+ mysql/innodb_index_stats
+ mysql/innodb_table_stats
+ mysql/transaction_registry