MDEV-29909 : SST fails when table is defined with DATA DIRECTORY='/pa…

mysql-test/suite/galera/r/galera_defaults.result

@@ -3,7 +3,7 @@ connection node_1;
 # Correct Galera library found
 SELECT COUNT(*) `expect 49` FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME LIKE 'wsrep_%';
 expect 49
-49
+51
 SELECT VARIABLE_NAME, VARIABLE_VALUE
 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES
 WHERE VARIABLE_NAME LIKE 'wsrep_%'
@@ -57,6 +57,7 @@ WSREP_SST_AUTH	********
 WSREP_SST_DONOR	
 WSREP_SST_DONOR_REJECTS_QUERIES	OFF
 WSREP_SST_METHOD	rsync
+WSREP_SST_TMP_DIR	
 WSREP_STATUS_FILE	
 WSREP_SYNC_WAIT	15
 WSREP_TRX_FRAGMENT_SIZE	0

mysql-test/suite/galera/r/galera_sst_tmp_directory.result

@@ -0,0 +1,22 @@
+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+create table t1(id int not null primary key, b varchar(32)) engine=InnoDB;
+insert into t1 SELECT seq, md5(rand()) from seq_1_to_50000;
+connection node_2;
+Shutting down server ...
+connection node_1;
+connection node_2;
+Starting server ...
+connection node_1;
+SELECT COUNT(*) AS EXPECT_5000 FROM t1;
+EXPECT_5000
+50000
+connection node_2;
+SELECT COUNT(*) AS EXPECT_5000 FROM t1;
+EXPECT_5000
+50000
+DROP TABLE t1;
+disconnect node_2;
+disconnect node_1;

mysql-test/suite/galera/t/galera_sst_tmp_directory.cnf

@@ -0,0 +1,19 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=1
+innodb_fast_shutdown=0
+innodb_undo_tablespaces=0
+
+[mysqld.1]
+wsrep_provider_options='pc.ignore_sb=true;[email protected].#galera_port'
+
+[mysqld.2]
+wsrep_provider_options='pc.ignore_sb=true;[email protected].#galera_port'
+[email protected]_VARDIR/tmp
+
+[sst]
+[email protected]_GALERA_TFMT
+streamfmt=mbstream

mysql-test/suite/galera/t/galera_sst_tmp_directory.combinations

@@ -0,0 +1,5 @@
+[binlogon]
+log-bin
+log-slave-updates
+
+[binlogoff]

mysql-test/suite/galera/t/galera_sst_tmp_directory.test

@@ -0,0 +1,59 @@
+--source include/galera_cluster.inc
+--source include/have_sequence.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+#
+# TODO : This would be correct way to test but as CREATE TABLE is TOI
+# exactly the same string is executed on node_2 also leading to problem
+# where t1.ibd file would be in same location on both node_1 and node_2
+# and this leads to fact that CREATE TABLE fails on node_2 as it
+# file already exists.
+#
+# EVAL create table t1(id int not null primary key, b varchar(32)) engine=innodb DATA DIRECTORY='$MYSQL_TMP_DIR';
+# insert into t1 SELECT seq, md5(rand()) from seq_1_to_50000;
+# --list_files $MYSQL_TMP_DIR/test
+# --list_files $MYSQLD_DATADIR/test
+
+create table t1(id int not null primary key, b varchar(32)) engine=InnoDB;
+insert into t1 SELECT seq, md5(rand()) from seq_1_to_50000;
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'
+--source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 50000 FROM t1
+--source include/wait_condition.inc
+
+--echo Shutting down server ...
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+--connection node_2
+# enforce SST
+--exec rm -rf $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
+
+--echo Starting server ...
+--source include/start_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+SELECT COUNT(*) AS EXPECT_5000 FROM t1;
+
+--connection node_2
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+SELECT COUNT(*) AS EXPECT_5000 FROM t1;
+
+# Restore original auto_increment_offset values.
+--source include/auto_increment_offset_restore.inc
+
+DROP TABLE t1;
+
+--source include/galera_end.inc

mysql-test/suite/sys_vars/r/sysvars_wsrep.result

@@ -691,6 +691,21 @@ ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 GLOBAL_VALUE_PATH	NULL
+VARIABLE_NAME	WSREP_SST_TMP_DIR
+SESSION_VALUE	NULL
+GLOBAL_VALUE	
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	VARCHAR
+VARIABLE_COMMENT	Path to SST temp root directory
+NUMERIC_MIN_VALUE	NULL
+NUMERIC_MAX_VALUE	NULL
+NUMERIC_BLOCK_SIZE	NULL
+ENUM_VALUE_LIST	NULL
+READ_ONLY	YES
+COMMAND_LINE_ARGUMENT	REQUIRED
+GLOBAL_VALUE_PATH	NULL
 VARIABLE_NAME	WSREP_START_POSITION
 SESSION_VALUE	NULL
 GLOBAL_VALUE	00000000-0000-0000-0000-000000000000:-1

mysql-test/suite/wsrep/r/variables.result

@@ -44,6 +44,7 @@ wsrep_commit_window	#
 wsrep_local_state	#
 wsrep_local_state_comment	#
 wsrep_cert_index_size	#
+wsrep_causal_reads	#
 wsrep_cert_interval	#
 wsrep_open_transactions	#
 wsrep_open_connections	#
@@ -91,6 +92,7 @@ wsrep_thread_count	2
 SELECT VARIABLE_NAME FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE VARIABLE_NAME LIKE "wsrep%" ORDER BY VARIABLE_NAME;
 VARIABLE_NAME
 WSREP_ALLOWLIST
+WSREP_APPLIER_RETRY_COUNT
 WSREP_AUTO_INCREMENT_CONTROL
 WSREP_CERTIFICATION_RULES
 WSREP_CERTIFY_NONPK
@@ -135,6 +137,7 @@ WSREP_SST_DONOR
 WSREP_SST_DONOR_REJECTS_QUERIES
 WSREP_SST_METHOD
 WSREP_SST_RECEIVE_ADDRESS
+WSREP_SST_TMP_DIR
 WSREP_START_POSITION
 WSREP_STATUS_FILE
 WSREP_SYNC_WAIT

scripts/wsrep_sst_common.sh

@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2024 MariaDB
+# Copyright (C) 2017-2025 MariaDB
 # Copyright (C) 2012-2015 Codership Oy
 #
 # This program is free software; you can redistribute it and/or modify
@@ -189,6 +189,7 @@ WSREP_SST_OPT_BINLOG=""
 WSREP_SST_OPT_BINLOG_INDEX=""
 WSREP_SST_OPT_LOG_BASENAME=""
 WSREP_SST_OPT_DATA=""
+WSREP_SST_OPT_TMP_DIR=""
 WSREP_SST_OPT_AUTH="${WSREP_SST_OPT_AUTH:-}"
 WSREP_SST_OPT_USER="${WSREP_SST_OPT_USER:-}"
 WSREP_SST_OPT_PSWD="${WSREP_SST_OPT_PSWD:-}"
@@ -306,6 +307,10 @@ case "$1" in
         readonly WSREP_SST_OPT_DATA=$(trim_dir "$2")
         shift
         ;;
+    '--sst-tmp-dir')
+	readonly WSREP_SST_OPT_TMP_DIR=$(trim_dir "$2")
+	shift
+	;;
     '--aria-log-dir-path')
         # Let's remove the trailing slash:
         readonly ARIA_LOG_DIR=$(trim_dir "$2")
@@ -648,6 +653,12 @@ case "$1" in
                        fi
                        skip_mysqld_arg=1
                        ;;
+		   '--sst-temp-dir')
+		       if [ -z "$WSREP_SST_OPT_TMP_DIR" ]; then
+			   MYSQLD_OPT_SST_TMP_DIR=$(trim_dir "$value")
+		       fi
+		       skip_mysqld_arg=1
+		       ;;
                esac
                if [ $skip_mysqld_arg -eq 0 ]; then
                    original_cmd="$original_cmd '$1'"
@@ -727,6 +738,10 @@ if [ -n "${MYSQLD_OPT_LOG_BASENAME:-}" -a \
      -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then
     readonly WSREP_SST_OPT_LOG_BASENAME="$MYSQLD_OPT_LOG_BASENAME"
 fi
+if [ -n "${MYSQLD_OPT_SST_TMP_DIR:-}" -a \
+     -z "$WSREP_SST_OPT_TMP_DIR" ]; then
+    readonly WSREP_SST_OPT_TMP_DIR="$MYSQLD_OPT_SST_TMP_DIR"
+fi
 
 # If the --log-bin option is present without a value, then
 # set WSREP_SST_OPT_BINLOG value using other arguments:

scripts/wsrep_sst_mariabackup.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
 
 set -ue
-
-# Copyright (C) 2017-2024 MariaDB
+# For script debugging
+# set -uex
+#
+# Copyright (C) 2017-2025 MariaDB
 # Copyright (C) 2013 Percona Inc
 #
 # This program is free software; you can redistribute it and/or modify
@@ -84,6 +86,7 @@ STATDIR=""
 tmpopts=""
 itmpdir=""
 xtmpdir=""
+sstdir=""
 
 scomp=""
 sdecomp=""
@@ -1315,14 +1318,23 @@ else # joiner
             fi
         fi
 
-        if [ -d "$DATA/.sst" ]; then
+	# If user has configured temporary sst directory use it
+	if [ -n "$WSREP_SST_OPT_TMP_DIR" ]; then
+	    sstdir="$WSREP_SST_OPT_TMP_DIR/.sst"
+	    wsrep_log_info "Using user configured SST directory $sstdir"
+	else
+	    sstdir="$DATA/.sst"
+	    wsrep_log_info "Using default SST directory $sstdir"
+	fi
+
+        if [ -d "$sstdir"]; then
             wsrep_log_info \
                 "WARNING: Stale temporary SST directory:" \
-                "'$DATA/.sst' from previous state transfer, removing..."
-            rm -rf "$DATA/.sst"
+                "'$sstdir' from previous state transfer, removing..."
+            rm -rf "$sstdir"
         fi
-        mkdir -p "$DATA/.sst"
-        (recv_joiner "$DATA/.sst" "$stagemsg-SST" 0 0 0) &
+        mkdir -p "$sstdir"
+        (recv_joiner "$sstdir" "$stagemsg-SST" 0 0 0) &
         BACKUP_PID=$!
         wsrep_log_info "Proceeding with SST"
 
@@ -1383,7 +1395,7 @@ else # joiner
         [ -f "$DATA/mariadb_backup_binlog_pos_innodb" ] && rm -f "$DATA/mariadb_backup_binlog_pos_innodb"
 
         TDATA="$DATA"
-        DATA="$DATA/.sst"
+        DATA="$sstdir"
         MAGIC_FILE="$DATA/$INFO_FILE"
 
         wsrep_log_info "Waiting for SST streaming to complete!"
@@ -1543,6 +1555,7 @@ else # joiner
             cd "$OLD_PWD"
         fi
 
+	cd "$TDATA"
         MAGIC_FILE="$TDATA/$INFO_FILE"
         DONOR_MAGIC_FILE="$TDATA/$DONOR_INFO_FILE"
 

sql/mysqld.cc

@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2015, Oracle and/or its affiliates.
-   Copyright (c) 2008, 2023, MariaDB
+   Copyright (c) 2008, 2025, MariaDB plc
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by

sql/sys_vars.cc

@@ -1,5 +1,5 @@
 /* Copyright (c) 2002, 2015, Oracle and/or its affiliates.
-   Copyright (c) 2012, 2022, MariaDB Corporation.
+   Copyright (c) 2012, 2025, MariaDB plc.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -6777,6 +6777,11 @@ static Sys_var_uint Sys_wsrep_applier_retry_count (
        VALID_RANGE(0, UINT_MAX), DEFAULT(0), BLOCK_SIZE(1),
        NO_MUTEX_GUARD, NOT_IN_BINLOG);
 
+static Sys_var_charptr_fscs Sys_wsrep_sst_tmp_dir(
+       "wsrep_sst_tmp_dir", "Path to SST temp root directory",
+       READ_ONLY GLOBAL_VAR(wsrep_sst_tmp_dir), CMD_LINE(REQUIRED_ARG),
+       DEFAULT(0), NO_MUTEX_GUARD, NOT_IN_BINLOG);
+
 #endif /* WITH_WSREP */
 
 static bool fix_host_cache_size(sys_var *, THD *, enum_var_type)

sql/wsrep_mysqld.cc

@@ -1,5 +1,5 @@
 /* Copyright (c) 2008, 2025, Codership Oy <http://www.codership.com>
-   Copyright (c) 2020, 2025, MariaDB
+   Copyright (c) 2020, 2025, MariaDB Corporation
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -85,6 +85,8 @@ const char *wsrep_dbug_option;
 const char *wsrep_notify_cmd;
 const char *wsrep_status_file;
 const char *wsrep_allowlist;
+const char *wsrep_sst_tmp_dir;
+char *wsrep_sst_tmp_dir_real=nullptr;
 
 ulong   wsrep_debug;                            // Debug level logging
 my_bool wsrep_convert_LOCK_to_trx;              // Convert locking sessions to trx
@@ -901,6 +903,9 @@ int wsrep_init()
   if (!wsrep_data_home_dir || strlen(wsrep_data_home_dir) == 0)
     wsrep_data_home_dir= mysql_real_data_home;
 
+  /* In case of errors, SST tmp dir is not set */
+  wsrep_sst_tmp_dir_check();
+
   Wsrep_server_state::init_provider_services();
   if (Wsrep_server_state::instance().load_provider(
       wsrep_provider,
@@ -937,6 +942,7 @@ int wsrep_init()
 
   WSREP_DEBUG("SR storage init for: %s",
               (wsrep_SR_store_type == WSREP_SR_STORE_TABLE) ? "table" : "void");
+
   return 0;
 }
 
@@ -1057,6 +1063,12 @@ void wsrep_deinit(bool free_options)
 
   wsrep_inited= 0;
 
+  if (wsrep_sst_tmp_dir_real)
+  {
+    my_free(wsrep_sst_tmp_dir_real);
+    wsrep_sst_tmp_dir_real= nullptr;
+  }
+
   if (wsrep_provider_capabilities != NULL)
   {
     char* p= wsrep_provider_capabilities;

sql/wsrep_mysqld.h

@@ -1,4 +1,5 @@
-/* Copyright 2008-2023 Codership Oy <http://www.codership.com>
+/* Copyright 2008-2025 Codership Oy <http://www.codership.com>
+   Copyright 2025 MariaDB plc
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -93,6 +94,8 @@ extern uint32      wsrep_gtid_domain_id;
 extern std::atomic <bool > wsrep_thread_create_failed;
 extern ulonglong   wsrep_mode;
 extern uint        wsrep_applier_retry_count;
+extern char *wsrep_sst_tmp_dir_real;
+extern const char *wsrep_sst_tmp_dir;
 
 enum enum_wsrep_reject_types {
   WSREP_REJECT_NONE,    /* nothing rejected */