Add configuration for database connection collation

config/global.ini.php

@@ -45,6 +45,14 @@
 ; Matomo should work correctly without this setting but we recommend to have a charset set.
 charset = utf8
 
+; In some database setups the collation used for queries and creating tables can have unexpected
+; values, or change after a database version upgrade.
+; If you encounter "Illegal mix of collation" errors, setting this config to the value matching
+; your existing database tables can help.
+; This setting will only be used if "charset" is also set.
+; Matomo should work correctly without this setting but we recommend to have a collation set.
+collation =
+
 ; Database error codes to ignore during updates
 ;
 ;ignore_error_codes[] = 1105
@@ -84,6 +92,7 @@
 type = InnoDB
 schema = Mysql
 charset = utf8mb4
+collation = utf8mb4_general_ci
 enable_ssl = 0
 ssl_ca =
 ssl_cert =

core/Db.php

@@ -186,6 +186,7 @@ public static function createReaderDatabaseObject($dbConfig = null)
         $dbConfig['type'] = $masterDbConfig['type'];
         $dbConfig['tables_prefix'] = $masterDbConfig['tables_prefix'];
         $dbConfig['charset'] = $masterDbConfig['charset'];
+        $dbConfig['collation'] = $masterDbConfig['collation'] ?? null;
 
         $db = @Adapter::factory($dbConfig['adapter'], $dbConfig);
 

core/Db/Schema.php

@@ -89,6 +89,17 @@ private function getSchema(): SchemaInterface
         return $this->schema;
     }
 
+    /**
+     * Returns the default collation for a charset.
+     *
+     * @param string $charset
+     * @return string
+     */
+    public function getDefaultCollationForCharset(string $charset): string
+    {
+        return $this->getSchema()->getDefaultCollationForCharset($charset);
+    }
+
     /**
      * Get the table options to use for a CREATE TABLE statement.
      *

core/Db/Schema/Mysql.php

@@ -669,6 +669,31 @@ public function supportsComplexColumnUpdates(): bool
         return true;
     }
 
+    /**
+     * Returns the default collation for a charset.
+     *
+     * @param string $charset
+     *
+     * @return string
+     * @throws Exception
+     */
+    public function getDefaultCollationForCharset(string $charset): string
+    {
+        $result = $this->getDb()->fetchRow(
+            'SHOW COLLATION WHERE `Default` = "Yes" AND `Charset` = ?',
+            [$charset]
+        );
+
+        if (!isset($result['Collation'])) {
+            throw new Exception(sprintf(
+                'Failed to detect default collation for character set "%s"',
+                $charset
+            ));
+        }
+
+        return $result['Collation'];
+    }
+
     public function getDefaultPort(): int
     {
         return 3306;
@@ -678,10 +703,15 @@ public function getTableCreateOptions(): string
     {
         $engine = $this->getTableEngine();
         $charset = $this->getUsedCharset();
+        $collation = $this->getUsedCollation();
         $rowFormat = $this->getTableRowFormat();
 
         $options = "ENGINE=$engine DEFAULT CHARSET=$charset";
 
+        if ('' !== $collation) {
+            $options .= " COLLATE=$collation";
+        }
+
         if ('' !== $rowFormat) {
             $options .= " $rowFormat";
         }
@@ -765,8 +795,9 @@ public function getSupportedReadIsolationTransactionLevel(): string
     protected function getDatabaseCreateOptions(): string
     {
         $charset = DbHelper::getDefaultCharset();
+        $collation = $this->getDefaultCollationForCharset($charset);
 
-        return "DEFAULT CHARACTER SET $charset";
+        return "DEFAULT CHARACTER SET $charset COLLATE $collation";
     }
 
     protected function getTableEngine()
@@ -784,6 +815,11 @@ protected function getUsedCharset(): string
         return $this->getDbSettings()->getUsedCharset();
     }
 
+    protected function getUsedCollation(): string
+    {
+        return $this->getDbSettings()->getUsedCollation();
+    }
+
     private function getTablePrefix()
     {
         return $this->getDbSettings()->getTablePrefix();

core/Db/Schema/Tidb.php

@@ -9,8 +9,6 @@
 
 namespace Piwik\Db\Schema;
 
-use Piwik\DbHelper;
-
 /**
  * Mariadb schema
  */
@@ -27,6 +25,18 @@ public function supportsComplexColumnUpdates(): bool
         return false;
     }
 
+    public function getDefaultCollationForCharset(string $charset): string
+    {
+        $collation = parent::getDefaultCollationForCharset($charset);
+
+        if ('utf8mb4' === $charset && 'utf8mb4_bin' === $collation) {
+            // replace the TiDB default "utf8mb4_bin" with a better default
+            return 'utf8mb4_0900_ai_ci';
+        }
+
+        return $collation;
+    }
+
     public function getDefaultPort(): int
     {
         return 4000;
@@ -36,12 +46,17 @@ public function getTableCreateOptions(): string
     {
         $engine = $this->getTableEngine();
         $charset = $this->getUsedCharset();
+        $collation = $this->getUsedCollation();
         $rowFormat = $this->getTableRowFormat();
 
+        if ('utf8mb4' === $charset && '' === $collation) {
+            $collation = 'utf8mb4_0900_ai_ci';
+        }
+
         $options = "ENGINE=$engine DEFAULT CHARSET=$charset";
 
-        if ('utf8mb4' === $charset) {
-            $options .= ' COLLATE=utf8mb4_0900_ai_ci';
+        if ('' !== $collation) {
+            $options .= " COLLATE=$collation";
         }
 
         if ('' !== $rowFormat) {
@@ -73,16 +88,4 @@ public function getSupportedReadIsolationTransactionLevel(): string
         // TiDB doesn't support READ UNCOMMITTED
         return 'READ COMMITTED';
     }
-
-    protected function getDatabaseCreateOptions(): string
-    {
-        $charset = DbHelper::getDefaultCharset();
-        $options = "DEFAULT CHARACTER SET $charset";
-
-        if ('utf8mb4' === $charset) {
-            $options .= ' COLLATE=utf8mb4_0900_ai_ci';
-        }
-
-        return $options;
-    }
 }

core/Db/SchemaInterface.php

@@ -135,6 +135,15 @@ public function addMaxExecutionTimeHintToQuery(string $sql, float $limit): strin
      */
     public function supportsComplexColumnUpdates(): bool;
 
+    /**
+     * Returns the default collation for a charset used by this database engine.
+     *
+     * @param string $charset
+     *
+     * @return string
+     */
+    public function getDefaultCollationForCharset(string $charset): string;
+
     /**
      * Return the default port used by this database engine
      *

core/Db/Settings.php

@@ -33,6 +33,11 @@ public function getUsedCharset()
         return strtolower($this->getDbSetting('charset'));
     }
 
+    public function getUsedCollation()
+    {
+        return strtolower($this->getDbSetting('collation') ?? '');
+    }
+
     public function getRowFormat()
     {
         return $this->getUsedCharset() === 'utf8mb4' ? 'ROW_FORMAT=DYNAMIC' : '';

core/DbHelper.php

@@ -210,7 +210,7 @@ public static function tableHasIndex($table, $indexName)
      * @return string
      * @throws Tracker\Db\DbException
      */
-    public static function getDefaultCharset()
+    public static function getDefaultCharset(): string
     {
         $result = Db::get()->fetchRow("SHOW CHARACTER SET LIKE 'utf8mb4'");
 
@@ -233,6 +233,19 @@ public static function getDefaultCharset()
         return 'utf8mb4';
     }
 
+    /**
+     * Returns the default collation for a charset.
+     *
+     * @param string $charset
+     *
+     * @return string
+     * @throws Exception
+     */
+    public static function getDefaultCollationForCharset(string $charset): string
+    {
+        return Schema::getInstance()->getDefaultCollationForCharset($charset);
+    }
+
     /**
      * Returns sql queries to convert all installed tables to utf8mb4
      *

core/Tracker/Db/Mysqli.php

@@ -26,6 +26,7 @@ class Mysqli extends Db
     protected $username;
     protected $password;
     protected $charset;
+    protected $collation;
     protected $activeTransaction = false;
 
     protected $enable_ssl;
@@ -57,11 +58,12 @@ public function __construct($dbInfo, $driverName = 'mysql')
             $this->port = (int)$dbInfo['port'];
             $this->socket = null;
         }
+
         $this->dbname = $dbInfo['dbname'];
         $this->username = $dbInfo['username'];
         $this->password = $dbInfo['password'];
-        $this->charset = isset($dbInfo['charset']) ? $dbInfo['charset'] : null;
-
+        $this->charset = $dbInfo['charset'] ?? null;
+        $this->collation = $dbInfo['collation'] ?? null;
 
         if (!empty($dbInfo['enable_ssl'])) {
             $this->enable_ssl = $dbInfo['enable_ssl'];
@@ -133,8 +135,17 @@ public function connect()
             throw new DbException("Connect failed: " . mysqli_connect_error());
         }
 
-        if ($this->charset && !mysqli_set_charset($this->connection, $this->charset)) {
-            throw new DbException("Set Charset failed: " . mysqli_error($this->connection));
+        if ($this->charset && $this->collation) {
+            // mysqli_set_charset does not support setting a collation
+            $query = "SET NAMES '" . $this->charset . "' COLLATE '" . $this->collation . "'";
+
+            if (!mysqli_query($this->connection, $query)) {
+                throw new DbException("Set charset/connection collation failed: " . mysqli_error($this->connection));
+            }
+        } elseif ($this->charset) {
+            if (!mysqli_set_charset($this->connection, $this->charset)) {
+                throw new DbException("Set Charset failed: " . mysqli_error($this->connection));
+            }
         }
 
         $this->password = '';

core/Tracker/Db/Pdo/Mysql.php

@@ -26,13 +26,33 @@ class Mysql extends Db
      * @var PDO
      */
     protected $connection = null;
+
+    /**
+     * @var string
+     */
     protected $dsn;
+
+    /**
+     * @var string
+     */
     private $username;
+
+    /**
+     * @var string
+     */
     private $password;
+
+    /**
+     * @var string|null
+     */
     protected $charset;
 
-    protected $mysqlOptions = array();
+    /**
+     * @var string|null
+     */
+    private $collation;
 
+    protected $mysqlOptions = [];
 
     protected $activeTransaction = false;
 
@@ -58,8 +78,11 @@ public function __construct($dbInfo, $driverName = 'mysql')
         if (isset($dbInfo['charset'])) {
             $this->charset = $dbInfo['charset'];
             $this->dsn .= ';charset=' . $this->charset;
-        }
 
+            if (!empty($dbInfo['collation'])) {
+                $this->collation = $dbInfo['collation'];
+            }
+        }
 
         if (isset($dbInfo['enable_ssl']) && $dbInfo['enable_ssl']) {
             if (!empty($dbInfo['ssl_key'])) {
@@ -409,6 +432,11 @@ private function establishConnection(): void
          */
         if (!empty($this->charset)) {
             $sql = "SET NAMES '" . $this->charset . "'";
+
+            if (!empty($this->collation)) {
+                $sql .= " COLLATE '" . $this->collation . "'";
+            }
+
             $this->connection->exec($sql);
         }
     }