fix: update authentication plugin of MySQL users in v8.4

closes overhangio#1095
- mysql_native_password is disabled in MySQL v8.4
- Users created in MySQL v5.7 use the mysql_native_password authentication plugin
- This plugin is not updated when MySQL is upgraded
- We therefore manually upgrade these users when upgrading to v8.4
- Upgrading directly from MySQL v5.7 -> v8.4 is also not allowed
- We therefore first jump to v8.1, update the authentication plugin, then go to v8.4

changelog.d/20240718_171945_danyal.faheem_mysql_authentication_plugin_change.md

@@ -0,0 +1,4 @@
+- [Bugfix] Manually update the authentication plugin of MySQL users from mysql_native_password to caching_sha2_password when upgrading from quince. (by @Danyal-Faheem)
+   This process should be automatic for most users. However, if you are running a third-party MySQL (i.e., RUN_MYSQL=false), you are expected to perform this process yourself. Please refer to the third-party provider's documentation for detailed instructions. Ensuring that your MySQL version is up-to-date is crucial for maintaining compatibility and security.
+- [Bugfix] Do not directly upgrade MySQL from v5.7 to v8.4 when upgrading from quince as MySQL does not allow that. First, upgrade to v8.1 and then to v8.4. (by @Danyal-Faheem)
+  This process should be automatic for most users. However, if you are running a third-party MySQL (i.e., RUN_MYSQL=false), you are expected to perform this process yourself. Please refer to the third-party provider's documentation for detailed instructions. Ensuring that your MySQL version is up-to-date is crucial for maintaining compatibility and security. 

tutor/commands/upgrade/common.py

@@ -43,6 +43,29 @@ def upgrade_from_nutmeg(context: click.Context, config: Config) -> None:
     )
 
 
+def get_mysql_change_authentication_plugin_query(config: Config) -> str:
+    """Helper function to generate queries depending on the loaded plugins"""
+    loaded_plugins = list(plugins.iter_loaded())
+
+    def generate_mysql_authentication_plugin_update_query(username: str, password: str, host: str) -> str:
+        return f"ALTER USER '{username}'@'{host}' IDENTIFIED with caching_sha2_password BY '{password}';"
+
+    host = "%"
+    query = ""
+    query += generate_mysql_authentication_plugin_update_query(
+        config["MYSQL_ROOT_USERNAME"], config["MYSQL_ROOT_PASSWORD"], host)
+    query += generate_mysql_authentication_plugin_update_query(
+        config["OPENEDX_MYSQL_USERNAME"], config["OPENEDX_MYSQL_PASSWORD"], host)
+
+    for plugin in loaded_plugins:
+        plugin_uppercase = plugin.upper()
+        if f"{plugin_uppercase}_MYSQL_USERNAME" in config and f"{plugin_uppercase}_MYSQL_PASSWORD" in config:
+            query += generate_mysql_authentication_plugin_update_query(
+                config[f"{plugin_uppercase}_MYSQL_USERNAME"], config[f"{plugin_uppercase}_MYSQL_PASSWORD"], host)
+
+    return query
+
+
 def get_mongo_upgrade_parameters(
     docker_version: str, compatibility_version: str
 ) -> tuple[int, dict[str, int | str]]:

tutor/commands/upgrade/compose.py

@@ -1,6 +1,7 @@
 from time import sleep
 
 import click
+import shlex
 
 from tutor import config as tutor_config
 from tutor import env as tutor_env
@@ -165,6 +166,48 @@ def upgrade_from_quince(context: click.Context, config: Config) -> None:
     upgrade_mongodb(context, config, "6.0.14", "6.0")
     upgrade_mongodb(context, config, "7.0.7", "7.0")
 
+    if not config["RUN_MYSQL"]:
+        fmt.echo_info(
+            "You are not running MySQL (RUN_MYSQL=false). It is your "
+            "responsibility to upgrade your MySQL instance to v8.4. There is "
+            "nothing left to do to upgrade from Quince."
+        )
+        return
+
+    # Revert the MySQL image first to build data dictionary on v8.1
+    old_mysql_docker_image = "docker.io/mysql:8.1.0"
+    new_mysql_docker_image = config["DOCKER_IMAGE_MYSQL"]
+    config["DOCKER_IMAGE_MYSQL"] = old_mysql_docker_image
+    click.echo(
+        fmt.title(f"Upgrading MySQL to v{new_mysql_docker_image.split(':')[1]}"))
+    tutor_env.save(context.obj.root, config)
+    context.invoke(compose.start, detach=True, services=["mysql"])
+    fmt.echo_info("Waiting for MySQL to boot...")
+    sleep(30)
+
+    query = common_upgrade.get_mysql_change_authentication_plugin_query(config)
+
+    # Update the authentication plugin in v8.1 as it is disabled in v8.4
+    context.invoke(
+        compose.execute,
+        args=[
+            "mysql",
+            "bash",
+            "-e",
+            "-c",
+            f"mysql --user={config['MYSQL_ROOT_USERNAME']} --password='{config['MYSQL_ROOT_PASSWORD']}' --host={config['MYSQL_HOST']} --port={config['MYSQL_PORT']} --skip-column-names --silent " + shlex.join([
+                                                                                                                                                                                                                f"--database={config['OPENEDX_MYSQL_DATABASE']}", "-e", query])
+
+        ],
+    )
+
+    # Upgrade back to v8.4
+    config["DOCKER_IMAGE_MYSQL"] = new_mysql_docker_image
+    tutor_env.save(context.obj.root, config)
+    context.invoke(compose.start, detach=True, services=["mysql"])
+    fmt.echo_info("Waiting for MySQL to boot...")
+    sleep(30)
+
 
 def upgrade_mongodb(
     context: click.Context,

tutor/commands/upgrade/k8s.py

@@ -171,6 +171,36 @@ def upgrade_from_quince(config: Config) -> None:
     upgrade_mongodb(config, "5.0.26", "5.0")
     upgrade_mongodb(config, "6.0.14", "6.0")
     upgrade_mongodb(config, "7.0.7", "7.0")
+
+    if not config["RUN_MYSQL"]:
+        fmt.echo_info(
+            "You are not running MySQL (RUN_MYSQL=false). It is your "
+            "responsibility to upgrade your MySQL instance to v8.4. There is "
+            "nothing left to do to upgrade from Quince."
+        )
+        return
+
+    query = common_upgrade.get_mysql_change_authentication_plugin_query(config)
+
+    # We need to first revert MySQL back to v8.1 to build the data dictionary on it
+    # And also to update the authentication plugin as it is disabled on v8.4
+    message = f"""Automatic release upgrade is unsupported in Kubernetes. If you are upgrading from Olive or an earlier release to Redwood, you
+    should upgrade the authentication plugin of your users. To upgrade, run something similar to:
+
+    tutor k8s stop
+    tutor config save --set DOCKER_IMAGE_MYSQL=docker.io/mysql:8.1.0
+    tutor k8s start
+    tutor k8s exec mysql mysql \
+    --user=$(tutor config printvalue MYSQL_ROOT_USERNAME) \
+    --password=$(tutor config printvalue MYSQL_ROOT_PASSWORD) \
+    --host=$(tutor config printvalue MYSQL_HOST) \
+    --port=$(tutor config printvalue MYSQL_PORT) \
+    --database=$(tutor config printvalue OPENEDX_MYSQL_DATABASE) \
+    -e "{query}"
+    tutor config save --unset DOCKER_IMAGE_MYSQL
+    tutor k8s start
+    """
+    fmt.echo_info(message)
 
 
 def upgrade_mongodb(

tutor/templates/k8s/deployments.yml

@@ -397,7 +397,6 @@ spec:
             - "--character-set-server=utf8mb4"
             - "--collation-server=utf8mb4_unicode_ci"
             - "--binlog-expire-logs-seconds=259200"
-            - "--mysql-native-password=ON"
           env:
             - name: MYSQL_ROOT_PASSWORD
               value: "{{ MYSQL_ROOT_PASSWORD }}"

tutor/templates/local/docker-compose.yml

@@ -44,7 +44,6 @@ services:
       --character-set-server=utf8mb4
       --collation-server=utf8mb4_unicode_ci
       --binlog-expire-logs-seconds=259200
-      --mysql-native-password=ON
     restart: unless-stopped
     user: "999:999"
     volumes: