From d5a8eec4e671f34014acd8b17201c9d9d4fa14ae Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 29 Mar 2023 14:59:42 +0200 Subject: [PATCH] feat: move default MySQL charset to utf8mb4 --- src/DIRAC/Core/Utilities/MySQL.py | 4 ++-- .../DatasetManager/DatasetManager.py | 2 +- .../DataManagementSystem/DB/FileCatalogDB.sql | 18 ++++++++--------- .../DB/FileCatalogWithFkAndPsDB.sql | 18 ++++++++--------- src/DIRAC/FrameworkSystem/DB/AuthDB.py | 6 +++--- .../DB/InstalledComponentsDB.py | 8 ++++---- src/DIRAC/FrameworkSystem/DB/TokenDB.py | 2 +- .../ProductionSystem/DB/ProductionDB.sql | 8 ++++---- .../DB/ResourceManagementDB.py | 16 +++++++-------- .../DB/ResourceStatusDB.py | 4 ++-- .../DB/TransformationDB.sql | 18 ++++++++--------- .../DB/ElasticJobParametersDB.py | 2 +- .../WorkloadManagementSystem/DB/JobDB.sql | 20 +++++++++---------- .../DB/PilotAgentsDB.sql | 6 +++--- 14 files changed, 66 insertions(+), 66 deletions(-) diff --git a/src/DIRAC/Core/Utilities/MySQL.py b/src/DIRAC/Core/Utilities/MySQL.py index 3f594862ee4..24200094459 100755 --- a/src/DIRAC/Core/Utilities/MySQL.py +++ b/src/DIRAC/Core/Utilities/MySQL.py @@ -896,7 +896,7 @@ def _createTables(self, tableDict, force=False): index is the list of fields to be indexed. This indexes will declared unique. "Engine": use the given DB engine, InnoDB is the default if not present. - "Charset": use the given character set. Default is latin1 + "Charset": use the given character set. Default is utf8mb4 force: if True, requested tables are DROP if they exist. if False, returned with S_ERROR if table exist. @@ -1013,7 +1013,7 @@ def _createTables(self, tableDict, force=False): ) engine = thisTable.get("Engine", "InnoDB") - charset = thisTable.get("Charset", "latin1") + charset = thisTable.get("Charset", "utf8mb4") cmd = "CREATE TABLE `{}` (\n{}\n) ENGINE={} DEFAULT CHARSET={}".format( table, diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py index 4fb95b4ed26..f4bb88da11a 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py @@ -12,7 +12,7 @@ class DatasetManager: _tables["FC_MetaDatasets"] = { "Fields": { "DatasetID": "INT AUTO_INCREMENT", - "DatasetName": "VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL", + "DatasetName": "VARCHAR(128) CHARACTER SET utfmb4 COLLATE utf8mb4_bin NOT NULL", "MetaQuery": "VARCHAR(512)", "DirID": "INT NOT NULL DEFAULT 0", "TotalSize": "BIGINT UNSIGNED NOT NULL", diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql index 54063a99969..e6a7a8fbae3 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql @@ -23,7 +23,7 @@ CREATE TABLE FC_Files( UID SMALLINT UNSIGNED NOT NULL, GID TINYINT UNSIGNED NOT NULL, Status SMALLINT UNSIGNED NOT NULL, - FileName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + FileName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, INDEX (DirID), INDEX (UID,GID), INDEX (Status), @@ -119,7 +119,7 @@ CREATE TABLE FC_Users ( CREATE TABLE FC_StorageElements ( SEID INTEGER AUTO_INCREMENT PRIMARY KEY, - SEName VARCHAR(127) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + SEName VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, AliasName VARCHAR(127) DEFAULT '', UNIQUE KEY (SEName) ) ENGINE = INNODB; @@ -159,7 +159,7 @@ CREATE TABLE FC_DirectoryInfo ( CREATE TABLE FC_DirMeta ( DirID INTEGER NOT NULL, - MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname', + MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname', MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname', PRIMARY KEY (DirID,MetaKey) ) ENGINE = INNODB; @@ -168,7 +168,7 @@ CREATE TABLE FC_DirMeta ( CREATE TABLE FC_FileMeta ( FileID INTEGER NOT NULL, - MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname', + MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname', MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname', PRIMARY KEY (FileID,MetaKey) ) ENGINE = INNODB; @@ -177,7 +177,7 @@ CREATE TABLE FC_FileMeta ( CREATE TABLE FC_DirectoryTree ( DirID INT AUTO_INCREMENT PRIMARY KEY, - DirName VARCHAR(1024) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + DirName VARCHAR(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, Parent INT NOT NULL DEFAULT 0, INDEX (Parent), INDEX (DirName) @@ -187,7 +187,7 @@ CREATE TABLE FC_DirectoryTree ( CREATE TABLE FC_DirectoryTreeM ( DirID INT AUTO_INCREMENT PRIMARY KEY, - DirName VARCHAR(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + DirName VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, Parent INT NOT NULL DEFAULT 0, Level INT NOT NULL, INDEX (Level), @@ -199,7 +199,7 @@ CREATE TABLE FC_DirectoryTreeM ( CREATE TABLE FC_DirectoryLevelTree ( DirID INT AUTO_INCREMENT PRIMARY KEY, - DirName VARCHAR(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + DirName VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, Parent INT NOT NULL DEFAULT 0, Level INT NOT NULL, LPATH1 INT NOT NULL DEFAULT 0, @@ -240,7 +240,7 @@ CREATE TABLE FC_DirectoryUsage( CREATE TABLE FC_MetaFields ( MetaID INT AUTO_INCREMENT PRIMARY KEY, - MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaType VARCHAR(128) NOT NULL ) ENGINE = INNODB; @@ -248,7 +248,7 @@ CREATE TABLE FC_MetaFields ( CREATE TABLE FC_FileMetaFields ( MetaID INT AUTO_INCREMENT PRIMARY KEY, - MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaType VARCHAR(128) NOT NULL ) ENGINE = INNODB; diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql index c27e91ef3aa..8d65d1b2d6f 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql @@ -36,7 +36,7 @@ INSERT INTO FC_Statuses (StatusID, Status) values (1, 'FakeStatus'); CREATE TABLE FC_StorageElements ( SEID INTEGER AUTO_INCREMENT, - SEName VARCHAR(127) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + SEName VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, AliasName VARCHAR(127) DEFAULT '', PRIMARY KEY (SEID), @@ -79,7 +79,7 @@ INSERT INTO FC_Users (UID, UserName) values (1, 'root'); -- -- create table FC_DirectoryList ( -- DirID INT NOT NULL AUTO_INCREMENT, --- Name varchar(255)CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, +-- Name varchar(255)CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, -- -- PRIMARY KEY (DirID), -- @@ -116,7 +116,7 @@ create table FC_DirectoryList ( ModificationDate DATETIME, Mode SMALLINT UNSIGNED NOT NULL DEFAULT 775, Status INTEGER NOT NULL DEFAULT 0, - Name varchar(255)CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + Name varchar(255)CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, PRIMARY KEY (DirID), FOREIGN KEY (UID) REFERENCES FC_Users(UID), @@ -158,7 +158,7 @@ CREATE TABLE FC_Files( Mode SMALLINT UNSIGNED NOT NULL DEFAULT 775, ChecksumType ENUM('Adler32','MD5'), Checksum VARCHAR(32), - FileName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + FileName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, PRIMARY KEY (FileID), FOREIGN KEY (DirID) REFERENCES FC_DirectoryList(DirID) ON DELETE CASCADE, @@ -223,7 +223,7 @@ CREATE TABLE FC_DirectoryUsage( CREATE TABLE FC_DirMeta ( DirID INTEGER NOT NULL, - MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname', + MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname', MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname', PRIMARY KEY (DirID,MetaKey) ) ENGINE = INNODB; @@ -232,7 +232,7 @@ CREATE TABLE FC_DirMeta ( CREATE TABLE FC_FileMeta ( FileID INTEGER NOT NULL, - MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname', + MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname', MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname', PRIMARY KEY (FileID,MetaKey) ) ENGINE = INNODB; @@ -242,7 +242,7 @@ CREATE TABLE FC_FileMeta ( CREATE TABLE FC_MetaFields ( MetaID INT AUTO_INCREMENT PRIMARY KEY, - MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaType VARCHAR(128) NOT NULL ) ENGINE = INNODB; @@ -250,7 +250,7 @@ CREATE TABLE FC_MetaFields ( CREATE TABLE FC_FileMetaFields ( MetaID INT AUTO_INCREMENT PRIMARY KEY, - MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaType VARCHAR(128) NOT NULL ) ENGINE = INNODB; @@ -280,7 +280,7 @@ CREATE TABLE FC_FileAncestors ( CREATE TABLE FC_MetaDatasets ( DatasetID INT AUTO_INCREMENT, - DatasetName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + DatasetName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaQuery VARCHAR(512), DirID INT NOT NULL DEFAULT 0, TotalSize BIGINT UNSIGNED NOT NULL, diff --git a/src/DIRAC/FrameworkSystem/DB/AuthDB.py b/src/DIRAC/FrameworkSystem/DB/AuthDB.py index 66803122d4c..0ea16d4fc75 100644 --- a/src/DIRAC/FrameworkSystem/DB/AuthDB.py +++ b/src/DIRAC/FrameworkSystem/DB/AuthDB.py @@ -22,7 +22,7 @@ class RefreshToken(Model): __tablename__ = "RefreshToken" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} jti = Column(String(255), nullable=False, primary_key=True) issued_at = Column(Integer, nullable=False, default=0) access_token = Column(Text, nullable=False) @@ -31,7 +31,7 @@ class RefreshToken(Model): class JWK(Model): __tablename__ = "JWK" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} kid = Column(String(255), unique=True, primary_key=True, nullable=False) key = Column(Text, nullable=False) expires_at = Column(Integer, nullable=False, default=0) @@ -39,7 +39,7 @@ class JWK(Model): class AuthSession(Model): __tablename__ = "AuthSession" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} id = Column(String(255), unique=True, primary_key=True, nullable=False) uri = Column(String(255)) state = Column(String(255)) diff --git a/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py b/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py index a06e87f7fc5..9796bebddcc 100644 --- a/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py +++ b/src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py @@ -27,7 +27,7 @@ class Component(componentsBase): """ __tablename__ = "Components" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} componentID = Column("ComponentID", Integer, primary_key=True) system = Column("DIRACSystem", String(32), nullable=False) @@ -87,7 +87,7 @@ class Host(componentsBase): """ __tablename__ = "Hosts" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} hostID = Column("HostID", Integer, primary_key=True) hostName = Column("HostName", String(32), nullable=False) @@ -138,7 +138,7 @@ class InstalledComponent(componentsBase): """ __tablename__ = "InstalledComponents" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} componentID = Column("ComponentID", Integer, ForeignKey("Components.ComponentID"), primary_key=True) hostID = Column("HostID", Integer, ForeignKey("Hosts.HostID"), primary_key=True) @@ -217,7 +217,7 @@ class HostLogging(componentsBase): """ __tablename__ = "HostLogging" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} hostName = Column("HostName", String(32), nullable=False, primary_key=True) # status diff --git a/src/DIRAC/FrameworkSystem/DB/TokenDB.py b/src/DIRAC/FrameworkSystem/DB/TokenDB.py index 424625d6c16..274dbf195e7 100644 --- a/src/DIRAC/FrameworkSystem/DB/TokenDB.py +++ b/src/DIRAC/FrameworkSystem/DB/TokenDB.py @@ -24,7 +24,7 @@ class Token(Model, OAuth2TokenMixin): """This class describes token fields""" __tablename__ = "Token" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} # access_token too large for varchar(255) # 767 bytes is the stated prefix limitation for InnoDB tables in MySQL version 5.6 # https://stackoverflow.com/questions/1827063/mysql-error-key-specification-without-a-key-length diff --git a/src/DIRAC/ProductionSystem/DB/ProductionDB.sql b/src/DIRAC/ProductionSystem/DB/ProductionDB.sql index c12d3343330..a33cc5cf8f7 100755 --- a/src/DIRAC/ProductionSystem/DB/ProductionDB.sql +++ b/src/DIRAC/ProductionSystem/DB/ProductionDB.sql @@ -30,7 +30,7 @@ CREATE TABLE Productions( Status CHAR(32) DEFAULT 'New', PRIMARY KEY(ProductionID), INDEX(ProductionName) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS ProductionSteps; @@ -50,7 +50,7 @@ CREATE TABLE ProductionSteps( InsertedTime DATETIME, PRIMARY KEY(StepID), UNIQUE INDEX(StepID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS ProductionTransformations; @@ -62,7 +62,7 @@ CREATE TABLE ProductionTransformations( PRIMARY KEY(ProductionID, TransformationID), UNIQUE INDEX(TransformationID), FOREIGN KEY(ProductionID) REFERENCES Productions(ProductionID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS ProductionTransformationLinks; @@ -74,6 +74,6 @@ CREATE TABLE ProductionTransformationLinks( INDEX(TransformationID), FOREIGN KEY(ProductionID) REFERENCES Productions(ProductionID), FOREIGN KEY(TransformationID) REFERENCES ProductionTransformations(TransformationID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; SET FOREIGN_KEY_CHECKS = 1; diff --git a/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py b/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py index c58c643304f..36f2034e6f6 100644 --- a/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py +++ b/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py @@ -45,7 +45,7 @@ class AccountingCache(rmsBase): """AccountingCache table""" __tablename__ = "AccountingCache" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} name = Column("Name", String(64), nullable=False, primary_key=True) plotname = Column("PlotName", String(64), nullable=False, primary_key=True) @@ -88,7 +88,7 @@ class DowntimeCache(rmsBase): """DowntimeCache table""" __tablename__ = "DowntimeCache" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} downtimeid = Column("DowntimeID", String(127), nullable=False, primary_key=True) name = Column("Name", String(64), nullable=False) @@ -150,7 +150,7 @@ class GGUSTicketsCache(rmsBase): """GGUSTicketsCache table""" __tablename__ = "GGUSTicketsCache" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} gocsite = Column("GocSite", String(64), nullable=False, primary_key=True) tickets = Column("Tickets", String(1024), nullable=False) @@ -186,7 +186,7 @@ class JobCache(rmsBase): """JobCache table""" __tablename__ = "JobCache" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} site = Column("Site", String(64), nullable=False, primary_key=True) status = Column("Status", String(16), nullable=False) @@ -219,7 +219,7 @@ class PilotCache(rmsBase): """PilotCache table""" __tablename__ = "PilotCache" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} site = Column("Site", String(64), nullable=False, primary_key=True) ce = Column("CE", String(64), nullable=False, primary_key=True) @@ -261,7 +261,7 @@ class PolicyResult(rmsBase): """PolicyResult table""" __tablename__ = "PolicyResult" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} policyname = Column("PolicyName", String(64), nullable=False, primary_key=True) statustype = Column("StatusType", String(16), nullable=False, server_default="", primary_key=True) @@ -320,7 +320,7 @@ class SpaceTokenOccupancyCache(rmsBase): """SpaceTokenOccupancyCache table""" __tablename__ = "SpaceTokenOccupancyCache" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} endpoint = Column("Endpoint", String(128), nullable=False, primary_key=True) token = Column("Token", String(64), nullable=False, primary_key=True) @@ -358,7 +358,7 @@ class TransferCache(rmsBase): """TransferCache table""" __tablename__ = "TransferCache" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} sourcename = Column("SourceName", String(64), nullable=False, primary_key=True) destinationname = Column("DestinationName", String(64), nullable=False, primary_key=True) diff --git a/src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py b/src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py index a36e0b26e24..f3f1b67619e 100644 --- a/src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py +++ b/src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py @@ -50,7 +50,7 @@ class ResourceStatusCache(rssBase): """ __tablename__ = "ResourceStatusCache" - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} id = Column("ID", BigInteger, nullable=False, autoincrement=True, primary_key=True) sitename = Column("SiteName", String(64), nullable=False) @@ -89,7 +89,7 @@ class ElementStatusBase: Prototype for tables. """ - __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"} + __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"} name = Column("Name", String(64), nullable=False, primary_key=True) statustype = Column("StatusType", String(128), nullable=False, server_default="all", primary_key=True) diff --git a/src/DIRAC/TransformationSystem/DB/TransformationDB.sql b/src/DIRAC/TransformationSystem/DB/TransformationDB.sql index 04be76b9c2f..6c6739a45ee 100755 --- a/src/DIRAC/TransformationSystem/DB/TransformationDB.sql +++ b/src/DIRAC/TransformationSystem/DB/TransformationDB.sql @@ -45,7 +45,7 @@ CREATE TABLE Transformations( PRIMARY KEY(TransformationID), INDEX(TransformationName), INDEX(TransformationFamily) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS DataFiles; @@ -56,7 +56,7 @@ CREATE TABLE DataFiles( INDEX(Status), INDEX(LFN), PRIMARY KEY(FileID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS AdditionalParameters; @@ -67,7 +67,7 @@ CREATE TABLE AdditionalParameters( ParameterType VARCHAR(32) DEFAULT 'StringType', PRIMARY KEY(TransformationID, ParameterName), FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS TransformationLog; @@ -81,7 +81,7 @@ CREATE TABLE TransformationLog( INDEX(TransformationID), INDEX(MessageDate), FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS TransformationTasks; @@ -97,7 +97,7 @@ CREATE TABLE TransformationTasks( INDEX(ExternalStatus), INDEX(TransformationID,ExternalStatus), FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- This is required to mimic the AUTO_INCREMENT behavior of TaskID which was possible with MyISAM: CREATE TRIGGER `TaskID_Generator` BEFORE INSERT ON TransformationTasks @@ -122,7 +122,7 @@ CREATE TABLE TransformationFiles( INDEX(TransformationID,Status), FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID), FOREIGN KEY(FileID) REFERENCES DataFiles(FileID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS TransformationFileTasks; @@ -134,7 +134,7 @@ CREATE TABLE TransformationFileTasks( FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID), FOREIGN KEY(TransformationID, FileID) REFERENCES TransformationFiles(TransformationID, FileID), FOREIGN KEY(TransformationID, TaskID) REFERENCES TransformationTasks(TransformationID, TaskID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS TaskInputs; @@ -144,7 +144,7 @@ CREATE TABLE TaskInputs( InputVector MEDIUMTEXT, PRIMARY KEY(TransformationID, TaskID), FOREIGN KEY(TransformationID, TaskID) REFERENCES TransformationTasks(TransformationID, TaskID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; -- ------------------------------------------------------------------------------- DROP TABLE IF EXISTS TransformationMetaQueries; @@ -156,6 +156,6 @@ CREATE TABLE TransformationMetaQueries( QueryType ENUM('Input', 'Output') DEFAULT 'Input', PRIMARY KEY(TransformationID, MetaDataName, QueryType), FOREIGN KEY(TransformationID) REFERENCES Transformations(TransformationID) -) ENGINE = InnoDB DEFAULT CHARSET = utf8; +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; SET FOREIGN_KEY_CHECKS = 1; diff --git a/src/DIRAC/WorkloadManagementSystem/DB/ElasticJobParametersDB.py b/src/DIRAC/WorkloadManagementSystem/DB/ElasticJobParametersDB.py index 407a7a5bde9..82468cbffb9 100644 --- a/src/DIRAC/WorkloadManagementSystem/DB/ElasticJobParametersDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/ElasticJobParametersDB.py @@ -9,7 +9,7 @@ `Value` TEXT NOT NULL, PRIMARY KEY (`JobID`,`Name`), FOREIGN KEY (`JobID`) REFERENCES `Jobs`(`JobID`) - ) ENGINE=InnoDB DEFAULT CHARSET=latin1; + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; Here we define a dynamic mapping with the constant fields:: diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql index b5736313bad..55b3118ceef 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql @@ -28,7 +28,7 @@ CREATE TABLE `JobJDLs` ( `JobRequirements` TEXT NOT NULL, `OriginalJDL` MEDIUMTEXT NOT NULL, PRIMARY KEY (`JobID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ------------------------------------------------------------------------------ DROP TABLE IF EXISTS `Jobs`; @@ -78,7 +78,7 @@ CREATE TABLE `Jobs` ( KEY `ApplicationStatus` (`ApplicationStatus`), KEY `StatusSite` (`Status`,`Site`), KEY `LastUpdateTime` (`LastUpdateTime`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ------------------------------------------------------------------------------ DROP TABLE IF EXISTS `InputData`; @@ -88,7 +88,7 @@ CREATE TABLE `InputData` ( `Status` VARCHAR(32) NOT NULL DEFAULT 'AprioriGood', PRIMARY KEY (`JobID`,`LFN`), FOREIGN KEY (`JobID`) REFERENCES `Jobs`(`JobID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ------------------------------------------------------------------------------ DROP TABLE IF EXISTS `JobParameters`; @@ -98,7 +98,7 @@ CREATE TABLE `JobParameters` ( `Value` TEXT NOT NULL, PRIMARY KEY (`JobID`,`Name`), FOREIGN KEY (`JobID`) REFERENCES `Jobs`(`JobID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ------------------------------------------------------------------------------ DROP TABLE IF EXISTS `OptimizerParameters`; @@ -108,7 +108,7 @@ CREATE TABLE `OptimizerParameters` ( `Value` MEDIUMTEXT NOT NULL, PRIMARY KEY (`JobID`,`Name`), FOREIGN KEY (`JobID`) REFERENCES `Jobs`(`JobID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ------------------------------------------------------------------------------ DROP TABLE IF EXISTS `AtticJobParameters`; @@ -119,7 +119,7 @@ CREATE TABLE `AtticJobParameters` ( `RescheduleCycle` INT(11) UNSIGNED NOT NULL, PRIMARY KEY (`JobID`,`Name`,`RescheduleCycle`), FOREIGN KEY (`JobID`) REFERENCES `Jobs`(`JobID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ------------------------------------------------------------------------------ DROP TABLE IF EXISTS `SiteMask`; @@ -130,7 +130,7 @@ CREATE TABLE `SiteMask` ( `Author` VARCHAR(255) NOT NULL, `Comment` TEXT NOT NULL, PRIMARY KEY (`Site`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `SiteMaskLogging`; CREATE TABLE `SiteMaskLogging` ( @@ -140,7 +140,7 @@ CREATE TABLE `SiteMaskLogging` ( `Author` VARCHAR(255) NOT NULL, `Comment` TEXT NOT NULL, PRIMARY KEY (`Site`,`UpdateTime`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ------------------------------------------------------------------------------ DROP TABLE IF EXISTS `HeartBeatLoggingInfo`; @@ -151,7 +151,7 @@ CREATE TABLE `HeartBeatLoggingInfo` ( `HeartBeatTime` DATETIME NOT NULL, PRIMARY KEY (`JobID`,`Name`,`HeartBeatTime`), FOREIGN KEY (`JobID`) REFERENCES `Jobs`(`JobID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- ------------------------------------------------------------------------------ DROP TABLE IF EXISTS `JobCommands`; @@ -164,4 +164,4 @@ CREATE TABLE `JobCommands` ( `ExecutionTime` DATETIME DEFAULT NULL, PRIMARY KEY (`JobID`,`Arguments`,`ReceptionTime`), FOREIGN KEY (`JobID`) REFERENCES `Jobs`(`JobID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql index afddcdd7b11..6b5026cd6d7 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.sql @@ -52,7 +52,7 @@ CREATE TABLE `PilotAgents` ( KEY `PilotJobReference` (`PilotJobReference`), KEY `Status` (`Status`), KEY `Statuskey` (`GridSite`,`DestinationSite`,`Status`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `JobToPilotMapping`; @@ -62,7 +62,7 @@ CREATE TABLE `JobToPilotMapping` ( `StartTime` DATETIME NOT NULL, KEY `JobID` (`JobID`), KEY `PilotID` (`PilotID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; DROP TABLE IF EXISTS `PilotOutput`; CREATE TABLE `PilotOutput` ( @@ -70,4 +70,4 @@ CREATE TABLE `PilotOutput` ( `StdOutput` MEDIUMTEXT, `StdError` MEDIUMTEXT, PRIMARY KEY (`PilotID`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;