From 30f49d0d347c4cad635c8a6eb568b2657870b9c5 Mon Sep 17 00:00:00 2001 From: zhengya Date: Fri, 31 May 2024 17:13:36 +0800 Subject: [PATCH] [CONTROLLER/DB] optimized deleting org performence --- .../db/mysql/migrator/common/migrator.go | 11 +- .../controller/db/mysql/migrator/migrator.go | 6 + server/controller/db/mysql/org.go | 108 +++++++++--------- server/controller/http/service/org_data.go | 6 +- 4 files changed, 73 insertions(+), 58 deletions(-) diff --git a/server/controller/db/mysql/migrator/common/migrator.go b/server/controller/db/mysql/migrator/common/migrator.go index bd8a77942c3..6e73e736b65 100644 --- a/server/controller/db/mysql/migrator/common/migrator.go +++ b/server/controller/db/mysql/migrator/common/migrator.go @@ -58,24 +58,28 @@ func (dc *DBConfig) SetConfig(c config.MySqlConfig) { } func LogDBName(databaseName string, format string, a ...any) string { - return fmt.Sprintf("db: %s, ", databaseName) + fmt.Sprintf(format, a...) + return fmt.Sprintf("[DBName-%s] ", databaseName) + fmt.Sprintf(format, a...) } func DropDatabase(dc *DBConfig) error { log.Infof(LogDBName(dc.Config.Database, "drop database")) + defer log.Infof(LogDBName(dc.Config.Database, "dropped database")) + var databaseName string + log.Infof(LogDBName(dc.Config.Database, "drop database, check if database exists")) dc.DB.Raw(fmt.Sprintf("SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='%s'", dc.Config.Database)).Scan(&databaseName) if databaseName == dc.Config.Database { + log.Infof(LogDBName(dc.Config.Database, "drop database, database exists")) return dc.DB.Exec(fmt.Sprintf("DROP DATABASE %s", dc.Config.Database)).Error } else { - log.Infof(LogDBName(dc.Config.Database, "database doesn't exist")) + log.Infof(LogDBName(dc.Config.Database, "drop database, database doesn't exist")) return nil } } func CreateDatabase(dc *DBConfig) error { log.Infof(LogDBName(dc.Config.Database, "create database")) - log.Infof("%#v", dc.DB) + defer log.Infof(LogDBName(dc.Config.Database, "created database")) return dc.DB.Exec(fmt.Sprintf("CREATE DATABASE %s", dc.Config.Database)).Error } @@ -118,6 +122,7 @@ func initCEORGTables(dc *DBConfig) error { return err } err = dc.DB.Exec(string(initSQL)).Error + log.Info(LogDBName(dc.Config.Database, "read CE org file successfully")) if err != nil { log.Error(LogDBName(dc.Config.Database, "failed to initialize CE org tables: %s", err.Error())) return err diff --git a/server/controller/db/mysql/migrator/migrator.go b/server/controller/db/mysql/migrator/migrator.go index 35a467ac443..4bf41477b2b 100644 --- a/server/controller/db/mysql/migrator/migrator.go +++ b/server/controller/db/mysql/migrator/migrator.go @@ -84,10 +84,12 @@ func upgradeDatabase(cfg config.MySqlConfig, run bool) error { } func CreateDatabase(cfg config.MySqlConfig) (databaseExisted bool, err error) { + log.Infof(common.LogDBName(cfg.Database, "create database, get db session without name")) db, err := common.GetSessionWithoutName(cfg) if err != nil { return } + log.Infof(common.LogDBName(cfg.Database, "create database, got db session without name")) dc := common.NewDBConfig(db, cfg) databaseExisted, err = common.CreateDatabaseIfNotExists(dc) if err != nil { @@ -95,10 +97,12 @@ func CreateDatabase(cfg config.MySqlConfig) (databaseExisted bool, err error) { return } if !databaseExisted { + log.Infof(common.LogDBName(cfg.Database, "create database, get db session with name %s", cfg.Database)) db, err = common.GetSessionWithName(cfg) if err != nil { return } + log.Infof(common.LogDBName(cfg.Database, "create database, got db session with name %s", cfg.Database)) dc.SetDB(db) err = table.DropDatabaseIfInitTablesFailed(dc) } @@ -106,9 +110,11 @@ func CreateDatabase(cfg config.MySqlConfig) (databaseExisted bool, err error) { } func DropDatabase(cfg config.MySqlConfig) error { + log.Infof("drop database, get db session with name %s", cfg.Database) db, err := common.GetSessionWithName(cfg) if err != nil { return err } + log.Infof("drop database, got db session with name %s", cfg.Database) return common.DropDatabase(common.NewDBConfig(db, cfg)) } diff --git a/server/controller/db/mysql/org.go b/server/controller/db/mysql/org.go index a2c6902ea63..20a3ffa2930 100644 --- a/server/controller/db/mysql/org.go +++ b/server/controller/db/mysql/org.go @@ -18,12 +18,12 @@ package mysql import ( "fmt" - "reflect" + // "reflect" "strings" "github.com/deepflowio/deepflow/server/controller/db/mysql/common" - "gorm.io/gorm" - "gorm.io/gorm/clause" + // "gorm.io/gorm" + // "gorm.io/gorm/clause" ) const ORG_TABLE = "org" @@ -72,60 +72,60 @@ func SyncDefaultOrgData[T any](data []T) error { return nil } - // get fields to update - dataType := reflect.TypeOf(data[0]) - var fields []string - for i := 0; i < dataType.NumField(); i++ { - field := dataType.Field(i) - dbTag := field.Tag.Get("gorm") - if dbTag != "" { - columnName := GetColumnNameFromTag(dbTag) - if columnName != "" { - fields = append(fields, columnName) - } - } - } + // // get fields to update + // dataType := reflect.TypeOf(data[0]) + // var fields []string + // for i := 0; i < dataType.NumField(); i++ { + // field := dataType.Field(i) + // dbTag := field.Tag.Get("gorm") + // if dbTag != "" { + // columnName := GetColumnNameFromTag(dbTag) + // if columnName != "" { + // fields = append(fields, columnName) + // } + // } + // } - for orgID, db := range dbs.orgIDToDB { - if orgID == DefaultDB.ORGID { - continue - } + // for orgID, db := range dbs.orgIDToDB { + // if orgID == DefaultDB.ORGID { + // continue + // } - err := db.Transaction(func(tx *gorm.DB) error { - // delete - var existingIDs []int - var t T - if err := db.Model(&t).Pluck("id", &existingIDs).Error; err != nil { - return err - } - existingIDMap := make(map[int]bool) - for _, id := range existingIDs { - existingIDMap[id] = true - } - for _, item := range data { - id := reflect.ValueOf(item).FieldByName("ID").Int() - existingIDMap[int(id)] = false - } - for id, exists := range existingIDMap { - if exists { - if err := db.Where("id = ?", id).Delete(&t).Error; err != nil { - return err - } - } - } + // err := db.Transaction(func(tx *gorm.DB) error { + // // delete + // var existingIDs []int + // var t T + // if err := db.Model(&t).Pluck("id", &existingIDs).Error; err != nil { + // return err + // } + // existingIDMap := make(map[int]bool) + // for _, id := range existingIDs { + // existingIDMap[id] = true + // } + // for _, item := range data { + // id := reflect.ValueOf(item).FieldByName("ID").Int() + // existingIDMap[int(id)] = false + // } + // for id, exists := range existingIDMap { + // if exists { + // if err := db.Where("id = ?", id).Delete(&t).Error; err != nil { + // return err + // } + // } + // } - // add or update - if err := db.Clauses(clause.OnConflict{ - DoUpdates: clause.AssignmentColumns(fields), // `UpdateAll: true,` can not update time - }).Save(&data).Error; err != nil { - return fmt.Errorf("failed to sync data: %v", err) - } - return nil - }) - if err != nil { - log.Errorf("org(id:%d, name:%s) error: %s", db.ORGID, db.Name, err.Error()) - } - } + // // add or update + // if err := db.Clauses(clause.OnConflict{ + // DoUpdates: clause.AssignmentColumns(fields), // `UpdateAll: true,` can not update time + // }).Save(&data).Error; err != nil { + // return fmt.Errorf("failed to sync data: %v", err) + // } + // return nil + // }) + // if err != nil { + // log.Errorf("org(id:%d, name:%s) error: %s", db.ORGID, db.Name, err.Error()) + // } + // } return nil } diff --git a/server/controller/http/service/org_data.go b/server/controller/http/service/org_data.go index 54a3f04623d..a4909834b31 100644 --- a/server/controller/http/service/org_data.go +++ b/server/controller/http/service/org_data.go @@ -69,11 +69,15 @@ func CreateORGData(dataCreate model.ORGDataCreate, mysqlCfg mysqlcfg.MySqlConfig } func DeleteORGData(orgID int, mysqlCfg mysqlcfg.MySqlConfig) (err error) { - log.Infof("delete org (id: %d) data", orgID) + log.Infof("drop org (id: %d) data", orgID) + defer log.Infof("drop org (id: %d) data, dropped", orgID) + cfg := common.ReplaceConfigDatabaseName(mysqlCfg, orgID) + log.Infof("drop org, replaced database name: %s", cfg.Database) if err = migrator.DropDatabase(cfg); err != nil { return err } + log.Infof("drop org, database dropped") return servercommon.DropOrg(uint16(orgID)) }