Skip to content

Commit

Permalink
[CONTROLLER/DB] optimized deleting org performence
Browse files Browse the repository at this point in the history
  • Loading branch information
zhengya committed Jun 4, 2024
1 parent 743e28e commit 30f49d0
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 58 deletions.
11 changes: 8 additions & 3 deletions server/controller/db/mysql/migrator/common/migrator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,24 +58,28 @@ func (dc *DBConfig) SetConfig(c config.MySqlConfig) {
}

func LogDBName(databaseName string, format string, a ...any) string {
return fmt.Sprintf("db: %s, ", databaseName) + fmt.Sprintf(format, a...)
return fmt.Sprintf("[DBName-%s] ", databaseName) + fmt.Sprintf(format, a...)
}

func DropDatabase(dc *DBConfig) error {
log.Infof(LogDBName(dc.Config.Database, "drop database"))
defer log.Infof(LogDBName(dc.Config.Database, "dropped database"))

var databaseName string
log.Infof(LogDBName(dc.Config.Database, "drop database, check if database exists"))
dc.DB.Raw(fmt.Sprintf("SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='%s'", dc.Config.Database)).Scan(&databaseName)
if databaseName == dc.Config.Database {
log.Infof(LogDBName(dc.Config.Database, "drop database, database exists"))
return dc.DB.Exec(fmt.Sprintf("DROP DATABASE %s", dc.Config.Database)).Error
} else {
log.Infof(LogDBName(dc.Config.Database, "database doesn't exist"))
log.Infof(LogDBName(dc.Config.Database, "drop database, database doesn't exist"))
return nil
}
}

func CreateDatabase(dc *DBConfig) error {
log.Infof(LogDBName(dc.Config.Database, "create database"))
log.Infof("%#v", dc.DB)
defer log.Infof(LogDBName(dc.Config.Database, "created database"))
return dc.DB.Exec(fmt.Sprintf("CREATE DATABASE %s", dc.Config.Database)).Error
}

Expand Down Expand Up @@ -118,6 +122,7 @@ func initCEORGTables(dc *DBConfig) error {
return err
}
err = dc.DB.Exec(string(initSQL)).Error
log.Info(LogDBName(dc.Config.Database, "read CE org file successfully"))
if err != nil {
log.Error(LogDBName(dc.Config.Database, "failed to initialize CE org tables: %s", err.Error()))
return err
Expand Down
6 changes: 6 additions & 0 deletions server/controller/db/mysql/migrator/migrator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,31 +84,37 @@ func upgradeDatabase(cfg config.MySqlConfig, run bool) error {
}

func CreateDatabase(cfg config.MySqlConfig) (databaseExisted bool, err error) {
log.Infof(common.LogDBName(cfg.Database, "create database, get db session without name"))
db, err := common.GetSessionWithoutName(cfg)
if err != nil {
return
}
log.Infof(common.LogDBName(cfg.Database, "create database, got db session without name"))
dc := common.NewDBConfig(db, cfg)
databaseExisted, err = common.CreateDatabaseIfNotExists(dc)
if err != nil {
log.Error(common.LogDBName(cfg.Database, "database is not ready: %v", err))
return
}
if !databaseExisted {
log.Infof(common.LogDBName(cfg.Database, "create database, get db session with name %s", cfg.Database))
db, err = common.GetSessionWithName(cfg)
if err != nil {
return
}
log.Infof(common.LogDBName(cfg.Database, "create database, got db session with name %s", cfg.Database))
dc.SetDB(db)
err = table.DropDatabaseIfInitTablesFailed(dc)
}
return
}

func DropDatabase(cfg config.MySqlConfig) error {
log.Infof("drop database, get db session with name %s", cfg.Database)
db, err := common.GetSessionWithName(cfg)
if err != nil {
return err
}
log.Infof("drop database, got db session with name %s", cfg.Database)
return common.DropDatabase(common.NewDBConfig(db, cfg))
}
108 changes: 54 additions & 54 deletions server/controller/db/mysql/org.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ package mysql

import (
"fmt"
"reflect"
// "reflect"
"strings"

"github.com/deepflowio/deepflow/server/controller/db/mysql/common"
"gorm.io/gorm"
"gorm.io/gorm/clause"
// "gorm.io/gorm"
// "gorm.io/gorm/clause"
)

const ORG_TABLE = "org"
Expand Down Expand Up @@ -72,60 +72,60 @@ func SyncDefaultOrgData[T any](data []T) error {
return nil
}

// get fields to update
dataType := reflect.TypeOf(data[0])
var fields []string
for i := 0; i < dataType.NumField(); i++ {
field := dataType.Field(i)
dbTag := field.Tag.Get("gorm")
if dbTag != "" {
columnName := GetColumnNameFromTag(dbTag)
if columnName != "" {
fields = append(fields, columnName)
}
}
}
// // get fields to update
// dataType := reflect.TypeOf(data[0])
// var fields []string
// for i := 0; i < dataType.NumField(); i++ {
// field := dataType.Field(i)
// dbTag := field.Tag.Get("gorm")
// if dbTag != "" {
// columnName := GetColumnNameFromTag(dbTag)
// if columnName != "" {
// fields = append(fields, columnName)
// }
// }
// }

for orgID, db := range dbs.orgIDToDB {
if orgID == DefaultDB.ORGID {
continue
}
// for orgID, db := range dbs.orgIDToDB {
// if orgID == DefaultDB.ORGID {
// continue
// }

err := db.Transaction(func(tx *gorm.DB) error {
// delete
var existingIDs []int
var t T
if err := db.Model(&t).Pluck("id", &existingIDs).Error; err != nil {
return err
}
existingIDMap := make(map[int]bool)
for _, id := range existingIDs {
existingIDMap[id] = true
}
for _, item := range data {
id := reflect.ValueOf(item).FieldByName("ID").Int()
existingIDMap[int(id)] = false
}
for id, exists := range existingIDMap {
if exists {
if err := db.Where("id = ?", id).Delete(&t).Error; err != nil {
return err
}
}
}
// err := db.Transaction(func(tx *gorm.DB) error {
// // delete
// var existingIDs []int
// var t T
// if err := db.Model(&t).Pluck("id", &existingIDs).Error; err != nil {
// return err
// }
// existingIDMap := make(map[int]bool)
// for _, id := range existingIDs {
// existingIDMap[id] = true
// }
// for _, item := range data {
// id := reflect.ValueOf(item).FieldByName("ID").Int()
// existingIDMap[int(id)] = false
// }
// for id, exists := range existingIDMap {
// if exists {
// if err := db.Where("id = ?", id).Delete(&t).Error; err != nil {
// return err
// }
// }
// }

// add or update
if err := db.Clauses(clause.OnConflict{
DoUpdates: clause.AssignmentColumns(fields), // `UpdateAll: true,` can not update time
}).Save(&data).Error; err != nil {
return fmt.Errorf("failed to sync data: %v", err)
}
return nil
})
if err != nil {
log.Errorf("org(id:%d, name:%s) error: %s", db.ORGID, db.Name, err.Error())
}
}
// // add or update
// if err := db.Clauses(clause.OnConflict{
// DoUpdates: clause.AssignmentColumns(fields), // `UpdateAll: true,` can not update time
// }).Save(&data).Error; err != nil {
// return fmt.Errorf("failed to sync data: %v", err)
// }
// return nil
// })
// if err != nil {
// log.Errorf("org(id:%d, name:%s) error: %s", db.ORGID, db.Name, err.Error())
// }
// }
return nil
}

Expand Down
6 changes: 5 additions & 1 deletion server/controller/http/service/org_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,15 @@ func CreateORGData(dataCreate model.ORGDataCreate, mysqlCfg mysqlcfg.MySqlConfig
}

func DeleteORGData(orgID int, mysqlCfg mysqlcfg.MySqlConfig) (err error) {
log.Infof("delete org (id: %d) data", orgID)
log.Infof("drop org (id: %d) data", orgID)
defer log.Infof("drop org (id: %d) data, dropped", orgID)

cfg := common.ReplaceConfigDatabaseName(mysqlCfg, orgID)
log.Infof("drop org, replaced database name: %s", cfg.Database)
if err = migrator.DropDatabase(cfg); err != nil {
return err
}
log.Infof("drop org, database dropped")
return servercommon.DropOrg(uint16(orgID))
}

Expand Down

0 comments on commit 30f49d0

Please sign in to comment.