Skip to content

Commit

Permalink
Refactor config
Browse files Browse the repository at this point in the history
  • Loading branch information
@iychoi
iychoi committed Sep 20, 2024
1 parent 3bb3936 commit ee6acb8
Show file tree
Hide file tree
Showing 36 changed files with 537 additions and 898 deletions.
168 changes: 123 additions & 45 deletions cmd/flag/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"os"

irodsclient_config "github.com/cyverse/go-irodsclient/config"
"github.com/cyverse/gocommands/commons"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -73,6 +74,31 @@ func GetCommonFlagValues(command *cobra.Command) *CommonFlagValues {
return &commonFlagValues
}

func getLogrusLogLevel(irodsLogLevel int) log.Level {
switch irodsLogLevel {
case 0:
return log.PanicLevel
case 1:
return log.FatalLevel
case 2, 3:
return log.ErrorLevel
case 4, 5, 6:
return log.WarnLevel
case 7:
return log.InfoLevel
case 8:
return log.DebugLevel
case 9, 10:
return log.TraceLevel
}

if irodsLogLevel < 0 {
return log.PanicLevel
}

return log.TraceLevel
}

func setLogLevel(command *cobra.Command) {
myCommonFlagValues := GetCommonFlagValues(command)

Expand Down Expand Up @@ -108,59 +134,122 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) {
return false, nil // stop here
}

// init config
err := commons.InitEnvironmentManager()
if err != nil {
return false, xerrors.Errorf("failed to init environment manager: %w", err)
}

environmentManager := commons.GetEnvironmentManager()

logger.Debugf("use sessionID - %d", myCommonFlagValues.SessionID)
commons.SetSessionID(myCommonFlagValues.SessionID)
environmentManager.SetPPID(myCommonFlagValues.SessionID)

readConfig := false
configFilePath := ""

// find config file location from env
if irodsEnvironmentFileEnvVal, ok := os.LookupEnv(IRODSEnvironmentFileEnvKey); ok {
if len(irodsEnvironmentFileEnvVal) > 0 {
configFilePath = irodsEnvironmentFileEnvVal
}
}

// user defined config file
if len(myCommonFlagValues.ConfigFilePath) > 0 {
// user defined config file
err := commons.LoadConfigFromFile(myCommonFlagValues.ConfigFilePath)
configFilePath = myCommonFlagValues.ConfigFilePath
}

// load config
if len(configFilePath) > 0 {
configFilePath, err = commons.ExpandHomeDir(configFilePath)
if err != nil {
return false, xerrors.Errorf("failed to load config from file %q: %w", myCommonFlagValues.ConfigFilePath, err) // stop here
return false, xerrors.Errorf("failed to expand home directory for %q: %w", configFilePath, err)
}

readConfig = true
} else {
// read config path from env
// then read config
if irodsEnvironmentFileEnvVal, ok := os.LookupEnv(IRODSEnvironmentFileEnvKey); ok {
if len(irodsEnvironmentFileEnvVal) > 0 {
err := commons.LoadConfigFromFile(irodsEnvironmentFileEnvVal)
if err != nil {
return false, xerrors.Errorf("failed to load config file %q: %w", irodsEnvironmentFileEnvVal, err) // stop here
}

readConfig = true
status, err := os.Stat(configFilePath)
if err != nil {
if os.IsNotExist(err) {
return false, xerrors.Errorf("config path %q does not exist", configFilePath)
}

return false, xerrors.Errorf("failed to stat %q: %w", configFilePath, err)
}

// read config from default icommands config path
if !readConfig {
// auto detect
err := commons.LoadConfigFromFile("~/.irods")
if status.IsDir() {
// config root
err = environmentManager.SetEnvironmentDirPath(configFilePath)
if err != nil {
return false, xerrors.Errorf("failed to set configuration root directory %q: %w", configFilePath, err)
}

// load
err = environmentManager.Load()
if err != nil {
logger.Debug(err)
// ignore error
return false, xerrors.Errorf("failed to load configuration file %q: %w", environmentManager.EnvironmentFilePath, err)
}
} else {
// config file
if commons.IsYAMLFile(configFilePath) {
// yaml
yamlConfig, err := irodsclient_config.NewConfigFromYAMLFile(configFilePath)
if err != nil {
return false, xerrors.Errorf("failed to load YAML config from file %q: %w", configFilePath, err)
}

// load
err = environmentManager.Load()
if err != nil {
return false, xerrors.Errorf("failed to load configuration file %q: %w", environmentManager.EnvironmentFilePath, err)
}

// overwrite
environmentManager.Environment = yamlConfig
} else {
readConfig = true
// json
jsonConfig, err := irodsclient_config.NewConfigFromJSONFile(configFilePath)
if err != nil {
return false, xerrors.Errorf("failed to load JSON config from file %q: %w", configFilePath, err)
}

environmentManager.Environment = jsonConfig
environmentManager.EnvironmentFilePath = configFilePath

// load
err = environmentManager.Load()
if err != nil {
return false, xerrors.Errorf("failed to load configuration file %q: %w", environmentManager.EnvironmentFilePath, err)
}
}
}
} else {
// default
// load
err = environmentManager.Load()
if err != nil {
return false, xerrors.Errorf("failed to load configuration file %q: %w", environmentManager.EnvironmentFilePath, err)
}
}

// set default config
if !readConfig {
commons.SetDefaultConfigIfEmpty()
// load config from env
envConfig, err := irodsclient_config.OverwriteConfigFromEnv(environmentManager.Environment)
if err != nil {
return false, xerrors.Errorf("failed to load config from environment: %w", err)
}

// re-configure level
setLogLevel(command)
// overwrite
environmentManager.Environment = envConfig

err := commons.LoadAndOverwriteConfigFromEnv()
sessionConfig, err := environmentManager.GetSessionConfig()
if err != nil {
return false, xerrors.Errorf("failed to load config from environment: %w", err) // stop here
return false, xerrors.Errorf("failed to get session config: %w", err)
}

if sessionConfig.LogLevel > 0 {
// set log level
log.SetLevel(getLogrusLogLevel(sessionConfig.LogLevel))
}

// re-configure level
// prioritize log level user set via command-line argument
setLogLevel(command)

if retryFlagValues.RetryChild {
Expand All @@ -171,20 +260,9 @@ func ProcessCommonFlags(command *cobra.Command) (bool, error) {
}
}

appConfig := commons.GetConfig()

syncAccount := false
if myCommonFlagValues.ResourceUpdated {
appConfig.DefaultResource = myCommonFlagValues.Resource
logger.Debugf("use default resource server %q", appConfig.DefaultResource)
syncAccount = true
}

if syncAccount {
err := commons.SyncAccount()
if err != nil {
return false, err
}
environmentManager.Environment.DefaultResource = myCommonFlagValues.Resource
logger.Debugf("use default resource server %q", myCommonFlagValues.Resource)
}

return true, nil // contiue
Expand Down
4 changes: 2 additions & 2 deletions cmd/subcmd/addmeta.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func (addMeta *AddMetaCommand) Process() error {
}

// Create a file system
addMeta.account = commons.GetAccount()
addMeta.account = commons.GetSessionConfig().ToIRODSAccount()
addMeta.filesystem, err = commons.GetIRODSFSClient(addMeta.account)
if err != nil {
return xerrors.Errorf("failed to get iRODS FS Client: %w", err)
Expand Down Expand Up @@ -125,7 +125,7 @@ func (addMeta *AddMetaCommand) addMetaToPath(targetPath string, attribute string

cwd := commons.GetCWD()
home := commons.GetHomeDir()
zone := commons.GetZone()
zone := addMeta.account.ClientZone
targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath)

logger.Debugf("add metadata to path %q (attr %q, value %q, unit %q)", targetPath, attribute, value, unit)
Expand Down
4 changes: 2 additions & 2 deletions cmd/subcmd/bclean.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func (bclean *BcleanCommand) Process() error {
}

// Create a file system
bclean.account = commons.GetAccount()
bclean.account = commons.GetSessionConfig().ToIRODSAccount()
bclean.filesystem, err = commons.GetIRODSFSClient(bclean.account)
if err != nil {
return xerrors.Errorf("failed to get iRODS FS Client: %w", err)
Expand Down Expand Up @@ -124,7 +124,7 @@ func (bclean *BcleanCommand) cleanOne(targetPath string) {

cwd := commons.GetCWD()
home := commons.GetHomeDir()
zone := commons.GetZone()
zone := bclean.account.ClientZone
targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath)

if commons.IsStagingDirInTargetPath(targetPath) {
Expand Down
61 changes: 31 additions & 30 deletions cmd/subcmd/bput.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ func (bput *BputCommand) Process() error {
}

// Create a file system
bput.account = commons.GetAccount()
bput.account = commons.GetSessionConfig().ToIRODSAccount()
bput.filesystem, err = commons.GetIRODSFSClientAdvanced(bput.account, bput.maxConnectionNum, bput.parallelTransferFlagValues.TCPBufferSize)
if err != nil {
return xerrors.Errorf("failed to get iRODS FS Client: %w", err)
Expand Down Expand Up @@ -212,7 +212,7 @@ func (bput *BputCommand) Process() error {
}

// bundle transfer manager
bput.bundleTransferManager = commons.NewBundleTransferManager(bput.filesystem, bput.transferReportManager, bput.targetPath, localBundleRootPath, bput.bundleTransferFlagValues.MinFileNum, bput.bundleTransferFlagValues.MaxFileNum, bput.bundleTransferFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleThread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTransferFlagValues.LocalTempPath, stagingDirPath, bput.bundleTransferFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath)
bput.bundleTransferManager = commons.NewBundleTransferManager(bput.account, bput.filesystem, bput.transferReportManager, bput.targetPath, localBundleRootPath, bput.bundleTransferFlagValues.MinFileNum, bput.bundleTransferFlagValues.MaxFileNum, bput.bundleTransferFlagValues.MaxFileSize, bput.parallelTransferFlagValues.SingleThread, bput.parallelTransferFlagValues.ThreadNumber, bput.parallelTransferFlagValues.RedirectToResource, bput.parallelTransferFlagValues.Icat, bput.bundleTransferFlagValues.LocalTempPath, stagingDirPath, bput.bundleTransferFlagValues.NoBulkRegistration, bput.progressFlagValues.ShowProgress, bput.progressFlagValues.ShowFullPath)
bput.bundleTransferManager.Start()

// run
Expand Down Expand Up @@ -257,7 +257,7 @@ func (bput *BputCommand) Process() error {
func (bput *BputCommand) ensureTargetIsDir(targetPath string) error {
cwd := commons.GetCWD()
home := commons.GetHomeDir()
zone := commons.GetZone()
zone := bput.account.ClientZone
targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath)

targetEntry, err := bput.filesystem.Stat(targetPath)
Expand Down Expand Up @@ -286,7 +286,7 @@ func (bput *BputCommand) getStagingDir(targetPath string) (string, error) {

cwd := commons.GetCWD()
home := commons.GetHomeDir()
zone := commons.GetZone()
zone := bput.account.ClientZone
targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath)

if len(bput.bundleTransferFlagValues.IRODSTempPath) > 0 {
Expand Down Expand Up @@ -503,10 +503,10 @@ func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) erro
SourcePath: sourcePath,
SourceSize: sourceStat.Size(),

DestPath: targetEntry.Path,
DestSize: targetEntry.Size,
ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm),
Notes: []string{"differential", "no_hash", "same file size", "skip"},
DestPath: targetEntry.Path,
DestSize: targetEntry.Size,
DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm),
Notes: []string{"differential", "no_hash", "same file size", "skip"},
}

bput.transferReportManager.AddFile(reportFile)
Expand All @@ -528,17 +528,18 @@ func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) erro
// skip
now := time.Now()
reportFile := &commons.TransferReportFile{
Method: commons.TransferMethodPut,
StartAt: now,
EndAt: now,
SourcePath: sourcePath,
SourceSize: sourceStat.Size(),
SourceChecksum: hex.EncodeToString(localChecksum),
DestPath: targetEntry.Path,
DestSize: targetEntry.Size,
DestChecksum: hex.EncodeToString(targetEntry.CheckSum),
ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm),
Notes: []string{"differential", "same checksum", "skip"},
Method: commons.TransferMethodPut,
StartAt: now,
EndAt: now,
SourcePath: sourcePath,
SourceSize: sourceStat.Size(),
SourceChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm),
SourceChecksum: hex.EncodeToString(localChecksum),
DestPath: targetEntry.Path,
DestSize: targetEntry.Size,
DestChecksum: hex.EncodeToString(targetEntry.CheckSum),
DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm),
Notes: []string{"differential", "same checksum", "skip"},
}

bput.transferReportManager.AddFile(reportFile)
Expand All @@ -558,16 +559,16 @@ func (bput *BputCommand) putFile(sourceStat fs.FileInfo, sourcePath string) erro
// skip
now := time.Now()
reportFile := &commons.TransferReportFile{
Method: commons.TransferMethodPut,
StartAt: now,
EndAt: now,
SourcePath: sourcePath,
SourceSize: sourceStat.Size(),
DestPath: targetEntry.Path,
DestSize: targetEntry.Size,
DestChecksum: hex.EncodeToString(targetEntry.CheckSum),
ChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm),
Notes: []string{"no_overwrite", "skip"},
Method: commons.TransferMethodPut,
StartAt: now,
EndAt: now,
SourcePath: sourcePath,
SourceSize: sourceStat.Size(),
DestPath: targetEntry.Path,
DestSize: targetEntry.Size,
DestChecksum: hex.EncodeToString(targetEntry.CheckSum),
DestChecksumAlgorithm: string(targetEntry.CheckSumAlgorithm),
Notes: []string{"no_overwrite", "skip"},
}

bput.transferReportManager.AddFile(reportFile)
Expand Down Expand Up @@ -726,7 +727,7 @@ func (bput *BputCommand) deleteOnSuccess(sourcePath string) error {
func (bput *BputCommand) deleteExtra(targetPath string) error {
cwd := commons.GetCWD()
home := commons.GetHomeDir()
zone := commons.GetZone()
zone := bput.account.ClientZone
targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath)

return bput.deleteExtraInternal(targetPath)
Expand Down
4 changes: 2 additions & 2 deletions cmd/subcmd/bun.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func (bun *BunCommand) Process() error {
}

// Create a file system
bun.account = commons.GetAccount()
bun.account = commons.GetSessionConfig().ToIRODSAccount()
bun.filesystem, err = commons.GetIRODSFSClient(bun.account)
if err != nil {
return xerrors.Errorf("failed to get iRODS FS Client: %w", err)
Expand Down Expand Up @@ -150,7 +150,7 @@ func (bun *BunCommand) extractOne(sourcePath string, targetPath string) error {

cwd := commons.GetCWD()
home := commons.GetHomeDir()
zone := commons.GetZone()
zone := bun.account.ClientZone
sourcePath = commons.MakeIRODSPath(cwd, home, zone, sourcePath)
targetPath = commons.MakeIRODSPath(cwd, home, zone, targetPath)

Expand Down
Loading

0 comments on commit ee6acb8

Please sign in to comment.