Configurable Timezone for CEEMS DB (#253)

* feat: Make time zone for CEEMS API server DB configurable. Use the configured time zone in storing date strings in DB

* feat: Add timezone query parameter for API server. For API server, always convert the timestamp to date time in the location configured for DB. `timezone` query parameter for API server is only used to convert datetime strings to that time zone before sending response

* refactor: Improve handling of SLURM cli execution mode

* chore: Ensure to set retention period when creating new TSDB backend

* Add more kv pairs in logging to help debugging

* Fix API server's slurm cli config documentation

* test: Update unit and e2e tests

* docs: Update docs

---------

Signed-off-by: Mahendra Paipuri <[email protected]>

build/config/ceems_api_server/ceems_api_server.yml

@@ -34,6 +34,14 @@ ceems_api_server:
     #
     retention_period: 30d
 
+    # Time zone to be used when storing times of different events in the DB.
+    # It takes a value defined in IANA (https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
+    # like `Europe/Paris`
+    # 
+    # A special value `Local` can be used to use server local time zone.
+    #
+    time_zone: Local
+
     # CEEMS API server is capable of creating DB backups using SQLite backup API. Created
     # DB backups will be saved to this path. NOTE that for huge DBs, this backup can take 
     # a considerable amount of time. 
@@ -266,19 +274,17 @@ clusters: []
   #   # When SLURM resource manager is configured to fetch job data using `sacct` command,
   #   # execution mode of the command will be decided as follows:
   #   #
-  #   #  - If the current user running `ceems_api_server` is `root` or `slurm` user, `sacct`
-  #   #    command will be executed natively as that user.
+  #   #  - If the current user running `ceems_api_server` is `root`, `sacct`
+  #   #    command will be executed as that user in a security context.
   #   #
-  #   #  - If above check fails, `sacct` command will be attempted to execute as `slurm` user.
-  #   #    If the `ceems_api_server` process have enough privileges setup using Linux capabilities
-  #   #    in the systemd unit file, this will succeed and `sacct` will be always executed 
-  #   #    as `slurm` user.
+  #   #  - If the `ceems_api_server` process has `CAP_SETUID` and `CAP_SETGID` capabilities, `sacct` 
+  #   #    command will be executed as `root` user in a security context.
   #   #
-  #   #  - If above check fails as well, we attempt to execute `sacct` with `sudo` prefix. If
+  #   #  - As a last attempt, we attempt to execute `sacct` with `sudo` prefix. If
   #   #    the current user running `ceems_api_server` is in the list of sudoers, this check
   #   #    will pass and `sacct` will be always executed as `sudo sacct <args>` to fetch jobs.
   #   #
-  #   # If none of the above checks, pass, `sacct` will be executed as the current user 
+  #   # If none of the above conditions are true, `sacct` will be executed as the current user 
   #   # which might not give job data of _all_ users in the cluster.
   #   #
   #   # If the operators are unsure which method to use, there is a default systemd

internal/common/helpers.go

@@ -35,7 +35,7 @@ func Round(value int64, nearest int64) int64 {
 // TimeTrack tracks execution time of each function.
 func TimeTrack(start time.Time, name string, logger *slog.Logger) {
 	elapsed := time.Since(start)
-	logger.Debug(name, "elapsed_time", elapsed)
+	logger.Debug(name, "duration", elapsed)
 }
 
 // SanitizeFloat replaces +/-Inf and NaN with zero.

pkg/api/base/base.go

@@ -55,6 +55,9 @@ var (
 // DatetimeLayout to be used in the package.
 var DatetimeLayout = fmt.Sprintf("%sT%s", time.DateOnly, time.TimeOnly)
 
+// DatetimezoneLayout to be used in the package.
+var DatetimezoneLayout = DatetimeLayout + "-0700"
+
 // CLI args with global scope.
 var (
 	ConfigFilePath string

pkg/api/cli/cli.go

@@ -60,6 +60,7 @@ func (c *CEEMSAPIAppConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
 				RetentionPeriod: model.Duration(30 * 24 * time.Hour),
 				UpdateInterval:  model.Duration(15 * time.Minute),
 				BackupInterval:  model.Duration(24 * time.Hour),
+				TimeLocation:    ceems_db.TimeLocation{Location: time.Local},
 				LastUpdateTime:  todayMidnight,
 			},
 			Admin: ceems_db.AdminConfig{

pkg/api/db/db.go

@@ -46,6 +46,30 @@ type AdminConfig struct {
 	Grafana common.GrafanaWebConfig `yaml:"grafana"`
 }
 
+type TimeLocation struct {
+	*time.Location
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (t *TimeLocation) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var tmp string
+
+	err := unmarshal(&tmp)
+	if err != nil {
+		return err
+	}
+
+	// Attempt to create location from string
+	loc, err := time.LoadLocation(tmp)
+	if err != nil {
+		return err
+	}
+
+	*t = TimeLocation{loc}
+
+	return nil
+}
+
 // DataConfig is the container for the data related config.
 type DataConfig struct {
 	Path               string         `yaml:"path"`
@@ -54,6 +78,7 @@ type DataConfig struct {
 	UpdateInterval     model.Duration `yaml:"update_interval"`
 	BackupInterval     model.Duration `yaml:"backup_interval"`
 	LastUpdateTime     time.Time      `yaml:"update_from"`
+	TimeLocation       TimeLocation   `yaml:"time_zone"`
 	SkipDeleteOldUnits bool
 }
 
@@ -72,14 +97,15 @@ type storageConfig struct {
 	dbBackupPath       string
 	retentionPeriod    time.Duration
 	lastUpdateTime     time.Time
+	timeLocation       *time.Location
 	skipDeleteOldUnits bool
 }
 
 // String implements Stringer interface for storageConfig.
 func (s *storageConfig) String() string {
 	return fmt.Sprintf(
-		"DB File Path: %s; Retention Period: %s; Last Updated At: %s",
-		s.dbPath, s.retentionPeriod, s.lastUpdateTime,
+		"DB File Path: %s; Retention Period: %s; Location: %s; Last Updated At: %s",
+		s.dbPath, s.retentionPeriod, s.timeLocation, s.lastUpdateTime,
 	)
 }
 
@@ -191,7 +217,7 @@ func New(c *Config) (*stats, error) {
 		c.Data.LastUpdateTime.Minute(),
 		c.Data.LastUpdateTime.Second(),
 		c.Data.LastUpdateTime.Nanosecond(),
-		time.Now().Location(),
+		c.Data.TimeLocation.Location,
 	)
 	c.Logger.Info("DB will be updated from", "time", c.Data.LastUpdateTime)
 
@@ -220,6 +246,7 @@ func New(c *Config) (*stats, error) {
 		dbBackupPath:       c.Data.BackupPath,
 		retentionPeriod:    time.Duration(c.Data.RetentionPeriod),
 		lastUpdateTime:     c.Data.LastUpdateTime,
+		timeLocation:       c.Data.TimeLocation.Location,
 		skipDeleteOldUnits: c.Data.SkipDeleteOldUnits,
 	}
 
@@ -259,7 +286,7 @@ func (s *stats) Collect(ctx context.Context) error {
 	// Measure elapsed time
 	defer common.TimeTrack(time.Now(), "Data collection", s.logger)
 
-	currentTime := time.Now()
+	currentTime := time.Now().In(s.storage.timeLocation)
 
 	// If duration is less than 1 day do single update
 	if currentTime.Sub(s.storage.lastUpdateTime) < 24*time.Hour {
@@ -733,7 +760,7 @@ func (s *stats) createBackup(ctx context.Context) error {
 	backupDBFileName := fmt.Sprintf(
 		"%s-%s.db",
 		strings.Split(base.CEEMSDBName, ".")[0],
-		time.Now().Format("200601021504"),
+		time.Now().In(s.storage.timeLocation).Format("200601021504"),
 	)
 
 	backupDBFilePath := filepath.Join(filepath.Dir(s.storage.dbPath), backupDBFileName)

pkg/api/db/db_test.go

@@ -570,6 +570,7 @@ func prepareMockConfig(tmpDir string) (*Config, error) {
 			BackupPath:      dataBackupDir,
 			LastUpdateTime:  time.Now(),
 			RetentionPeriod: model.Duration(24 * time.Hour),
+			TimeLocation:    TimeLocation{Location: time.UTC},
 		},
 		Admin: AdminConfig{
 			Users: []string{"adm1", "adm2"},

pkg/api/helper/helper.go

@@ -109,7 +109,7 @@ func NodelistParser(nodelistExp string) []string {
 // TimeToTimestamp converts a date in a given layout to unix timestamp of the date.
 func TimeToTimestamp(layout string, date string) int64 {
 	if t, err := time.Parse(layout, date); err == nil {
-		return t.Local().UnixMilli()
+		return t.UnixMilli()
 	}
 
 	return 0