From 50ab69fe7ba3a018a192b64a6f868a4cc2017700 Mon Sep 17 00:00:00 2001 From: Jian Zeng Date: Tue, 2 Jan 2024 12:47:40 +0800 Subject: [PATCH] refactor: cleanup Signed-off-by: Jian Zeng --- pkg/cron/cron.go | 20 -------- pkg/cron/cron_test.go | 2 +- pkg/model/repo.go | 2 +- pkg/server/config.go | 83 +------------------------------- pkg/server/config_test.go | 28 +++++------ pkg/server/main.go | 66 ++++++++++++++----------- pkg/server/main_test.go | 2 +- pkg/server/repo_handlers_test.go | 2 +- pkg/server/utils.go | 23 ++++++++- test/integration/sync_test.go | 22 +++------ 10 files changed, 84 insertions(+), 166 deletions(-) diff --git a/pkg/cron/cron.go b/pkg/cron/cron.go index 1729324..ad8375c 100644 --- a/pkg/cron/cron.go +++ b/pkg/cron/cron.go @@ -17,11 +17,6 @@ type FuncJob = cron.FuncJob var scheduledJobs sync.Map -// Parse parses job specification. -func Parse(spec string) (cron.Schedule, error) { - return cron.ParseStandard(spec) -} - // New returns an instance of Cron. func New() *Cron { c := cron.New() @@ -29,10 +24,6 @@ func New() *Cron { return &Cron{c} } -func (c *Cron) Entries() []cron.Entry { - return c.inner.Entries() -} - // Jobs returns a map of job names to job. func (c *Cron) Jobs() map[string]cron.Entry { ret := map[string]cron.Entry{} @@ -54,11 +45,6 @@ func (c *Cron) Jobs() map[string]cron.Entry { return ret } -func (c *Cron) AddFunc(spec string, cmd func()) error { - _, err := c.inner.AddFunc(spec, cmd) - return err -} - // AddJob removes the job with the same name first and adds a new job. func (c *Cron) AddJob(name, spec string, cmd FuncJob) error { c.RemoveJob(name) @@ -70,12 +56,6 @@ func (c *Cron) AddJob(name, spec string, cmd FuncJob) error { return nil } -// HasJob returns whether the given job exists. -func (c *Cron) HasJob(name string) bool { - _, ok := scheduledJobs.Load(name) - return ok -} - // RemoveJob remove the job with the given name. func (c *Cron) RemoveJob(name string) { if v, ok := scheduledJobs.Load(name); ok { diff --git a/pkg/cron/cron_test.go b/pkg/cron/cron_test.go index 3558fa0..f949a58 100644 --- a/pkg/cron/cron_test.go +++ b/pkg/cron/cron_test.go @@ -15,7 +15,7 @@ func TestCron(t *testing.T) { next := now.Add(time.Second * 4) wg.Add(1) cron := New() - err := cron.AddFunc("@every 3s", func() { + err := cron.AddJob("test", "@every 3s", func() { wg.Done() }) require.NoError(t, err) diff --git a/pkg/model/repo.go b/pkg/model/repo.go index f409ef1..ef45069 100644 --- a/pkg/model/repo.go +++ b/pkg/model/repo.go @@ -6,7 +6,7 @@ type StringMap map[string]string type Repo struct { Name string `gorm:"primaryKey" json:"name" validate:"required"` Cron string `json:"cron" validate:"required,cron"` - Image string `json:"image" validate:"required,containsrune=:"` + Image string `json:"image" validate:"required"` StorageDir string `json:"storageDir" validate:"required,dir"` User string `json:"user"` BindIP string `json:"bindIP" validate:"omitempty,ip"` diff --git a/pkg/server/config.go b/pkg/server/config.go index f1832f1..148975d 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -2,17 +2,11 @@ package server import ( "fmt" - "log/slog" "os" "time" - - "github.com/go-playground/validator/v10" - "github.com/spf13/viper" - - "github.com/ustclug/Yuki/pkg/fs" ) -type AppConfig struct { +type Config struct { Debug bool `mapstructure:"debug"` DbURL string `mapstructure:"db_url" validate:"required"` FileSystem string `mapstructure:"fs" validate:"oneof=xfs zfs default"` @@ -31,26 +25,7 @@ type AppConfig struct { SeccompProfile string `mapstructure:"seccomp_profile" validate:"omitempty,filepath"` } -type Config struct { - Debug bool - DbURL string - DockerEndpoint string - Owner string - LogFile string - RepoLogsDir string - RepoConfigDir []string - LogLevel slog.Level - ListenAddr string - BindIP string - NamePrefix string - PostSync []string - ImagesUpgradeInterval time.Duration - SyncTimeout time.Duration - SeccompProfile string - GetSizer fs.GetSizer -} - -var DefaultAppConfig = AppConfig{ +var DefaultConfig = Config{ FileSystem: "default", DockerEndpoint: "unix:///var/run/docker.sock", LogFile: "/dev/stderr", @@ -61,57 +36,3 @@ var DefaultAppConfig = AppConfig{ LogLevel: "info", ImagesUpgradeInterval: time.Hour, } - -func loadConfig(configPath string) (*Config, error) { - v := viper.New() - v.SetConfigFile(configPath) - if err := v.ReadInConfig(); err != nil { - return nil, err - } - appCfg := DefaultAppConfig - if err := v.Unmarshal(&appCfg); err != nil { - return nil, err - } - validate := validator.New() - if err := validate.Struct(&appCfg); err != nil { - return nil, err - } - cfg := Config{ - Debug: appCfg.Debug, - DbURL: appCfg.DbURL, - DockerEndpoint: appCfg.DockerEndpoint, - Owner: appCfg.Owner, - LogFile: appCfg.LogFile, - RepoConfigDir: appCfg.RepoConfigDir, - RepoLogsDir: appCfg.RepoLogsDir, - ListenAddr: appCfg.ListenAddr, - BindIP: appCfg.BindIP, - NamePrefix: appCfg.NamePrefix, - PostSync: appCfg.PostSync, - ImagesUpgradeInterval: appCfg.ImagesUpgradeInterval, - SyncTimeout: appCfg.SyncTimeout, - SeccompProfile: appCfg.SeccompProfile, - } - - switch appCfg.FileSystem { - case "zfs": - cfg.GetSizer = fs.New(fs.ZFS) - case "xfs": - cfg.GetSizer = fs.New(fs.XFS) - default: - cfg.GetSizer = fs.New(fs.DEFAULT) - } - - switch appCfg.LogLevel { - case "debug": - cfg.LogLevel = slog.LevelDebug - case "warn": - cfg.LogLevel = slog.LevelWarn - case "error": - cfg.LogLevel = slog.LevelError - default: - cfg.LogLevel = slog.LevelInfo - } - - return &cfg, nil -} diff --git a/pkg/server/config_test.go b/pkg/server/config_test.go index b65cecf..b6dfa8c 100644 --- a/pkg/server/config_test.go +++ b/pkg/server/config_test.go @@ -10,25 +10,19 @@ import ( testutils "github.com/ustclug/Yuki/test/utils" ) -func TestLoadSyncTimeoutConfig(t *testing.T) { - f, err := os.CreateTemp("", "sync_timeout*.toml") +func TestLoadConfig(t *testing.T) { + tmp, err := os.CreateTemp("", "TestLoadConfig*.toml") require.NoError(t, err) - t.Cleanup(func() { - _ = f.Close() - _ = os.Remove(f.Name()) - }) - - testutils.WriteFile(t, f.Name(), ` -db_url = "test" - -repo_logs_dir = "/tmp/log_yuki/" - -repo_config_dir = "/tmp/config_yuki" - + defer os.Remove(tmp.Name()) + defer tmp.Close() + testutils.WriteFile(t, tmp.Name(), ` +db_url = ":memory:" +repo_logs_dir = "/tmp" +repo_config_dir = "/tmp" sync_timeout = "15s" `) - - config, err := loadConfig(f.Name()) + srv, err := New(tmp.Name()) require.NoError(t, err) - require.EqualValues(t, time.Second*15, config.SyncTimeout) + require.EqualValues(t, time.Second*15, srv.config.SyncTimeout) + require.EqualValues(t, "/tmp", srv.config.RepoConfigDir[0]) } diff --git a/pkg/server/main.go b/pkg/server/main.go index a8ecafa..10a92bc 100644 --- a/pkg/server/main.go +++ b/pkg/server/main.go @@ -8,18 +8,19 @@ import ( "log/slog" "net/http" "os" - "strings" "sync" "time" "github.com/go-playground/validator/v10" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" + "github.com/spf13/viper" "gorm.io/driver/sqlite" "gorm.io/gorm" "github.com/ustclug/Yuki/pkg/cron" "github.com/ustclug/Yuki/pkg/docker" + "github.com/ustclug/Yuki/pkg/fs" "github.com/ustclug/Yuki/pkg/model" ) @@ -28,7 +29,7 @@ type Server struct { e *echo.Echo dockerCli docker.Client - config *Config + config Config cron *cron.Cron db *gorm.DB logger *slog.Logger @@ -36,32 +37,24 @@ type Server struct { } func New(configPath string) (*Server, error) { - cfg, err := loadConfig(configPath) + v := viper.New() + v.SetConfigFile(configPath) + err := v.ReadInConfig() if err != nil { return nil, err } + cfg := DefaultConfig + if err := v.Unmarshal(&cfg); err != nil { + return nil, err + } + validate := validator.New() + if err := validate.Struct(&cfg); err != nil { + return nil, err + } return NewWithConfig(cfg) } -func newSlogger(writer io.Writer, addSource bool, level slog.Leveler) *slog.Logger { - return slog.New(slog.NewTextHandler(writer, &slog.HandlerOptions{ - AddSource: addSource, - Level: level, - ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { - // Taken from https://gist.github.com/HalCanary/6bd335057c65f3b803088cc55b9ebd2b - if a.Key == slog.SourceKey { - source, _ := a.Value.Any().(*slog.Source) - if source != nil { - _, after, _ := strings.Cut(source.File, "Yuki") - source.File = after - } - } - return a - }, - })) -} - -func NewWithConfig(cfg *Config) (*Server, error) { +func NewWithConfig(cfg Config) (*Server, error) { db, err := gorm.Open(sqlite.Open(cfg.DbURL), &gorm.Config{ QueryFields: true, }) @@ -80,7 +73,20 @@ func NewWithConfig(cfg *Config) (*Server, error) { if err != nil { return nil, err } - slogger := newSlogger(logfile, cfg.Debug, cfg.LogLevel) + + var logLvl slog.Level + switch cfg.LogLevel { + case "debug": + logLvl = slog.LevelDebug + case "warn": + logLvl = slog.LevelWarn + case "error": + logLvl = slog.LevelError + default: + logLvl = slog.LevelInfo + } + + slogger := newSlogger(logfile, cfg.Debug, logLvl) s := Server{ e: echo.New(), @@ -89,12 +95,18 @@ func NewWithConfig(cfg *Config) (*Server, error) { logger: slogger, dockerCli: dockerCli, config: cfg, - - getSize: cfg.GetSizer.GetSize, + } + switch cfg.FileSystem { + case "zfs": + s.getSize = fs.New(fs.ZFS).GetSize + case "xfs": + s.getSize = fs.New(fs.XFS).GetSize + default: + s.getSize = fs.New(fs.DEFAULT).GetSize } - v := validator.New() - s.e.Validator = echoValidator(v.Struct) + validate := validator.New() + s.e.Validator = echoValidator(validate.Struct) s.e.Debug = cfg.Debug s.e.HideBanner = true s.e.Logger.SetOutput(io.Discard) diff --git a/pkg/server/main_test.go b/pkg/server/main_test.go index 22b0788..33e0c08 100644 --- a/pkg/server/main_test.go +++ b/pkg/server/main_test.go @@ -52,6 +52,7 @@ func NewTestEnv(t *testing.T) *TestEnv { dbFile, err := os.CreateTemp("", "yukid*.db") require.NoError(t, err) t.Cleanup(func() { + _ = dbFile.Close() _ = os.Remove(dbFile.Name()) }) db, err := gorm.Open(sqlite.Open(dbFile.Name()), &gorm.Config{ @@ -65,7 +66,6 @@ func NewTestEnv(t *testing.T) *TestEnv { db: db, cron: cron.New(), logger: slogger, - config: &Config{}, dockerCli: fakedocker.NewClient(), getSize: fs.New(fs.DEFAULT).GetSize, } diff --git a/pkg/server/repo_handlers_test.go b/pkg/server/repo_handlers_test.go index b5100b5..e8d18a0 100644 --- a/pkg/server/repo_handlers_test.go +++ b/pkg/server/repo_handlers_test.go @@ -44,7 +44,7 @@ func TestHandlerReloadAllRepos(t *testing.T) { t.Cleanup(func() { _ = os.RemoveAll(stateDir) }) - te.server.config = &Config{ + te.server.config = Config{ RepoLogsDir: filepath.Join(stateDir, "logs"), RepoConfigDir: []string{"/no/such/dir", stateDir}, } diff --git a/pkg/server/utils.go b/pkg/server/utils.go index e63e6cd..a6337a4 100644 --- a/pkg/server/utils.go +++ b/pkg/server/utils.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "log/slog" "net/http" "os/exec" @@ -272,7 +273,7 @@ func (s *Server) waitRunningContainers() error { func (s *Server) upgradeImages() { db := s.db logger := s.logger - logger.Info("Upgrading images") + logger.Debug("Upgrading images") var images []string err := db.Model(&model.Repo{}). @@ -298,7 +299,7 @@ func (s *Server) upgradeImages() { } _ = eg.Wait() - logger.Info("Removing dangling images") + logger.Debug("Removing dangling images") err = s.dockerCli.RemoveDanglingImages() if err != nil { @@ -467,3 +468,21 @@ func (s *Server) syncRepo(ctx context.Context, name string, debug bool) error { return nil } + +func newSlogger(writer io.Writer, addSource bool, level slog.Leveler) *slog.Logger { + return slog.New(slog.NewTextHandler(writer, &slog.HandlerOptions{ + AddSource: addSource, + Level: level, + ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { + // Taken from https://gist.github.com/HalCanary/6bd335057c65f3b803088cc55b9ebd2b + if a.Key == slog.SourceKey { + source, _ := a.Value.Any().(*slog.Source) + if source != nil { + _, after, _ := strings.Cut(source.File, "Yuki") + source.File = after + } + } + return a + }, + })) +} diff --git a/test/integration/sync_test.go b/test/integration/sync_test.go index d3a6528..0888a96 100644 --- a/test/integration/sync_test.go +++ b/test/integration/sync_test.go @@ -2,7 +2,6 @@ package integration import ( "context" - "log/slog" "os" "path/filepath" "testing" @@ -11,7 +10,6 @@ import ( "github.com/go-resty/resty/v2" "github.com/stretchr/testify/require" - "github.com/ustclug/Yuki/pkg/fs" testutils "github.com/ustclug/Yuki/test/utils" "github.com/ustclug/Yuki/pkg/api" @@ -28,19 +26,13 @@ func TestSyncRepo(t *testing.T) { cfgDir := filepath.Join(tmpdir, "config") os.MkdirAll(logDir, 0755) os.MkdirAll(cfgDir, 0755) - cfg := server.Config{ - DbURL: filepath.Join(tmpdir, "yukid.db"), - DockerEndpoint: server.DefaultAppConfig.DockerEndpoint, - Owner: server.DefaultAppConfig.Owner, - LogFile: server.DefaultAppConfig.LogFile, - RepoLogsDir: logDir, - RepoConfigDir: []string{cfgDir}, - LogLevel: slog.LevelInfo, - ListenAddr: "127.0.0.1:0", - GetSizer: fs.New(fs.DEFAULT), - } - srv, err := server.NewWithConfig(&cfg) + cfg := server.DefaultConfig + cfg.DbURL = filepath.Join(tmpdir, "yukid.db") + cfg.RepoConfigDir = []string{cfgDir} + cfg.RepoLogsDir = logDir + cfg.ListenAddr = "127.0.0.1:0" + srv, err := server.NewWithConfig(cfg) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -52,7 +44,7 @@ func TestSyncRepo(t *testing.T) { cancel() }() - testutils.WriteFile(t, filepath.Join(tmpdir, "config/foo.yaml"), ` + testutils.WriteFile(t, filepath.Join(cfgDir, "foo.yaml"), ` name: "foo" cron: "@every 1h" image: "ustcmirror/test:latest"