From c720770cf8707c70ab032f2a9b637e92c311b7a2 Mon Sep 17 00:00:00 2001 From: Michal-Leszczynski <74614433+Michal-Leszczynski@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:24:18 +0200 Subject: [PATCH] feat(restore): allow for unpinning agent from CPU (#4073) * chore(go.mod): bump sub pkg versions * feat(agent): allow for setting multiple CPUs in config This commit makes it possible to specify multiple CPUs in 'scylla-manager-agent.yaml' config to which agent will be pinned. The 'cpu' field in config now allows for both single int and array of ints values. The default behavior remains the same. * refactor(agent): extract cpu pinning logic to separate function Except for the extraction, this commit also increases log level (from DEBUG to INFO) of message about missing cpuset file. It also makes this function return an error, which is going to be useful when this function is going to be used inside the cpu pinning agent endpoints. * feat(agent): implement cpu pinning endpoints This commit implement 3 cpu pinning endpoints allowing to query (GET), pin (POST) and unpin (DELETE) agent cpus. * feat(scyllaclient): implement cpu pinning methods * feat(restore): optionally unpin agent from CPUs This commit adds UnpinAgentCPU field to Target. It optionally allows to unpin agent from CPUs for the time of the restore. Fixes #3951 * feat(restore): always enable auto compaction after restore This should be done just because of safety. It prevents rare edge cases like: - restore runs with --allow-compaction=false - restore is paused - restore is modified with --allow-compaction=true - restore is resumed * feat(command/restore): add --unpin-agent-cpu flag This commit allows user to control whether agent should be pinned to CPUs during restore. * feat(restore_test): extend TestRestoreTablesPreparationIntegration with cpu pinning This way this test also checks cpu pinning before and after backup. It also checks cpu pinning before, in the middle, when paused, when resumed, and after restore. --- dist/etc/scylla-manager-agent.yaml | 1 + .../sctool/partials/sctool_restore.yaml | 3 + .../partials/sctool_restore_update.yaml | 3 + go.mod | 9 +- go.sum | 6 - pkg/cmd/agent/agent.go | 29 ++- pkg/cmd/agent/router.go | 2 +- pkg/cmd/agent/server.go | 73 ++++--- pkg/command/restore/cmd.go | 6 + pkg/command/restore/res.yaml | 4 + pkg/config/agent/agent.go | 37 +++- pkg/config/agent/agent_test.go | 5 + .../agent/testdata/multiple_cpus.golden.yaml | 189 ++++++++++++++++++ .../agent/testdata/multiple_cpus.input.yaml | 4 + pkg/scyllaclient/client_agent.go | 30 +++ pkg/service/restore/model.go | 1 + .../restore/restore_integration_test.go | 69 +++++-- pkg/service/restore/tables_worker.go | 42 +++- .../scylla-manager-agent-ipv6.yaml | 2 + .../scylla-manager-agent.yaml | 2 + .../v3/pkg/managerclient/LICENSE | 62 ------ .../scylla-manager/v3/pkg/util/LICENSE | 62 ------ .../v3/pkg/util/cpuset/cpuset_linux.go | 25 +++ .../scylla-manager/v3/swagger/LICENSE | 62 ------ .../client/operations/get_cpu_parameters.go | 113 +++++++++++ .../client/operations/get_cpu_responses.go | 135 +++++++++++++ .../client/operations/operations_client.go | 111 ++++++++++ .../client/operations/pin_cpu_parameters.go | 113 +++++++++++ .../client/operations/pin_cpu_responses.go | 133 ++++++++++++ .../client/operations/unpin_cpu_parameters.go | 113 +++++++++++ .../client/operations/unpin_cpu_responses.go | 133 ++++++++++++ .../v3/swagger/gen/agent/models/cpus.go | 43 ++++ vendor/modules.txt | 9 +- 33 files changed, 1379 insertions(+), 252 deletions(-) create mode 100644 pkg/config/agent/testdata/multiple_cpus.golden.yaml create mode 100644 pkg/config/agent/testdata/multiple_cpus.input.yaml delete mode 100644 vendor/github.com/scylladb/scylla-manager/v3/pkg/managerclient/LICENSE delete mode 100644 vendor/github.com/scylladb/scylla-manager/v3/pkg/util/LICENSE delete mode 100644 vendor/github.com/scylladb/scylla-manager/v3/swagger/LICENSE create mode 100644 vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/get_cpu_parameters.go create mode 100644 vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/get_cpu_responses.go create mode 100644 vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/pin_cpu_parameters.go create mode 100644 vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/pin_cpu_responses.go create mode 100644 vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/unpin_cpu_parameters.go create mode 100644 vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/unpin_cpu_responses.go create mode 100644 vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models/cpus.go diff --git a/dist/etc/scylla-manager-agent.yaml b/dist/etc/scylla-manager-agent.yaml index c2e8a6302c..6d3e8c29a6 100644 --- a/dist/etc/scylla-manager-agent.yaml +++ b/dist/etc/scylla-manager-agent.yaml @@ -40,6 +40,7 @@ # CPU to run Scylla Manager Agent on. By default the agent would read Scylla # configuration at /etc/scylla.d/cpuset.conf and try to find a core not used by # Scylla. If that's not possible user can specify a core to run agent on. +# It's possible to specify a list of CPUs instead of a single value. #cpu: 0 # Logging configuration. diff --git a/docs/source/sctool/partials/sctool_restore.yaml b/docs/source/sctool/partials/sctool_restore.yaml index 18d0620b10..d0efdc23db 100644 --- a/docs/source/sctool/partials/sctool_restore.yaml +++ b/docs/source/sctool/partials/sctool_restore.yaml @@ -169,6 +169,9 @@ options: Sets the amount of file transfers to run in parallel when downloading files from backup location to Scylla node. Set to 0 for the fastest download (results in setting transfers to 2*node_shard_count). Set to -1 for using the transfers value defined in node's 'scylla-manager-agent.yaml' config file. + - name: unpin-agent-cpu + default_value: "false" + usage: "Defines if ScyllaDB Manager Agent should be unpinned from CPUs during restore.\nThis might significantly improve download speed at the cost of decreasing streaming speed. \n" - name: window default_value: '[]' usage: | diff --git a/docs/source/sctool/partials/sctool_restore_update.yaml b/docs/source/sctool/partials/sctool_restore_update.yaml index 44b79e79c9..3f07946071 100644 --- a/docs/source/sctool/partials/sctool_restore_update.yaml +++ b/docs/source/sctool/partials/sctool_restore_update.yaml @@ -167,6 +167,9 @@ options: Sets the amount of file transfers to run in parallel when downloading files from backup location to Scylla node. Set to 0 for the fastest download (results in setting transfers to 2*node_shard_count). Set to -1 for using the transfers value defined in node's 'scylla-manager-agent.yaml' config file. + - name: unpin-agent-cpu + default_value: "false" + usage: "Defines if ScyllaDB Manager Agent should be unpinned from CPUs during restore.\nThis might significantly improve download speed at the cost of decreasing streaming speed. \n" - name: window default_value: '[]' usage: | diff --git a/go.mod b/go.mod index acd9d2ab87..00f607d5c3 100644 --- a/go.mod +++ b/go.mod @@ -29,9 +29,9 @@ require ( github.com/scylladb/go-reflectx v1.0.1 github.com/scylladb/go-set v1.0.2 github.com/scylladb/gocqlx/v2 v2.8.0 - github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241015081800-ee47f3d10478 - github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241015081800-ee47f3d10478 - github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241015081800-ee47f3d10478 + github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241017140216-d044b1738488 + github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241017140216-d044b1738488 + github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241017140216-d044b1738488 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stoewer/go-strcase v1.3.0 @@ -128,5 +128,8 @@ require ( replace ( github.com/gocql/gocql => github.com/scylladb/gocql v1.12.0 github.com/rclone/rclone => github.com/scylladb/rclone v1.54.1-0.20240312172628-afe1fd2aa65e + github.com/scylladb/scylla-manager/v3/pkg/managerclient => ./v3/pkg/managerclient + github.com/scylladb/scylla-manager/v3/pkg/util => ./v3/pkg/util + github.com/scylladb/scylla-manager/v3/swagger => ./v3/swagger google.golang.org/api v0.114.0 => github.com/scylladb/google-api-go-client v0.34.1-patched ) diff --git a/go.sum b/go.sum index 8335697adc..d450d949e5 100644 --- a/go.sum +++ b/go.sum @@ -1053,12 +1053,6 @@ github.com/scylladb/google-api-go-client v0.34.1-patched h1:DW+T0HA+74o6FDr3TFzV github.com/scylladb/google-api-go-client v0.34.1-patched/go.mod h1:RriRmS2wJXH+2yd9PRTEcR380U9AXmurWwznqVhzsSc= github.com/scylladb/rclone v1.54.1-0.20240312172628-afe1fd2aa65e h1:lJRphCtu+nKd+mfo8whOTeFkgjMWvk8iCSlqgibKSa8= github.com/scylladb/rclone v1.54.1-0.20240312172628-afe1fd2aa65e/go.mod h1:JGZp4EvCUK+6AM1Fe1dye5xvihTc/Bk0WnHHSCJOePM= -github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241015081800-ee47f3d10478 h1:nI+3aLInCdLRYsR+iEuZkzFblgvAdGhWJT/Wqt7cKnU= -github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241015081800-ee47f3d10478/go.mod h1:6cKRPGgPPFrPSh+M90k+bp0oIFsB5SZsdI6hWeXnXOY= -github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241015081800-ee47f3d10478 h1:QEhE1TL21CZglbImBdDkjo6ZzPnc3OkOMcDYmo6QwIk= -github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241015081800-ee47f3d10478/go.mod h1:+sPCx2oaOXmMpy/ODNNEDGJ7vCghBeKP4S7xEfMI+eA= -github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241015081800-ee47f3d10478 h1:cL8e+sW/7MZcq9guSBleR0bU+lEGxw1ZUjAf6vajHdE= -github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241015081800-ee47f3d10478/go.mod h1:Oxfuz1XcXi9iV4ggSGfQdn+p6gPz6djPOegRMMe/6/s= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4 h1:8qmTC5ByIXO3GP/IzBkxcZ/99VITvnIETDhdFz/om7A= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= diff --git a/pkg/cmd/agent/agent.go b/pkg/cmd/agent/agent.go index d8e164aa41..f387cad358 100644 --- a/pkg/cmd/agent/agent.go +++ b/pkg/cmd/agent/agent.go @@ -10,13 +10,40 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/render" "github.com/pkg/errors" + "github.com/scylladb/go-log" "github.com/scylladb/scylla-manager/v3/pkg/config/agent" + "github.com/scylladb/scylla-manager/v3/pkg/util/cpuset" + "github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models" ) -func newAgentHandler(c agent.Config, rclone http.Handler) *chi.Mux { +func newAgentHandler(c agent.Config, rclone http.Handler, logger log.Logger) *chi.Mux { m := chi.NewMux() m.Get("/node_info", newNodeInfoHandler(c).getNodeInfo) + m.Get("/pin_cpu", func(writer http.ResponseWriter, request *http.Request) { + cpus, err := cpuset.SchedGetAffinity() + if err != nil { + render.Status(request, http.StatusInternalServerError) + render.Respond(writer, request, err) + } + casted := make([]int64, len(cpus)) + for i := range cpus { + casted[i] = int64(cpus[i]) + } + render.Respond(writer, request, &models.Cpus{CPU: casted}) + }) + m.Post("/pin_cpu", func(writer http.ResponseWriter, request *http.Request) { + if err := findAndPinCPUs(request.Context(), c, logger); err != nil { + render.Status(request, http.StatusInternalServerError) + render.Respond(writer, request, err) + } + }) + m.Delete("/pin_cpu", func(writer http.ResponseWriter, request *http.Request) { + if err := unpinFromCPUs(); err != nil { + render.Status(request, http.StatusInternalServerError) + render.Respond(writer, request, err) + } + }) m.Post("/terminate", selfSigterm()) m.Post("/free_os_memory", func(writer http.ResponseWriter, request *http.Request) { debug.FreeOSMemory() diff --git a/pkg/cmd/agent/router.go b/pkg/cmd/agent/router.go index b6d21c5a1b..c24e56d4bc 100644 --- a/pkg/cmd/agent/router.go +++ b/pkg/cmd/agent/router.go @@ -34,7 +34,7 @@ func newRouter(c agent.Config, metrics AgentMetrics, rclone http.Handler, logger auth.ValidateToken(c.AuthToken, time.Second, unauthorizedErrorBody), ) // Agent specific endpoints - priv.Mount("/agent", newAgentHandler(c, rclone)) + priv.Mount("/agent", newAgentHandler(c, rclone, logger.Named("agent"))) // Scylla prometheus proxy priv.Mount("/metrics", promProxy(c)) // Fallback to Scylla API proxy diff --git a/pkg/cmd/agent/server.go b/pkg/cmd/agent/server.go index 2710a3e715..8a9b9768cd 100644 --- a/pkg/cmd/agent/server.go +++ b/pkg/cmd/agent/server.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "runtime" + "slices" "time" "github.com/pkg/errors" @@ -83,32 +84,7 @@ func (s *server) init(ctx context.Context) error { ) } - // Try to get CPUs to pin to - var cpus []int - if s.config.CPU != agent.NoCPU { - cpus = []int{s.config.CPU} - } else if free, err := findFreeCPUs(); err != nil { - if os.IsNotExist(errors.Cause(err)) || errors.Is(err, cpuset.ErrNoCPUSetConfig) { - // Ignore if there is no cpuset file - s.logger.Debug(ctx, "Failed to find CPUs to pin to", "error", err) - } else { - s.logger.Error(ctx, "Failed to find CPUs to pin to", "error", err) - } - } else { - cpus = free - } - // Pin to CPUs if possible - if len(cpus) == 0 { - s.logger.Info(ctx, "Running on all CPUs") - runtime.GOMAXPROCS(1) - } else { - if err := pinToCPUs(cpus); err != nil { - s.logger.Error(ctx, "Failed to pin to CPUs", "cpus", cpus, "error", err) - } else { - s.logger.Info(ctx, "Running on CPUs", "cpus", cpus) - } - runtime.GOMAXPROCS(len(cpus)) - } + _ = findAndPinCPUs(ctx, s.config, s.logger) // nolint: errcheck // Log memory limit if l, err := cgroupMemoryLimit(); err != nil { @@ -134,6 +110,51 @@ func (s *server) init(ctx context.Context) error { ) } +func findAndPinCPUs(ctx context.Context, cfg agent.Config, logger log.Logger) error { + var aggregatedErr error + // Try to get CPUs to pin to + var cpus []int + if !slices.Contains(cfg.CPU, agent.NoCPU) { + cpus = cfg.CPU + } else if free, err := findFreeCPUs(); err != nil { + if os.IsNotExist(errors.Cause(err)) || errors.Is(err, cpuset.ErrNoCPUSetConfig) { + // Ignore if there is no cpuset file + logger.Info(ctx, "Failed to find CPUs to pin to", "error", err) + } else { + logger.Error(ctx, "Failed to find CPUs to pin to", "error", err) + aggregatedErr = multierr.Append(aggregatedErr, err) + } + } else { + cpus = free + } + // Pin to CPUs if possible + if len(cpus) == 0 { + logger.Info(ctx, "Running on all CPUs") + runtime.GOMAXPROCS(1) + } else { + if err := pinToCPUs(cpus); err != nil { + logger.Error(ctx, "Failed to pin to CPUs", "cpus", cpus, "error", err) + aggregatedErr = multierr.Append(aggregatedErr, err) + } else { + logger.Info(ctx, "Running on CPUs", "cpus", cpus) + } + runtime.GOMAXPROCS(len(cpus)) + } + return aggregatedErr +} + +func unpinFromCPUs() error { + cpus := make([]int, runtime.NumCPU()) + for i := range cpus { + cpus[i] = i + } + if err := pinToCPUs(cpus); err != nil { + return errors.Wrapf(err, "pin to cpus %v", cpus) + } + runtime.GOMAXPROCS(len(cpus)) + return nil +} + func (s *server) makeServers(ctx context.Context) error { tlsConfig, err := s.tlsConfig(ctx) if err != nil { diff --git a/pkg/command/restore/cmd.go b/pkg/command/restore/cmd.go index dcd82dda33..834f132a89 100644 --- a/pkg/command/restore/cmd.go +++ b/pkg/command/restore/cmd.go @@ -32,6 +32,7 @@ type command struct { transfers int rateLimit []string allowCompaction bool + unpinAgentCPU bool restoreSchema bool restoreTables bool dryRun bool @@ -84,6 +85,7 @@ func (cmd *command) init() { w.Unwrap().IntVar(&cmd.transfers, "transfers", 0, "") w.Unwrap().StringSliceVar(&cmd.rateLimit, "rate-limit", nil, "") w.Unwrap().BoolVar(&cmd.allowCompaction, "allow-compaction", false, "") + w.Unwrap().BoolVar(&cmd.unpinAgentCPU, "unpin-agent-cpu", false, "") w.Unwrap().BoolVar(&cmd.restoreSchema, "restore-schema", false, "") w.Unwrap().BoolVar(&cmd.restoreTables, "restore-tables", false, "") w.Unwrap().BoolVar(&cmd.dryRun, "dry-run", false, "") @@ -162,6 +164,10 @@ func (cmd *command) run(args []string) error { props["allow_compaction"] = cmd.allowCompaction ok = true } + if cmd.Flag("unpin-agent-cpu").Changed { + props["unpin_agent_cpu"] = cmd.unpinAgentCPU + ok = true + } if cmd.Flag("restore-schema").Changed { if cmd.Update() { return wrapper("restore-schema") diff --git a/pkg/command/restore/res.yaml b/pkg/command/restore/res.yaml index a780671f84..eaeb4bd569 100644 --- a/pkg/command/restore/res.yaml +++ b/pkg/command/restore/res.yaml @@ -48,6 +48,10 @@ allow-compaction: | Defines if auto compactions should be running on Scylla nodes during restore. Disabling auto compactions decreases restore time duration, but increases compaction workload after the restore is done. +unpin-agent-cpu: | + Defines if ScyllaDB Manager Agent should be unpinned from CPUs during restore. + This might significantly improve download speed at the cost of decreasing streaming speed. + restore-schema: | Specifies restore type (alternative to '--restore-tables' flag). Restore will recreate schema by applying the backed up output of DESCRIBE SCHEMA WITH INTERNALS via CQL. diff --git a/pkg/config/agent/agent.go b/pkg/config/agent/agent.go index 7e7fccb86c..622fc6d0e9 100644 --- a/pkg/config/agent/agent.go +++ b/pkg/config/agent/agent.go @@ -51,7 +51,7 @@ type Config struct { TLSKeyFile string `yaml:"tls_key_file"` Prometheus string `yaml:"prometheus"` Debug string `yaml:"debug"` - CPU int `yaml:"cpu"` + CPU CPUs `yaml:"cpu"` Logger config.LogConfig `yaml:"logger"` Scylla ScyllaConfig `yaml:"scylla"` Rclone rclone.GlobalOptions `yaml:"rclone"` @@ -66,7 +66,7 @@ func DefaultConfig() Config { TLSVersion: config.TLSv12, Prometheus: ":5090", Debug: "127.0.0.1:5112", - CPU: NoCPU, + CPU: []int{NoCPU}, Logger: DefaultLogConfig(), Scylla: ScyllaConfig{ APIAddress: "0.0.0.0", @@ -121,3 +121,36 @@ func Obfuscate(c Config) Config { } return c } + +// CPUs represents a list of CPUs. +// It supports custom marshalling methods which allow +// for specifying 'cpu' in 'scylla-manager-agent.yaml' config +// as both a single int or an array of ints. +type CPUs []int + +// UnmarshalYAML is a custom unmarshaller accepting both +// single ints and int array. +func (c *CPUs) UnmarshalYAML(unmarshal func(interface{}) error) error { + var v int + errV := unmarshal(&v) + if errV == nil { + *c = []int{v} + return nil + } + var arr []int + errArr := unmarshal(&arr) + if errArr == nil { + *c = arr + return nil + } + return errors.Wrap(multierr.Append(errV, errArr), "parse CPU as an int or int array") +} + +// MarshalYAML is a custom marshaller returning +// single int is possible, and int array otherwise. +func (c CPUs) MarshalYAML() (interface{}, error) { + if len(c) == 1 { + return (c)[0], nil + } + return c, nil +} diff --git a/pkg/config/agent/agent_test.go b/pkg/config/agent/agent_test.go index 8bb1f18622..2e743c074a 100644 --- a/pkg/config/agent/agent_test.go +++ b/pkg/config/agent/agent_test.go @@ -66,6 +66,11 @@ func TestParseConfig(t *testing.T) { Input: []string{"./testdata/structs_empty.input.yaml"}, Golden: "./testdata/structs_empty.golden.yaml", }, + { + Name: "multiple cpus", + Input: []string{"./testdata/multiple_cpus.input.yaml"}, + Golden: "./testdata/multiple_cpus.golden.yaml", + }, } s := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/config/agent/testdata/multiple_cpus.golden.yaml b/pkg/config/agent/testdata/multiple_cpus.golden.yaml new file mode 100644 index 0000000000..f39d31ffd1 --- /dev/null +++ b/pkg/config/agent/testdata/multiple_cpus.golden.yaml @@ -0,0 +1,189 @@ +auth_token: "" +https: 192.168.100.11:10001 +https_port: 10001 +tls_version: TLSv1.2 +tls_cert_file: "" +tls_key_file: "" +prometheus: :5090 +debug: 127.0.0.1:5112 +cpu: +- 0 +- 2 +- 4 +logger: + mode: stderr + level: info + sampling: + initial: 1 + thereafter: 100 + encoding: json + development: false +scylla: + api_address: 0.0.0.0 + api_port: "10000" + listenaddress: 192.168.100.11 + prometheusaddress: 192.168.100.11 + prometheusport: "9180" + datadirectory: /var/lib/scylla/data + broadcastrpcaddress: "" + rpcaddress: "" +rclone: + log_level: 7 + stats_log_level: 6 + use_json_log: false + dry_run: false + interactive: false + check_sum: true + size_only: false + ignore_times: false + ignore_existing: false + ignore_errors: true + modify_window: 1ns + checkers: 100 + transfers: 2 + connect_timeout: 1m0s + timeout: 5m0s + expect_continue_timeout: 1s + dump: 0 + insecure_skip_verify: false + delete_mode: 0 + max_delete: -1 + track_renames: false + track_renames_strategy: hash + low_level_retries: 20 + update_older: false + no_gzip: false + max_depth: -1 + ignore_size: false + ignore_checksum: true + ignore_case_sync: false + no_traverse: true + check_first: false + no_check_dest: false + no_unicode_normalization: false + no_update_mod_time: true + data_rate_unit: bytes + compare_dest: "" + copy_dest: "" + backup_dir: "" + suffix: "" + suffix_keep_extension: false + use_list_r: false + buffer_size: 16777216 + bw_limit: [] + bw_limit_file: [] + tps_limit: 5000 + tps_limit_burst: 1 + bind_addr: "" + disable_features: [] + user_agent: Scylla Manager Agent Snapshot + immutable: false + auto_confirm: false + streaming_upload_cutoff: 102400 + stats_file_name_length: 45 + ask_password: true + password_command: [] + use_server_mod_time: true + max_transfer: -1 + max_duration: 0s + cutoff_mode: 0 + max_backlog: -1 + max_stats_groups: 1000 + stats_one_line: false + stats_one_line_date: false + stats_one_line_date_format: "" + error_on_no_transfer: false + progress: false + progress_terminal_title: false + cookie: false + use_mmap: true + ca_cert: "" + client_cert: "" + client_key: "" + multi_thread_cutoff: 262144000 + multi_thread_streams: 4 + multi_thread_set: false + order_by: "" + upload_headers: [] + download_headers: [] + headers: [] + refresh_times: false + no_console: false +s3: + provider: AWS + env_auth: "true" + access_key_id: "" + secret_access_key: "" + region: "" + endpoint: "" + location_constraint: "" + acl: "" + bucket_acl: "" + requester_pays: "" + server_side_encryption: "" + sse_customer_algorithm: "" + sse_kms_key_id: "" + sse_customer_key: "" + sse_customer_key_md5: "" + storage_class: "" + upload_cutoff: "" + chunk_size: 50M + max_upload_parts: "" + copy_cutoff: "" + disable_checksum: "true" + shared_credentials_file: "" + profile: "" + session_token: "" + upload_concurrency: "2" + force_path_style: "" + v2_auth: "" + use_accelerate_endpoint: "" + leave_parts_on_error: "" + list_chunk: "" + no_check_bucket: "true" + no_head: "" + encoding: "" + memory_pool_flush_time: 5m + memory_pool_use_mmap: "true" + disable_http2: "" +gcs: + client_id: "" + client_secret: "" + token: "" + auth_url: "" + token_url: "" + project_number: "" + service_account_file: "" + service_account_credentials: "" + anonymous: "" + object_acl: "" + bucket_acl: "" + bucket_policy_only: "true" + location: "" + storage_class: "" + encoding: "" + memory_pool_flush_time: 5m + memory_pool_use_mmap: "true" + chunk_size: 50M + list_chunk: "" + allow_create_bucket: "false" +azure: + account: "" + service_principal_file: "" + key: "" + sas_url: "" + use_msi: "" + msi_object_id: "" + msi_client_id: "" + msi_mi_res_id: "" + use_emulator: "" + endpoint: "" + upload_cutoff: "" + chunk_size: 50M + list_chunk: "" + access_tier: "" + archive_tier_delete: "" + disable_checksum: "true" + memory_pool_flush_time: 5m + memory_pool_use_mmap: "true" + encoding: "" diff --git a/pkg/config/agent/testdata/multiple_cpus.input.yaml b/pkg/config/agent/testdata/multiple_cpus.input.yaml new file mode 100644 index 0000000000..cb99447eb6 --- /dev/null +++ b/pkg/config/agent/testdata/multiple_cpus.input.yaml @@ -0,0 +1,4 @@ +cpu: + - 0 + - 2 + - 4 \ No newline at end of file diff --git a/pkg/scyllaclient/client_agent.go b/pkg/scyllaclient/client_agent.go index df036c67ea..68eb87e5ab 100644 --- a/pkg/scyllaclient/client_agent.go +++ b/pkg/scyllaclient/client_agent.go @@ -40,6 +40,36 @@ func (c *Client) AnyNodeInfo(ctx context.Context) (*NodeInfo, error) { return (*NodeInfo)(resp.Payload), nil } +// GetPinnedCPU returns CPUs to which agent is pinned. +func (c *Client) GetPinnedCPU(ctx context.Context, host string) ([]int64, error) { + p := operations.GetCPUParams{ + Context: forceHost(ctx, host), + } + resp, err := c.agentOps.GetCPU(&p) + if err != nil { + return nil, errors.Wrap(err, "get CPUs") + } + return resp.Payload.CPU, nil +} + +// UnpinFromCPU allows agent to use all CPUs. +func (c *Client) UnpinFromCPU(ctx context.Context, host string) error { + p := operations.UnpinCPUParams{ + Context: forceHost(ctx, host), + } + _, err := c.agentOps.UnpinCPU(&p) + return err +} + +// PinCPU pins agent to CPUs according to the start-up logic. +func (c *Client) PinCPU(ctx context.Context, host string) error { + p := operations.PinCPUParams{ + Context: forceHost(ctx, host), + } + _, err := c.agentOps.PinCPU(&p) + return err +} + // CQLAddr returns CQL address from NodeInfo. // Scylla can have separate rpc_address (CQL), listen_address and respectfully // broadcast_rpc_address and broadcast_address if some 3rd party routing diff --git a/pkg/service/restore/model.go b/pkg/service/restore/model.go index da616b515d..965c633913 100644 --- a/pkg/service/restore/model.go +++ b/pkg/service/restore/model.go @@ -31,6 +31,7 @@ type Target struct { Transfers int `json:"transfers"` RateLimit []DCLimit `json:"rate_limit,omitempty"` AllowCompaction bool `json:"allow_compaction,omitempty"` + UnpinAgentCPU bool `json:"unpin_agent_cpu"` RestoreSchema bool `json:"restore_schema,omitempty"` RestoreTables bool `json:"restore_tables,omitempty"` Continue bool `json:"continue"` diff --git a/pkg/service/restore/restore_integration_test.go b/pkg/service/restore/restore_integration_test.go index 318377cd58..300dc20a2d 100644 --- a/pkg/service/restore/restore_integration_test.go +++ b/pkg/service/restore/restore_integration_test.go @@ -10,6 +10,8 @@ import ( "encoding/json" "fmt" "net/http" + "runtime" + "slices" "strings" "sync/atomic" "testing" @@ -498,7 +500,7 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { Print("Fill setup") fillTable(t, h.srcCluster.rootSession, 100, ks, tab) - validateState := func(ch clusterHelper, tombstone string, compaction bool, transfers int, rateLimit int) { + validateState := func(ch clusterHelper, tombstone string, compaction bool, transfers int, rateLimit int, cpus []int64) { // Validate tombstone_gc mode if got := tombstoneGCMode(t, ch.rootSession, ks, tab); tombstone != got { t.Errorf("expected tombstone_gc=%s, got %s", tombstone, got) @@ -537,6 +539,24 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { t.Errorf("expected rate_limit=%s, got=%s on host %s", rawLimit, got, host) } } + // Validate cpu pinning + for _, host := range ch.Client.Config().Hosts { + got, err := ch.Client.GetPinnedCPU(context.Background(), host) + if err != nil { + t.Fatal(errors.Wrapf(err, "check transfers on host %s", host)) + } + slices.Sort(cpus) + slices.Sort(got) + if !slices.Equal(cpus, got) { + t.Errorf("expected cpus=%v, got=%v on host %s", cpus, got, host) + } + } + } + + pinnedCPU := []int64{0} // Taken from scylla-manager-agent.yaml used for testing + unpinnedCPU := make([]int64, runtime.NumCPU()) + for i := range unpinnedCPU { + unpinnedCPU[i] = int64(i) } shardCnt, err := h.dstCluster.Client.ShardCount(context.Background(), ManagedClusterHost()) @@ -545,7 +565,7 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { } transfers0 := 2 * int(shardCnt) - setTransfersAndRateLimit := func(ch clusterHelper, transfers int, rateLimit int) { + setTransfersAndRateLimitAndPinnedCPU := func(ch clusterHelper, transfers int, rateLimit int, pin bool) { for _, host := range ch.Client.Config().Hosts { err := ch.Client.RcloneSetTransfers(context.Background(), host, transfers) if err != nil { @@ -555,15 +575,26 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { if err != nil { t.Fatal(errors.Wrapf(err, "set rate limit on host %s", host)) } + if pin { + err = ch.Client.PinCPU(context.Background(), host) + if err != nil { + t.Fatal(errors.Wrapf(err, "pin CPUs on host %s", host)) + } + } else { + err = ch.Client.UnpinFromCPU(context.Background(), host) + if err != nil { + t.Fatal(errors.Wrapf(err, "unpin CPUs on host %s", host)) + } + } } } Print("Set initial transfers and rate limit") - setTransfersAndRateLimit(h.srcCluster, 10, 99) - setTransfersAndRateLimit(h.dstCluster, 10, 99) + setTransfersAndRateLimitAndPinnedCPU(h.srcCluster, 10, 99, true) + setTransfersAndRateLimitAndPinnedCPU(h.dstCluster, 10, 99, true) Print("Validate state before backup") - validateState(h.srcCluster, "repair", true, 10, 99) + validateState(h.srcCluster, "repair", true, 10, 99, pinnedCPU) Print("Run backup") loc := []Location{testLocation("preparation", "")} @@ -577,18 +608,19 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { }) Print("Validate state after backup") - validateState(h.srcCluster, "repair", true, 3, 88) + validateState(h.srcCluster, "repair", true, 3, 88, pinnedCPU) runRestore := func(ctx context.Context, finishedRestore chan error) { grantRestoreTablesPermissions(t, h.dstCluster.rootSession, ksFilter, h.dstUser) h.dstCluster.RunID = uuid.NewTime() rawProps, err := json.Marshal(map[string]any{ - "location": loc, - "keyspace": ksFilter, - "snapshot_tag": tag, - "transfers": 0, - "rate_limit": []string{"0"}, - "restore_tables": true, + "location": loc, + "keyspace": ksFilter, + "snapshot_tag": tag, + "transfers": 0, + "rate_limit": []string{"0"}, + "unpin_agent_cpu": true, + "restore_tables": true, }) if err != nil { finishedRestore <- err @@ -621,7 +653,7 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { makeLASHang(reachedDataStageChan, hangLAS) Print("Validate state before restore") - validateState(h.dstCluster, "repair", true, 10, 99) + validateState(h.dstCluster, "repair", true, 10, 99, pinnedCPU) Print("Run restore") finishedRestore := make(chan error) @@ -636,7 +668,7 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { } Print("Validate state during restore data") - validateState(h.dstCluster, "disabled", false, transfers0, 0) + validateState(h.dstCluster, "disabled", false, transfers0, 0, unpinnedCPU) Print("Pause restore") restoreCancel() @@ -651,11 +683,10 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { } Print("Validate state during pause") - validateState(h.dstCluster, "disabled", true, transfers0, 0) + validateState(h.dstCluster, "disabled", true, transfers0, 0, pinnedCPU) Print("Change transfers and rate limit during pause") - setTransfersAndRateLimit(h.srcCluster, 9, 55) - setTransfersAndRateLimit(h.dstCluster, 9, 55) + setTransfersAndRateLimitAndPinnedCPU(h.dstCluster, 9, 55, false) reachedDataStageChan = make(chan struct{}) hangLAS = make(chan struct{}) @@ -674,7 +705,7 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { } Print("Validate state during restore data after pause") - validateState(h.dstCluster, "disabled", false, transfers0, 0) + validateState(h.dstCluster, "disabled", false, transfers0, 0, unpinnedCPU) Print("Release LAS") close(hangLAS) @@ -686,7 +717,7 @@ func TestRestoreTablesPreparationIntegration(t *testing.T) { } Print("Validate state after restore success") - validateState(h.dstCluster, "repair", true, transfers0, 0) + validateState(h.dstCluster, "repair", true, transfers0, 0, pinnedCPU) Print("Validate table contents") h.validateIdenticalTables(t, []table{{ks: ks, tab: tab}}) diff --git a/pkg/service/restore/tables_worker.go b/pkg/service/restore/tables_worker.go index e35409f70a..1067ba59b0 100644 --- a/pkg/service/restore/tables_worker.go +++ b/pkg/service/restore/tables_worker.go @@ -188,17 +188,32 @@ func (w *tablesWorker) stageRestoreData(ctx context.Context) error { w.logger.Info(ctx, "Host shard count", "host", h, "shards", sh) } + // This defer is outside of target field check for improved safety. + // We always want to enable auto compaction outside the restore. + defer func() { + if err := w.setAutoCompaction(context.Background(), hosts, true); err != nil { + w.logger.Error(ctx, "Couldn't enable auto compaction", "error", err) + } + }() if !w.target.AllowCompaction { - defer func() { - if err := w.setAutoCompaction(context.Background(), hosts, true); err != nil { - w.logger.Error(ctx, "Couldn't enable auto compaction", "error", err) - } - }() if err := w.setAutoCompaction(ctx, hosts, false); err != nil { return errors.Wrapf(err, "disable auto compaction") } } + // Same as above. + // We always want to pin agent to CPUs outside the restore. + defer func() { + if err := w.pinAgentCPU(context.Background(), hosts, true); err != nil { + w.logger.Error(ctx, "Couldn't re-pin agent to CPUs", "error", err) + } + }() + if w.target.UnpinAgentCPU { + if err := w.pinAgentCPU(ctx, hosts, false); err != nil { + return errors.Wrapf(err, "unpin agent from CPUs") + } + } + bd := newBatchDispatcher(workload, w.target.BatchSize, hostToShard, w.target.locationHosts) f := func(n int) (err error) { @@ -349,6 +364,23 @@ func (w *tablesWorker) setAutoCompaction(ctx context.Context, hosts []string, en return nil } +// Pins/unpins all provided hosts to/from CPUs. +func (w *tablesWorker) pinAgentCPU(ctx context.Context, hosts []string, pin bool) error { + err := parallel.Run(len(hosts), parallel.NoLimit, + func(i int) error { + if pin { + return w.client.PinCPU(ctx, hosts[i]) + } + return w.client.UnpinFromCPU(ctx, hosts[i]) + }, func(i int, err error) { + w.logger.Error(ctx, "Failed to change agent CPU pinning", + "host", hosts[i], + "pinned", pin, + "error", err) + }) + return errors.Wrapf(err, "set agent CPU pinning") +} + func (w *tablesWorker) hostInfo(host, dc string, shards uint) HostInfo { return HostInfo{ Host: host, diff --git a/testing/scylla-manager-agent/scylla-manager-agent-ipv6.yaml b/testing/scylla-manager-agent/scylla-manager-agent-ipv6.yaml index d7ee3d4f06..2f4398284d 100644 --- a/testing/scylla-manager-agent/scylla-manager-agent-ipv6.yaml +++ b/testing/scylla-manager-agent/scylla-manager-agent-ipv6.yaml @@ -1,5 +1,7 @@ auth_token: token +cpu: 0 + debug: :5112 logger: diff --git a/testing/scylla-manager-agent/scylla-manager-agent.yaml b/testing/scylla-manager-agent/scylla-manager-agent.yaml index 9075bbc833..83431b07dd 100644 --- a/testing/scylla-manager-agent/scylla-manager-agent.yaml +++ b/testing/scylla-manager-agent/scylla-manager-agent.yaml @@ -1,5 +1,7 @@ auth_token: token +cpu: 0 + debug: :5112 logger: diff --git a/vendor/github.com/scylladb/scylla-manager/v3/pkg/managerclient/LICENSE b/vendor/github.com/scylladb/scylla-manager/v3/pkg/managerclient/LICENSE deleted file mode 100644 index a155a625fc..0000000000 --- a/vendor/github.com/scylladb/scylla-manager/v3/pkg/managerclient/LICENSE +++ /dev/null @@ -1,62 +0,0 @@ -Scylla SOURCE AVAILABLE LICENSE (SSAL) AGREEMENT Last Update: June 30, 2021 - -This Agreement sets forth the terms on which the Licensor makes available the Software. BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. - -If you are receiving the Software on behalf of a legal entity, you represent and warrant that you have the actual authority to agree to the terms and conditions of this agreement on behalf of such entity. - -The terms below have the meanings set forth below for purposes of this Agreement: - -Agreement: this Scylla Source Available License Agreement. - -Database Product: any of the following products or services: - -(a) Scylla Manager; - -(b) a product or service exposing the Scylla Manager API - -License: the Scylla Source Available License described in Section 1. - -Licensor: as indicated in the source code license. - -Modification: a modification of the Software made by You under the License, Section 1.1(c). - -Scylla: the open source Scylla software as described in scylladb.com - -Software: certain software components designed to work with Scylla NoSQL Database and provided to you under this Agreement. - -You: the recipient of this Software, an individual, or the entity on whose behalf you are receiving the Software. - -Your Application: an application developed by or for You, where such application is not a Database Product. - -LICENSE GRANT AND CONDITIONS -1.1 Subject to the terms and conditions of this Section 1, Licensor hereby grants to You a non-exclusive, royalty-free, worldwide, non-transferable license during the term of this Agreement to: - -a) distribute or make available the Software or your Modifications under the terms of this -Agreement, only as part of Your Application, so long as you include the following notice on any copy you distribute: “This software is subject to the terms of the Scylla Source Available License Agreement”. - -(b) use the Software, or your Modifications, only as part of Your Application, but not in connection - -with any Database Product that is distributed or otherwise made available by any third party. - -(c) modify the Software, provided that Modifications remain subject to the terms of this License. - -(d) reproduce the Software as necessary for the above. - -1.3. Notices. - -On all copies of the Software that you make, you must retain all copyright or other proprietary notices. - -TERM AND TERMINATION. -This Agreement will continue unless and until earlier terminated as set forth herein. If You breach any of its conditions or obligations under this Agreement, this Agreement will terminate automatically and the licenses granted herein will terminate automatically. - -INTELLECTUAL PROPERTY. -As between the parties, Licensor retains all right, title, and interest in the Software, and to Scylla or other Licensor trademarks or service marks, and all intellectual property rights therein. Licensor hereby reserves all rights not expressly granted to You in this Agreement. - -DISCLAIMER. -TO THE EXTENT ALLOWABLE UNDER LAW, LICENSOR HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, WITH RESPECT TO THE SOFTWARE. Licensor has no obligation to support the Software. - -LIMITATION OF LIABILITY. TO THE EXTENT ALLOWABLE UNDER LAW, LICENSOR WILL NOT BE LIABLE FOR ANY DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO, LOST PROFITS OR ANY CONSEQUENTIAL, SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, ARISING OUT OF OR RELATING TO THIS AGREEMENT. -GENERAL. You are not authorized to assign Your rights under this Agreement to any third party. Licensor may freely assign its rights under this Agreement to any third party. This Agreement is the entire agreement between the parties on the subject matter hereof. No amendment or modification hereof will be valid or binding upon the parties unless made in writing and signed by the duly authorized representatives of both parties. In the event that any provision, including without limitation any condition, of this Agreement is held to be unenforceable, this Agreement and all licenses and rights granted hereunder will immediately terminate. Failure by Licensor to exercise any right hereunder will not be construed as a waiver of any subsequent breach of that right or as a waiver of any other right. This Agreement will be governed by and interpreted in accordance with the laws of the state of California, without reference to its conflict of laws principles. If You are located within the United States, all disputes arising out of this Agreement are subject to the exclusive jurisdiction of courts located in Santa Clara County, California. USA. If You are located outside of the United States, any dispute, controversy or claim arising out of or relating to this Agreement will be referred to and finally determined by arbitration in accordance with the JAMS before a single arbitrator in Santa Clara County, California. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction thereof. - -This license is base on REDIS SOURCE AVAILABLE LICENSE (RSAL) AGREEMENT with some updates -https://redislabs.com/wp-content/uploads/2019/09/redis-source-available-license.pdf diff --git a/vendor/github.com/scylladb/scylla-manager/v3/pkg/util/LICENSE b/vendor/github.com/scylladb/scylla-manager/v3/pkg/util/LICENSE deleted file mode 100644 index a155a625fc..0000000000 --- a/vendor/github.com/scylladb/scylla-manager/v3/pkg/util/LICENSE +++ /dev/null @@ -1,62 +0,0 @@ -Scylla SOURCE AVAILABLE LICENSE (SSAL) AGREEMENT Last Update: June 30, 2021 - -This Agreement sets forth the terms on which the Licensor makes available the Software. BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. - -If you are receiving the Software on behalf of a legal entity, you represent and warrant that you have the actual authority to agree to the terms and conditions of this agreement on behalf of such entity. - -The terms below have the meanings set forth below for purposes of this Agreement: - -Agreement: this Scylla Source Available License Agreement. - -Database Product: any of the following products or services: - -(a) Scylla Manager; - -(b) a product or service exposing the Scylla Manager API - -License: the Scylla Source Available License described in Section 1. - -Licensor: as indicated in the source code license. - -Modification: a modification of the Software made by You under the License, Section 1.1(c). - -Scylla: the open source Scylla software as described in scylladb.com - -Software: certain software components designed to work with Scylla NoSQL Database and provided to you under this Agreement. - -You: the recipient of this Software, an individual, or the entity on whose behalf you are receiving the Software. - -Your Application: an application developed by or for You, where such application is not a Database Product. - -LICENSE GRANT AND CONDITIONS -1.1 Subject to the terms and conditions of this Section 1, Licensor hereby grants to You a non-exclusive, royalty-free, worldwide, non-transferable license during the term of this Agreement to: - -a) distribute or make available the Software or your Modifications under the terms of this -Agreement, only as part of Your Application, so long as you include the following notice on any copy you distribute: “This software is subject to the terms of the Scylla Source Available License Agreement”. - -(b) use the Software, or your Modifications, only as part of Your Application, but not in connection - -with any Database Product that is distributed or otherwise made available by any third party. - -(c) modify the Software, provided that Modifications remain subject to the terms of this License. - -(d) reproduce the Software as necessary for the above. - -1.3. Notices. - -On all copies of the Software that you make, you must retain all copyright or other proprietary notices. - -TERM AND TERMINATION. -This Agreement will continue unless and until earlier terminated as set forth herein. If You breach any of its conditions or obligations under this Agreement, this Agreement will terminate automatically and the licenses granted herein will terminate automatically. - -INTELLECTUAL PROPERTY. -As between the parties, Licensor retains all right, title, and interest in the Software, and to Scylla or other Licensor trademarks or service marks, and all intellectual property rights therein. Licensor hereby reserves all rights not expressly granted to You in this Agreement. - -DISCLAIMER. -TO THE EXTENT ALLOWABLE UNDER LAW, LICENSOR HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, WITH RESPECT TO THE SOFTWARE. Licensor has no obligation to support the Software. - -LIMITATION OF LIABILITY. TO THE EXTENT ALLOWABLE UNDER LAW, LICENSOR WILL NOT BE LIABLE FOR ANY DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO, LOST PROFITS OR ANY CONSEQUENTIAL, SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, ARISING OUT OF OR RELATING TO THIS AGREEMENT. -GENERAL. You are not authorized to assign Your rights under this Agreement to any third party. Licensor may freely assign its rights under this Agreement to any third party. This Agreement is the entire agreement between the parties on the subject matter hereof. No amendment or modification hereof will be valid or binding upon the parties unless made in writing and signed by the duly authorized representatives of both parties. In the event that any provision, including without limitation any condition, of this Agreement is held to be unenforceable, this Agreement and all licenses and rights granted hereunder will immediately terminate. Failure by Licensor to exercise any right hereunder will not be construed as a waiver of any subsequent breach of that right or as a waiver of any other right. This Agreement will be governed by and interpreted in accordance with the laws of the state of California, without reference to its conflict of laws principles. If You are located within the United States, all disputes arising out of this Agreement are subject to the exclusive jurisdiction of courts located in Santa Clara County, California. USA. If You are located outside of the United States, any dispute, controversy or claim arising out of or relating to this Agreement will be referred to and finally determined by arbitration in accordance with the JAMS before a single arbitrator in Santa Clara County, California. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction thereof. - -This license is base on REDIS SOURCE AVAILABLE LICENSE (RSAL) AGREEMENT with some updates -https://redislabs.com/wp-content/uploads/2019/09/redis-source-available-license.pdf diff --git a/vendor/github.com/scylladb/scylla-manager/v3/pkg/util/cpuset/cpuset_linux.go b/vendor/github.com/scylladb/scylla-manager/v3/pkg/util/cpuset/cpuset_linux.go index 865c1b32ca..5c7b379e2a 100644 --- a/vendor/github.com/scylladb/scylla-manager/v3/pkg/util/cpuset/cpuset_linux.go +++ b/vendor/github.com/scylladb/scylla-manager/v3/pkg/util/cpuset/cpuset_linux.go @@ -53,6 +53,31 @@ func schedSetAffinityToMany(pids []int, set *unix.CPUSet) (err error) { return } +// SchedGetAffinity gets a union of CPUs used by this process threads. +func SchedGetAffinity() ([]int, error) { + pids, err := osTasks(os.Getpid()) + if err != nil { + return nil, errors.Wrap(err, "get tasks") + } + + union := make(map[int]struct{}) + for _, pid := range pids { + var cpus unix.CPUSet + if err := unix.SchedGetaffinity(pid, &cpus); err != nil { + return nil, errors.Wrap(err, "get affinity") + } + for _, cpu := range cpulist(&cpus) { + union[cpu] = struct{}{} + } + } + + var out []int + for cpu := range union { + out = append(out, cpu) + } + return out, nil +} + func cpuset(cpus []int) *unix.CPUSet { set := &unix.CPUSet{} set.Zero() diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/LICENSE b/vendor/github.com/scylladb/scylla-manager/v3/swagger/LICENSE deleted file mode 100644 index a155a625fc..0000000000 --- a/vendor/github.com/scylladb/scylla-manager/v3/swagger/LICENSE +++ /dev/null @@ -1,62 +0,0 @@ -Scylla SOURCE AVAILABLE LICENSE (SSAL) AGREEMENT Last Update: June 30, 2021 - -This Agreement sets forth the terms on which the Licensor makes available the Software. BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. - -If you are receiving the Software on behalf of a legal entity, you represent and warrant that you have the actual authority to agree to the terms and conditions of this agreement on behalf of such entity. - -The terms below have the meanings set forth below for purposes of this Agreement: - -Agreement: this Scylla Source Available License Agreement. - -Database Product: any of the following products or services: - -(a) Scylla Manager; - -(b) a product or service exposing the Scylla Manager API - -License: the Scylla Source Available License described in Section 1. - -Licensor: as indicated in the source code license. - -Modification: a modification of the Software made by You under the License, Section 1.1(c). - -Scylla: the open source Scylla software as described in scylladb.com - -Software: certain software components designed to work with Scylla NoSQL Database and provided to you under this Agreement. - -You: the recipient of this Software, an individual, or the entity on whose behalf you are receiving the Software. - -Your Application: an application developed by or for You, where such application is not a Database Product. - -LICENSE GRANT AND CONDITIONS -1.1 Subject to the terms and conditions of this Section 1, Licensor hereby grants to You a non-exclusive, royalty-free, worldwide, non-transferable license during the term of this Agreement to: - -a) distribute or make available the Software or your Modifications under the terms of this -Agreement, only as part of Your Application, so long as you include the following notice on any copy you distribute: “This software is subject to the terms of the Scylla Source Available License Agreement”. - -(b) use the Software, or your Modifications, only as part of Your Application, but not in connection - -with any Database Product that is distributed or otherwise made available by any third party. - -(c) modify the Software, provided that Modifications remain subject to the terms of this License. - -(d) reproduce the Software as necessary for the above. - -1.3. Notices. - -On all copies of the Software that you make, you must retain all copyright or other proprietary notices. - -TERM AND TERMINATION. -This Agreement will continue unless and until earlier terminated as set forth herein. If You breach any of its conditions or obligations under this Agreement, this Agreement will terminate automatically and the licenses granted herein will terminate automatically. - -INTELLECTUAL PROPERTY. -As between the parties, Licensor retains all right, title, and interest in the Software, and to Scylla or other Licensor trademarks or service marks, and all intellectual property rights therein. Licensor hereby reserves all rights not expressly granted to You in this Agreement. - -DISCLAIMER. -TO THE EXTENT ALLOWABLE UNDER LAW, LICENSOR HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, WITH RESPECT TO THE SOFTWARE. Licensor has no obligation to support the Software. - -LIMITATION OF LIABILITY. TO THE EXTENT ALLOWABLE UNDER LAW, LICENSOR WILL NOT BE LIABLE FOR ANY DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO, LOST PROFITS OR ANY CONSEQUENTIAL, SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, ARISING OUT OF OR RELATING TO THIS AGREEMENT. -GENERAL. You are not authorized to assign Your rights under this Agreement to any third party. Licensor may freely assign its rights under this Agreement to any third party. This Agreement is the entire agreement between the parties on the subject matter hereof. No amendment or modification hereof will be valid or binding upon the parties unless made in writing and signed by the duly authorized representatives of both parties. In the event that any provision, including without limitation any condition, of this Agreement is held to be unenforceable, this Agreement and all licenses and rights granted hereunder will immediately terminate. Failure by Licensor to exercise any right hereunder will not be construed as a waiver of any subsequent breach of that right or as a waiver of any other right. This Agreement will be governed by and interpreted in accordance with the laws of the state of California, without reference to its conflict of laws principles. If You are located within the United States, all disputes arising out of this Agreement are subject to the exclusive jurisdiction of courts located in Santa Clara County, California. USA. If You are located outside of the United States, any dispute, controversy or claim arising out of or relating to this Agreement will be referred to and finally determined by arbitration in accordance with the JAMS before a single arbitrator in Santa Clara County, California. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction thereof. - -This license is base on REDIS SOURCE AVAILABLE LICENSE (RSAL) AGREEMENT with some updates -https://redislabs.com/wp-content/uploads/2019/09/redis-source-available-license.pdf diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/get_cpu_parameters.go b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/get_cpu_parameters.go new file mode 100644 index 0000000000..1e48d3ae8c --- /dev/null +++ b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/get_cpu_parameters.go @@ -0,0 +1,113 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "net/http" + "time" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + cr "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" +) + +// NewGetCPUParams creates a new GetCPUParams object +// with the default values initialized. +func NewGetCPUParams() *GetCPUParams { + + return &GetCPUParams{ + + timeout: cr.DefaultTimeout, + } +} + +// NewGetCPUParamsWithTimeout creates a new GetCPUParams object +// with the default values initialized, and the ability to set a timeout on a request +func NewGetCPUParamsWithTimeout(timeout time.Duration) *GetCPUParams { + + return &GetCPUParams{ + + timeout: timeout, + } +} + +// NewGetCPUParamsWithContext creates a new GetCPUParams object +// with the default values initialized, and the ability to set a context for a request +func NewGetCPUParamsWithContext(ctx context.Context) *GetCPUParams { + + return &GetCPUParams{ + + Context: ctx, + } +} + +// NewGetCPUParamsWithHTTPClient creates a new GetCPUParams object +// with the default values initialized, and the ability to set a custom HTTPClient for a request +func NewGetCPUParamsWithHTTPClient(client *http.Client) *GetCPUParams { + + return &GetCPUParams{ + HTTPClient: client, + } +} + +/* +GetCPUParams contains all the parameters to send to the API endpoint +for the get Cpu operation typically these are written to a http.Request +*/ +type GetCPUParams struct { + timeout time.Duration + Context context.Context + HTTPClient *http.Client +} + +// WithTimeout adds the timeout to the get Cpu params +func (o *GetCPUParams) WithTimeout(timeout time.Duration) *GetCPUParams { + o.SetTimeout(timeout) + return o +} + +// SetTimeout adds the timeout to the get Cpu params +func (o *GetCPUParams) SetTimeout(timeout time.Duration) { + o.timeout = timeout +} + +// WithContext adds the context to the get Cpu params +func (o *GetCPUParams) WithContext(ctx context.Context) *GetCPUParams { + o.SetContext(ctx) + return o +} + +// SetContext adds the context to the get Cpu params +func (o *GetCPUParams) SetContext(ctx context.Context) { + o.Context = ctx +} + +// WithHTTPClient adds the HTTPClient to the get Cpu params +func (o *GetCPUParams) WithHTTPClient(client *http.Client) *GetCPUParams { + o.SetHTTPClient(client) + return o +} + +// SetHTTPClient adds the HTTPClient to the get Cpu params +func (o *GetCPUParams) SetHTTPClient(client *http.Client) { + o.HTTPClient = client +} + +// WriteToRequest writes these params to a swagger request +func (o *GetCPUParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { + + if err := r.SetTimeout(o.timeout); err != nil { + return err + } + var res []error + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/get_cpu_responses.go b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/get_cpu_responses.go new file mode 100644 index 0000000000..aba30ba14b --- /dev/null +++ b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/get_cpu_responses.go @@ -0,0 +1,135 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/go-openapi/runtime" + "github.com/go-openapi/strfmt" + + "github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models" +) + +// GetCPUReader is a Reader for the GetCPU structure. +type GetCPUReader struct { + formats strfmt.Registry +} + +// ReadResponse reads a server response into the received o. +func (o *GetCPUReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { + switch response.Code() { + case 200: + result := NewGetCPUOK() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return result, nil + default: + result := NewGetCPUDefault(response.Code()) + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + if response.Code()/100 == 2 { + return result, nil + } + return nil, result + } +} + +// NewGetCPUOK creates a GetCPUOK with default headers values +func NewGetCPUOK() *GetCPUOK { + return &GetCPUOK{} +} + +/* +GetCPUOK handles this case with default header values. + +CPUs to which agent is pinned +*/ +type GetCPUOK struct { + Payload *models.Cpus + JobID int64 +} + +func (o *GetCPUOK) GetPayload() *models.Cpus { + return o.Payload +} + +func (o *GetCPUOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + o.Payload = new(models.Cpus) + + // response payload + if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { + return err + } + + if jobIDHeader := response.GetHeader("x-rclone-jobid"); jobIDHeader != "" { + jobID, err := strconv.ParseInt(jobIDHeader, 10, 64) + if err != nil { + return err + } + + o.JobID = jobID + } + return nil +} + +// NewGetCPUDefault creates a GetCPUDefault with default headers values +func NewGetCPUDefault(code int) *GetCPUDefault { + return &GetCPUDefault{ + _statusCode: code, + } +} + +/* +GetCPUDefault handles this case with default header values. + +Server error +*/ +type GetCPUDefault struct { + _statusCode int + + Payload *models.ErrorResponse + JobID int64 +} + +// Code gets the status code for the get Cpu default response +func (o *GetCPUDefault) Code() int { + return o._statusCode +} + +func (o *GetCPUDefault) GetPayload() *models.ErrorResponse { + return o.Payload +} + +func (o *GetCPUDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + o.Payload = new(models.ErrorResponse) + + // response payload + if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { + return err + } + + if jobIDHeader := response.GetHeader("x-rclone-jobid"); jobIDHeader != "" { + jobID, err := strconv.ParseInt(jobIDHeader, 10, 64) + if err != nil { + return err + } + + o.JobID = jobID + } + return nil +} + +func (o *GetCPUDefault) Error() string { + return fmt.Sprintf("agent [HTTP %d] %s", o._statusCode, strings.TrimRight(o.Payload.Message, ".")) +} diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/operations_client.go b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/operations_client.go index 4c3f1546bb..d64a8fe645 100644 --- a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/operations_client.go +++ b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/operations_client.go @@ -35,6 +35,8 @@ type ClientService interface { FreeOSMemory(params *FreeOSMemoryParams) (*FreeOSMemoryOK, error) + GetCPU(params *GetCPUParams) (*GetCPUOK, error) + JobInfo(params *JobInfoParams) (*JobInfoOK, error) JobProgress(params *JobProgressParams) (*JobProgressOK, error) @@ -61,6 +63,8 @@ type ClientService interface { OperationsPurge(params *OperationsPurgeParams) (*OperationsPurgeOK, error) + PinCPU(params *PinCPUParams) (*PinCPUOK, error) + Reload(params *ReloadParams) (*ReloadOK, error) SyncCopyDir(params *SyncCopyDirParams) (*SyncCopyDirOK, error) @@ -69,6 +73,8 @@ type ClientService interface { SyncMoveDir(params *SyncMoveDirParams) (*SyncMoveDirOK, error) + UnpinCPU(params *UnpinCPUParams) (*UnpinCPUOK, error) + SetTransport(transport runtime.ClientTransport) } @@ -247,6 +253,41 @@ func (a *Client) FreeOSMemory(params *FreeOSMemoryParams) (*FreeOSMemoryOK, erro return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) } +/* +GetCPU gets c p us to which agent is pinned + +Get CPUs to which agent is pinned +*/ +func (a *Client) GetCPU(params *GetCPUParams) (*GetCPUOK, error) { + // TODO: Validate the params before sending + if params == nil { + params = NewGetCPUParams() + } + + result, err := a.transport.Submit(&runtime.ClientOperation{ + ID: "GetCpu", + Method: "GET", + PathPattern: "/pin_cpu", + ProducesMediaTypes: []string{"application/json"}, + ConsumesMediaTypes: []string{"application/json"}, + Schemes: []string{"http"}, + Params: params, + Reader: &GetCPUReader{formats: a.formats}, + Context: params.Context, + Client: params.HTTPClient, + }) + if err != nil { + return nil, err + } + success, ok := result.(*GetCPUOK) + if ok { + return success, nil + } + // unexpected success response + unexpectedSuccess := result.(*GetCPUDefault) + return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) +} + /* JobInfo transfers stats about the job @@ -702,6 +743,41 @@ func (a *Client) OperationsPurge(params *OperationsPurgeParams) (*OperationsPurg return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) } +/* +PinCPU pins agent to c p us according to start up logic + +Pin agent to CPUs according to start-up logic +*/ +func (a *Client) PinCPU(params *PinCPUParams) (*PinCPUOK, error) { + // TODO: Validate the params before sending + if params == nil { + params = NewPinCPUParams() + } + + result, err := a.transport.Submit(&runtime.ClientOperation{ + ID: "PinCpu", + Method: "POST", + PathPattern: "/pin_cpu", + ProducesMediaTypes: []string{"application/json"}, + ConsumesMediaTypes: []string{"application/json"}, + Schemes: []string{"http"}, + Params: params, + Reader: &PinCPUReader{formats: a.formats}, + Context: params.Context, + Client: params.HTTPClient, + }) + if err != nil { + return nil, err + } + success, ok := result.(*PinCPUOK) + if ok { + return success, nil + } + // unexpected success response + unexpectedSuccess := result.(*PinCPUDefault) + return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) +} + /* Reload reloads agent config @@ -842,6 +918,41 @@ func (a *Client) SyncMoveDir(params *SyncMoveDirParams) (*SyncMoveDirOK, error) return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) } +/* +UnpinCPU unpins agent from c p us + +Unpin agent from CPUs +*/ +func (a *Client) UnpinCPU(params *UnpinCPUParams) (*UnpinCPUOK, error) { + // TODO: Validate the params before sending + if params == nil { + params = NewUnpinCPUParams() + } + + result, err := a.transport.Submit(&runtime.ClientOperation{ + ID: "UnpinCpu", + Method: "DELETE", + PathPattern: "/pin_cpu", + ProducesMediaTypes: []string{"application/json"}, + ConsumesMediaTypes: []string{"application/json"}, + Schemes: []string{"http"}, + Params: params, + Reader: &UnpinCPUReader{formats: a.formats}, + Context: params.Context, + Client: params.HTTPClient, + }) + if err != nil { + return nil, err + } + success, ok := result.(*UnpinCPUOK) + if ok { + return success, nil + } + // unexpected success response + unexpectedSuccess := result.(*UnpinCPUDefault) + return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code()) +} + // SetTransport changes the transport on the client func (a *Client) SetTransport(transport runtime.ClientTransport) { a.transport = transport diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/pin_cpu_parameters.go b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/pin_cpu_parameters.go new file mode 100644 index 0000000000..7fa59de8a2 --- /dev/null +++ b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/pin_cpu_parameters.go @@ -0,0 +1,113 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "net/http" + "time" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + cr "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" +) + +// NewPinCPUParams creates a new PinCPUParams object +// with the default values initialized. +func NewPinCPUParams() *PinCPUParams { + + return &PinCPUParams{ + + timeout: cr.DefaultTimeout, + } +} + +// NewPinCPUParamsWithTimeout creates a new PinCPUParams object +// with the default values initialized, and the ability to set a timeout on a request +func NewPinCPUParamsWithTimeout(timeout time.Duration) *PinCPUParams { + + return &PinCPUParams{ + + timeout: timeout, + } +} + +// NewPinCPUParamsWithContext creates a new PinCPUParams object +// with the default values initialized, and the ability to set a context for a request +func NewPinCPUParamsWithContext(ctx context.Context) *PinCPUParams { + + return &PinCPUParams{ + + Context: ctx, + } +} + +// NewPinCPUParamsWithHTTPClient creates a new PinCPUParams object +// with the default values initialized, and the ability to set a custom HTTPClient for a request +func NewPinCPUParamsWithHTTPClient(client *http.Client) *PinCPUParams { + + return &PinCPUParams{ + HTTPClient: client, + } +} + +/* +PinCPUParams contains all the parameters to send to the API endpoint +for the pin Cpu operation typically these are written to a http.Request +*/ +type PinCPUParams struct { + timeout time.Duration + Context context.Context + HTTPClient *http.Client +} + +// WithTimeout adds the timeout to the pin Cpu params +func (o *PinCPUParams) WithTimeout(timeout time.Duration) *PinCPUParams { + o.SetTimeout(timeout) + return o +} + +// SetTimeout adds the timeout to the pin Cpu params +func (o *PinCPUParams) SetTimeout(timeout time.Duration) { + o.timeout = timeout +} + +// WithContext adds the context to the pin Cpu params +func (o *PinCPUParams) WithContext(ctx context.Context) *PinCPUParams { + o.SetContext(ctx) + return o +} + +// SetContext adds the context to the pin Cpu params +func (o *PinCPUParams) SetContext(ctx context.Context) { + o.Context = ctx +} + +// WithHTTPClient adds the HTTPClient to the pin Cpu params +func (o *PinCPUParams) WithHTTPClient(client *http.Client) *PinCPUParams { + o.SetHTTPClient(client) + return o +} + +// SetHTTPClient adds the HTTPClient to the pin Cpu params +func (o *PinCPUParams) SetHTTPClient(client *http.Client) { + o.HTTPClient = client +} + +// WriteToRequest writes these params to a swagger request +func (o *PinCPUParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { + + if err := r.SetTimeout(o.timeout); err != nil { + return err + } + var res []error + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/pin_cpu_responses.go b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/pin_cpu_responses.go new file mode 100644 index 0000000000..13e3e79c45 --- /dev/null +++ b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/pin_cpu_responses.go @@ -0,0 +1,133 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/go-openapi/runtime" + "github.com/go-openapi/strfmt" + + "github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models" +) + +// PinCPUReader is a Reader for the PinCPU structure. +type PinCPUReader struct { + formats strfmt.Registry +} + +// ReadResponse reads a server response into the received o. +func (o *PinCPUReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { + switch response.Code() { + case 200: + result := NewPinCPUOK() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return result, nil + default: + result := NewPinCPUDefault(response.Code()) + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + if response.Code()/100 == 2 { + return result, nil + } + return nil, result + } +} + +// NewPinCPUOK creates a PinCPUOK with default headers values +func NewPinCPUOK() *PinCPUOK { + return &PinCPUOK{} +} + +/* +PinCPUOK handles this case with default header values. + +Empty object +*/ +type PinCPUOK struct { + Payload interface{} + JobID int64 +} + +func (o *PinCPUOK) GetPayload() interface{} { + return o.Payload +} + +func (o *PinCPUOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + // response payload + if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { + return err + } + + if jobIDHeader := response.GetHeader("x-rclone-jobid"); jobIDHeader != "" { + jobID, err := strconv.ParseInt(jobIDHeader, 10, 64) + if err != nil { + return err + } + + o.JobID = jobID + } + return nil +} + +// NewPinCPUDefault creates a PinCPUDefault with default headers values +func NewPinCPUDefault(code int) *PinCPUDefault { + return &PinCPUDefault{ + _statusCode: code, + } +} + +/* +PinCPUDefault handles this case with default header values. + +Server error +*/ +type PinCPUDefault struct { + _statusCode int + + Payload *models.ErrorResponse + JobID int64 +} + +// Code gets the status code for the pin Cpu default response +func (o *PinCPUDefault) Code() int { + return o._statusCode +} + +func (o *PinCPUDefault) GetPayload() *models.ErrorResponse { + return o.Payload +} + +func (o *PinCPUDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + o.Payload = new(models.ErrorResponse) + + // response payload + if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { + return err + } + + if jobIDHeader := response.GetHeader("x-rclone-jobid"); jobIDHeader != "" { + jobID, err := strconv.ParseInt(jobIDHeader, 10, 64) + if err != nil { + return err + } + + o.JobID = jobID + } + return nil +} + +func (o *PinCPUDefault) Error() string { + return fmt.Sprintf("agent [HTTP %d] %s", o._statusCode, strings.TrimRight(o.Payload.Message, ".")) +} diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/unpin_cpu_parameters.go b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/unpin_cpu_parameters.go new file mode 100644 index 0000000000..968e1b092e --- /dev/null +++ b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/unpin_cpu_parameters.go @@ -0,0 +1,113 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "net/http" + "time" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + cr "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" +) + +// NewUnpinCPUParams creates a new UnpinCPUParams object +// with the default values initialized. +func NewUnpinCPUParams() *UnpinCPUParams { + + return &UnpinCPUParams{ + + timeout: cr.DefaultTimeout, + } +} + +// NewUnpinCPUParamsWithTimeout creates a new UnpinCPUParams object +// with the default values initialized, and the ability to set a timeout on a request +func NewUnpinCPUParamsWithTimeout(timeout time.Duration) *UnpinCPUParams { + + return &UnpinCPUParams{ + + timeout: timeout, + } +} + +// NewUnpinCPUParamsWithContext creates a new UnpinCPUParams object +// with the default values initialized, and the ability to set a context for a request +func NewUnpinCPUParamsWithContext(ctx context.Context) *UnpinCPUParams { + + return &UnpinCPUParams{ + + Context: ctx, + } +} + +// NewUnpinCPUParamsWithHTTPClient creates a new UnpinCPUParams object +// with the default values initialized, and the ability to set a custom HTTPClient for a request +func NewUnpinCPUParamsWithHTTPClient(client *http.Client) *UnpinCPUParams { + + return &UnpinCPUParams{ + HTTPClient: client, + } +} + +/* +UnpinCPUParams contains all the parameters to send to the API endpoint +for the unpin Cpu operation typically these are written to a http.Request +*/ +type UnpinCPUParams struct { + timeout time.Duration + Context context.Context + HTTPClient *http.Client +} + +// WithTimeout adds the timeout to the unpin Cpu params +func (o *UnpinCPUParams) WithTimeout(timeout time.Duration) *UnpinCPUParams { + o.SetTimeout(timeout) + return o +} + +// SetTimeout adds the timeout to the unpin Cpu params +func (o *UnpinCPUParams) SetTimeout(timeout time.Duration) { + o.timeout = timeout +} + +// WithContext adds the context to the unpin Cpu params +func (o *UnpinCPUParams) WithContext(ctx context.Context) *UnpinCPUParams { + o.SetContext(ctx) + return o +} + +// SetContext adds the context to the unpin Cpu params +func (o *UnpinCPUParams) SetContext(ctx context.Context) { + o.Context = ctx +} + +// WithHTTPClient adds the HTTPClient to the unpin Cpu params +func (o *UnpinCPUParams) WithHTTPClient(client *http.Client) *UnpinCPUParams { + o.SetHTTPClient(client) + return o +} + +// SetHTTPClient adds the HTTPClient to the unpin Cpu params +func (o *UnpinCPUParams) SetHTTPClient(client *http.Client) { + o.HTTPClient = client +} + +// WriteToRequest writes these params to a swagger request +func (o *UnpinCPUParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { + + if err := r.SetTimeout(o.timeout); err != nil { + return err + } + var res []error + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/unpin_cpu_responses.go b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/unpin_cpu_responses.go new file mode 100644 index 0000000000..b4a31d1c47 --- /dev/null +++ b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations/unpin_cpu_responses.go @@ -0,0 +1,133 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package operations + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/go-openapi/runtime" + "github.com/go-openapi/strfmt" + + "github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models" +) + +// UnpinCPUReader is a Reader for the UnpinCPU structure. +type UnpinCPUReader struct { + formats strfmt.Registry +} + +// ReadResponse reads a server response into the received o. +func (o *UnpinCPUReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { + switch response.Code() { + case 200: + result := NewUnpinCPUOK() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return result, nil + default: + result := NewUnpinCPUDefault(response.Code()) + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + if response.Code()/100 == 2 { + return result, nil + } + return nil, result + } +} + +// NewUnpinCPUOK creates a UnpinCPUOK with default headers values +func NewUnpinCPUOK() *UnpinCPUOK { + return &UnpinCPUOK{} +} + +/* +UnpinCPUOK handles this case with default header values. + +Empty object +*/ +type UnpinCPUOK struct { + Payload interface{} + JobID int64 +} + +func (o *UnpinCPUOK) GetPayload() interface{} { + return o.Payload +} + +func (o *UnpinCPUOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + // response payload + if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { + return err + } + + if jobIDHeader := response.GetHeader("x-rclone-jobid"); jobIDHeader != "" { + jobID, err := strconv.ParseInt(jobIDHeader, 10, 64) + if err != nil { + return err + } + + o.JobID = jobID + } + return nil +} + +// NewUnpinCPUDefault creates a UnpinCPUDefault with default headers values +func NewUnpinCPUDefault(code int) *UnpinCPUDefault { + return &UnpinCPUDefault{ + _statusCode: code, + } +} + +/* +UnpinCPUDefault handles this case with default header values. + +Server error +*/ +type UnpinCPUDefault struct { + _statusCode int + + Payload *models.ErrorResponse + JobID int64 +} + +// Code gets the status code for the unpin Cpu default response +func (o *UnpinCPUDefault) Code() int { + return o._statusCode +} + +func (o *UnpinCPUDefault) GetPayload() *models.ErrorResponse { + return o.Payload +} + +func (o *UnpinCPUDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + o.Payload = new(models.ErrorResponse) + + // response payload + if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { + return err + } + + if jobIDHeader := response.GetHeader("x-rclone-jobid"); jobIDHeader != "" { + jobID, err := strconv.ParseInt(jobIDHeader, 10, 64) + if err != nil { + return err + } + + o.JobID = jobID + } + return nil +} + +func (o *UnpinCPUDefault) Error() string { + return fmt.Sprintf("agent [HTTP %d] %s", o._statusCode, strings.TrimRight(o.Payload.Message, ".")) +} diff --git a/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models/cpus.go b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models/cpus.go new file mode 100644 index 0000000000..9f463b04e2 --- /dev/null +++ b/vendor/github.com/scylladb/scylla-manager/v3/swagger/gen/agent/models/cpus.go @@ -0,0 +1,43 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package models + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" +) + +// Cpus cpus +// +// swagger:model Cpus +type Cpus struct { + + // List of cpus + CPU []int64 `json:"cpu"` +} + +// Validate validates this cpus +func (m *Cpus) Validate(formats strfmt.Registry) error { + return nil +} + +// MarshalBinary interface implementation +func (m *Cpus) MarshalBinary() ([]byte, error) { + if m == nil { + return nil, nil + } + return swag.WriteJSON(m) +} + +// UnmarshalBinary interface implementation +func (m *Cpus) UnmarshalBinary(b []byte) error { + var res Cpus + if err := swag.ReadJSON(b, &res); err != nil { + return err + } + *m = res + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b01ba617d1..ebc34418b5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -395,11 +395,11 @@ github.com/scylladb/gocqlx/v2/dbutil github.com/scylladb/gocqlx/v2/migrate github.com/scylladb/gocqlx/v2/qb github.com/scylladb/gocqlx/v2/table -# github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241015081800-ee47f3d10478 +# github.com/scylladb/scylla-manager/v3/pkg/managerclient v0.0.0-20241017140216-d044b1738488 => ./v3/pkg/managerclient ## explicit; go 1.21.1 github.com/scylladb/scylla-manager/v3/pkg/managerclient github.com/scylladb/scylla-manager/v3/pkg/managerclient/table -# github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241015081800-ee47f3d10478 +# github.com/scylladb/scylla-manager/v3/pkg/util v0.0.0-20241017140216-d044b1738488 => ./v3/pkg/util ## explicit; go 1.21.1 github.com/scylladb/scylla-manager/v3/pkg/util github.com/scylladb/scylla-manager/v3/pkg/util/certutil @@ -431,7 +431,7 @@ github.com/scylladb/scylla-manager/v3/pkg/util/timeutc github.com/scylladb/scylla-manager/v3/pkg/util/uuid github.com/scylladb/scylla-manager/v3/pkg/util/version github.com/scylladb/scylla-manager/v3/pkg/util/workerpool -# github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241015081800-ee47f3d10478 +# github.com/scylladb/scylla-manager/v3/swagger v0.0.0-20241017140216-d044b1738488 => ./v3/swagger ## explicit; go 1.21.1 github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client github.com/scylladb/scylla-manager/v3/swagger/gen/agent/client/operations @@ -758,3 +758,6 @@ gopkg.in/yaml.v2 gopkg.in/yaml.v3 # github.com/gocql/gocql => github.com/scylladb/gocql v1.12.0 # github.com/rclone/rclone => github.com/scylladb/rclone v1.54.1-0.20240312172628-afe1fd2aa65e +# github.com/scylladb/scylla-manager/v3/pkg/managerclient => ./v3/pkg/managerclient +# github.com/scylladb/scylla-manager/v3/pkg/util => ./v3/pkg/util +# github.com/scylladb/scylla-manager/v3/swagger => ./v3/swagger