diff --git a/cmd/attack/attack.go b/cmd/attack/attack.go index b7ed71a2..eca0be49 100644 --- a/cmd/attack/attack.go +++ b/cmd/attack/attack.go @@ -42,6 +42,7 @@ func NewAttackCommand() *cobra.Command { NewHTTPAttackCommand(&uid), NewVMAttackCommand(&uid), NewUserDefinedCommand(&uid), + NewPatroniAttackCommand(&uid), ) return cmd diff --git a/cmd/attack/patroni.go b/cmd/attack/patroni.go new file mode 100644 index 00000000..bb7849dd --- /dev/null +++ b/cmd/attack/patroni.go @@ -0,0 +1,99 @@ +// Copyright 2020 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package attack + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + "go.uber.org/fx" + + "github.com/chaos-mesh/chaosd/cmd/server" + "github.com/chaos-mesh/chaosd/pkg/core" + "github.com/chaos-mesh/chaosd/pkg/server/chaosd" + "github.com/chaos-mesh/chaosd/pkg/utils" +) + +func NewPatroniAttackCommand(uid *string) *cobra.Command { + options := core.NewPatroniCommand() + dep := fx.Options( + server.Module, + fx.Provide(func() *core.PatroniCommand { + options.UID = *uid + return options + }), + ) + + cmd := &cobra.Command{ + Use: "patroni ", + Short: "Patroni attack related commands", + } + + cmd.AddCommand( + NewPatroniSwitchoverCommand(dep, options), + NewPatroniFailoverCommand(dep, options), + ) + + cmd.PersistentFlags().StringVarP(&options.User, "user", "u", "patroni", "patroni cluster user") + cmd.PersistentFlags().StringVar(&options.Password, "password", "p", "patroni cluster password") + cmd.PersistentFlags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts") + cmd.PersistentFlags().BoolVarP(&options.LocalMode, "local-mode", "l", false, "execute patronictl on host with chaosd. User with privileges required.") + cmd.PersistentFlags().BoolVarP(&options.RemoteMode, "remote-mode", "r", false, "execute patroni command by REST API") + + return cmd +} + +func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { + cmd := &cobra.Command{ + Use: "switchover", + Short: "exec switchover command. Warning! Command is not recover!", + Run: func(*cobra.Command, []string) { + options.Action = core.SwitchoverAction + utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() + }, + } + cmd.Flags().StringVarP(&options.Candidate, "candidate", "c", "", "switchover candidate, default random unit for replicas") + cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprint(time.Now().Add(time.Second*60).Format(time.RFC3339)), `scheduled switchover, + default now()+1 minute by remote mode`) + + return cmd +} + +func NewPatroniFailoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command { + cmd := &cobra.Command{ + Use: "failover", + Short: "Exec failover command. Warning! Command is not recover!", + Run: func(*cobra.Command, []string) { + options.Action = core.FailoverAction + utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run() + }, + } + + cmd.Flags().StringVarP(&options.Candidate, "leader", "c", "", "failover new leader, default random unit for replicas") + return cmd +} + +func PatroniAttackF(options *core.PatroniCommand, chaos *chaosd.Server) { + if err := options.Validate(); err != nil { + utils.ExitWithError(utils.ExitBadArgs, err) + } + + uid, err := chaos.ExecuteAttack(chaosd.PatroniAttack, options, core.CommandMode) + if err != nil { + utils.ExitWithError(utils.ExitError, err) + } + + utils.NormalExit(fmt.Sprintf("Attack %s successfully to patroni address %s, uid: %s", options.Action, options.Address, uid)) +} diff --git a/go.mod b/go.mod index 1df2d751..42225ab8 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe github.com/swaggo/gin-swagger v1.5.0 github.com/swaggo/swag v1.8.3 + github.com/tidwall/gjson v1.14.4 go.uber.org/fx v1.17.1 go.uber.org/zap v1.21.0 google.golang.org/grpc v1.40.0 @@ -125,6 +126,8 @@ require ( github.com/romana/ipset v1.0.0 // indirect github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/ugorji/go/codec v1.2.7 // indirect diff --git a/go.sum b/go.sum index ed93fc4b..744982b3 100644 --- a/go.sum +++ b/go.sum @@ -1040,6 +1040,12 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= diff --git a/pkg/core/experiment.go b/pkg/core/experiment.go index 5d209a0e..13827cf7 100644 --- a/pkg/core/experiment.go +++ b/pkg/core/experiment.go @@ -44,6 +44,7 @@ const ( FileAttack = "file" HTTPAttack = "http" VMAttack = "vm" + PatroniAttack = "patroni" UserDefinedAttack = "userDefined" ) @@ -128,6 +129,8 @@ func GetAttackByKind(kind string) *AttackConfig { attackConfig = &VMOption{} case UserDefinedAttack: attackConfig = &UserDefinedOption{} + case PatroniAttack: + attackConfig = &PatroniCommand{} default: return nil } diff --git a/pkg/core/patroni.go b/pkg/core/patroni.go new file mode 100644 index 00000000..bed91264 --- /dev/null +++ b/pkg/core/patroni.go @@ -0,0 +1,82 @@ +// Copyright 2020 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package core + +import ( + "encoding/json" + + "github.com/pingcap/errors" +) + +const ( + SwitchoverAction = "switchover" + FailoverAction = "failover" +) + +var _ AttackConfig = &PatroniCommand{} + +type PatroniCommand struct { + CommonAttackConfig + + Address string `json:"address,omitempty"` + Candidate string `json:"candidate,omitempty"` + Leader string `json:"leader,omitempty"` + User string `json:"user,omitempty"` + Password string `json:"password,omitempty"` + Scheduled_at string `json:"scheduled_at,omitempty"` + LocalMode bool `json:"local_mode,omitempty"` + RemoteMode bool `json:"remote_mode,omitempty"` + RecoverCmd string `json:"recoverCmd,omitempty"` +} + +func (p *PatroniCommand) Validate() error { + if err := p.CommonAttackConfig.Validate(); err != nil { + return err + } + + if !p.RemoteMode && !p.LocalMode { + return errors.New("local or remote mode required") + } + + if p.RemoteMode { + + if len(p.Address) == 0 { + return errors.New("address not provided") + } + + if len(p.User) == 0 { + return errors.New("patroni user not provided") + } + + if len(p.Password) == 0 { + return errors.New("patroni password not provided") + } + } + + return nil +} + +func (p PatroniCommand) RecoverData() string { + data, _ := json.Marshal(p) + + return string(data) +} + +func NewPatroniCommand() *PatroniCommand { + return &PatroniCommand{ + CommonAttackConfig: CommonAttackConfig{ + Kind: PatroniAttack, + }, + } +} diff --git a/pkg/server/chaosd/patroni.go b/pkg/server/chaosd/patroni.go new file mode 100644 index 00000000..698bc101 --- /dev/null +++ b/pkg/server/chaosd/patroni.go @@ -0,0 +1,193 @@ +// Copyright 2023 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package chaosd + +import ( + "encoding/json" + "fmt" + "math/rand" + "net/http" + "os/exec" + "strings" + + "github.com/pingcap/errors" + "github.com/pingcap/log" + + "github.com/chaos-mesh/chaosd/pkg/core" + "github.com/chaos-mesh/chaosd/pkg/server/utils" +) + +type patroniAttack struct{} + +var PatroniAttack AttackType = patroniAttack{} + +func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error { + attack := options.(*core.PatroniCommand) + + var responce []byte + + var address string + + var err error + + values := make(map[string]string) + + if attack.RemoteMode { + address = attack.Address + } else if attack.LocalMode { + address, err = utils.GetLocalHostname() + if err != nil { + return errors.WithStack(err) + } + } + + patroniInfo, err := utils.GetPatroniInfo(address) + if err != nil { + err = errors.Errorf("failed to get patroni info for %v: %v", options.String(), err) + return errors.WithStack(err) + } + + if len(patroniInfo.Replicas) == 0 && len(patroniInfo.SyncStandby) == 0 { + err = errors.Errorf("failed to get available candidates. Please, check your cluster") + return errors.WithStack(err) + } + + sync_mode_check, err := isSynchronousClusterMode(address, attack.User, attack.Password) + if err != nil { + err = errors.Errorf("failed to check cluster synchronous mode for %v: %v", options.String(), err) + return errors.WithStack(err) + } + + if attack.Candidate == "" { + if sync_mode_check { + values["candidate"] = patroniInfo.SyncStandby[rand.Intn(len(patroniInfo.SyncStandby))] + } else { + values["candidate"] = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))] + } + + } + + if attack.Leader == "" { + values["leader"] = patroniInfo.Master + } + + values["scheduled_at"] = attack.Scheduled_at + + cmd := options.String() + + switch cmd { + case "switchover": + + log.Info(fmt.Sprintf("Switchover will be done from %v to %v in %v", values["leader"], values["candidate"], values["scheduled_at"])) + + case "failover": + + log.Info(fmt.Sprintf("Failover will be done from %v to %v", values["leader"], values["candidate"])) + + } + + if attack.RemoteMode { + responce, err = execPatroniAttackByRemoteMode(address, attack.User, attack.Password, cmd, values) + if err != nil { + return err + } + } else if attack.LocalMode { + responce, err = execPatroniAttackByLocalMode(cmd, values) + if err != nil { + return err + } + } + + if attack.RemoteMode { + log.S().Infof("Execute %v successfully: %v", cmd, string(responce)) + } + + if attack.LocalMode { + log.S().Infof("Execute %v successfully", cmd) + fmt.Println(string(responce)) + } + + return nil +} + +func execPatroniAttackByRemoteMode(patroniAddr string, user string, password string, cmd string, values map[string]string) ([]byte, error) { + + data, err := json.Marshal(values) + if err != nil { + err = errors.Errorf("failed to marshal data: %v", values) + return nil, errors.WithStack(err) + } + + buf, err := utils.MakeHTTPRequest(http.MethodPost, patroniAddr, 8008, cmd, data, user, password) + if err != nil { + return nil, errors.WithStack(err) + } + + return buf, nil +} + +func execPatroniAttackByLocalMode(cmd string, values map[string]string) ([]byte, error) { + var cmdTemplate string + + if cmd == "failover" { + cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --force", cmd, values["leader"], values["candidate"]) + } else if cmd == "switchover" { + cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --scheduled %v --force", cmd, values["leader"], values["candidate"], values["scheduled_at"]) + } + + execCmd := exec.Command("bash", "-c", cmdTemplate) + output, err := execCmd.CombinedOutput() + if err != nil { + log.S().Errorf(fmt.Sprintf("failed to %v: %v", cmdTemplate, string(output))) + return nil, err + } + + if strings.Contains(string(output), "failed") { + err = errors.New(string(output)) + return nil, err + } + + return output, nil +} + +func isSynchronousClusterMode(patroniAddr string, user string, password string) (bool, error) { + + buf, err := utils.MakeHTTPRequest(http.MethodGet, patroniAddr, 8008, "config", []byte{}, user, password) + if err != nil { + return false, err + } + + patroni_responce := make(map[string]interface{}) + + err = json.Unmarshal(buf, &patroni_responce) + if err != nil { + return false, fmt.Errorf("bad request %v %v", err.Error(), http.StatusBadRequest) + } + + mode_check, ok := patroni_responce["synchronous_mode"].(bool) + if !ok { + return false, fmt.Errorf("failed to cast synchronous_mode field from patroni responce") + } + + if mode_check { + return true, nil + } + + return false, nil + +} + +func (patroniAttack) Recover(exp core.Experiment, _ Environment) error { + return nil +} diff --git a/pkg/server/chaosd/recover.go b/pkg/server/chaosd/recover.go index ea532d42..ccb5b8cf 100644 --- a/pkg/server/chaosd/recover.go +++ b/pkg/server/chaosd/recover.go @@ -79,6 +79,8 @@ func (s *Server) RecoverAttack(uid string) error { attackType = VMAttack case core.UserDefinedAttack: attackType = UserDefinedAttack + case core.PatroniAttack: + attackType = PatroniAttack default: return perr.Errorf("chaos experiment kind %s not found", exp.Kind) } diff --git a/pkg/server/utils/http_requests.go b/pkg/server/utils/http_requests.go new file mode 100644 index 00000000..87ff03af --- /dev/null +++ b/pkg/server/utils/http_requests.go @@ -0,0 +1,135 @@ +// Copyright 2023 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "bytes" + "fmt" + "io" + "net/http" + + "github.com/pingcap/log" + "github.com/pkg/errors" + "github.com/tidwall/gjson" +) + +type PatroniInfo struct { + Master string + Replicas []string + SyncStandby []string + Status []string +} + +func GetPatroniInfo(address string) (PatroniInfo, error) { + res, err := http.Get(fmt.Sprintf("http://%v:8008/cluster", address)) + if err != nil { + err = errors.Errorf("failed to get patroni status: %v", err) + return PatroniInfo{}, errors.WithStack(err) + } + + defer res.Body.Close() + + buf, err := io.ReadAll(res.Body) + if err != nil { + err = errors.Errorf("failed to read responce: %v", err) + return PatroniInfo{}, errors.WithStack(err) + } + + data := string(buf) + + patroniInfo := PatroniInfo{} + + members := gjson.Get(data, "members") + + for _, member := range members.Array() { + switch member.Get("role").Str { + case "leader": + patroniInfo.Master = member.Get("name").Str + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + case "replica": + patroniInfo.Replicas = append(patroniInfo.Replicas, member.Get("name").Str) + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + case "sync_standby": + patroniInfo.SyncStandby = append(patroniInfo.SyncStandby, member.Get("name").Str) + patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str) + + } + } + + log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, sync_standy %s, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas, + patroniInfo.SyncStandby, patroniInfo.Status)) + + return patroniInfo, nil + +} + +func MakeHTTPRequest(method string, address string, port int64, path string, body []byte, user string, password string) ([]byte, error) { + + url := fmt.Sprintf("http://%v:%v/%v", address, port, path) + + var request *http.Request + + var resp *http.Response + + var err error + + switch method { + case http.MethodPost: + request, err = http.NewRequest("POST", url, bytes.NewBuffer(body)) + if err != nil { + err = errors.Errorf("failed to post request %v: %v", url, err) + return nil, errors.WithStack(err) + } + + case http.MethodGet: + request, err = http.NewRequest("GET", url, nil) + if err != nil { + err = errors.Errorf("failed to get request %v: %v", url, err) + return nil, errors.WithStack(err) + } + } + + if user != "" && password != "" { + request.Header.Set("Content-Type", "application/json") + request.SetBasicAuth(user, password) + } + + client := &http.Client{} + resp, err = client.Do(request) + if err != nil { + err = errors.Errorf("failed to exec %v request %v: %v", method, url, err) + return nil, errors.WithStack(err) + } + + defer resp.Body.Close() + + if resp.StatusCode != 200 && resp.StatusCode != 202 { + //to simplify diagnostics + buf, err := io.ReadAll(resp.Body) + if err != nil { + err = errors.Errorf("failed to read from %s responce: status code %v, responce %v, error %v", path, resp.StatusCode, resp.Body, err) + return nil, err + } + err = errors.Errorf("failed to exec %v request: status code %v, responce %v", path, resp.StatusCode, buf) + return nil, errors.WithStack(err) + } + + buf, err := io.ReadAll(resp.Body) + if err != nil { + err = errors.Errorf("failed to read %v from %s responce: %v", resp.Body, path, err) + return nil, errors.WithStack(err) + } + + return buf, nil +} diff --git a/pkg/server/utils/system_unix.go b/pkg/server/utils/system_unix.go new file mode 100644 index 00000000..abe6acae --- /dev/null +++ b/pkg/server/utils/system_unix.go @@ -0,0 +1,27 @@ +// Copyright 2023 Chaos Mesh Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "os" +) + +func GetLocalHostname() (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", err + } + + return hostname, nil +}