Skip to content

Commit

Permalink
implement non-graceful shutdown
Browse files Browse the repository at this point in the history
  • Loading branch information
YZ775 committed Feb 6, 2025
1 parent 1c079b9 commit 42b10c7
Show file tree
Hide file tree
Showing 6 changed files with 453 additions and 25 deletions.
20 changes: 14 additions & 6 deletions etc/cke-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,15 @@ repair:
command_retries: 2
command_interval: 30
watch_seconds: 600
- repair_command: ["sh", "neco nonGracefulNodeShutdown shutdown $1", "nonGracefulNodeShutdown"]
command_timeout_seconds: 300
command_retries: 3
command_interval: 30
need_drain: true
- repair_command: ["neco", "bmc", "repair", "dell", "discharge"]
command_timeout_seconds: 60 # "discharge" is a compound command and requires a little more time
command_retries: 2
command_interval: 30
need_drain: true
watch_seconds: 360 # too short watch for discharging; this is to power on machine 6 minutes after
- repair_command: ["sh", "-c", "neco power start $1 || true", "power-start"]
command_timeout_seconds: 30
Expand All @@ -56,18 +60,19 @@ repair:
watch_seconds: 1500
health_check_command: ["sh", "-c", "check-machine-state $1 healthy retired", "check-machine-state"]
command_timeout_seconds: 10
success_command: ["sh", "-c", "neco nonGracefulNodeShutdown cleanup $1", "nonGracefulNodeShutdown"]
success_timeout_seconds: 300
- operation: unreachable
repair_steps:
- repair_command: ["neco", "bmc", "repair", "dell", "reset-idrac"]
command_timeout_seconds: 30
command_retries: 2
- repair_command: ["sh", "neco nonGracefulNodeShutdown shutdown $1", "nonGracefulNodeShutdown"]
command_timeout_seconds: 300
command_retries: 3
command_interval: 30
watch_seconds: 600
need_drain: false
- repair_command: ["neco", "bmc", "repair", "dell", "discharge"]
command_timeout_seconds: 60 # "discharge" is a compound command and requires a little more time
command_retries: 2
command_interval: 30
need_drain: false
watch_seconds: 360 # too short watch for discharging; this is to power on machine 6 minutes after
- repair_command: ["sh", "-c", "neco power start $1 || true", "power-start"]
command_timeout_seconds: 30
Expand All @@ -82,6 +87,8 @@ repair:
watch_seconds: 600
health_check_command: ["sh", "-c", "check-machine-state $1 healthy retired", "check-machine-state"]
command_timeout_seconds: 10
success_command: ["sh", "-c", "neco nonGracefulNodeShutdown cleanup $1", "nonGracefulNodeShutdown"]
success_timeout_seconds: 300
max_concurrent_repairs: 1
evict_retries: 30
evict_interval: 10
Expand Down Expand Up @@ -152,6 +159,7 @@ options:
- "--leader-elect-renew-deadline=15s"
- "--leader-elect-lease-duration=20s"
- "--kube-api-qps=50"
- "--disable-force-detach-on-timeout=true"
kube-proxy:
disable: true
# For local-proxy running on boot servers
Expand Down
15 changes: 9 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ go 1.23.3
replace github.com/armon/go-metrics v0.5.3 => github.com/hashicorp/go-metrics v0.5.3

require (
github.com/csi-addons/kubernetes-csi-addons v0.11.0
github.com/cybozu-go/cke v1.30.0
github.com/cybozu-go/etcdutil v1.6.9
github.com/cybozu-go/log v1.7.0
Expand Down Expand Up @@ -40,9 +41,10 @@ require (
golang.org/x/oauth2 v0.24.0
golang.org/x/sync v0.10.0
golang.org/x/term v0.27.0
k8s.io/api v0.31.0
k8s.io/apimachinery v0.31.0
k8s.io/client-go v0.31.0
k8s.io/api v0.31.2
k8s.io/apimachinery v0.31.2
k8s.io/client-go v0.31.2
sigs.k8s.io/controller-runtime v0.19.1
sigs.k8s.io/yaml v1.4.0
)

Expand All @@ -65,6 +67,7 @@ require (
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.1 // indirect
Expand Down Expand Up @@ -95,7 +98,7 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/logutils v1.0.0 // indirect
github.com/hashicorp/memberlist v0.5.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
Expand Down Expand Up @@ -142,12 +145,12 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/grpc v1.68.0 // indirect
google.golang.org/protobuf v1.35.1 // indirect
google.golang.org/protobuf v1.35.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/component-base v0.30.5 // indirect
k8s.io/component-base v0.31.0 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
k8s.io/kube-proxy v0.30.5 // indirect
Expand Down
38 changes: 25 additions & 13 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/csi-addons/kubernetes-csi-addons v0.11.0 h1:0f6AIXcpu68Vj0Q1IKij1l6arJfKFiaTZ9GwHuvLm/o=
github.com/csi-addons/kubernetes-csi-addons v0.11.0/go.mod h1:HJd3znD4i5D92/2PKqzrwBg5Q7Ur2me20VYakdBHzpk=
github.com/cybozu-go/cke v1.30.0 h1:K2x++xboeW75vxy7z1/4Zd2il61cPC+ilubJ27Iq/CM=
github.com/cybozu-go/cke v1.30.0/go.mod h1:kI1KcZIvZwDMHKIUNhcM6El+Tns7ORSkzYWmvxKK444=
github.com/cybozu-go/etcdutil v1.6.9 h1:3ZlKlZYrb8MfWRlx1q94yM4+SaFerRKyOGBccIleB2I=
Expand All @@ -80,6 +82,8 @@ github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryef
github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
Expand All @@ -101,6 +105,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
Expand Down Expand Up @@ -198,8 +204,8 @@ github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY
github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
github.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/pj5DA=
github.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8=
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
Expand Down Expand Up @@ -496,13 +502,15 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0=
google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA=
google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
Expand All @@ -512,22 +520,24 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo=
k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE=
k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc=
k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8=
k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU=
k8s.io/component-base v0.30.5 h1:O6W8GfdBuyctVy7lu7I0yo8kB6bYgzGzjCyaagb2BR0=
k8s.io/component-base v0.30.5/go.mod h1:eliJtfE7RG18UHMWrqPQWodf1GnQVFGA6McNOHYi11g=
k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0=
k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk=
k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk=
k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk=
k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc=
k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs=
k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs=
k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo=
k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
Expand All @@ -540,6 +550,8 @@ k8s.io/kubelet v0.30.5 h1:7jsJo1MkmGC3VEE0cf1tHiFKzi8DMBVNih3/OttSBaw=
k8s.io/kubelet v0.30.5/go.mod h1:vzllyQKrbFpTq0WOHV6yX0gUs6KqTwCBAO13cAearMc=
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA=
k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
sigs.k8s.io/controller-runtime v0.19.1 h1:Son+Q40+Be3QWb+niBXAg2vFiYWolDjjRfO8hn/cxOk=
sigs.k8s.io/controller-runtime v0.19.1/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
Expand Down
101 changes: 101 additions & 0 deletions pkg/neco/cmd/non_graceful_shutdown.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package cmd

import (
"context"
"errors"
"os"
"os/exec"
"strings"

csiaddonsv1alpha1 "github.com/csi-addons/kubernetes-csi-addons/api/csiaddons/v1alpha1"
"github.com/cybozu-go/neco"
"github.com/spf13/cobra"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var (
nonGracefulNodeShutdownConfig = "/tmp/non-graceful-node-shutdown-config"
)

var nonGracefulNodeShutdownCmd = &cobra.Command{
Use: "non-graceful-node-shutdown",
Short: "non-Graceful Node Shutdown related commands",
Long: `non-Graceful Node Shutdown related commands.`,
}

type CephCluster struct {
Name string
NameSpace string
}

func issueAndLoadKubeconfigForNonGracefulNodeShutdown() (client.Client, error) {
scheme := runtime.NewScheme()
clientgoscheme.AddToScheme(scheme)
csiaddonsv1alpha1.AddToScheme(scheme)

stdout, err := os.Create(nonGracefulNodeShutdownConfig)
if err != nil {
return nil, err
}
defer stdout.Close()
issueCmd := exec.Command(neco.CKECLIBin, "kubernetes", "issue")
issueCmd.Stdout = stdout
err = issueCmd.Run()
if err != nil {
return nil, err
}
stdout.Sync()

config, err := clientcmd.BuildConfigFromFlags("", nonGracefulNodeShutdownConfig)
if err != nil {
return nil, err
}

kubeClient, err := client.New(config, client.Options{Scheme: scheme})
if err != nil {
return nil, err
}
return kubeClient, nil
}

func listRBDCephClusters(ctx context.Context, kubeClient client.Client) ([]CephCluster, error) {
cephClusters := []CephCluster{}
scs := &storagev1.StorageClassList{}
err := kubeClient.List(ctx, scs)
if err != nil {
return nil, err
}
cephClusterIDs := []string{}
for _, sc := range scs.Items {
if strings.HasSuffix(sc.Provisioner, "rbd.csi.ceph.com") {
cephClusterIDs = append(cephClusterIDs, sc.Parameters["clusterID"])
}
}
for _, cephClusterID := range cephClusterIDs {
cephCluster := &unstructured.UnstructuredList{}
cephCluster.SetAPIVersion("ceph.rook.io/v1")
cephCluster.SetKind("CephCluster")
err = kubeClient.List(ctx, cephCluster, &client.ListOptions{Namespace: cephClusterID})
if err != nil {
return nil, err
}
if len(cephCluster.Items) != 1 {
return nil, errors.New("cephCluster is not found or multiple cephClusters are found")
}
cephClusters = append(cephClusters, CephCluster{Name: cephCluster.Items[0].GetName(), NameSpace: cephCluster.Items[0].GetNamespace()})
}
return cephClusters, nil
}

func generateFenceName(clusterName, node string) string {
return clusterName + "-" + strings.Replace(node, ".", "-", -1)
}

func init() {
rootCmd.AddCommand(nonGracefulNodeShutdownCmd)
}
Loading

0 comments on commit 42b10c7

Please sign in to comment.