diff --git a/.gitignore b/.gitignore index 014e817..d0282b8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ # Go workspace file go.work - # editor and IDE paraphernalia .idea *.swp @@ -33,3 +32,10 @@ go.work /kubeconfig .dockerignore .docker + +*.yaml +*.iso + +bin/ + +env.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1698e21 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +GO_GCFLAGS ?= -gcflags=all='-N -l' +GO=GO111MODULE=on go +GO_BUILD_RECIPE=CGO_ENABLED=0 $(GO) build $(GO_GCFLAGS) + +OUT_DIR ?= bin + +all: build + +.PHONY: build +build: + $(GO_BUILD_RECIPE) -o $(OUT_DIR)/hypershift-agent-automation . + +clean: + rm -rf $(OUT_DIR)/* diff --git a/README.md b/README.md index d9f0ec6..c9080e9 100644 --- a/README.md +++ b/README.md @@ -1 +1,97 @@ -# hypershift-agent-automation \ No newline at end of file +# Hypershift Agent Automation + +Cli tool to create and destroy agent based hypershift hosted cluster on PowerVC platform. + +## Setup: +This tool is designed only to create and destroy the agent cluster. + +### Pre-Req: +- **Management Cluster** - An OCP cluster which will be used to host the agent based hosted cluster. Only x86 type of management cluster is supported as of now. Need to set up below things in management cluster. + - **MCE** - Refer this [link](https://github.com/hypershift-on-power/hack/wiki/Agent-based-Hosted-Cluster-on-PowerVM-using-MCE-with-Assisted-Service-and-Hypershift#install-the-mce-operator) to install the MCE operator. + - **AgentServiceConfig** - Refer this [link](https://github.com/hypershift-on-power/hack/wiki/Agent-based-Hosted-Cluster-on-PowerVM-using-MCE-with-Assisted-Service-and-Hypershift#create-agentserviceconfig) to configure agentserviceconfig. +- A valid [pull secret](https://cloud.redhat.com/openshift/install/aws/installer-provisioned) file. +- The OpenShift CLI (oc) or Kubernetes CLI (kubectl). +- **Hypershift** +```shell + git clone https://github.com/openshift/hypershift.git + cd hypershift + make build + sudo install -m 0755 bin/hypershift /usr/local/bin/hypershift +``` +- **Hypershift Agent Automation** +```shell + git clone https://github.com/ppc64le-cloud/hypershift-agent-automation + cd hypershift-agent-automation + make build + sudo install -m 0755 bin/hypershift-agent-automation /usr/local/bin/hypershift-agent-automation +``` + +### Infra: +Need to set below env vars to connect HMC, VIOS and PowerVC. + +```shell +# To get authenticated with PowerVC(OpenStack Client) +export OS_USERNAME='' +export OS_PASSWORD='' +export OS_IDENTITY_API_VERSION='' +export OS_AUTH_URL='' +export OS_CACERT='' +export OS_REGION_NAME='' +export OS_PROJECT_DOMAIN_NAME='' +export OS_PROJECT_NAME='' +export OS_TENANT_NAME='' +export OS_USER_DOMAIN_NAME='' + +# Required PowerVC resource names +export POWERVC_STORAGE_TEMPLATE='' +export POWERVC_HOST='' +export POWERVC_NETWORK_NAME='' + +# HMC details +export HMC_IP='' +export HMC_USERNAME='' +export HMC_PASSWORD='' + +# VIOS details +export VIOS_IP='' +export VIOS_USERNAME='' +export VIOS_PASSWORD='' +export VIOS_HOMEDIR='' +``` + + + +## Commands: + +### Create Agent Cluster: +```shell +hypershift-agent-automation cluster create \ +--name $CLUSTER_NAME \ +--base-domain $BASE_DOMAIN \ +--pull-secret $PULL_SECRET \ +--release-image $RELEASE_IMAGE \ +--ssh-key $SSH_KEY_FILE \ +--node-count $NODE_COUNT +``` + +### Destroy Agent Cluster: +```shell +hypershift-agent-automation cluster destroy \ +--name $CLUSTER_NAME \ +--base-domain $BASE_DOMAIN \ +--pull-secret $PULL_SECRET \ +--release-image $RELEASE_IMAGE \ +--ssh-key $SSH_KEY_FILE \ +--node-count $NODE_COUNT +``` + +### Running e2e: +```shell +hypershift-agent-automation e2e \ +--name $CLUSTER_NAME \ +--base-domain $BASE_DOMAIN \ +--pull-secret $PULL_SECRET \ +--release-image $RELEASE_IMAGE \ +--ssh-key $SSH_KEY_FILE \ +--node-count $NODE_COUNT +``` \ No newline at end of file diff --git a/cmd/cluster/cluster.go b/cmd/cluster/cluster.go new file mode 100644 index 0000000..f80da5d --- /dev/null +++ b/cmd/cluster/cluster.go @@ -0,0 +1,22 @@ +package cluster + +import ( + "github.com/spf13/cobra" + + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/cluster/create" + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/cluster/destroy" + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/options" +) + +func Command(options options.ClusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "cluster", + Short: "Commands to interact with hypershift agent cluster", + SilenceUsage: true, + } + + cmd.AddCommand(create.Command(options)) + cmd.AddCommand(destroy.Command(options)) + + return cmd +} diff --git a/cmd/cluster/create/create.go b/cmd/cluster/create/create.go new file mode 100644 index 0000000..84acae4 --- /dev/null +++ b/cmd/cluster/create/create.go @@ -0,0 +1,146 @@ +package create + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/options" + cmdUtil "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/util" + "github.com/ppc64le-cloud/hypershift-agent-automation/log" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/cluster" + "github.com/ppc64le-cloud/hypershift-agent-automation/util" +) + +func SetupPreReq(c *cluster.Cluster) (string, string, string, int, error) { + volumeID, err := c.PowerVC.SetupEmptyBootVol(c.Name) + if err != nil { + log.Logger.Errorf("error setup empty boot volume: %v", err) + return "", "", "", -1, fmt.Errorf("error setup empty boot volume: %v", err) + } + log.Logger.Infof("%s volume id will be used", volumeID) + + imageID, err := c.PowerVC.SetupPreReqImage(c.Name, volumeID) + if err != nil { + log.Logger.Errorf("error setup image %v", err) + return "", "", "", -1, fmt.Errorf("error setup image: %v", err) + } + log.Logger.Infof("%s image id will be used", imageID) + + if err = c.PowerVC.SetupFlavor(c.Name); err != nil { + log.Logger.Errorf("error setup flavor: %v", err) + return "", "", "", -1, fmt.Errorf("error setup flavor: %v", err) + } + log.Logger.Infof("%s flavor id will be used", util.GenerateFlavourID(c.Name)) + + networkID, gatewayIP, prefix, err := c.PowerVC.GetNetworkID() + if err != nil { + log.Logger.Errorf("unable to retrieve id for network, error: %v", err) + return "", "", "", -1, fmt.Errorf("unable to retrieve id for network, error: %v", err) + } + log.Logger.Infof("%s network id will be used", networkID) + + return imageID, networkID, gatewayIP, prefix, nil +} + +func SetupCluster(c *cluster.Cluster, imageID, networkID, gatewayIP string, prefix int) error { + agents, err := c.PowerVC.SetupAgents(c.GetWorkerName(), imageID, networkID, c.NodeCount) + if err != nil { + return fmt.Errorf("error setup agents: %v", err) + } + log.Logger.Infof("agent setup done. agent details: %+v", agents) + + nmStateLabel := fmt.Sprintf("label: nmstate-config-%s", c.Name) + if err = os.Mkdir(util.GetManifestDir(c.Name), 0750); err != nil && !os.IsExist(err) { + log.Logger.Error("error creating output dir for manifests", err) + } + if err = c.SetupHC(); err != nil { + return fmt.Errorf("error setup hosted cluster: %v", err) + } + log.Logger.Info("hosted cluster setup done") + if err = c.SetupNMStateConfig(agents, prefix, gatewayIP, nmStateLabel); err != nil { + return fmt.Errorf("error setup nmstate config: %v", err) + } + log.Logger.Info("nmstate config setup done") + + if err = c.SetupCISDNSRecords(agents[0].IP); err != nil { + return fmt.Errorf("error update cis dns records: %v", err) + } + log.Logger.Info("update cis dns records done") + + if err = c.SetupInfraEnv(nmStateLabel); err != nil { + return fmt.Errorf("error setup infraenv: %v", err) + } + log.Logger.Info("infraenv setup done") + + if err = c.DownloadISO(); err != nil { + return fmt.Errorf("error download iso: %v", err) + } + log.Logger.Info("download discovery iso done") + + if err = c.CopyAndMountISO(agents); err != nil { + return fmt.Errorf("error copy iso: %v", err) + } + log.Logger.Info("mount iso on agents done") + + if err = c.PowerVC.RestartAgents(agents); err != nil { + return fmt.Errorf("error restarting vm: %v", err) + } + log.Logger.Info("agents restarted") + + if err = c.ApproveAgents(agents); err != nil { + return err + } + log.Logger.Info("agents approved") + + if err = c.ScaleNodePool(); err != nil { + return err + } + log.Logger.Info("node pool scaled") + + if err = c.DownloadKubeConfig(); err != nil { + return fmt.Errorf("error downloading kubeconfig: %v", err) + } + log.Logger.Info("kubeconfig downloaded") + + if err = c.SetupIngressControllerNodeSelector(agents[0].Name); err != nil { + return err + } + + if err = c.MonitorHC(); err != nil { + return fmt.Errorf("error monitor hosted cluster to reach completed state: %v", err) + } + log.Logger.Info("hosted cluster reached completed state") + + return nil +} + +func Command(opts options.ClusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "create", + Short: "Command to create a hypershift agent cluster", + SilenceUsage: true, + } + + cmd.RunE = func(_ *cobra.Command, args []string) error { + c, err := cmdUtil.CreateClusterClient(opts) + if err != nil { + return fmt.Errorf("error create clients: %v", err) + } + imageID, networkID, gatewayIP, prefix, err := SetupPreReq(c) + if err != nil { + return fmt.Errorf("error setup pre req: %v", err) + } + log.Logger.Infof("retrieved prereq resource info imageID: %s, networkID: %s, gatewayIP: %s, prefix: %d", imageID, networkID, gatewayIP, prefix) + + if err = SetupCluster(c, imageID, networkID, gatewayIP, prefix); err != nil { + return fmt.Errorf("error setup cluster: %v", err) + } + log.Logger.Info("setup cluster done") + + return nil + } + + return cmd +} diff --git a/cmd/cluster/destroy/destroy.go b/cmd/cluster/destroy/destroy.go new file mode 100644 index 0000000..00df681 --- /dev/null +++ b/cmd/cluster/destroy/destroy.go @@ -0,0 +1,116 @@ +package destroy + +import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/options" + cmdUtil "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/util" + "github.com/ppc64le-cloud/hypershift-agent-automation/log" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/client/powervc" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/cluster" +) + +func DestroyPreReq(clusterName string, powervcClient *powervc.Client) error { + var errs []error + if err := powervcClient.CleanUpBootImage(clusterName); err != nil { + errs = append(errs, err) + } else { + log.Logger.Info("boot image cleaned") + } + if err := powervcClient.CleanUpBootVolume(clusterName); err != nil { + errs = append(errs, err) + } else { + log.Logger.Info("boot volume cleaned") + } + if err := powervcClient.CleanUpFlavor(clusterName); err != nil { + errs = append(errs, err) + } else { + log.Logger.Info("flavor cleaned") + } + + if len(errs) > 1 { + return errors.Join(errs...) + } + + return nil +} + +func DestroyCluster(c *cluster.Cluster) error { + var errs []error + if err := c.DescaleNodePool(); err != nil { + errs = append(errs, err) + } else { + log.Logger.Info("node pool descaled") + } + + if err := c.DestroyHC(); err != nil { + errs = append(errs, err) + } else { + log.Logger.Info("hosted cluster destroyed") + } + + if err := DestroyPreReq(c.Name, c.PowerVC); err != nil { + errs = append(errs, err) + } else { + log.Logger.Info("prereq resources destroyed") + } + + if err := c.PowerVC.DestroyAgents(c.GetWorkerName()); err != nil { + errs = append(errs, fmt.Errorf("error destroying agents: %v", err)) + } else { + log.Logger.Info("agents destroyed") + } + + if err := c.CleanupISOsInVIOS(); err != nil { + errs = append(errs, fmt.Errorf("error cleaning up iso in hmc: %v", err)) + } else { + log.Logger.Info("iso clean up done") + } + + if err := c.RemoveCISDNSRecords(); err != nil { + errs = append(errs, fmt.Errorf("error removing cis dns records: %v", err)) + } else { + log.Logger.Info("cis records deleted") + } + + if err := c.CleanupHCManifestDir(); err != nil { + errs = append(errs, fmt.Errorf("error removing hc manifest dir: %v", err)) + } else { + log.Logger.Infof("cleaned local manifest dir") + } + + if len(errs) > 1 { + return errors.Join(errs...) + } + + log.Logger.Info("destroying cluster completed") + + return nil +} + +func Command(opts options.ClusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "destroy", + Short: "Command to destroy the hypershift agent cluster", + SilenceUsage: true, + } + + cmd.RunE = func(_ *cobra.Command, args []string) error { + c, err := cmdUtil.CreateClusterClient(opts) + if err != nil { + return fmt.Errorf("error create clients: %v", err) + } + + if err = DestroyCluster(c); err != nil { + return fmt.Errorf("error destroy cluster: %v", err) + } + log.Logger.Info("destroy cluster done") + + return nil + } + + return cmd +} diff --git a/cmd/e2e/e2e.go b/cmd/e2e/e2e.go new file mode 100644 index 0000000..145fb07 --- /dev/null +++ b/cmd/e2e/e2e.go @@ -0,0 +1,50 @@ +package e2e + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/cluster/create" + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/cluster/destroy" + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/options" + cmdUtil "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/util" + "github.com/ppc64le-cloud/hypershift-agent-automation/log" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/cluster" +) + +func Run(c *cluster.Cluster) error { + imageID, networkID, gatewayIP, prefix, err := create.SetupPreReq(c) + if err != nil { + return fmt.Errorf("error setup pre req: %v", err) + } + log.Logger.Infof("retrieved prereq resource info imageID: %s, networkID: %s, gatewayIP: %s, prefix: %d", imageID, networkID, gatewayIP, prefix) + + if err = create.SetupCluster(c, imageID, networkID, gatewayIP, prefix); err != nil { + return fmt.Errorf("error setup cluster: %v", err) + } + log.Logger.Info("setup cluster done") + + if err = destroy.DestroyCluster(c); err != nil { + return fmt.Errorf("error destroying cluster: %v", err) + } + log.Logger.Info("destroying cluster completed") + + return nil +} + +func Command(opts options.ClusterOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "e2e", + Short: "Command to run e2e test on hypershift agent cluster", + SilenceUsage: true, + } + cmd.RunE = func(_ *cobra.Command, args []string) error { + c, err := cmdUtil.CreateClusterClient(opts) + if err != nil { + return fmt.Errorf("error create clients: %v", err) + } + return Run(c) + } + return cmd +} diff --git a/cmd/options/options.go b/cmd/options/options.go new file mode 100644 index 0000000..f8e1f78 --- /dev/null +++ b/cmd/options/options.go @@ -0,0 +1,10 @@ +package options + +type ClusterOptions struct { + Name string + BaseDomain string + PullSecretFile string + SSHKeyFile string + ReleaseImage string + NodeCount int +} diff --git a/cmd/util/client.go b/cmd/util/client.go new file mode 100644 index 0000000..1d98609 --- /dev/null +++ b/cmd/util/client.go @@ -0,0 +1,35 @@ +package util + +import ( + "fmt" + + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/options" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/client/hmc" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/client/ibmcloud" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/client/powervc" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/cluster" +) + +func CreateClusterClient(opts options.ClusterOptions) (*cluster.Cluster, error) { + hmcClient, err := hmc.NewClient() + if err != nil { + return nil, fmt.Errorf("error create powervc client: %v", err) + } + + ibmCloudClient, err := ibmcloud.NewClient() + if err != nil { + return nil, fmt.Errorf("error create ibmcloud client: %v", err) + } + + powerVCClient, err := powervc.NewClient() + if err != nil { + return nil, fmt.Errorf("error create powervc client: %v", err) + } + + c, err := cluster.New(opts, hmcClient, ibmCloudClient, powerVCClient) + if err != nil { + return nil, fmt.Errorf("error get new cluster: %v", err) + } + + return c, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..af1eaf1 --- /dev/null +++ b/go.mod @@ -0,0 +1,30 @@ +module github.com/ppc64le-cloud/hypershift-agent-automation + +go 1.20 + +require ( + github.com/3th1nk/cidr v0.2.0 + github.com/bramvdbogaerde/go-scp v1.2.1 + github.com/gophercloud/gophercloud v1.3.0 + github.com/gophercloud/utils v0.0.0-20230418172808-6eab72e966e1 + github.com/spf13/cobra v1.7.0 + go.uber.org/zap v1.24.0 + golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/apimachinery v0.27.2 +) + +require ( + github.com/go-logr/logr v1.2.3 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7940e93 --- /dev/null +++ b/go.sum @@ -0,0 +1,89 @@ +github.com/3th1nk/cidr v0.2.0 h1:81jjEknszD8SHPLVTPPk+BZjNVqq1ND2YXLSChl6Lrs= +github.com/3th1nk/cidr v0.2.0/go.mod h1:XsSQnS4rEYyB2veDfnIGgViulFpIITPKtp3f0VxpiLw= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/bramvdbogaerde/go-scp v1.2.1 h1:BKTqrqXiQYovrDlfuVFaEGz0r4Ou6EED8L7jCXw6Buw= +github.com/bramvdbogaerde/go-scp v1.2.1/go.mod h1:s4ZldBoRAOgUg8IrRP2Urmq5qqd2yPXQTPshACY8vQ0= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/gophercloud/gophercloud v1.3.0 h1:RUKyCMiZoQR3VlVR5E3K7PK1AC3/qppsWYo6dtBiqs8= +github.com/gophercloud/gophercloud v1.3.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gophercloud/utils v0.0.0-20230418172808-6eab72e966e1 h1:vJyXd9+MB5vAKxpOo4z/PDSiPgKmEyJwHIDOdV4Y0KY= +github.com/gophercloud/utils v0.0.0-20230418172808-6eab72e966e1/go.mod h1:VSalo4adEk+3sNkmVJLnhHoOyOYYS8sTWLG4mv5BKto= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= +k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= +k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= diff --git a/log/log.go b/log/log.go new file mode 100644 index 0000000..f35b88d --- /dev/null +++ b/log/log.go @@ -0,0 +1,8 @@ +package log + +import "go.uber.org/zap" + +var ( + zapLogger, _ = zap.NewProduction() + Logger = zapLogger.Sugar() +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..756e57c --- /dev/null +++ b/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/cluster" + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/e2e" + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/options" +) + +func main() { + cmd := &cobra.Command{ + Use: "hypershift-agent-automation", + SilenceUsage: true, + TraverseChildren: true, + + Run: func(cmd *cobra.Command, args []string) { + cmd.Help() + os.Exit(1) + }, + } + opts := options.ClusterOptions{ + NodeCount: 2, + } + + cmd.PersistentFlags().StringVar(&opts.Name, "name", opts.Name, "A name for the cluster") + cmd.PersistentFlags().StringVar(&opts.BaseDomain, "base-domain", opts.BaseDomain, "The ingress base domain for the cluster") + cmd.PersistentFlags().StringVar(&opts.ReleaseImage, "release-image", opts.ReleaseImage, "The OCP release image for the cluster") + cmd.PersistentFlags().StringVar(&opts.PullSecretFile, "pull-secret", opts.PullSecretFile, "File path to a pull secret.") + cmd.PersistentFlags().StringVar(&opts.SSHKeyFile, "ssh-key", opts.SSHKeyFile, "Path to an SSH key file") + cmd.PersistentFlags().IntVar(&opts.NodeCount, "node-count", opts.NodeCount, "Number of nodes in the the cluster") + + cmd.MarkFlagRequired("name") + cmd.MarkFlagRequired("base-domain") + cmd.MarkFlagRequired("release-image") + cmd.MarkFlagRequired("pull-secret") + cmd.MarkFlagRequired("ssh-key") + + cmd.AddCommand(e2e.Command(opts)) + cmd.AddCommand(cluster.Command(opts)) + + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} diff --git a/pkg/client/hmc/hmc.go b/pkg/client/hmc/hmc.go new file mode 100644 index 0000000..40bbf38 --- /dev/null +++ b/pkg/client/hmc/hmc.go @@ -0,0 +1,152 @@ +package hmc + +import ( + "errors" + "fmt" + "golang.org/x/crypto/ssh" + "os" + "strings" + + "github.com/ppc64le-cloud/hypershift-agent-automation/util" +) + +type VIOS struct { + SSHClient *ssh.Client + IP string + UserName string + Password string + HomeDir string +} + +type Client struct { + SSHClient *ssh.Client + VIOS *VIOS + IP string + UserName string + Password string +} + +func LoadEnv(c *Client) error { + var set bool + var errs []error + + c.IP, set = os.LookupEnv("HMC_IP") + if !set { + errs = append(errs, fmt.Errorf("HMC_IP env var not set")) + } + c.UserName, set = os.LookupEnv("HMC_USERNAME") + if !set { + errs = append(errs, fmt.Errorf("HMC_USERNAME env var not set")) + } + c.Password, set = os.LookupEnv("HMC_PASSWORD") + if !set { + errs = append(errs, fmt.Errorf("HMC_PASSWORD env var not set")) + } + c.VIOS.IP, set = os.LookupEnv("VIOS_IP") + if !set { + errs = append(errs, fmt.Errorf("VIOS_IP env var not set")) + } + c.VIOS.UserName, set = os.LookupEnv("VIOS_USERNAME") + if !set { + errs = append(errs, fmt.Errorf("VIOS_USERNAME env var not set")) + } + c.VIOS.Password, set = os.LookupEnv("VIOS_PASSWORD") + if !set { + errs = append(errs, fmt.Errorf("VIOS_PASSWORD env var not set")) + } + c.VIOS.HomeDir, set = os.LookupEnv("VIOS_HOMEDIR") + if !set { + errs = append(errs, fmt.Errorf("VIOS_HOMEDIR env var not set")) + } + + return errors.Join(errs...) +} + +func NewClient() (*Client, error) { + c := &Client{VIOS: &VIOS{}} + if err := LoadEnv(c); err != nil { + return nil, fmt.Errorf("error reading env for hmc client: %v", err) + } + + var err error + c.SSHClient, err = util.CreateSSHClient(c.IP, c.UserName, c.Password) + if err != nil { + return nil, fmt.Errorf("error create hmc ssh client: %v", err) + } + + c.VIOS.SSHClient, err = util.CreateSSHClient(c.VIOS.IP, c.VIOS.UserName, c.VIOS.Password) + if err != nil { + return nil, fmt.Errorf("error create vios ssh client: %v", err) + } + + return c, nil +} + +func (hmc Client) GetLPARID(host, lparName string) (string, error) { + lparIDCommand := fmt.Sprintf("lshwres -m %s -r virtualio --rsubtype scsi --filter \"lpar_names=%s\"", host, lparName) + out, _, err := util.ExecuteRemoteCommand(hmc.SSHClient, lparIDCommand) + if err != nil { + return "", fmt.Errorf("error executing command to retrieve lpar_id %v", err) + } + for _, item := range strings.Split(out, ",") { + if strings.Contains(item, "lpar_id") { + return strings.Split(item, "=")[1], nil + } + } + + return "", fmt.Errorf("not able to retrieve lpar_id command, output: %s", out) +} + +func (hmc Client) GetVHOST(lparID string) (string, error) { + vhostCommand := fmt.Sprintf("ioscli lsmap -all -dec -cpid %s | awk 'NR==3{ print $1 }'", lparID) + out, _, err := util.ExecuteRemoteCommand(hmc.VIOS.SSHClient, vhostCommand) + if err != nil { + return "", fmt.Errorf("error executing command to retrieve vhost: %v", err) + } + if out == "" { + return "", fmt.Errorf("not able to retrieve vhost, command used: %s", vhostCommand) + } + + return out, nil +} + +func (hmc Client) CreateVOpt(voptName, isoPath string) error { + mkvoptCommand := fmt.Sprintf("ioscli mkvopt -name %s -file %s/%s", voptName, hmc.VIOS.HomeDir, isoPath) + _, e, err := util.ExecuteRemoteCommand(hmc.VIOS.SSHClient, mkvoptCommand) + if err != nil || e != "" { + return fmt.Errorf("error executing command to create vopt: %v, e: %s", err, e) + } + return nil +} + +func (hmc Client) MapVOptToVTOpt(vhost string, vopt string) error { + mkvdevCommand := fmt.Sprintf("ioscli mkvdev -fbo -vadapter %s", vhost) + out, _, err := util.ExecuteRemoteCommand(hmc.VIOS.SSHClient, mkvdevCommand) + if err != nil { + return fmt.Errorf("error executing command to create vopt %v", err) + } + + var vtopt string + if strings.Contains(out, "Available") { + vtopt = strings.Split(out, " ")[0] + } + if vtopt == "" { + return fmt.Errorf("error retrieving available vtopt for vhost: %s, error: %v", vhost, err) + } + + loadoptCommand := fmt.Sprintf("ioscli loadopt -vtd %s -disk %s", vtopt, vopt) + if _, _, err = util.ExecuteRemoteCommand(hmc.VIOS.SSHClient, loadoptCommand); err != nil { + return fmt.Errorf("error executing loadopt command: %v", err) + } + + return nil +} + +func (hmc Client) SetupBootString(host, partitionName string) error { + chsyscfgcmd := fmt.Sprintf("chsyscfg -r lpar -m %s -i name=%s,boot_string=/vdevice/v-scsi@30000002/disk@8200000000000000", host, partitionName) + _, e, err := util.ExecuteRemoteCommand(hmc.SSHClient, chsyscfgcmd) + if err != nil || e != "" { + return fmt.Errorf("error executing command to configuring boot_string: %v, e: %s", err, e) + } + return nil +} diff --git a/pkg/client/ibmcloud/ibmcloud.go b/pkg/client/ibmcloud/ibmcloud.go new file mode 100644 index 0000000..7da8521 --- /dev/null +++ b/pkg/client/ibmcloud/ibmcloud.go @@ -0,0 +1,106 @@ +package ibmcloud + +import ( + "encoding/json" + "errors" + "fmt" + "os" + + "github.com/ppc64le-cloud/hypershift-agent-automation/log" + "github.com/ppc64le-cloud/hypershift-agent-automation/util" +) + +type Client struct { + CISDomain string + CISDomainID string +} + +var DNSRecordNotExist = func(name string) error { return fmt.Errorf("no dns record exists with name: %s", name) } + +func LoadEnv(c *Client) error { + var set bool + var errs []error + + _, set = os.LookupEnv("IBMCLOUD_API_KEY") + if !set { + errs = append(errs, fmt.Errorf("IBMCLOUD_API_KEY env var not set")) + } + + c.CISDomain, set = os.LookupEnv("BASE_DOMAIN") + if !set { + errs = append(errs, fmt.Errorf("BASE_DOMAIN env var not set")) + } + + return errors.Join(errs...) +} + +func NewClient() (*Client, error) { + c := &Client{} + var err error + if err = LoadEnv(c); err != nil { + return nil, fmt.Errorf("error reading env for ibmcloud client: %v", err) + } + + var out, e string + if out, e, err = util.ExecuteCommand("ibmcloud", []string{"login", "--no-region", "--quiet"}); err != nil || e != "" { + return nil, fmt.Errorf("error login ibmcloud cli, out: %v, e: %v, err: %v", out, e, err) + } + log.Logger.Infof("out: %v, e: %v, err: %v", out, e, err) + + c.CISDomainID, err = util.GetCISDomainID(c.CISDomain) + if err != nil { + return nil, fmt.Errorf("error retrieve cis domain id: %v", err) + } + + return c, nil +} + +func (c Client) CreateDNSRecord(rType, name, content string) error { + args := []string{"cis", "dns-record-create", c.CISDomainID, "--type", rType, "--name", name, "--content", content} + _, e, err := util.ExecuteCommand("ibmcloud", args) + if err != nil || e != "" { + return fmt.Errorf("error creating dns record, e: %v, err: %v", e, err) + } + + return nil +} + +func (c Client) UpdateDNSRecord(dnsRecordID, content string) error { + args := []string{"cis", "dns-record-update", c.CISDomainID, dnsRecordID, "--content", content} + _, e, err := util.ExecuteCommand("ibmcloud", args) + if err != nil || e != "" { + return fmt.Errorf("error updating dns record, e: %v, err: %v", e, err) + } + + return nil +} + +func (c Client) DeleteDNSRecord(dnsRecordID string) error { + args := []string{"cis", "dns-record-delete", c.CISDomainID, dnsRecordID} + _, e, err := util.ExecuteCommand("ibmcloud", args) + if err != nil || e != "" { + return fmt.Errorf("error deleting dns record, e: %v, err: %v", e, err) + } + + return nil +} + +func (c Client) GetDNSRecordID(name string) (string, error) { + args := []string{"cis", "dns-records", c.CISDomainID, "--name", fmt.Sprintf("%s.%s", name, c.CISDomain), "--output", "JSON"} + out, e, err := util.ExecuteCommand("ibmcloud", args) + if err != nil || e != "" { + return "", fmt.Errorf("error retrieving dns record, e: %v, err: %v", e, err) + } + dnsRecords := make([]map[string]interface{}, 0) + if err = json.Unmarshal([]byte(out), &dnsRecords); err != nil { + return "", err + } + var dnsRecordID string + if len(dnsRecords) > 0 { + dnsRecordID = dnsRecords[0]["id"].(string) + } else { + return "", DNSRecordNotExist(name) + } + + return dnsRecordID, nil +} diff --git a/pkg/client/powervc/powervc.go b/pkg/client/powervc/powervc.go new file mode 100644 index 0000000..0f39eb0 --- /dev/null +++ b/pkg/client/powervc/powervc.go @@ -0,0 +1,483 @@ +package powervc + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/3th1nk/cidr" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/gophercloud/gophercloud" + "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes" + "github.com/gophercloud/gophercloud/openstack/blockstorage/v3/volumetypes" + "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors" + "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" + "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" + "github.com/gophercloud/gophercloud/openstack/imageservice/v2/imagedata" + "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" + "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" + "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" + "github.com/gophercloud/utils/openstack/clientconfig" + + "github.com/ppc64le-cloud/hypershift-agent-automation/log" + "github.com/ppc64le-cloud/hypershift-agent-automation/util" +) + +type Client struct { + computeClient *gophercloud.ServiceClient + imageClient *gophercloud.ServiceClient + networkClient *gophercloud.ServiceClient + volumeClient *gophercloud.ServiceClient + StorageTemplate string + NetworkName string + Host string +} + +type Agent struct { + ID string + Name string + PartitionName string + IP string + MAC string +} + +const ( + DiskSize = 120 +) + +var tags = []string{"purpose:hypershift-bm-agent-ci"} + +var VolumeNotFound = func(name string) error { return fmt.Errorf("volume not found by the name %s", name) } +var ImageNotFound = func(name string) error { return fmt.Errorf("image not found by the name %s", name) } + +func LoadEnv(c *Client) error { + var set bool + var errs []error + + c.StorageTemplate, set = os.LookupEnv("POWERVC_STORAGE_TEMPLATE") + if !set { + errs = append(errs, fmt.Errorf("POWERVC_STORAGE_TEMPLATE env var not set")) + } + c.NetworkName, set = os.LookupEnv("POWERVC_NETWORK_NAME") + if !set { + errs = append(errs, fmt.Errorf("POWERVC_NETWORK_NAME env var not set")) + } + c.Host, set = os.LookupEnv("POWERVC_HOST") + if !set { + errs = append(errs, fmt.Errorf("POWERVC_HOST env var not set")) + } + + return errors.Join(errs...) +} + +func NewClient() (*Client, error) { + options := &clientconfig.ClientOpts{} + imageClient, err := clientconfig.NewServiceClient("image", options) + if err != nil { + return nil, err + } + + computeClient, err := clientconfig.NewServiceClient("compute", options) + if err != nil { + return nil, err + } + + networkClient, err := clientconfig.NewServiceClient("network", options) + if err != nil { + return nil, err + } + + volumeClient, err := clientconfig.NewServiceClient("volume", options) + if err != nil { + return nil, err + } + + c := &Client{ + imageClient: imageClient, + computeClient: computeClient, + networkClient: networkClient, + volumeClient: volumeClient, + } + + if err = LoadEnv(c); err != nil { + return nil, fmt.Errorf("error loading env var of powervc: %v", err) + } + + return c, nil +} + +func (c Client) GetVolumeIDByName(name string) (string, error) { + volumeListPage, err := volumes.List(c.volumeClient, volumes.ListOpts{Name: name}).AllPages() + if err != nil { + return "", err + } + + volumeList := volumeListPage.GetBody().(map[string][]interface{})["volumes"] + for _, volume := range volumeList { + if volume.(map[string]interface{})["name"].(string) == name { + return volume.(map[string]interface{})["id"].(string), nil + } + } + + return "", VolumeNotFound(name) +} + +func (c Client) SetupEmptyBootVol(clusterName string) (string, error) { + var volumeTypeID string + + volumeTypeListPage, err := volumetypes.List(c.volumeClient, volumetypes.ListOpts{}).AllPages() + if err != nil { + return "", err + } + + volumeTypeList := volumeTypeListPage.GetBody().(map[string][]interface{})["volume_types"] + for _, volumeType := range volumeTypeList { + if volumeType.(map[string]interface{})["name"].(string) == c.StorageTemplate { + volumeTypeID = volumeType.(map[string]interface{})["id"].(string) + } + } + volumeID, err := c.GetVolumeIDByName(clusterName) + if err != nil && err.Error() != VolumeNotFound(clusterName).Error() { + return "", fmt.Errorf("error checking volume before creation: %v", err) + } + if volumeID != "" { + return volumeID, nil + } + + options := volumes.CreateOpts{ + Name: clusterName, + Size: DiskSize, + VolumeType: volumeTypeID, + Metadata: map[string]string{"is_image_volume": "True", "is_boot_volume": "True"}, + } + + volume, err := volumes.Create(c.volumeClient, options).Extract() + if err != nil { + return "", err + } + + return volume.ID, nil +} + +func (c Client) CleanUpBootVolume(clusterName string) error { + volumeID, err := c.GetVolumeIDByName(clusterName) + if err != nil && err.Error() != VolumeNotFound(clusterName).Error() { + return fmt.Errorf("error checking volume before creation: %v", err) + } + if volumeID == "" { + log.Logger.Infof("volume %s is cleaned", clusterName) + return nil + } + + if res := volumes.Delete(c.volumeClient, volumeID, nil); res.Err != nil { + return fmt.Errorf("error deleting volume %s: %v", clusterName, res.PrettyPrintJSON()) + } + + return nil +} + +func (c Client) GetImageIDByName(name string) (string, error) { + imageListResult, err := images.List(c.imageClient, &images.ListOpts{Name: name, Tags: tags}).AllPages() + if err != nil { + return "", err + } + + isImageExist, err := imageListResult.IsEmpty() + if err != nil { + return "", err + } + if !isImageExist { + imageList := imageListResult.GetBody().(map[string][]interface{})["images"] + return imageList[0].(map[string]interface{})["id"].(string), nil + } + + return "", ImageNotFound(name) +} + +func (c Client) CleanUpBootImage(clusterName string) error { + imageID, err := c.GetImageIDByName(clusterName) + if err != nil && ImageNotFound(clusterName).Error() != err.Error() { + return fmt.Errorf("error getting imageID : %v", err) + } + if imageID == "" { + log.Logger.Infof("image %s cleaned up", clusterName) + return nil + } + if res := images.Delete(c.imageClient, imageID); res.Err != nil { + return fmt.Errorf("error deleting image %s: %v", clusterName, res.PrettyPrintJSON()) + } + + return nil +} + +func (c Client) SetupPreReqImage(clusterName, volumeID string) (string, error) { + imageID, err := c.GetImageIDByName(clusterName) + if err != nil && ImageNotFound(clusterName).Error() != err.Error() { + return "", fmt.Errorf("error checking image before creation: %v", err) + } + if imageID != "" { + log.Logger.Infof("Image %s already exist", clusterName) + return imageID, nil + } + + visibility := images.ImageVisibilityPrivate + blockDeviceMapping := fmt.Sprintf("[{\"guest_format\":null,\"boot_index\":0,\"no_device\":null,\"image_id\":null,\"volume_id\":\"%v\",\"disk_bus\":null,\"volume_size\":null,\"source_type\":\"volume\",\"device_type\":\"disk\",\"snapshot_id\":null,\"destination_type\":\"volume\",\"delete_on_termination\":true}]", volumeID) + createOpts := images.CreateOpts{ + Name: clusterName, + ContainerFormat: "bare", + Visibility: &visibility, + DiskFormat: "raw", + MinDisk: 1, + Tags: tags, + Properties: map[string]string{ + "os_distro": "coreos", + "endianness": "little-endian", + "architecture": "ppc64", + "hypervisor_type": "phyp", + "root_device_name": "/dev/sda", + "block_device_mapping": blockDeviceMapping, + "bdm_v2": "true", + }, + } + + image, err := images.Create(c.imageClient, createOpts).Extract() + if err != nil { + return "", err + } + + err = imagedata.Upload(c.imageClient, image.ID, strings.NewReader("")).Err + + return image.ID, err +} + +func (c Client) CleanUpFlavor(clusterName string) error { + flavorID := util.GenerateFlavourID(clusterName) + if err := flavors.Delete(c.computeClient, flavorID).Err; err != nil { + return fmt.Errorf("error deleting flavorID %s: %v", flavorID, err) + } + + return nil +} + +func (c Client) SetupFlavor(clusterName string) error { + flavorID := util.GenerateFlavourID(clusterName) + res := flavors.Get(c.computeClient, flavorID) + if res.Err == nil { + return nil + } + + disk := 0 + flavorCreateOpts := flavors.CreateOpts{ + Name: clusterName, + RAM: 16384, + VCPUs: 1, + Disk: &disk, + ID: flavorID, + } + if err := flavors.Create(c.computeClient, flavorCreateOpts).Err; err != nil { + return err + } + + extraSpecs := flavors.ExtraSpecsOpts{ + "powervm:processor_compatibility": "default", + "powervm:srr_capability": "false", + "powervm:min_vcpu": "1", + "powervm:max_vcpu": "1", + "powervm:min_mem": "4096", + "powervm:max_mem": "16384", + "powervm:availability_priority": "127", + "powervm:enable_lpar_metric": "false", + "powervm:enforce_affinity_check": "false", + "powervm:secure_boot": "0", + "powervm:proc_units": "0.5", + "powervm:min_proc_units": "0.5", + "powervm:max_proc_units": "1", + "powervm:dedicated_proc": "false", + "powervm:shared_proc_pool_name": "DefaultPool", + "powervm:uncapped": "true", + "powervm:shared_weight": "128", + "powervm:ame_expansion_factor": "0", + } + err := flavors.CreateExtraSpecs(c.computeClient, flavorID, extraSpecs).Err + + return err +} + +func (c Client) GetNetworkID() (string, string, int, error) { + networkPages, err := networks.List(c.networkClient, networks.ListOpts{Name: c.NetworkName}).AllPages() + if err != nil { + return "", "", 0, err + } + networkList := networkPages.GetBody().(map[string][]interface{})["networks"] + if len(networkList) < 1 { + return "", "", 0, fmt.Errorf("network %s not exist", c.NetworkName) + } + network := networkList[0].(map[string]interface{}) + + subnetPages, err := subnets.List(c.networkClient, subnets.ListOpts{NetworkID: network["id"].(string)}).AllPages() + if err != nil { + return "", "", 0, err + } + + subnetL, err := subnets.ExtractSubnets(subnetPages) + if err != nil { + return "", "", 0, err + } + if len(subnetL) < 0 { + return "", "", 0, fmt.Errorf("network does not contain any subnets") + } + + cidrParsed, err := cidr.Parse(subnetL[0].CIDR) + if err != nil { + return "", "", 0, err + } + ones, _ := cidrParsed.MaskSize() + + return network["id"].(string), subnetL[0].GatewayIP, ones, nil +} + +func (c Client) GetHypervisorHostMTMS(hostDisplayName string) (string, error) { + hypervisorsListPages, err := hypervisors.List(c.computeClient, hypervisors.ListOpts{}).AllPages() + if err != nil { + return "", err + } + + hypervisorsList := hypervisorsListPages.GetBody().(map[string]interface{})["hypervisors"].([]interface{}) + var hypHostName string + for _, hv := range hypervisorsList { + hypervisor := hv.(map[string]interface{}) + if hypervisor["service"].(map[string]interface{})["host_display_name"].(string) == hostDisplayName { + hypHostName = hypervisor["hypervisor_hostname"].(string) + } + } + + if hypHostName == "" { + return "", fmt.Errorf("no host found with name %s", hostDisplayName) + } + return hypHostName, nil +} + +func (c Client) CreateServer(clusterName, host, imageID, networkID string, nodeCount int) error { + hostMTMS, err := c.GetHypervisorHostMTMS(host) + if err != nil { + return fmt.Errorf("error retrieving mtms for the host %s %v", host, err) + } + createOpts := servers.CreateOpts{ + Name: clusterName, + ImageRef: imageID, + AvailabilityZone: fmt.Sprintf(":%s", hostMTMS), + FlavorRef: util.GenerateFlavourID(clusterName), + Networks: []servers.Network{{UUID: networkID}}, + Metadata: map[string]string{"primary_network": networkID}, + Min: nodeCount, + } + + p, _ := json.Marshal(createOpts) + log.Logger.Infof("create server with payload %s", string(p)) + + return servers.Create(c.computeClient, createOpts).Err +} + +func (c Client) SetupAgents(workerName, imageID, networkID string, nodeCount int) ([]Agent, error) { + + serverPages, err := servers.List(c.computeClient, servers.ListOpts{Name: workerName}).AllPages() + if err != nil { + return nil, err + } + + serverList := serverPages.GetBody().(map[string][]interface{})["servers"] + if len(serverList) < 1 { + if err = c.CreateServer(workerName, c.Host, imageID, networkID, nodeCount); err != nil { + return nil, err + } + + serverPages, err = servers.List(c.computeClient, servers.ListOpts{Name: workerName}).AllPages() + if err != nil { + return nil, err + } + + serverList = serverPages.GetBody().(map[string][]interface{})["servers"] + } + + monitorServer := func(id string) (map[string]interface{}, error) { + var server map[string]interface{} + f := func() (bool, error) { + server = servers.Get(c.computeClient, id).Body.(map[string]interface{})["server"].(map[string]interface{}) + if err != nil { + return false, err + } + currentState := server["OS-EXT-STS:vm_state"].(string) + log.Logger.Infof("waiting for agent to reach active state, current state: %s", currentState) + if currentState == "active" { + return true, nil + } + + if currentState == "failed" || currentState == "error" { + details, _ := json.Marshal(server) + return false, fmt.Errorf("agent %s is in failed or error state, details %v", server["name"].(string), string(details)) + } + + return false, nil + } + + err = wait.PollImmediate(time.Second*30, time.Minute*10, f) + return server, err + } + + var serverL []map[string]interface{} + for _, s := range serverList { + server, err := monitorServer(s.(map[string]interface{})["id"].(string)) + if err != nil { + return nil, err + } + serverL = append(serverL, server) + } + + var agentList []Agent + for _, server := range serverL { + name := server["name"].(string) + partitionName := server["OS-EXT-SRV-ATTR:instance_name"].(string) + id := server["id"].(string) + addr := server["addresses"].(map[string]interface{})[c.NetworkName].([]interface{})[0].(map[string]interface{}) + mac := addr["OS-EXT-IPS-MAC:mac_addr"].(string) + ip := addr["addr"].(string) + + agentList = append(agentList, Agent{ID: id, Name: name, PartitionName: partitionName, IP: ip, MAC: mac}) + } + + return agentList, nil +} + +func (c Client) RestartAgents(agents []Agent) error { + for _, agent := range agents { + //time.Sleep(time.Minute * 2) + if err := servers.Reboot(c.computeClient, agent.ID, servers.RebootOpts{Type: servers.SoftReboot}).Err; err != nil { + return fmt.Errorf("error rebooting agent %s: %v ", agent.PartitionName, err) + } + log.Logger.Infof("rebooted %s", agent.PartitionName) + } + + return nil +} + +func (c Client) DestroyAgents(agentName string) error { + serverPages, err := servers.List(c.computeClient, servers.ListOpts{Name: agentName}).AllPages() + if err != nil { + return err + } + serverList := serverPages.GetBody().(map[string][]interface{})["servers"] + + for _, s := range serverList { + serverID := s.(map[string]interface{})["id"].(string) + log.Logger.Infof("deleting %s", s.(map[string]interface{})["name"].(string)) + if err = servers.Delete(c.computeClient, serverID).Err; err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go new file mode 100644 index 0000000..eaad408 --- /dev/null +++ b/pkg/cluster/cluster.go @@ -0,0 +1,595 @@ +package cluster + +import ( + "bytes" + "encoding/json" + "fmt" + "gopkg.in/yaml.v3" + "os" + "strconv" + "strings" + "text/template" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/ppc64le-cloud/hypershift-agent-automation/cmd/options" + "github.com/ppc64le-cloud/hypershift-agent-automation/log" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/client/hmc" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/client/ibmcloud" + "github.com/ppc64le-cloud/hypershift-agent-automation/pkg/client/powervc" + "github.com/ppc64le-cloud/hypershift-agent-automation/util" +) + +const ( + hcDesiredServiceSpec = `[ + { + "service": "APIServer", + "servicePublishingStrategy": { + "type": "LoadBalancer" + } + }, + { + "service": "OAuthServer", + "servicePublishingStrategy": { + "type": "Route" + } + }, + { + "service": "OIDC", + "servicePublishingStrategy": { + "type": "None" + } + }, + { + "service": "Konnectivity", + "servicePublishingStrategy": { + "type": "Route" + } + }, + { + "service": "Ignition", + "servicePublishingStrategy": { + "type": "Route" + } + }, + { + "service": "OVNSbDb", + "servicePublishingStrategy": { + "type": "Route" + } + } + ]` + + nmStateConfigTemplateFile = "nmstate-config-template.yaml" + infraEnvTemplateFile = "infraenv-template.yaml" + + infraEnvFile = "infraenv.yaml" +) + +var discoveryISOFile = func(hcName string) string { return fmt.Sprintf("%s-discovery.iso", hcName) } + +type Cluster struct { + Name string + Namespace string + PullSecretFile string + BaseDomain string + SSHKeyFile string + ReleaseImage string + NodeCount int + HMC *hmc.Client + IBMCloud *ibmcloud.Client + PowerVC *powervc.Client +} + +func New(opts options.ClusterOptions, hmc *hmc.Client, ibmcloud *ibmcloud.Client, powervc *powervc.Client) (*Cluster, error) { + c := &Cluster{ + Name: opts.Name, + BaseDomain: opts.BaseDomain, + ReleaseImage: opts.ReleaseImage, + PullSecretFile: opts.PullSecretFile, + SSHKeyFile: opts.SSHKeyFile, + NodeCount: opts.NodeCount, + HMC: hmc, + IBMCloud: ibmcloud, + PowerVC: powervc, + } + + return c, nil +} + +func (c Cluster) getCPNamespace() string { return fmt.Sprintf("%s-%s", c.Namespace, c.Name) } +func (c Cluster) GetWorkerName() string { return fmt.Sprintf("%s-worker", c.Name) } + +func (c Cluster) createHC() error { + args := []string{"create", "cluster", "agent", + "--name", c.Name, + "--agent-namespace", c.getCPNamespace(), + "--pull-secret", c.PullSecretFile, + "--base-domain", c.BaseDomain, + "--ssh-key", c.SSHKeyFile, + "--release-image", c.ReleaseImage, + "--render", + } + + out, e, err := util.ExecuteCommand("hypershift", args) + + if err != nil { + return fmt.Errorf("error create cluster agent %v, stdout: %s, stderr: %s", err, out, e) + } + log.Logger.Infof("out: %v", out) + + ogManifestStr := out + ogManifestL := strings.Split(ogManifestStr, "---") + + log.Logger.Infof("ogL", ogManifestL) + var hcManifestS string + var hcManifestIndex int + for i, m := range ogManifestL { + if strings.Contains(m, "kind: HostedCluster") { + hcManifestS = m + hcManifestIndex = i + } + } + + log.Logger.Infof("hcmanifest: %v", hcManifestS) + hcManifest := map[string]interface{}{} + + if err = yaml.Unmarshal([]byte(hcManifestS), hcManifest); err != nil { + return err + } + + var desiredService []map[string]interface{} + if err = json.Unmarshal([]byte(hcDesiredServiceSpec), &desiredService); err != nil { + return err + } + + spec := hcManifest["spec"].(map[string]interface{}) + spec["services"] = desiredService + hcManifest["spec"] = spec + + desiredHCSpec, err := yaml.Marshal(hcManifest) + if err != nil { + return err + } + + ogManifestL[hcManifestIndex] = "\n" + string(desiredHCSpec) + clusterManifestLoc := fmt.Sprintf("%s/clusters.yaml", util.GetManifestDir(c.Name)) + f, err := os.Create(clusterManifestLoc) + if err != nil { + return err + } + + if _, err = f.Write([]byte(strings.Join(ogManifestL, "---"))); err != nil { + return err + } + + args = []string{"apply", "-f", clusterManifestLoc} + out, e, err = util.ExecuteCommand("oc", args) + if err != nil || e != "" { + return fmt.Errorf("error applying clusters manifest, stderr: %s, error: %v", e, err) + } + + log.Logger.Info("hosted cluster manifests applied") + return nil +} + +func (c Cluster) SetupHC() error { + args := []string{"create", "namespace", c.getCPNamespace()} + _, e, err := util.ExecuteCommand("oc", args) + if err != nil && !strings.Contains(e, "AlreadyExists") { + return fmt.Errorf("error creating contorl plane namespace stderr: %s, error: %v", e, err) + } + + args = []string{"get", "hc", c.Name, "-n", c.Namespace} + _, e, err = util.ExecuteCommand("oc", args) + if err != nil || e != "" { + if strings.Contains(e, "NotFound") { + if err = c.createHC(); err != nil { + return fmt.Errorf("error create hc: %v", err) + } + } else { + return fmt.Errorf("error get hc: %v, e: %v", err, e) + } + } + + log.Logger.Info("waiting for hosted cluster status to become available") + if _, _, err = util.ExecuteCommand("oc", []string{"wait", "hc", c.Name, "-n", c.Namespace, "--for=condition=Available", "--timeout=10m"}); err != nil { + return err + } + + return nil +} + +func (c Cluster) SetupCISDNSRecords(workerIP string) error { + args := []string{"get", "service", "kube-apiserver", "-n", c.getCPNamespace(), "-o", "json"} + out, e, err := util.ExecuteCommand("oc", args) + + if err != nil || e != "" { + return fmt.Errorf("error retrieving kube apiserver details: %v, e: %v", err, e) + } + + apiServerSvc := map[string]interface{}{} + if err = json.Unmarshal([]byte(out), &apiServerSvc); err != nil { + return fmt.Errorf("error unmarshal kube apiserver response: %v", err) + } + + var apiServerHostname string + ingress := apiServerSvc["status"].(map[string]interface{})["loadBalancer"].(map[string]interface{})["ingress"].([]interface{}) + if len(ingress) < 0 { + return fmt.Errorf("hostname not generated for kube-apiserver") + } + apiServerHostname = ingress[0].(map[string]interface{})["hostname"].(string) + + createOrUpdateDNSRecord := func(rType, name, content string) error { + var dnsRecordID string + dnsRecordID, err = c.IBMCloud.GetDNSRecordID(name) + if err != nil && err.Error() != ibmcloud.DNSRecordNotExist(name).Error() { + return err + } + + if dnsRecordID != "" { + log.Logger.Infof("updating dns record name %s content %s", name, content) + if err = c.IBMCloud.UpdateDNSRecord(dnsRecordID, content); err != nil { + return err + } + } else { + log.Logger.Infof("creating dns record name %s content %s", name, content) + if err = c.IBMCloud.CreateDNSRecord(rType, name, content); err != nil { + return err + } + } + + return nil + } + + if err = createOrUpdateDNSRecord("CNAME", fmt.Sprintf("api.%s", c.Name), apiServerHostname); err != nil { + return fmt.Errorf("error setting up api cis record: %v", err) + } + + if err = createOrUpdateDNSRecord("CNAME", fmt.Sprintf("api-int.%s", c.Name), apiServerHostname); err != nil { + return fmt.Errorf("error setting up api-int cis record: %v", err) + } + + if err = createOrUpdateDNSRecord("A", fmt.Sprintf("*.apps.%s", c.Name), workerIP); err != nil { + return fmt.Errorf("error setting up *.apps cis record: %v", err) + } + + return nil +} + +func (c Cluster) SetupNMStateConfig(agents []powervc.Agent, prefix int, gateway, label string) error { + for _, agent := range agents { + templateConfig := map[string]string{ + "Name": fmt.Sprintf("%s-%s", c.Name, agent.PartitionName), + "Namespace": c.getCPNamespace(), + "Label": label, + "Prefix": strconv.Itoa(prefix), + "Gateway": gateway, + "MAC": agent.MAC, + "IP": agent.IP, + } + + nmStateConfigTemplateFileAbsPath, err := util.GetAbsoluteTemplatePath(nmStateConfigTemplateFile) + if err != nil { + return err + } + + templateCont, err := os.ReadFile(nmStateConfigTemplateFileAbsPath) + if err != nil { + return err + } + + t := template.Must(template.New("nmstate-config").Parse(string(templateCont))) + + nmStateConfigData := &bytes.Buffer{} + if err = t.Execute(nmStateConfigData, templateConfig); err != nil { + return err + } + + nmStateConfigFileLoc := fmt.Sprintf("%s/nmstate-config-%s.yaml", util.GetManifestDir(c.Name), agent.PartitionName) + f, err := os.Create(nmStateConfigFileLoc) + if err != nil { + return err + } + + if _, err = f.Write(nmStateConfigData.Bytes()); err != nil { + return err + } + + _, e, err := util.ExecuteCommand("oc", []string{"apply", "-f", nmStateConfigFileLoc}) + if err != nil || e != "" { + return fmt.Errorf("error applying nmstate config for agent: %s, e: %v, err: %v", agent, e, err) + } + + log.Logger.Infof("%s applied", nmStateConfigFileLoc) + } + + return nil +} + +func (c Cluster) SetupInfraEnv(label string) error { + sshPubKeyContent, err := os.ReadFile(c.SSHKeyFile) + if err != nil { + return err + } + + templateConfig := map[string]string{ + "Name": c.Name, + "Namespace": c.getCPNamespace(), + "Label": label, + "SSH_Pub_Key": string(sshPubKeyContent), + } + + infraEnvTemplateFileAbsPath, err := util.GetAbsoluteTemplatePath(infraEnvTemplateFile) + if err != nil { + return err + } + + templateCont, err := os.ReadFile(infraEnvTemplateFileAbsPath) + if err != nil { + return err + } + + t := template.Must(template.New("infraenv").Parse(string(templateCont))) + + infraEnvConfigData := &bytes.Buffer{} + if err = t.Execute(infraEnvConfigData, templateConfig); err != nil { + return err + } + + infraEnvFileLoc := fmt.Sprintf("%s/%s", util.GetManifestDir(c.Name), infraEnvFile) + f, err := os.Create(infraEnvFileLoc) + if err != nil { + return err + } + + if _, err = f.Write(infraEnvConfigData.Bytes()); err != nil { + return err + } + + _, e, err := util.ExecuteCommand("oc", []string{"apply", "-f", infraEnvFileLoc}) + if err != nil || e != "" { + return fmt.Errorf("error applying infraenv, e: %v, err: %v", e, err) + } + + log.Logger.Infof("%s applied", infraEnvFileLoc) + return nil +} + +func (c Cluster) DownloadISO() error { + cpNamespce := c.getCPNamespace() + _, e, err := util.ExecuteCommand("oc", []string{"wait", "--timeout=5m", "--for=condition=ImageCreated", "-n", cpNamespce, fmt.Sprintf("infraenv/%s", c.Name)}) + if err != nil || e != "" { + return fmt.Errorf("error waiting for discovery iso creation, e: %v, err: %v", e, err) + } + + out, e, err := util.ExecuteCommand("oc", []string{"get", "infraenv", c.Name, "-n", cpNamespce, "-o", "json"}) + if err != nil || e != "" { + return fmt.Errorf("error get infraenv, e: %v, err: %v", e, err) + } + + infraEnv := map[string]interface{}{} + if err = json.Unmarshal([]byte(out), &infraEnv); err != nil { + return err + } + isoDownloadURL := infraEnv["status"].(map[string]interface{})["isoDownloadURL"].(string) + + out, e, err = util.ExecuteCommand("curl", []string{isoDownloadURL, "--output", fmt.Sprintf("%s/%s", util.GetManifestDir(c.Name), discoveryISOFile(c.Name))}) + if err != nil { + return fmt.Errorf("error downloading iso, e: %v, err: %v", e, err) + } + + return nil +} + +func (c Cluster) CopyAndMountISO(agents []powervc.Agent) error { + + if err := util.SCPFile(c.HMC.VIOS.SSHClient, discoveryISOFile(c.Name), util.GetManifestDir(c.Name), c.HMC.VIOS.HomeDir); err != nil { + return fmt.Errorf("error scp file: %v", err) + } + log.Logger.Info("scp discovery file to vios done") + + for _, agent := range agents { + lparID, err := c.HMC.GetLPARID(c.PowerVC.Host, agent.PartitionName) + if err != nil { + return fmt.Errorf("error retrieving lpar id for host: %s, error: %v", agent.PartitionName, err) + } + log.Logger.Infof("%s lparID retrieved for agent: %s", lparID, agent.Name) + + vhost, err := c.HMC.GetVHOST(lparID) + if err != nil { + return fmt.Errorf("error retrieving vhost: %v", err) + } + log.Logger.Infof("%s vhost retrieved for agent: %s", vhost, agent.Name) + + voptName := fmt.Sprintf("%s-agent", agent.PartitionName) + if err = c.HMC.CreateVOpt(voptName, discoveryISOFile(c.Name)); err != nil { + return err + } + log.Logger.Infof("%s vopt created for agent %s", voptName, agent.Name) + + if err = c.HMC.MapVOptToVTOpt(vhost, voptName); err != nil { + return fmt.Errorf("error map vtopt to vopt: %v", err) + } + log.Logger.Infof("mounted iso on agent %s", agent.Name) + + if err = c.HMC.SetupBootString(c.PowerVC.Host, agent.PartitionName); err != nil { + return err + } + log.Logger.Infof("boot_string configured for %s", agent.Name) + } + + return nil +} + +func (c Cluster) ApproveAgents(agents []powervc.Agent) error { + var currentlyApproved int + f := func() (bool, error) { + + out, e, err := util.ExecuteCommand("oc", []string{"get", "agents", "-n", c.getCPNamespace(), "-o", "json"}) + if err != nil || e != "" { + return false, fmt.Errorf("error get agents, e: %v, err: %v", e, err) + } + var resp map[string]interface{} + if err = json.Unmarshal([]byte(out), &resp); err != nil { + return false, fmt.Errorf("error unmarshal agent list resp: %v", err) + } + + approveAgent := func(name, hostName string) error { + patchCont := fmt.Sprintf("{\"spec\":{\"approved\":true, \"hostname\": \"%s\"}}", hostName) + _, e, err = util.ExecuteCommand("oc", []string{"patch", "agent", name, "-n", c.getCPNamespace(), "-p", patchCont, "--type", "merge"}) + if err != nil || e != "" { + return fmt.Errorf("error approving primary agent, e: %v, err: %v", e, err) + } + return nil + } + + for _, a := range resp["items"].([]interface{}) { + agent := a.(map[string]interface{}) + approved := agent["spec"].(map[string]interface{})["approved"].(bool) + + if !approved { + rName := agent["metadata"].(map[string]interface{})["name"].(string) + log.Logger.Infof("agent: %v", agent) + inventory := agent["status"].(map[string]interface{})["inventory"] + if inventory == nil { + // still inventory not collected for the agent + continue + } + nwInterfaces := inventory.(map[string]interface{})["interfaces"].([]interface{}) + mac := nwInterfaces[0].(map[string]interface{})["macAddress"].(string) + for _, ag := range agents { + if ag.MAC == mac { + if err = approveAgent(rName, ag.Name); err != nil { + return false, fmt.Errorf("error approving agent %s: %v", rName, err) + } + log.Logger.Infof("Approved agent %s", rName) + currentlyApproved += 1 + break + } + } + } + } + if currentlyApproved < c.NodeCount { + log.Logger.Infof("still agents are not approved, currently approved %v", currentlyApproved) + return false, nil + } + + return true, nil + } + if err := wait.PollImmediate(time.Minute*1, time.Minute*30, f); err != nil { + return fmt.Errorf("error approving agents %v", err) + } + + return nil +} + +func (c Cluster) ScaleNodePool() error { + _, e, err := util.ExecuteCommand("oc", []string{"scale", "np", c.Name, "-n", c.Namespace, "--replicas", "2"}) + if err != nil || e != "" { + return fmt.Errorf("error scaling node pool, e: %v, err: %v", e, err) + } + return nil +} + +func (c Cluster) DownloadKubeConfig() error { + f, err := os.Create(util.KubeConfigFile(c.Name)) + if err != nil { + return fmt.Errorf("error creating kubeconfig file: %v", err) + } + + out, e, err := util.ExecuteCommand("hypershift", []string{"create", "kubeconfig", "--name", c.Name}) + if e != "" || err != nil { + return fmt.Errorf("error retrieving kubeconfig, e: %s, err: %v", e, err) + } + + _, err = f.Write([]byte(out)) + if err != nil { + return fmt.Errorf("error writing kubeconfig content to file: %v", err) + } + + return nil +} + +func (c Cluster) SetupIngressControllerNodeSelector(agentName string) error { + args := []string{"patch", "ingresscontroller", "default", "-n", "openshift-ingress-operator", "-p", fmt.Sprintf(`{"spec": {"nodePlacement": {"nodeSelector": { "matchLabels": { "kubernetes.io/hostname": "%s"}}, "tolerations": [{ "effect": "NoSchedule", "key": "kubernetes.io/hostname", "operator": "Exists"}]}}}`, agentName), "--type=merge", fmt.Sprintf("--kubeconfig=%s", util.KubeConfigFile(c.Name))} + _, e, err := util.ExecuteCommand("oc", args) + if e != "" || err != nil { + return fmt.Errorf("error configuring ingresscontroller node selector on agent cluster, e: %s, err: %v", e, err) + } + + return nil +} + +func (c Cluster) MonitorHC() error { + _, e, err := util.ExecuteCommand("oc", []string{"wait", "hc", c.Name, "-n", c.Namespace, "--for=condition=ClusterVersionAvailable=True", "--timeout=30m"}) + if err != nil || e != "" { + return fmt.Errorf("error wait for hosted cluster to reach completed state, e: %v, err: %v", e, err) + } + return nil +} + +func (c Cluster) DescaleNodePool() error { + _, e, err := util.ExecuteCommand("oc", []string{"scale", "np", c.Name, "-n", c.Namespace, "--replicas", "0"}) + if err != nil || e != "" { + return fmt.Errorf("error descaling node pool, e: %v, err: %v", e, err) + } + return nil +} + +func (c Cluster) CleanupISOsInVIOS() interface{} { + viosSSHClient, err := util.CreateSSHClient(c.HMC.VIOS.IP, c.HMC.VIOS.UserName, c.HMC.VIOS.Password) + if err != nil { + return fmt.Errorf("error create ssh client: %v", err) + } + + isoPath := fmt.Sprintf("%s/%s", c.HMC.VIOS.HomeDir, c.Name) + rmISOCommand := fmt.Sprintf("rm -f %s", isoPath) + if _, _, err = util.ExecuteRemoteCommand(viosSSHClient, rmISOCommand); err != nil { + return fmt.Errorf("error executing command to remove iso: %v", err) + } + + return nil +} + +func (c Cluster) DestroyHC() error { + _, e, err := util.ExecuteCommand("hypershift", []string{"destroy", "cluster", "agent", "--name", c.Name}) + if err != nil || e != "" { + return fmt.Errorf("error destroying hosted cluster, e: %v, err: %v", e, err) + } + return nil +} + +func (c Cluster) RemoveCISDNSRecords() error { + deleteDNSRecord := func(name string) error { + var dnsRecordID string + var err error + dnsRecordID, err = c.IBMCloud.GetDNSRecordID(name) + if err != nil { + return err + } + if err = c.IBMCloud.DeleteDNSRecord(dnsRecordID); err != nil { + return err + } + return nil + } + + if err := deleteDNSRecord(fmt.Sprintf("api.%s", c.Name)); err != nil { + return fmt.Errorf("error deleteing dns record %s, err: %v", fmt.Sprintf("api.%s", c.Name), err) + } + + if err := deleteDNSRecord(fmt.Sprintf("api-int.%s", c.Name)); err != nil { + return fmt.Errorf("error deleteing dns record %s, err: %v", fmt.Sprintf("api.%s", c.Name), err) + } + + if err := deleteDNSRecord(fmt.Sprintf("*.apps.%s", c.Name)); err != nil { + return fmt.Errorf("error deleteing dns record %s, err: %v", fmt.Sprintf("api.%s", c.Name), err) + } + + return nil +} + +func (c Cluster) CleanupHCManifestDir() error { + return os.RemoveAll(util.GetManifestDir(c.Name)) +} diff --git a/util/util.go b/util/util.go new file mode 100644 index 0000000..b932917 --- /dev/null +++ b/util/util.go @@ -0,0 +1,124 @@ +package util + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/bramvdbogaerde/go-scp" + "github.com/ppc64le-cloud/hypershift-agent-automation/log" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" + "os" + "os/exec" + "strings" +) + +var GetManifestDir = func(hcName string) string { return fmt.Sprintf(".%s", hcName) } +var KubeConfigFile = func(hcName string) string { return fmt.Sprintf("%s/kubeconfig", GetManifestDir(hcName)) } +var GenerateFlavourID = func(hcName string) string { return fmt.Sprintf("%s-flavor") } + +func CreateSSHClient(host, username, password string) (*ssh.Client, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + hostKeyCallback, err := knownhosts.New(fmt.Sprintf("%s/.ssh/known_hosts", homeDir)) + if err != nil { + return nil, err + } + + config := &ssh.ClientConfig{ + User: username, + Auth: []ssh.AuthMethod{ + ssh.Password(password), + }, + HostKeyCallback: hostKeyCallback, + } + conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:22", host), config) + if err != nil { + return nil, fmt.Errorf("error establishing ssh connection to %s %v", host, err) + } + + return conn, nil +} + +func ExecuteRemoteCommand(client *ssh.Client, command string) (string, string, error) { + session, err := client.NewSession() + if err != nil { + return "", "", fmt.Errorf("error creating ssh session %v", err) + } + var outBuff, errBuff bytes.Buffer + session.Stdout = &outBuff + session.Stderr = &errBuff + + if err = session.Run(command); err != nil { + return "", "", fmt.Errorf("error running command %s, %v", command, err) + } + + return outBuff.String(), errBuff.String(), nil +} + +func SCPFile(sshClient *ssh.Client, file, srcDir, destDir string) error { + client, err := scp.NewClientBySSH(sshClient) + if err != nil { + return fmt.Errorf("error creating new ssh session from existing connection: %v", err) + } + srcFilePath := fmt.Sprintf("%s/%s", srcDir, file) + f, err := os.Open(srcFilePath) + if err != nil { + return fmt.Errorf("error opening source file: %v", err) + } + defer client.Close() + log.Logger.Infof("scp file %s started", srcFilePath) + return client.CopyFromFile(context.Background(), *f, fmt.Sprintf("%s/%s", destDir, file), "0644") +} + +func ExecuteCommand(command string, args []string) (string, string, error) { + cmd := exec.Command(command, args...) + + var out strings.Builder + var e strings.Builder + cmd.Stdout = &out + cmd.Stderr = &e + cmd.Env = append(os.Environ()) + + err := cmd.Run() + + return out.String(), e.String(), err +} + +func GetAbsoluteTemplatePath(templateName string) (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + return fmt.Sprintf("%s/templates/%s", cwd, templateName), nil +} + +func GetCISDomainID(baseDomain string) (string, error) { + args := []string{"cis", "domains", "--output", "json"} + out, e, err := ExecuteCommand("ibmcloud", args) + if err != nil || e != "" { + return "", fmt.Errorf("error listing ibmcloud domains: %v, e: %v", err, e) + } + + domainList := make([]map[string]interface{}, 0) + if err = json.Unmarshal([]byte(out), &domainList); err != nil { + return "", err + } + if len(domainList) < 0 { + return "", fmt.Errorf("%s domain not exist", baseDomain) + } + + var domainID string + for _, domain := range domainList { + domainName := domain["name"].(string) + if domainName == baseDomain { + domainID = domain["id"].(string) + } + } + + return domainID, nil +}