diff --git a/.pipelines/e2e.yml b/.pipelines/e2e.yml index 90edc5c18ed..068d2b60df1 100644 --- a/.pipelines/e2e.yml +++ b/.pipelines/e2e.yml @@ -30,7 +30,7 @@ jobs: - script: | set -xe sudo rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - sudo dnf install -y openvpn make podman + sudo dnf install -y openvpn make podman jq displayName: Setup (Container) target: container @@ -84,6 +84,12 @@ jobs: . ./hack/e2e/run-rp-and-e2e.sh hack/get-admin-kubeconfig.sh /subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$CLUSTER/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER >admin.kubeconfig + displayName: Get admin kubeconfig for must-gather + condition: failed() + # must-gather collection must be run inside the container so it can access the VPN + - script: | + export CI=true + . ./hack/e2e/run-rp-and-e2e.sh export KUBECONFIG=admin.kubeconfig wget -nv https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/$(OpenShiftVersion)/openshift-client-linux-$(OpenShiftVersion).tar.gz @@ -91,6 +97,7 @@ jobs: ./oc adm must-gather tar cf must-gather.tar.gz must-gather.local.* displayName: Collect must-gather + target: container condition: failed() - publish: must-gather.tar.gz artifact: must-gather diff --git a/cmd/aro/update_ocp_versions.go b/cmd/aro/update_ocp_versions.go index 43e9a72a522..2b0d60d3893 100644 --- a/cmd/aro/update_ocp_versions.go +++ b/cmd/aro/update_ocp_versions.go @@ -21,6 +21,29 @@ import ( "github.com/Azure/ARO-RP/pkg/util/version" ) +func getInstallerImageDigests(envKey string) (map[string]string, error) { + var installerImageDigests map[string]string + var err error + + jsonData := []byte(os.Getenv(envKey)) + + // For Azure DevOps pipelines, the JSON data is Base64-encoded + // since it's embedded in JSON-formatted build artifacts. But + // let's not force that on local development mode. + if !env.IsLocalDevelopmentMode() { + jsonData, err = base64.StdEncoding.DecodeString(string(jsonData)) + if err != nil { + return nil, fmt.Errorf("%s: Failed to decode base64: %v", envKey, err) + } + } + + if err = json.Unmarshal(jsonData, &installerImageDigests); err != nil { + return nil, fmt.Errorf("%s: Failed to parse JSON: %v", envKey, err) + } + + return installerImageDigests, nil +} + func getLatestOCPVersions(ctx context.Context, log *logrus.Entry) ([]api.OpenShiftVersion, error) { env, err := env.NewCoreForCI(ctx, log) if err != nil { @@ -36,20 +59,9 @@ func getLatestOCPVersions(ctx context.Context, log *logrus.Entry) ([]api.OpenShi // the aro-installer wrapper digest. This allows us to utilize // Azure Safe Deployment Practices (SDP) instead of pushing the // version tag and deploying to all regions at once. - var installerImageDigests map[string]string - jsonData := []byte(os.Getenv("INSTALLER_IMAGE_DIGESTS")) - - // For Azure DevOps pipelines, the JSON data is Base64-encoded - // since it's embedded in JSON-formatted build artifacts. But - // let's not force that on local development mode. - if !env.IsLocalDevelopmentMode() { - jsonData, err = base64.StdEncoding.DecodeString(string(jsonData)) - if err != nil { - return nil, fmt.Errorf("INSTALLER_IMAGE_DIGESTS: Failed to decode base64: %v", err) - } - } - if err = json.Unmarshal(jsonData, &installerImageDigests); err != nil { - return nil, fmt.Errorf("INSTALLER_IMAGE_DIGESTS: Failed to parse JSON: %v", err) + installerImageDigests, err := getInstallerImageDigests("INSTALLER_IMAGE_DIGESTS") + if err != nil { + return nil, err } for _, vers := range version.HiveInstallStreams { diff --git a/pkg/cluster/generate.go b/pkg/cluster/generate.go index 0076f15d61b..2eb197fb00b 100644 --- a/pkg/cluster/generate.go +++ b/pkg/cluster/generate.go @@ -4,4 +4,6 @@ package cluster // Licensed under the Apache License 2.0. //go:generate go run ../../vendor/github.com/golang/mock/mockgen -destination=../util/mocks/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/$GOPACKAGE Interface +//go:generate go run ../../vendor/github.com/golang/mock/mockgen -destination=../util/mocks/samplesclient/versioned.go github.com/openshift/client-go/samples/clientset/versioned Interface +//go:generate go run ../../vendor/github.com/golang/mock/mockgen -destination=../util/mocks/samples/samples.go github.com/openshift/client-go/samples/clientset/versioned/typed/samples/v1 SamplesV1Interface,ConfigInterface //go:generate go run ../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../util/mocks/$GOPACKAGE/$GOPACKAGE.go diff --git a/pkg/cluster/samples.go b/pkg/cluster/samples.go index 61077b4ef5f..befa7962422 100644 --- a/pkg/cluster/samples.go +++ b/pkg/cluster/samples.go @@ -8,6 +8,7 @@ import ( configv1 "github.com/openshift/api/config/v1" operatorv1 "github.com/openshift/api/operator/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" ) @@ -19,17 +20,22 @@ func (m *manager) disableSamples(ctx context.Context) error { return nil } - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - c, err := m.samplescli.SamplesV1().Configs().Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return err - } + return retry.OnError( + retry.DefaultRetry, + func(err error) bool { + return errors.IsConflict(err) || errors.IsNotFound(err) + }, + func() error { + c, err := m.samplescli.SamplesV1().Configs().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } - c.Spec.ManagementState = operatorv1.Removed + c.Spec.ManagementState = operatorv1.Removed - _, err = m.samplescli.SamplesV1().Configs().Update(ctx, c, metav1.UpdateOptions{}) - return err - }) + _, err = m.samplescli.SamplesV1().Configs().Update(ctx, c, metav1.UpdateOptions{}) + return err + }) } // disableOperatorHubSources disables operator hub sources if there's no diff --git a/pkg/cluster/samples_test.go b/pkg/cluster/samples_test.go new file mode 100644 index 00000000000..1fb4ecb5981 --- /dev/null +++ b/pkg/cluster/samples_test.go @@ -0,0 +1,105 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "errors" + "testing" + + "github.com/golang/mock/gomock" + operatorv1 "github.com/openshift/api/operator/v1" + samplesv1 "github.com/openshift/api/samples/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/Azure/ARO-RP/pkg/api" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" + mock_samples "github.com/Azure/ARO-RP/pkg/util/mocks/samples" + mock_samplesclient "github.com/Azure/ARO-RP/pkg/util/mocks/samplesclient" + utilerror "github.com/Azure/ARO-RP/test/util/error" +) + +func Test_manager_disableSamples(t *testing.T) { + ctx := context.Background() + samplesConfig := &samplesv1.Config{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: samplesv1.ConfigSpec{}, + Status: samplesv1.ConfigStatus{}, + } + tests := []struct { + name string + samplesConfig *samplesv1.Config + samplesCRGetError error + samplesCRUpdateError error + expectedMinNumberOfGetCalls int + expectedMaxNumberOfGetCalls int + wantErr string + }{ + { + name: "samples cr is found and updated", + samplesConfig: samplesConfig, + expectedMinNumberOfGetCalls: 1, + expectedMaxNumberOfGetCalls: 1, + wantErr: "", + }, + { + name: "samples cr is not found and retried", + samplesCRGetError: kerrors.NewNotFound(schema.GroupResource{}, "samples"), + expectedMinNumberOfGetCalls: 2, + expectedMaxNumberOfGetCalls: 15, + wantErr: " \"samples\" not found", + }, + { + name: "samples cr update is conflicting and retried", + samplesConfig: samplesConfig, + expectedMinNumberOfGetCalls: 2, + expectedMaxNumberOfGetCalls: 15, + samplesCRUpdateError: kerrors.NewConflict(schema.GroupResource{}, "samples", errors.New("conflict")), + wantErr: "Operation cannot be fulfilled on \"samples\": conflict", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + env := mock_env.NewMockInterface(controller) + samplescli := mock_samplesclient.NewMockInterface(controller) + samplesInterface := mock_samples.NewMockSamplesV1Interface(controller) + configInterface := mock_samples.NewMockConfigInterface(controller) + + env.EXPECT().IsLocalDevelopmentMode().Return(false) + samplescli.EXPECT().SamplesV1().AnyTimes().Return(samplesInterface) + samplesInterface.EXPECT().Configs().AnyTimes().Return(configInterface) + configInterface.EXPECT().Get(gomock.Any(), "cluster", metav1.GetOptions{}). + MinTimes(tt.expectedMinNumberOfGetCalls). + MaxTimes(tt.expectedMaxNumberOfGetCalls). + Return(tt.samplesConfig, tt.samplesCRGetError) + + if tt.samplesConfig != nil { + samplesConfig.Spec.ManagementState = operatorv1.Removed + configInterface.EXPECT().Update(gomock.Any(), samplesConfig, metav1.UpdateOptions{}).AnyTimes().Return(samplesConfig, tt.samplesCRUpdateError) + } + + m := &manager{ + env: env, + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + ResourceGroupID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/clusterRGName", + }, + }, + }, + }, + samplescli: samplescli, + } + + err := m.disableSamples(ctx) + utilerror.AssertErrorMessage(t, err, tt.wantErr) + }) + } +} diff --git a/pkg/deploy/assets/gateway-production.json b/pkg/deploy/assets/gateway-production.json index 2112428c0d4..3ce7be4d58a 100644 --- a/pkg/deploy/assets/gateway-production.json +++ b/pkg/deploy/assets/gateway-production.json @@ -309,7 +309,7 @@ "autoUpgradeMinorVersion": true, "settings": {}, "protectedSettings": { - "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'ACRRESOURCEID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('acrResourceId')),''')\n','AZURECLOUDNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureCloudName')),''')\n','AZURESECPACKQUALYSURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackQualysUrl')),''')\n','AZURESECPACKVSATENANTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackVSATenantId')),''')\n','DATABASEACCOUNTNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('databaseAccountName')),''')\n','DBTOKENCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenClientId')),''')\n','DBTOKENURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenUrl')),''')\n','MDMFRONTENDURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdmFrontendUrl')),''')\n','MDSDENVIRONMENT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdsdEnvironment')),''')\n','FLUENTBITIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fluentbitImage')),''')\n','GATEWAYMDSDCONFIGVERSION=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayMdsdConfigVersion')),''')\n','GATEWAYDOMAINS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayDomains')),''')\n','GATEWAYFEATURES=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayFeatures')),''')\n','KEYVAULTDNSSUFFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultDNSSuffix')),''')\n','KEYVAULTPREFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultPrefix')),''')\n','RPIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpImage')),''')\n','RPMDMACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdmAccount')),''')\n','RPMDSDACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdAccount')),''')\n','RPMDSDNAMESPACE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdNamespace')),''')\n','MDMIMAGE=''/genevamdm:2.2023.609.2051-821f47-20230706t0953''\n','LOCATION=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().location),''')\n','SUBSCRIPTIONID=$(base64 -d \u003c\u003c\u003c''',base64(subscription().subscriptionId),''')\n','RESOURCEGROUPNAME=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().name),''')\n','\n',base64ToString('IyEvYmluL2Jhc2gKCmVjaG8gInNldHRpbmcgc3NoIHBhc3N3b3JkIGF1dGhlbnRpY2F0aW9uIgojIFdlIG5lZWQgdG8gbWFudWFsbHkgc2V0IFBhc3N3b3JkQXV0aGVudGljYXRpb24gdG8gdHJ1ZSBpbiBvcmRlciBmb3IgdGhlIFZNU1MgQWNjZXNzIEpJVCB0byB3b3JrCnNlZCAtaSAncy9QYXNzd29yZEF1dGhlbnRpY2F0aW9uIG5vL1Bhc3N3b3JkQXV0aGVudGljYXRpb24geWVzL2cnIC9ldGMvc3NoL3NzaGRfY29uZmlnCnN5c3RlbWN0bCByZWxvYWQgc3NoZC5zZXJ2aWNlCgplY2hvICJydW5uaW5nIFJIVUkgZml4Igp5dW0gdXBkYXRlIC15IC0tZGlzYWJsZXJlcG89JyonIC0tZW5hYmxlcmVwbz0ncmh1aS1taWNyb3NvZnQtYXp1cmUqJwoKZWNobyAicnVubmluZyB5dW0gdXBkYXRlIgp5dW0gLXkgLXggV0FMaW51eEFnZW50IC14IFdBTGludXhBZ2VudC11ZGV2IHVwZGF0ZSAtLWFsbG93ZXJhc2luZwoKZWNobyAiZXh0ZW5kaW5nIHBhcnRpdGlvbiB0YWJsZSIKIyBMaW51eCBibG9jayBkZXZpY2VzIGFyZSBpbmNvbnNpc3RlbnRseSBuYW1lZAojIGl0J3MgZGlmZmljdWx0IHRvIHRpZSB0aGUgbHZtIHB2IHRvIHRoZSBwaHlzaWNhbCBkaXNrIHVzaW5nIC9kZXYvZGlzayBmaWxlcywgd2hpY2ggaXMgd2h5IGx2cyBpcyB1c2VkIGhlcmUKcGh5c2ljYWxEaXNrPSIkKGx2cyAtbyBkZXZpY2VzIC1hIHwgaGVhZCAtbjIgfCB0YWlsIC1uMSB8IGN1dCAtZCAnICcgLWYgMyB8IGN1dCAtZCBcKCAtZiAxIHwgdHIgLWQgJ1s6ZGlnaXQ6XScpIgpncm93cGFydCAiJHBoeXNpY2FsRGlzayIgMgoKZWNobyAiZXh0ZW5kaW5nIGZpbGVzeXN0ZW1zIgpsdmV4dGVuZCAtbCArMjAlRlJFRSAvZGV2L3Jvb3R2Zy9yb290bHYKeGZzX2dyb3dmcyAvCgpsdmV4dGVuZCAtbCArMTAwJUZSRUUgL2Rldi9yb290dmcvdmFybHYKeGZzX2dyb3dmcyAvdmFyCgpycG0gLS1pbXBvcnQgaHR0cHM6Ly9kbC5mZWRvcmFwcm9qZWN0Lm9yZy9wdWIvZXBlbC9SUE0tR1BHLUtFWS1FUEVMLTgKcnBtIC0taW1wb3J0IGh0dHBzOi8vcGFja2FnZXMubWljcm9zb2Z0LmNvbS9rZXlzL21pY3Jvc29mdC5hc2MKCmZvciBhdHRlbXB0IGluIHsxLi41fTsgZG8KICB5dW0gLXkgaW5zdGFsbCBodHRwczovL2RsLmZlZG9yYXByb2plY3Qub3JnL3B1Yi9lcGVsL2VwZWwtcmVsZWFzZS1sYXRlc3QtOC5ub2FyY2gucnBtICYmIGJyZWFrCiAgaWYgW1sgJHthdHRlbXB0fSAtbHQgNSBdXTsgdGhlbiBzbGVlcCAxMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmVjaG8gImNvbmZpZ3VyaW5nIGxvZ3JvdGF0ZSIKY2F0ID4vZXRjL2xvZ3JvdGF0ZS5jb25mIDw8J0VPRicKIyBzZWUgIm1hbiBsb2dyb3RhdGUiIGZvciBkZXRhaWxzCiMgcm90YXRlIGxvZyBmaWxlcyB3ZWVrbHkKd2Vla2x5CgojIGtlZXAgMiB3ZWVrcyB3b3J0aCBvZiBiYWNrbG9ncwpyb3RhdGUgMgoKIyBjcmVhdGUgbmV3IChlbXB0eSkgbG9nIGZpbGVzIGFmdGVyIHJvdGF0aW5nIG9sZCBvbmVzCmNyZWF0ZQoKIyB1c2UgZGF0ZSBhcyBhIHN1ZmZpeCBvZiB0aGUgcm90YXRlZCBmaWxlCmRhdGVleHQKCiMgdW5jb21tZW50IHRoaXMgaWYgeW91IHdhbnQgeW91ciBsb2cgZmlsZXMgY29tcHJlc3NlZApjb21wcmVzcwoKIyBSUE0gcGFja2FnZXMgZHJvcCBsb2cgcm90YXRpb24gaW5mb3JtYXRpb24gaW50byB0aGlzIGRpcmVjdG9yeQppbmNsdWRlIC9ldGMvbG9ncm90YXRlLmQKCiMgbm8gcGFja2FnZXMgb3duIHd0bXAgYW5kIGJ0bXAgLS0gd2UnbGwgcm90YXRlIHRoZW0gaGVyZQovdmFyL2xvZy93dG1wIHsKICAgIG1vbnRobHkKICAgIGNyZWF0ZSAwNjY0IHJvb3QgdXRtcAogICAgICAgIG1pbnNpemUgMU0KICAgIHJvdGF0ZSAxCn0KCi92YXIvbG9nL2J0bXAgewogICAgbWlzc2luZ29rCiAgICBtb250aGx5CiAgICBjcmVhdGUgMDYwMCByb290IHV0bXAKICAgIHJvdGF0ZSAxCn0KRU9GCgplY2hvICJjb25maWd1cmluZyB5dW0gcmVwb3NpdG9yeSBhbmQgcnVubmluZyB5dW0gdXBkYXRlIgpjYXQgPi9ldGMveXVtLnJlcG9zLmQvYXp1cmUucmVwbyA8PCdFT0YnClthenVyZS1jbGldCm5hbWU9YXp1cmUtY2xpCmJhc2V1cmw9aHR0cHM6Ly9wYWNrYWdlcy5taWNyb3NvZnQuY29tL3l1bXJlcG9zL2F6dXJlLWNsaQplbmFibGVkPXllcwpncGdjaGVjaz15ZXMKClthenVyZWNvcmVdCm5hbWU9YXp1cmVjb3JlCmJhc2V1cmw9aHR0cHM6Ly9wYWNrYWdlcy5taWNyb3NvZnQuY29tL3l1bXJlcG9zL2F6dXJlY29yZQplbmFibGVkPXllcwpncGdjaGVjaz1ubwpFT0YKCnNlbWFuYWdlIGZjb250ZXh0IC1hIC10IHZhcl9sb2dfdCAiL3Zhci9sb2cvam91cm5hbCgvLiopPyIKbWtkaXIgLXAgL3Zhci9sb2cvam91cm5hbAoKZm9yIGF0dGVtcHQgaW4gezEuLjV9OyBkbwogIHl1bSAteSBpbnN0YWxsIGNsYW1hdiBhenNlYy1jbGFtYXYgYXpzZWMtbW9uaXRvciBhenVyZS1jbGkgYXp1cmUtbWRzZCBhenVyZS1zZWN1cml0eSBwb2RtYW4tZG9ja2VyIG9wZW5zc2wtcGVybCBweXRob24zICYmIGJyZWFrCiAgIyBoYWNrIC0gd2UgYXJlIGluc3RhbGxpbmcgcHl0aG9uMyBvbiBob3N0cyBkdWUgdG8gYW4gaXNzdWUgd2l0aCBBenVyZSBMaW51eCBFeHRlbnNpb25zIGh0dHBzOi8vZ2l0aHViLmNvbS9BenVyZS9henVyZS1saW51eC1leHRlbnNpb25zL3B1bGwvMTUwNQogIGlmIFtbICR7YXR0ZW1wdH0gLWx0IDUgXV07IHRoZW4gc2xlZXAgMTA7IGVsc2UgZXhpdCAxOyBmaQpkb25lCgplY2hvICJhcHBseWluZyBmaXJld2FsbCBydWxlcyIKIyBodHRwczovL2FjY2Vzcy5yZWRoYXQuY29tL3NlY3VyaXR5L2N2ZS9jdmUtMjAyMC0xMzQwMQpjYXQgPi9ldGMvc3lzY3RsLmQvMDItZGlzYWJsZS1hY2NlcHQtcmEuY29uZiA8PCdFT0YnCm5ldC5pcHY2LmNvbmYuYWxsLmFjY2VwdF9yYT0wCkVPRgoKY2F0ID4vZXRjL3N5c2N0bC5kLzAxLWRpc2FibGUtY29yZS5jb25mIDw8J0VPRicKa2VybmVsLmNvcmVfcGF0dGVybiA9IHwvYmluL3RydWUKRU9GCnN5c2N0bCAtLXN5c3RlbQoKZmlyZXdhbGwtY21kIC0tYWRkLXBvcnQ9ODAvdGNwIC0tcGVybWFuZW50CmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTgwODEvdGNwIC0tcGVybWFuZW50CmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTQ0My90Y3AgLS1wZXJtYW5lbnQKCmVjaG8gImxvZ2dpbmcgaW50byBwcm9kIGFjciIKZXhwb3J0IEFaVVJFX0NMT1VEX05BTUU9JEFaVVJFQ0xPVUROQU1FCmF6IGxvZ2luIC1pIC0tYWxsb3ctbm8tc3Vic2NyaXB0aW9ucwoKIyBUaGUgbWFuYWdlZCBpZGVudGl0eSB0aGF0IHRoZSBWTSBydW5zIGFzIG9ubHkgaGFzIGEgc2luZ2xlIHJvbGVhc3NpZ25tZW50LgojIFRoaXMgcm9sZSBhc3NpZ25tZW50IGlzIEFDUlB1bGwgd2hpY2ggaXMgbm90IG5lY2Vzc2FyaWx5IHByZXNlbnQgaW4gdGhlCiMgc3Vic2NyaXB0aW9uIHdlJ3JlIGRlcGxveWluZyBpbnRvLiAgSWYgdGhlIGlkZW50aXR5IGRvZXMgbm90IGhhdmUgYW55CiMgcm9sZSBhc3NpZ25tZW50cyBzY29wZWQgb24gdGhlIHN1YnNjcmlwdGlvbiB3ZSdyZSBkZXBsb3lpbmcgaW50bywgaXQgd2lsbAojIG5vdCBzaG93IG9uIGF6IGxvZ2luIC1pLCB3aGljaCBpcyB3aHkgdGhlIGJlbG93IGxpbmUgaXMgY29tbWVudGVkLgojIGF6IGFjY291bnQgc2V0IC1zICIkU1VCU0NSSVBUSU9OSUQiCgojIFN1cHByZXNzIGVtdWxhdGlvbiBvdXRwdXQgZm9yIHBvZG1hbiBpbnN0ZWFkIG9mIGRvY2tlciBmb3IgYXogYWNyIGNvbXBhdGFiaWxpdHkKbWtkaXIgLXAgL2V0Yy9jb250YWluZXJzLwp0b3VjaCAvZXRjL2NvbnRhaW5lcnMvbm9kb2NrZXIKCm1rZGlyIC1wIC9yb290Ly5kb2NrZXIKUkVHSVNUUllfQVVUSF9GSUxFPS9yb290Ly5kb2NrZXIvY29uZmlnLmpzb24gYXogYWNyIGxvZ2luIC0tbmFtZSAiJChzZWQgLWUgJ3N8LiovfHwnIDw8PCIkQUNSUkVTT1VSQ0VJRCIpIgoKTURNSU1BR0U9IiR7UlBJTUFHRSUlLyp9LyR7TURNSU1BR0UjIyovfSIKZG9ja2VyIHB1bGwgIiRNRE1JTUFHRSIKZG9ja2VyIHB1bGwgIiRSUElNQUdFIgpkb2NrZXIgcHVsbCAiJEZMVUVOVEJJVElNQUdFIgoKYXogbG9nb3V0CgplY2hvICJjb25maWd1cmluZyBmbHVlbnRiaXQgc2VydmljZSIKbWtkaXIgLXAgL2V0Yy9mbHVlbnRiaXQvCm1rZGlyIC1wIC92YXIvbGliL2ZsdWVudAoKY2F0ID4vZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZiA8PCdFT0YnCltJTlBVVF0KCU5hbWUgc3lzdGVtZAoJVGFnIGpvdXJuYWxkCglTeXN0ZW1kX0ZpbHRlciBfQ09NTT1hcm8KCURCIC92YXIvbGliL2ZsdWVudC9qb3VybmFsZGIKCltGSUxURVJdCglOYW1lIG1vZGlmeQoJTWF0Y2ggam91cm5hbGQKCVJlbW92ZV93aWxkY2FyZCBfCglSZW1vdmUgVElNRVNUQU1QCgpbT1VUUFVUXQoJTmFtZSBmb3J3YXJkCglNYXRjaCAqCglQb3J0IDI5MjMwCkVPRgoKZWNobyAiRkxVRU5UQklUSU1BR0U9JEZMVUVOVEJJVElNQUdFIiA+L2V0Yy9zeXNjb25maWcvZmx1ZW50Yml0CgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vZmx1ZW50Yml0LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldApTdGFydExpbWl0SW50ZXJ2YWxTZWM9MAoKW1NlcnZpY2VdClJlc3RhcnRTZWM9MXMKRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL2ZsdWVudGJpdApFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1zZWN1cml0eS1vcHQgbGFiZWw9ZGlzYWJsZSBcCiAgLS1lbnRyeXBvaW50IC9vcHQvdGQtYWdlbnQtYml0L2Jpbi90ZC1hZ2VudC1iaXQgXAogIC0tbmV0PWhvc3QgXAogIC0taG9zdG5hbWUgJUggXAogIC0tbmFtZSAlTiBcCiAgLS1ybSBcCiAgLS1jYXAtZHJvcCBuZXRfcmF3IFwKICAtdiAvZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZjovZXRjL2ZsdWVudGJpdC9mbHVlbnRiaXQuY29uZiBcCiAgLXYgL3Zhci9saWIvZmx1ZW50Oi92YXIvbGliL2ZsdWVudDp6IFwKICAtdiAvdmFyL2xvZy9qb3VybmFsOi92YXIvbG9nL2pvdXJuYWw6cm8gXAogIC12IC9ldGMvbWFjaGluZS1pZDovZXRjL21hY2hpbmUtaWQ6cm8gXAogICRGTFVFTlRCSVRJTUFHRSBcCiAgLWMgL2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmYKCkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wICVOClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9NQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKZWNobyAiY29uZmlndXJpbmcgbWRtIHNlcnZpY2UiCmNhdCA+L2V0Yy9zeXNjb25maWcvbWRtIDw8RU9GCk1ETUZST05URU5EVVJMPSckTURNRlJPTlRFTkRVUkwnCk1ETUlNQUdFPSckTURNSU1BR0UnCk1ETVNPVVJDRUVOVklST05NRU5UPSckTE9DQVRJT04nCk1ETVNPVVJDRVJPTEU9Z2F0ZXdheQpNRE1TT1VSQ0VST0xFSU5TVEFOQ0U9JyQoaG9zdG5hbWUpJwpFT0YKCm1rZGlyIC92YXIvZXR3CmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9tZG0uc2VydmljZSA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL21kbQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1lbnRyeXBvaW50IC91c3Ivc2Jpbi9NZXRyaWNzRXh0ZW5zaW9uIFwKICAtLWhvc3RuYW1lICVIIFwKICAtLW5hbWUgJU4gXAogIC0tcm0gXAogIC0tY2FwLWRyb3AgbmV0X3JhdyBcCiAgLW0gMmcgXAogIC12IC9ldGMvbWRtLnBlbTovZXRjL21kbS5wZW0gXAogIC12IC92YXIvZXR3Oi92YXIvZXR3OnogXAogICRNRE1JTUFHRSBcCiAgLUNlcnRGaWxlIC9ldGMvbWRtLnBlbSBcCiAgLUZyb250RW5kVXJsICRNRE1GUk9OVEVORFVSTCBcCiAgLUxvZ2dlciBDb25zb2xlIFwKICAtTG9nTGV2ZWwgV2FybmluZyBcCiAgLVByaXZhdGVLZXlGaWxlIC9ldGMvbWRtLnBlbSBcCiAgLVNvdXJjZUVudmlyb25tZW50ICRNRE1TT1VSQ0VFTlZJUk9OTUVOVCBcCiAgLVNvdXJjZVJvbGUgJE1ETVNPVVJDRVJPTEUgXAogIC1Tb3VyY2VSb2xlSW5zdGFuY2UgJE1ETVNPVVJDRVJPTEVJTlNUQU5DRQpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlTgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIGFyby1nYXRld2F5IHNlcnZpY2UiCmNhdCA+L2V0Yy9zeXNjb25maWcvYXJvLWdhdGV3YXkgPDxFT0YKQUNSX1JFU09VUkNFX0lEPSckQUNSUkVTT1VSQ0VJRCcKREFUQUJBU0VfQUNDT1VOVF9OQU1FPSckREFUQUJBU0VBQ0NPVU5UTkFNRScKQVpVUkVfREJUT0tFTl9DTElFTlRfSUQ9JyREQlRPS0VOQ0xJRU5USUQnCkRCVE9LRU5fVVJMPSckREJUT0tFTlVSTCcKTURNX0FDQ09VTlQ9IiRSUE1ETUFDQ09VTlQiCk1ETV9OQU1FU1BBQ0U9R2F0ZXdheQpHQVRFV0FZX0RPTUFJTlM9JyRHQVRFV0FZRE9NQUlOUycKR0FURVdBWV9GRUFUVVJFUz0nJEdBVEVXQVlGRUFUVVJFUycKUlBJTUFHRT0nJFJQSU1BR0UnCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Fyby1nYXRld2F5LnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9hcm8tZ2F0ZXdheQpFeGVjU3RhcnRQcmU9LS91c3IvYmluL2RvY2tlciBybSAtZiAlTgpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1lIEFDUl9SRVNPVVJDRV9JRCBcCiAgLWUgREFUQUJBU0VfQUNDT1VOVF9OQU1FIFwKICAtZSBBWlVSRV9EQlRPS0VOX0NMSUVOVF9JRCBcCiAgLWUgREJUT0tFTl9VUkwgXAogIC1lIEdBVEVXQVlfRE9NQUlOUyBcCiAgLWUgR0FURVdBWV9GRUFUVVJFUyBcCiAgLWUgTURNX0FDQ09VTlQgXAogIC1lIE1ETV9OQU1FU1BBQ0UgXAogIC1tIDJnIFwKICAtcCA4MDo4MDgwIFwKICAtcCA4MDgxOjgwODEgXAogIC1wIDQ0Mzo4NDQzIFwKICAtdiAvcnVuL3N5c3RlbWQvam91cm5hbDovcnVuL3N5c3RlbWQvam91cm5hbCBcCiAgLXYgL3Zhci9ldHc6L3Zhci9ldHc6eiBcCiAgJFJQSU1BR0UgXAogIGdhdGV3YXkKRXhlY1N0b3A9L3Vzci9iaW4vZG9ja2VyIHN0b3AgLXQgMzYwMCAlTgpUaW1lb3V0U3RvcFNlYz0zNjAwClJlc3RhcnQ9YWx3YXlzClJlc3RhcnRTZWM9MQpTdGFydExpbWl0SW50ZXJ2YWw9MAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKY2hjb24gLVIgc3lzdGVtX3U6b2JqZWN0X3I6dmFyX2xvZ190OnMwIC92YXIvb3B0L21pY3Jvc29mdC9saW51eG1vbmFnZW50Cgpta2RpciAtcCAvdmFyL2xpYi93YWFnZW50L01pY3Jvc29mdC5BenVyZS5LZXlWYXVsdC5TdG9yZQoKZWNobyAiY29uZmlndXJpbmcgbWRzZCBhbmQgbWRtIHNlcnZpY2VzIgpmb3IgdmFyIGluICJtZHNkIiAibWRtIjsgZG8KY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Rvd25sb2FkLSR2YXItY3JlZGVudGlhbHMuc2VydmljZSA8PEVPRgpbVW5pdF0KRGVzY3JpcHRpb249UGVyaW9kaWMgJHZhciBjcmVkZW50aWFscyByZWZyZXNoCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4ZWNTdGFydD0vdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaCAkdmFyCkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2Rvd25sb2FkLSR2YXItY3JlZGVudGlhbHMudGltZXIgPDxFT0YKW1VuaXRdCkRlc2NyaXB0aW9uPVBlcmlvZGljICR2YXIgY3JlZGVudGlhbHMgcmVmcmVzaApBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbVGltZXJdCk9uQm9vdFNlYz0wbWluCk9uQ2FsZW5kYXI9MC8xMjowMDowMApBY2N1cmFjeVNlYz01cwoKW0luc3RhbGxdCldhbnRlZEJ5PXRpbWVycy50YXJnZXQKRU9GCmRvbmUKCmNhdCA+L3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggPDxFT0YKIyEvYmluL2Jhc2gKc2V0IC1ldQoKQ09NUE9ORU5UPSJcJDEiCmVjaG8gIkRvd25sb2FkIFwkQ09NUE9ORU5UIGNyZWRlbnRpYWxzIgoKVEVNUF9ESVI9XCQobWt0ZW1wIC1kKQpleHBvcnQgQVpVUkVfQ09ORklHX0RJUj1cJChta3RlbXAgLWQpCgplY2hvICJMb2dnaW5nIGludG8gQXp1cmUuLi4iClJFVFJJRVM9Mwp3aGlsZSBbICJcJFJFVFJJRVMiIC1ndCAwIF07IGRvCiAgICBpZiBheiBsb2dpbiAtaSAtLWFsbG93LW5vLXN1YnNjcmlwdGlvbnMKICAgIHRoZW4KICAgICAgICBlY2hvICJheiBsb2dpbiBzdWNjZXNzZnVsIgogICAgICAgIGJyZWFrCiAgICBlbHNlCiAgICAgICAgZWNobyAiYXogbG9naW4gZmFpbGVkLiBSZXRyeWluZy4uLiIKICAgICAgICBsZXQgUkVUUklFUy09MQogICAgICAgIHNsZWVwIDUKICAgIGZpCmRvbmUKCnRyYXAgImNsZWFudXAiIEVYSVQKCmNsZWFudXAoKSB7CiAgYXogbG9nb3V0CiAgW1sgIlwkVEVNUF9ESVIiID1+IC90bXAvLisgXV0gJiYgcm0gLXJmIFwkVEVNUF9ESVIKICBbWyAiXCRBWlVSRV9DT05GSUdfRElSIiA9fiAvdG1wLy4rIF1dICYmIHJtIC1yZiBcJEFaVVJFX0NPTkZJR19ESVIKfQoKaWYgWyAiXCRDT01QT05FTlQiID0gIm1kbSIgXTsgdGhlbgogIENVUlJFTlRfQ0VSVF9GSUxFPSIvZXRjL21kbS5wZW0iCmVsaWYgWyAiXCRDT01QT05FTlQiID0gIm1kc2QiIF07IHRoZW4KICBDVVJSRU5UX0NFUlRfRklMRT0iL3Zhci9saWIvd2FhZ2VudC9NaWNyb3NvZnQuQXp1cmUuS2V5VmF1bHQuU3RvcmUvbWRzZC5wZW0iCmVsc2UKICBlY2hvIEludmFsaWQgdXNhZ2UgJiYgZXhpdCAxCmZpCgpTRUNSRVRfTkFNRT0iZ3d5LVwke0NPTVBPTkVOVH0iCk5FV19DRVJUX0ZJTEU9IlwkVEVNUF9ESVIvXCRDT01QT05FTlQucGVtIgpmb3IgYXR0ZW1wdCBpbiB7MS4uNX07IGRvCiAgYXoga2V5dmF1bHQgc2VjcmV0IGRvd25sb2FkIC0tZmlsZSBcJE5FV19DRVJUX0ZJTEUgLS1pZCAiaHR0cHM6Ly8kS0VZVkFVTFRQUkVGSVgtZ3d5LiRLRVlWQVVMVEROU1NVRkZJWC9zZWNyZXRzL1wkU0VDUkVUX05BTUUiICYmIGJyZWFrCiAgaWYgW1sgXCRhdHRlbXB0IC1sdCA1IF1dOyB0aGVuIHNsZWVwIDEwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKaWYgWyAtZiBcJE5FV19DRVJUX0ZJTEUgXTsgdGhlbgogIGlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZHNkIiBdOyB0aGVuCiAgICBjaG93biBzeXNsb2c6c3lzbG9nIFwkTkVXX0NFUlRfRklMRQogIGVsc2UKICAgIHNlZCAtaSAtbmUgJzEsL0VORCBDRVJUSUZJQ0FURS8gcCcgXCRORVdfQ0VSVF9GSUxFCiAgZmkKICBpZiAhIGRpZmYgJE5FV19DRVJUX0ZJTEUgJENVUlJFTlRfQ0VSVF9GSUxFID4vZGV2L251bGwgMj4mMTsgdGhlbgogICAgY2htb2QgMDYwMCBcJE5FV19DRVJUX0ZJTEUKICAgIG12IFwkTkVXX0NFUlRfRklMRSBcJENVUlJFTlRfQ0VSVF9GSUxFCiAgZmkKZWxzZQogIGVjaG8gRmFpbGVkIHRvIHJlZnJlc2ggY2VydGlmaWNhdGUgZm9yIFwkQ09NUE9ORU5UICYmIGV4aXQgMQpmaQpFT0YKCmNobW9kIHUreCAvdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaAoKc3lzdGVtY3RsIGVuYWJsZSBkb3dubG9hZC1tZHNkLWNyZWRlbnRpYWxzLnRpbWVyCnN5c3RlbWN0bCBlbmFibGUgZG93bmxvYWQtbWRtLWNyZWRlbnRpYWxzLnRpbWVyCgovdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaCBtZHNkCi91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoIG1kbQpNRFNEQ0VSVElGSUNBVEVTQU49JChvcGVuc3NsIHg1MDkgLWluIC92YXIvbGliL3dhYWdlbnQvTWljcm9zb2Z0LkF6dXJlLktleVZhdWx0LlN0b3JlL21kc2QucGVtIC1ub291dCAtc3ViamVjdCB8IHNlZCAtZSAncy8uKkNOID0gLy8nKQoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3dhdGNoLW1kbS1jcmVkZW50aWFscy5zZXJ2aWNlIDw8RU9GCltVbml0XQpEZXNjcmlwdGlvbj1XYXRjaCBmb3IgY2hhbmdlcyBpbiBtZG0ucGVtIGFuZCByZXN0YXJ0cyB0aGUgbWRtIHNlcnZpY2UKCltTZXJ2aWNlXQpUeXBlPW9uZXNob3QKRXhlY1N0YXJ0PS91c3IvYmluL3N5c3RlbWN0bCByZXN0YXJ0IG1kbS5zZXJ2aWNlCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vd2F0Y2gtbWRtLWNyZWRlbnRpYWxzLnBhdGggPDxFT0YKW1BhdGhdClBhdGhNb2RpZmllZD0vZXRjL21kbS5wZW0KCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCnN5c3RlbWN0bCBlbmFibGUgd2F0Y2gtbWRtLWNyZWRlbnRpYWxzLnBhdGgKc3lzdGVtY3RsIHN0YXJ0IHdhdGNoLW1kbS1jcmVkZW50aWFscy5wYXRoCgpta2RpciAvZXRjL3N5c3RlbWQvc3lzdGVtL21kc2Quc2VydmljZS5kCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9tZHNkLnNlcnZpY2UuZC9vdmVycmlkZS5jb25mIDw8J0VPRicKW1VuaXRdCkFmdGVyPW5ldHdvcmstb25saW5lLnRhcmdldApFT0YKCmNhdCA+L2V0Yy9kZWZhdWx0L21kc2QgPDxFT0YKTURTRF9ST0xFX1BSRUZJWD0vdmFyL3J1bi9tZHNkL2RlZmF1bHQKTURTRF9PUFRJT05TPSItQSAtZCAtciBcJE1EU0RfUk9MRV9QUkVGSVgiCgpleHBvcnQgTU9OSVRPUklOR19HQ1NfRU5WSVJPTk1FTlQ9JyRNRFNERU5WSVJPTk1FTlQnCmV4cG9ydCBNT05JVE9SSU5HX0dDU19BQ0NPVU5UPSckUlBNRFNEQUNDT1VOVCcKZXhwb3J0IE1PTklUT1JJTkdfR0NTX1JFR0lPTj0nJExPQ0FUSU9OJwpleHBvcnQgTU9OSVRPUklOR19HQ1NfQVVUSF9JRF9UWVBFPUF1dGhLZXlWYXVsdApleHBvcnQgTU9OSVRPUklOR19HQ1NfQVVUSF9JRD0nJE1EU0RDRVJUSUZJQ0FURVNBTicKZXhwb3J0IE1PTklUT1JJTkdfR0NTX05BTUVTUEFDRT0nJFJQTURTRE5BTUVTUEFDRScKZXhwb3J0IE1PTklUT1JJTkdfQ09ORklHX1ZFUlNJT049JyRHQVRFV0FZTURTRENPTkZJR1ZFUlNJT04nCmV4cG9ydCBNT05JVE9SSU5HX1VTRV9HRU5FVkFfQ09ORklHX1NFUlZJQ0U9dHJ1ZQoKZXhwb3J0IE1PTklUT1JJTkdfVEVOQU5UPSckTE9DQVRJT04nCmV4cG9ydCBNT05JVE9SSU5HX1JPTEU9Z2F0ZXdheQpleHBvcnQgTU9OSVRPUklOR19ST0xFX0lOU1RBTkNFPSckKGhvc3RuYW1lKScKCmV4cG9ydCBNRFNEX01TR1BBQ0tfU09SVF9DT0xVTU5TPTEKRU9GCgojIHNldHRpbmcgTU9OSVRPUklOR19HQ1NfQVVUSF9JRF9UWVBFPUF1dGhLZXlWYXVsdCBzZWVtcyB0byBoYXZlIGNhdXNlZCBtZHNkIG5vdAojIHRvIGhvbm91ciBTU0xfQ0VSVF9GSUxFIGFueSBtb3JlLCBoZWF2ZW4gb25seSBrbm93cyB3aHkuCm1rZGlyIC1wIC91c3IvbGliL3NzbC9jZXJ0cwpjc3BsaXQgLWYgL3Vzci9saWIvc3NsL2NlcnRzL2NlcnQtIC1iICUwM2QucGVtIC9ldGMvcGtpL3Rscy9jZXJ0cy9jYS1idW5kbGUuY3J0IC9eJC8xIHsqfSA+L2Rldi9udWxsCmNfcmVoYXNoIC91c3IvbGliL3NzbC9jZXJ0cwoKIyB3ZSBsZWF2ZSBjbGllbnRJZCBibGFuayBhcyBsb25nIGFzIG9ubHkgMSBtYW5hZ2VkIGlkZW50aXR5IGFzc2lnbmVkIHRvIHZtc3MKIyBpZiB3ZSBoYXZlIG1vcmUgdGhhbiAxLCB3ZSB3aWxsIG5lZWQgdG8gcG9wdWxhdGUgd2l0aCBjbGllbnRJZCB1c2VkIGZvciBvZmYtbm9kZSBzY2FubmluZwpjYXQgPi9ldGMvZGVmYXVsdC92c2Etbm9kZXNjYW4tYWdlbnQuY29uZmlnIDw8RU9GCnsKICAgICJOaWNlIjogMTksCiAgICAiVGltZW91dCI6IDEwODAwLAogICAgIkNsaWVudElkIjogIiIsCiAgICAiVGVuYW50SWQiOiAiJEFaVVJFU0VDUEFDS1ZTQVRFTkFOVElEIiwKICAgICJRdWFseXNTdG9yZUJhc2VVcmwiOiAiJEFaVVJFU0VDUEFDS1FVQUxZU1VSTCIsCiAgICAiUHJvY2Vzc1RpbWVvdXQiOiAzMDAsCiAgICAiQ29tbWFuZERlbGF5IjogMAogIH0KRU9GCgojIHdlIHN0YXJ0IGEgY3JvbiBqb2IgdG8gcnVuIGV2ZXJ5IGhvdXIgdG8gZW5zdXJlIHRoZSBzYWlkIGRpcmVjdG9yeSBpcyBhY2Nlc3NpYmxlCiMgYnkgdGhlIGNvcnJlY3QgdXNlciBhcyBpdCBnZXRzIGNyZWF0ZWQgYnkgcm9vdCBhbmQgbWF5IGNhdXNlIGEgcmFjZSBjb25kaXRpb24KIyB3aGVyZSByb290IG93bnMgdGhlIGRpciBpbnN0ZWFkIG9mIHN5c2xvZwojIFRPRE86IGh0dHBzOi8vbXNhenVyZS52aXN1YWxzdHVkaW8uY29tL0F6dXJlUmVkSGF0T3BlblNoaWZ0L193b3JraXRlbXMvZWRpdC8xMjU5MTIwNwpjYXQgPi9ldGMvY3Jvbi5kL21kc2QtY2hvd24td29ya2Fyb3VuZCA8PEVPRgpTSEVMTD0vYmluL2Jhc2gKUEFUSD0vYmluCjAgKiAqICogKiByb290IGNob3duIHN5c2xvZzpzeXNsb2cgL3Zhci9vcHQvbWljcm9zb2Z0L2xpbnV4bW9uYWdlbnQvZWgvRXZlbnROb3RpY2UvYXJvcnBsb2dzKgpFT0YKCmVjaG8gImVuYWJsaW5nIGFybyBzZXJ2aWNlcyIKZm9yIHNlcnZpY2UgaW4gYXJvLWdhdGV3YXkgYXVvbXMgYXpzZWNkIGF6c2VjbW9uZCBtZHNkIG1kbSBjaHJvbnlkIGZsdWVudGJpdDsgZG8KICBzeXN0ZW1jdGwgZW5hYmxlICRzZXJ2aWNlLnNlcnZpY2UKZG9uZQoKZm9yIHNjYW4gaW4gYmFzZWxpbmUgY2xhbWF2IHNvZnR3YXJlOyBkbwogIC91c3IvbG9jYWwvYmluL2F6c2VjZCBjb25maWcgLXMgJHNjYW4gLWQgUDFECmRvbmUKCmVjaG8gInJlYm9vdGluZyIKcmVzdG9yZWNvbiAtUkYgL3Zhci9sb2cvKgooc2xlZXAgMzA7IHJlYm9vdCkgJgo=')))]" + "script": "[base64(concat(base64ToString('c2V0IC1leAoK'),'ACRRESOURCEID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('acrResourceId')),''')\n','AZURECLOUDNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureCloudName')),''')\n','AZURESECPACKQUALYSURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackQualysUrl')),''')\n','AZURESECPACKVSATENANTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('azureSecPackVSATenantId')),''')\n','DATABASEACCOUNTNAME=$(base64 -d \u003c\u003c\u003c''',base64(parameters('databaseAccountName')),''')\n','DBTOKENCLIENTID=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenClientId')),''')\n','DBTOKENURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('dbtokenUrl')),''')\n','MDMFRONTENDURL=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdmFrontendUrl')),''')\n','MDSDENVIRONMENT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('mdsdEnvironment')),''')\n','FLUENTBITIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('fluentbitImage')),''')\n','GATEWAYMDSDCONFIGVERSION=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayMdsdConfigVersion')),''')\n','GATEWAYDOMAINS=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayDomains')),''')\n','GATEWAYFEATURES=$(base64 -d \u003c\u003c\u003c''',base64(parameters('gatewayFeatures')),''')\n','KEYVAULTDNSSUFFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultDNSSuffix')),''')\n','KEYVAULTPREFIX=$(base64 -d \u003c\u003c\u003c''',base64(parameters('keyvaultPrefix')),''')\n','RPIMAGE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpImage')),''')\n','RPMDMACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdmAccount')),''')\n','RPMDSDACCOUNT=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdAccount')),''')\n','RPMDSDNAMESPACE=$(base64 -d \u003c\u003c\u003c''',base64(parameters('rpMdsdNamespace')),''')\n','MDMIMAGE=''/genevamdm:2.2023.609.2051-821f47-20230706t0953''\n','LOCATION=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().location),''')\n','SUBSCRIPTIONID=$(base64 -d \u003c\u003c\u003c''',base64(subscription().subscriptionId),''')\n','RESOURCEGROUPNAME=$(base64 -d \u003c\u003c\u003c''',base64(resourceGroup().name),''')\n','\n',base64ToString('IyEvYmluL2Jhc2gKCmVjaG8gInNldHRpbmcgc3NoIHBhc3N3b3JkIGF1dGhlbnRpY2F0aW9uIgojIFdlIG5lZWQgdG8gbWFudWFsbHkgc2V0IFBhc3N3b3JkQXV0aGVudGljYXRpb24gdG8gdHJ1ZSBpbiBvcmRlciBmb3IgdGhlIFZNU1MgQWNjZXNzIEpJVCB0byB3b3JrCnNlZCAtaSAncy9QYXNzd29yZEF1dGhlbnRpY2F0aW9uIG5vL1Bhc3N3b3JkQXV0aGVudGljYXRpb24geWVzL2cnIC9ldGMvc3NoL3NzaGRfY29uZmlnCnN5c3RlbWN0bCByZWxvYWQgc3NoZC5zZXJ2aWNlCgplY2hvICJydW5uaW5nIFJIVUkgZml4Igp5dW0gdXBkYXRlIC15IC0tZGlzYWJsZXJlcG89JyonIC0tZW5hYmxlcmVwbz0ncmh1aS1taWNyb3NvZnQtYXp1cmUqJwoKZWNobyAicnVubmluZyB5dW0gdXBkYXRlIgp5dW0gLXkgLXggV0FMaW51eEFnZW50IC14IFdBTGludXhBZ2VudC11ZGV2IHVwZGF0ZSAtLWFsbG93ZXJhc2luZwoKZWNobyAiZXh0ZW5kaW5nIHBhcnRpdGlvbiB0YWJsZSIKIyBMaW51eCBibG9jayBkZXZpY2VzIGFyZSBpbmNvbnNpc3RlbnRseSBuYW1lZAojIGl0J3MgZGlmZmljdWx0IHRvIHRpZSB0aGUgbHZtIHB2IHRvIHRoZSBwaHlzaWNhbCBkaXNrIHVzaW5nIC9kZXYvZGlzayBmaWxlcywgd2hpY2ggaXMgd2h5IGx2cyBpcyB1c2VkIGhlcmUKcGh5c2ljYWxfZGlzaz0iJChsdnMgLW8gZGV2aWNlcyAtYSB8IGhlYWQgLW4yIHwgdGFpbCAtbjEgfCBjdXQgLWQgJyAnIC1mIDMgfCBjdXQgLWQgXCggLWYgMSB8IHRyIC1kICdbOmRpZ2l0Ol0nKSIKZ3Jvd3BhcnQgIiRwaHlzaWNhbF9kaXNrIiAyCgplY2hvICJleHRlbmRpbmcgZmlsZXN5c3RlbXMiCmx2ZXh0ZW5kIC1sICsyMCVGUkVFIC9kZXYvcm9vdHZnL3Jvb3Rsdgp4ZnNfZ3Jvd2ZzIC8KCmx2ZXh0ZW5kIC1sICsxMDAlRlJFRSAvZGV2L3Jvb3R2Zy92YXJsdgp4ZnNfZ3Jvd2ZzIC92YXIKCnJwbSAtLWltcG9ydCBodHRwczovL2RsLmZlZG9yYXByb2plY3Qub3JnL3B1Yi9lcGVsL1JQTS1HUEctS0VZLUVQRUwtOApycG0gLS1pbXBvcnQgaHR0cHM6Ly9wYWNrYWdlcy5taWNyb3NvZnQuY29tL2tleXMvbWljcm9zb2Z0LmFzYwoKZm9yIGF0dGVtcHQgaW4gezEuLjV9OyBkbwogIHl1bSAteSBpbnN0YWxsIGh0dHBzOi8vZGwuZmVkb3JhcHJvamVjdC5vcmcvcHViL2VwZWwvZXBlbC1yZWxlYXNlLWxhdGVzdC04Lm5vYXJjaC5ycG0gJiYgYnJlYWsKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA1IF1dOyB0aGVuIHNsZWVwIDEwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiY29uZmlndXJpbmcgbG9ncm90YXRlIgoKIyBnYXRld2F5X2xvZ2RpciBpcyBhIHJlYWRvbmx5IHZhcmlhYmxlIHRoYXQgc3BlY2lmaWVzIHRoZSBob3N0IHBhdGggbW91bnQgcG9pbnQgZm9yIHRoZSBnYXRld2F5IGNvbnRhaW5lciBsb2cgZmlsZQojIGZvciB0aGUgcHVycG9zZSBvZiByb3RhdGluZyB0aGUgZ2F0ZXdheSBsb2dzCmRlY2xhcmUgLXIgZ2F0ZXdheV9sb2dkaXI9Jy92YXIvbG9nL2Fyby1nYXRld2F5JwoKY2F0ID4vZXRjL2xvZ3JvdGF0ZS5jb25mIDw8RU9GCiMgc2VlICJtYW4gbG9ncm90YXRlIiBmb3IgZGV0YWlscwojIHJvdGF0ZSBsb2cgZmlsZXMgd2Vla2x5CndlZWtseQoKIyBrZWVwIDIgd2Vla3Mgd29ydGggb2YgYmFja2xvZ3MKcm90YXRlIDIKCiMgY3JlYXRlIG5ldyAoZW1wdHkpIGxvZyBmaWxlcyBhZnRlciByb3RhdGluZyBvbGQgb25lcwpjcmVhdGUKCiMgdXNlIGRhdGUgYXMgYSBzdWZmaXggb2YgdGhlIHJvdGF0ZWQgZmlsZQpkYXRlZXh0CgojIHVuY29tbWVudCB0aGlzIGlmIHlvdSB3YW50IHlvdXIgbG9nIGZpbGVzIGNvbXByZXNzZWQKY29tcHJlc3MKCiMgUlBNIHBhY2thZ2VzIGRyb3AgbG9nIHJvdGF0aW9uIGluZm9ybWF0aW9uIGludG8gdGhpcyBkaXJlY3RvcnkKaW5jbHVkZSAvZXRjL2xvZ3JvdGF0ZS5kCgojIG5vIHBhY2thZ2VzIG93biB3dG1wIGFuZCBidG1wIC0tIHdlJ2xsIHJvdGF0ZSB0aGVtIGhlcmUKL3Zhci9sb2cvd3RtcCB7CiAgICBtb250aGx5CiAgICBjcmVhdGUgMDY2NCByb290IHV0bXAKICAgICAgICBtaW5zaXplIDFNCiAgICByb3RhdGUgMQp9CgovdmFyL2xvZy9idG1wIHsKICAgIG1pc3NpbmdvawogICAgbW9udGhseQogICAgY3JlYXRlIDA2MDAgcm9vdCB1dG1wCiAgICByb3RhdGUgMQp9CgojIE1heGltdW0gbG9nIGRpcmVjdG9yeSBzaXplIGlzIDEwMEcgd2l0aCB0aGlzIGNvbmZpZ3VyYXRpb24KIyBTZXR0aW5nIGxpbWl0IHRvIDEwMEcgdG8gYWxsb3cgc3BhY2UgZm9yIG90aGVyIGxvZ2dpbmcgc2VydmljZXMKIyBjb3B5dHJ1bmNhdGUgaXMgYSBjcml0aWNhbCBvcHRpb24gdXNlZCB0byBwcmV2ZW50IGxvZ3MgZnJvbSBiZWluZyBzaGlwcGVkIHR3aWNlCiR7Z2F0ZXdheV9sb2dkaXJ9IHsKICAgIHNpemUgMjBHCiAgICByb3RhdGUgNQogICAgY3JlYXRlIDA2MDAgcm9vdCByb290CiAgICBjb3B5dHJ1bmNhdGUKICAgIG5vb2xkZGlyCiAgICBjb21wcmVzcwp9CkVPRgoKZWNobyAiY29uZmlndXJpbmcgeXVtIHJlcG9zaXRvcnkgYW5kIHJ1bm5pbmcgeXVtIHVwZGF0ZSIKY2F0ID4vZXRjL3l1bS5yZXBvcy5kL2F6dXJlLnJlcG8gPDwnRU9GJwpbYXp1cmUtY2xpXQpuYW1lPWF6dXJlLWNsaQpiYXNldXJsPWh0dHBzOi8vcGFja2FnZXMubWljcm9zb2Z0LmNvbS95dW1yZXBvcy9henVyZS1jbGkKZW5hYmxlZD15ZXMKZ3BnY2hlY2s9eWVzCgpbYXp1cmVjb3JlXQpuYW1lPWF6dXJlY29yZQpiYXNldXJsPWh0dHBzOi8vcGFja2FnZXMubWljcm9zb2Z0LmNvbS95dW1yZXBvcy9henVyZWNvcmUKZW5hYmxlZD15ZXMKZ3BnY2hlY2s9bm8KRU9GCgpzZW1hbmFnZSBmY29udGV4dCAtYSAtdCB2YXJfbG9nX3QgIi92YXIvbG9nL2pvdXJuYWwoLy4qKT8iCm1rZGlyIC1wIC92YXIvbG9nL2pvdXJuYWwKCmZvciBhdHRlbXB0IGluIHsxLi41fTsgZG8KICB5dW0gLXkgaW5zdGFsbCBjbGFtYXYgYXpzZWMtY2xhbWF2IGF6c2VjLW1vbml0b3IgYXp1cmUtY2xpIGF6dXJlLW1kc2QgYXp1cmUtc2VjdXJpdHkgcG9kbWFuLWRvY2tlciBvcGVuc3NsLXBlcmwgcHl0aG9uMyAmJiBicmVhawogICMgaGFjayAtIHdlIGFyZSBpbnN0YWxsaW5nIHB5dGhvbjMgb24gaG9zdHMgZHVlIHRvIGFuIGlzc3VlIHdpdGggQXp1cmUgTGludXggRXh0ZW5zaW9ucyBodHRwczovL2dpdGh1Yi5jb20vQXp1cmUvYXp1cmUtbGludXgtZXh0ZW5zaW9ucy9wdWxsLzE1MDUKICBpZiBbWyAke2F0dGVtcHR9IC1sdCA1IF1dOyB0aGVuIHNsZWVwIDEwOyBlbHNlIGV4aXQgMTsgZmkKZG9uZQoKZWNobyAiYXBwbHlpbmcgZmlyZXdhbGwgcnVsZXMiCiMgaHR0cHM6Ly9hY2Nlc3MucmVkaGF0LmNvbS9zZWN1cml0eS9jdmUvY3ZlLTIwMjAtMTM0MDEKY2F0ID4vZXRjL3N5c2N0bC5kLzAyLWRpc2FibGUtYWNjZXB0LXJhLmNvbmYgPDwnRU9GJwpuZXQuaXB2Ni5jb25mLmFsbC5hY2NlcHRfcmE9MApFT0YKCmNhdCA+L2V0Yy9zeXNjdGwuZC8wMS1kaXNhYmxlLWNvcmUuY29uZiA8PCdFT0YnCmtlcm5lbC5jb3JlX3BhdHRlcm4gPSB8L2Jpbi90cnVlCkVPRgpzeXNjdGwgLS1zeXN0ZW0KCmZpcmV3YWxsLWNtZCAtLWFkZC1wb3J0PTgwL3RjcCAtLXBlcm1hbmVudApmaXJld2FsbC1jbWQgLS1hZGQtcG9ydD04MDgxL3RjcCAtLXBlcm1hbmVudApmaXJld2FsbC1jbWQgLS1hZGQtcG9ydD00NDMvdGNwIC0tcGVybWFuZW50CgplY2hvICJsb2dnaW5nIGludG8gcHJvZCBhY3IiCmV4cG9ydCBBWlVSRV9DTE9VRF9OQU1FPSRBWlVSRUNMT1VETkFNRQpheiBsb2dpbiAtaSAtLWFsbG93LW5vLXN1YnNjcmlwdGlvbnMKCiMgVGhlIG1hbmFnZWQgaWRlbnRpdHkgdGhhdCB0aGUgVk0gcnVucyBhcyBvbmx5IGhhcyBhIHNpbmdsZSByb2xlYXNzaWdubWVudC4KIyBUaGlzIHJvbGUgYXNzaWdubWVudCBpcyBBQ1JQdWxsIHdoaWNoIGlzIG5vdCBuZWNlc3NhcmlseSBwcmVzZW50IGluIHRoZQojIHN1YnNjcmlwdGlvbiB3ZSdyZSBkZXBsb3lpbmcgaW50by4gIElmIHRoZSBpZGVudGl0eSBkb2VzIG5vdCBoYXZlIGFueQojIHJvbGUgYXNzaWdubWVudHMgc2NvcGVkIG9uIHRoZSBzdWJzY3JpcHRpb24gd2UncmUgZGVwbG95aW5nIGludG8sIGl0IHdpbGwKIyBub3Qgc2hvdyBvbiBheiBsb2dpbiAtaSwgd2hpY2ggaXMgd2h5IHRoZSBiZWxvdyBsaW5lIGlzIGNvbW1lbnRlZC4KIyBheiBhY2NvdW50IHNldCAtcyAiJFNVQlNDUklQVElPTklEIgoKIyBTdXBwcmVzcyBlbXVsYXRpb24gb3V0cHV0IGZvciBwb2RtYW4gaW5zdGVhZCBvZiBkb2NrZXIgZm9yIGF6IGFjciBjb21wYXRhYmlsaXR5Cm1rZGlyIC1wIC9ldGMvY29udGFpbmVycy8KdG91Y2ggL2V0Yy9jb250YWluZXJzL25vZG9ja2VyCgpta2RpciAtcCAvcm9vdC8uZG9ja2VyClJFR0lTVFJZX0FVVEhfRklMRT0vcm9vdC8uZG9ja2VyL2NvbmZpZy5qc29uIGF6IGFjciBsb2dpbiAtLW5hbWUgIiQoc2VkIC1lICdzfC4qL3x8JyA8PDwiJEFDUlJFU09VUkNFSUQiKSIKCk1ETUlNQUdFPSIke1JQSU1BR0UlJS8qfS8ke01ETUlNQUdFIyMqL30iCmRvY2tlciBwdWxsICIkTURNSU1BR0UiCmRvY2tlciBwdWxsICIkUlBJTUFHRSIKZG9ja2VyIHB1bGwgIiRGTFVFTlRCSVRJTUFHRSIKCmF6IGxvZ291dAoKZWNobyAiY29uZmlndXJpbmcgZmx1ZW50Yml0IHNlcnZpY2UiCm1rZGlyIC1wIC9ldGMvZmx1ZW50Yml0Lwpta2RpciAtcCAvdmFyL2xpYi9mbHVlbnQKCmNhdCA+L2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmYgPDwnRU9GJwpbSU5QVVRdCglOYW1lIHN5c3RlbWQKCVRhZyBqb3VybmFsZAoJU3lzdGVtZF9GaWx0ZXIgX0NPTU09YXJvCglEQiAvdmFyL2xpYi9mbHVlbnQvam91cm5hbGRiCgpbRklMVEVSXQoJTmFtZSBtb2RpZnkKCU1hdGNoIGpvdXJuYWxkCglSZW1vdmVfd2lsZGNhcmQgXwoJUmVtb3ZlIFRJTUVTVEFNUAoKW09VVFBVVF0KCU5hbWUgZm9yd2FyZAoJTWF0Y2ggKgoJUG9ydCAyOTIzMApFT0YKCmVjaG8gIkZMVUVOVEJJVElNQUdFPSRGTFVFTlRCSVRJTUFHRSIgPi9ldGMvc3lzY29uZmlnL2ZsdWVudGJpdAoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL2ZsdWVudGJpdC5zZXJ2aWNlIDw8J0VPRicKW1VuaXRdCkFmdGVyPW5ldHdvcmstb25saW5lLnRhcmdldApXYW50cz1uZXR3b3JrLW9ubGluZS50YXJnZXQKU3RhcnRMaW1pdEludGVydmFsU2VjPTAKCltTZXJ2aWNlXQpSZXN0YXJ0U2VjPTFzCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9mbHVlbnRiaXQKRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0tc2VjdXJpdHktb3B0IGxhYmVsPWRpc2FibGUgXAogIC0tZW50cnlwb2ludCAvb3B0L3RkLWFnZW50LWJpdC9iaW4vdGQtYWdlbnQtYml0IFwKICAtLW5ldD1ob3N0IFwKICAtLWhvc3RuYW1lICVIIFwKICAtLW5hbWUgJU4gXAogIC0tcm0gXAogIC0tY2FwLWRyb3AgbmV0X3JhdyBcCiAgLXYgL2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmY6L2V0Yy9mbHVlbnRiaXQvZmx1ZW50Yml0LmNvbmYgXAogIC12IC92YXIvbGliL2ZsdWVudDovdmFyL2xpYi9mbHVlbnQ6eiBcCiAgLXYgL3Zhci9sb2cvam91cm5hbDovdmFyL2xvZy9qb3VybmFsOnJvIFwKICAtdiAvZXRjL21hY2hpbmUtaWQ6L2V0Yy9tYWNoaW5lLWlkOnJvIFwKICAkRkxVRU5UQklUSU1BR0UgXAogIC1jIC9ldGMvZmx1ZW50Yml0L2ZsdWVudGJpdC5jb25mCgpFeGVjU3RvcD0vdXNyL2Jpbi9kb2NrZXIgc3RvcCAlTgpSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTUKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmVjaG8gImNvbmZpZ3VyaW5nIG1kbSBzZXJ2aWNlIgpjYXQgPi9ldGMvc3lzY29uZmlnL21kbSA8PEVPRgpNRE1GUk9OVEVORFVSTD0nJE1ETUZST05URU5EVVJMJwpNRE1JTUFHRT0nJE1ETUlNQUdFJwpNRE1TT1VSQ0VFTlZJUk9OTUVOVD0nJExPQ0FUSU9OJwpNRE1TT1VSQ0VST0xFPWdhdGV3YXkKTURNU09VUkNFUk9MRUlOU1RBTkNFPSckKGhvc3RuYW1lKScKRU9GCgpta2RpciAvdmFyL2V0dwpjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vbWRtLnNlcnZpY2UgPDwnRU9GJwpbVW5pdF0KQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1NlcnZpY2VdCkVudmlyb25tZW50RmlsZT0vZXRjL3N5c2NvbmZpZy9tZG0KRXhlY1N0YXJ0UHJlPS0vdXNyL2Jpbi9kb2NrZXIgcm0gLWYgJU4KRXhlY1N0YXJ0PS91c3IvYmluL2RvY2tlciBydW4gXAogIC0tZW50cnlwb2ludCAvdXNyL3NiaW4vTWV0cmljc0V4dGVuc2lvbiBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1tIDJnIFwKICAtdiAvZXRjL21kbS5wZW06L2V0Yy9tZG0ucGVtIFwKICAtdiAvdmFyL2V0dzovdmFyL2V0dzp6IFwKICAkTURNSU1BR0UgXAogIC1DZXJ0RmlsZSAvZXRjL21kbS5wZW0gXAogIC1Gcm9udEVuZFVybCAkTURNRlJPTlRFTkRVUkwgXAogIC1Mb2dnZXIgQ29uc29sZSBcCiAgLUxvZ0xldmVsIFdhcm5pbmcgXAogIC1Qcml2YXRlS2V5RmlsZSAvZXRjL21kbS5wZW0gXAogIC1Tb3VyY2VFbnZpcm9ubWVudCAkTURNU09VUkNFRU5WSVJPTk1FTlQgXAogIC1Tb3VyY2VSb2xlICRNRE1TT1VSQ0VST0xFIFwKICAtU291cmNlUm9sZUluc3RhbmNlICRNRE1TT1VSQ0VST0xFSU5TVEFOQ0UKRXhlY1N0b3A9L3Vzci9iaW4vZG9ja2VyIHN0b3AgJU4KUmVzdGFydD1hbHdheXMKUmVzdGFydFNlYz0xClN0YXJ0TGltaXRJbnRlcnZhbD0wCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgplY2hvICJjb25maWd1cmluZyBhcm8tZ2F0ZXdheSBzZXJ2aWNlIgpjYXQgPi9ldGMvc3lzY29uZmlnL2Fyby1nYXRld2F5IDw8RU9GCkFDUl9SRVNPVVJDRV9JRD0nJEFDUlJFU09VUkNFSUQnCkRBVEFCQVNFX0FDQ09VTlRfTkFNRT0nJERBVEFCQVNFQUNDT1VOVE5BTUUnCkFaVVJFX0RCVE9LRU5fQ0xJRU5UX0lEPSckREJUT0tFTkNMSUVOVElEJwpEQlRPS0VOX1VSTD0nJERCVE9LRU5VUkwnCk1ETV9BQ0NPVU5UPSIkUlBNRE1BQ0NPVU5UIgpNRE1fTkFNRVNQQUNFPUdhdGV3YXkKR0FURVdBWV9ET01BSU5TPSckR0FURVdBWURPTUFJTlMnCkdBVEVXQVlfRkVBVFVSRVM9JyRHQVRFV0FZRkVBVFVSRVMnClJQSU1BR0U9JyRSUElNQUdFJwpFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9hcm8tZ2F0ZXdheS5zZXJ2aWNlIDw8RU9GCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKV2FudHM9bmV0d29yay1vbmxpbmUudGFyZ2V0CgpbU2VydmljZV0KRW52aXJvbm1lbnRGaWxlPS9ldGMvc3lzY29uZmlnL2Fyby1nYXRld2F5CkV4ZWNTdGFydFByZT0tL3Vzci9iaW4vZG9ja2VyIHJtIC1mICVOCkV4ZWNTdGFydFByZT0vdXNyL2Jpbi9ta2RpciAtcCAke2dhdGV3YXlfbG9nZGlyfQpFeGVjU3RhcnQ9L3Vzci9iaW4vZG9ja2VyIHJ1biBcCiAgLS1ob3N0bmFtZSAlSCBcCiAgLS1uYW1lICVOIFwKICAtLXJtIFwKICAtLWNhcC1kcm9wIG5ldF9yYXcgXAogIC1lIEFDUl9SRVNPVVJDRV9JRCBcCiAgLWUgREFUQUJBU0VfQUNDT1VOVF9OQU1FIFwKICAtZSBBWlVSRV9EQlRPS0VOX0NMSUVOVF9JRCBcCiAgLWUgREJUT0tFTl9VUkwgXAogIC1lIEdBVEVXQVlfRE9NQUlOUyBcCiAgLWUgR0FURVdBWV9GRUFUVVJFUyBcCiAgLWUgTURNX0FDQ09VTlQgXAogIC1lIE1ETV9OQU1FU1BBQ0UgXAogIC1tIDJnIFwKICAtcCA4MDo4MDgwIFwKICAtcCA4MDgxOjgwODEgXAogIC1wIDQ0Mzo4NDQzIFwKICAtdiAvcnVuL3N5c3RlbWQvam91cm5hbDovcnVuL3N5c3RlbWQvam91cm5hbCBcCiAgLXYgL3Zhci9ldHc6L3Zhci9ldHc6eiBcCiAgLXYgL2N0ci5sb2c6JHtnYXRld2F5X2xvZ2Rpcn06eiBcCiAgXCRSUElNQUdFIFwKICBnYXRld2F5CkV4ZWNTdG9wPS91c3IvYmluL2RvY2tlciBzdG9wIC10IDM2MDAgJU4KVGltZW91dFN0b3BTZWM9MzYwMApSZXN0YXJ0PWFsd2F5cwpSZXN0YXJ0U2VjPTEKU3RhcnRMaW1pdEludGVydmFsPTAKCltJbnN0YWxsXQpXYW50ZWRCeT1tdWx0aS11c2VyLnRhcmdldApFT0YKCmNoY29uIC1SIHN5c3RlbV91Om9iamVjdF9yOnZhcl9sb2dfdDpzMCAvdmFyL29wdC9taWNyb3NvZnQvbGludXhtb25hZ2VudAoKbWtkaXIgLXAgL3Zhci9saWIvd2FhZ2VudC9NaWNyb3NvZnQuQXp1cmUuS2V5VmF1bHQuU3RvcmUKCmVjaG8gImNvbmZpZ3VyaW5nIG1kc2QgYW5kIG1kbSBzZXJ2aWNlcyIKZm9yIHZhciBpbiAibWRzZCIgIm1kbSI7IGRvCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9kb3dubG9hZC0kdmFyLWNyZWRlbnRpYWxzLnNlcnZpY2UgPDxFT0YKW1VuaXRdCkRlc2NyaXB0aW9uPVBlcmlvZGljICR2YXIgY3JlZGVudGlhbHMgcmVmcmVzaAoKW1NlcnZpY2VdClR5cGU9b25lc2hvdApFeGVjU3RhcnQ9L3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggJHZhcgpFT0YKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS9kb3dubG9hZC0kdmFyLWNyZWRlbnRpYWxzLnRpbWVyIDw8RU9GCltVbml0XQpEZXNjcmlwdGlvbj1QZXJpb2RpYyAkdmFyIGNyZWRlbnRpYWxzIHJlZnJlc2gKQWZ0ZXI9bmV0d29yay1vbmxpbmUudGFyZ2V0CldhbnRzPW5ldHdvcmstb25saW5lLnRhcmdldAoKW1RpbWVyXQpPbkJvb3RTZWM9MG1pbgpPbkNhbGVuZGFyPTAvMTI6MDA6MDAKQWNjdXJhY3lTZWM9NXMKCltJbnN0YWxsXQpXYW50ZWRCeT10aW1lcnMudGFyZ2V0CkVPRgpkb25lCgpjYXQgPi91c3IvbG9jYWwvYmluL2Rvd25sb2FkLWNyZWRlbnRpYWxzLnNoIDw8RU9GCiMhL2Jpbi9iYXNoCnNldCAtZXUKCkNPTVBPTkVOVD0iXCQxIgplY2hvICJEb3dubG9hZCBcJENPTVBPTkVOVCBjcmVkZW50aWFscyIKClRFTVBfRElSPVwkKG1rdGVtcCAtZCkKZXhwb3J0IEFaVVJFX0NPTkZJR19ESVI9XCQobWt0ZW1wIC1kKQoKZWNobyAiTG9nZ2luZyBpbnRvIEF6dXJlLi4uIgpSRVRSSUVTPTMKd2hpbGUgWyAiXCRSRVRSSUVTIiAtZ3QgMCBdOyBkbwogICAgaWYgYXogbG9naW4gLWkgLS1hbGxvdy1uby1zdWJzY3JpcHRpb25zCiAgICB0aGVuCiAgICAgICAgZWNobyAiYXogbG9naW4gc3VjY2Vzc2Z1bCIKICAgICAgICBicmVhawogICAgZWxzZQogICAgICAgIGVjaG8gImF6IGxvZ2luIGZhaWxlZC4gUmV0cnlpbmcuLi4iCiAgICAgICAgbGV0IFJFVFJJRVMtPTEKICAgICAgICBzbGVlcCA1CiAgICBmaQpkb25lCgp0cmFwICJjbGVhbnVwIiBFWElUCgpjbGVhbnVwKCkgewogIGF6IGxvZ291dAogIFtbICJcJFRFTVBfRElSIiA9fiAvdG1wLy4rIF1dICYmIHJtIC1yZiBcJFRFTVBfRElSCiAgW1sgIlwkQVpVUkVfQ09ORklHX0RJUiIgPX4gL3RtcC8uKyBdXSAmJiBybSAtcmYgXCRBWlVSRV9DT05GSUdfRElSCn0KCmlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZG0iIF07IHRoZW4KICBDVVJSRU5UX0NFUlRfRklMRT0iL2V0Yy9tZG0ucGVtIgplbGlmIFsgIlwkQ09NUE9ORU5UIiA9ICJtZHNkIiBdOyB0aGVuCiAgQ1VSUkVOVF9DRVJUX0ZJTEU9Ii92YXIvbGliL3dhYWdlbnQvTWljcm9zb2Z0LkF6dXJlLktleVZhdWx0LlN0b3JlL21kc2QucGVtIgplbHNlCiAgZWNobyBJbnZhbGlkIHVzYWdlICYmIGV4aXQgMQpmaQoKU0VDUkVUX05BTUU9Imd3eS1cJHtDT01QT05FTlR9IgpORVdfQ0VSVF9GSUxFPSJcJFRFTVBfRElSL1wkQ09NUE9ORU5ULnBlbSIKZm9yIGF0dGVtcHQgaW4gezEuLjV9OyBkbwogIGF6IGtleXZhdWx0IHNlY3JldCBkb3dubG9hZCAtLWZpbGUgXCRORVdfQ0VSVF9GSUxFIC0taWQgImh0dHBzOi8vJEtFWVZBVUxUUFJFRklYLWd3eS4kS0VZVkFVTFRETlNTVUZGSVgvc2VjcmV0cy9cJFNFQ1JFVF9OQU1FIiAmJiBicmVhawogIGlmIFtbIFwkYXR0ZW1wdCAtbHQgNSBdXTsgdGhlbiBzbGVlcCAxMDsgZWxzZSBleGl0IDE7IGZpCmRvbmUKCmlmIFsgLWYgXCRORVdfQ0VSVF9GSUxFIF07IHRoZW4KICBpZiBbICJcJENPTVBPTkVOVCIgPSAibWRzZCIgXTsgdGhlbgogICAgY2hvd24gc3lzbG9nOnN5c2xvZyBcJE5FV19DRVJUX0ZJTEUKICBlbHNlCiAgICBzZWQgLWkgLW5lICcxLC9FTkQgQ0VSVElGSUNBVEUvIHAnIFwkTkVXX0NFUlRfRklMRQogIGZpCiAgaWYgISBkaWZmICRORVdfQ0VSVF9GSUxFICRDVVJSRU5UX0NFUlRfRklMRSA+L2Rldi9udWxsIDI+JjE7IHRoZW4KICAgIGNobW9kIDA2MDAgXCRORVdfQ0VSVF9GSUxFCiAgICBtdiBcJE5FV19DRVJUX0ZJTEUgXCRDVVJSRU5UX0NFUlRfRklMRQogIGZpCmVsc2UKICBlY2hvIEZhaWxlZCB0byByZWZyZXNoIGNlcnRpZmljYXRlIGZvciBcJENPTVBPTkVOVCAmJiBleGl0IDEKZmkKRU9GCgpjaG1vZCB1K3ggL3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2gKCnN5c3RlbWN0bCBlbmFibGUgZG93bmxvYWQtbWRzZC1jcmVkZW50aWFscy50aW1lcgpzeXN0ZW1jdGwgZW5hYmxlIGRvd25sb2FkLW1kbS1jcmVkZW50aWFscy50aW1lcgoKL3Vzci9sb2NhbC9iaW4vZG93bmxvYWQtY3JlZGVudGlhbHMuc2ggbWRzZAovdXNyL2xvY2FsL2Jpbi9kb3dubG9hZC1jcmVkZW50aWFscy5zaCBtZG0KTURTRENFUlRJRklDQVRFU0FOPSQob3BlbnNzbCB4NTA5IC1pbiAvdmFyL2xpYi93YWFnZW50L01pY3Jvc29mdC5BenVyZS5LZXlWYXVsdC5TdG9yZS9tZHNkLnBlbSAtbm9vdXQgLXN1YmplY3QgfCBzZWQgLWUgJ3MvLipDTiA9IC8vJykKCmNhdCA+L2V0Yy9zeXN0ZW1kL3N5c3RlbS93YXRjaC1tZG0tY3JlZGVudGlhbHMuc2VydmljZSA8PEVPRgpbVW5pdF0KRGVzY3JpcHRpb249V2F0Y2ggZm9yIGNoYW5nZXMgaW4gbWRtLnBlbSBhbmQgcmVzdGFydHMgdGhlIG1kbSBzZXJ2aWNlCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4ZWNTdGFydD0vdXNyL2Jpbi9zeXN0ZW1jdGwgcmVzdGFydCBtZG0uc2VydmljZQoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIudGFyZ2V0CkVPRgoKY2F0ID4vZXRjL3N5c3RlbWQvc3lzdGVtL3dhdGNoLW1kbS1jcmVkZW50aWFscy5wYXRoIDw8RU9GCltQYXRoXQpQYXRoTW9kaWZpZWQ9L2V0Yy9tZG0ucGVtCgpbSW5zdGFsbF0KV2FudGVkQnk9bXVsdGktdXNlci50YXJnZXQKRU9GCgpzeXN0ZW1jdGwgZW5hYmxlIHdhdGNoLW1kbS1jcmVkZW50aWFscy5wYXRoCnN5c3RlbWN0bCBzdGFydCB3YXRjaC1tZG0tY3JlZGVudGlhbHMucGF0aAoKbWtkaXIgL2V0Yy9zeXN0ZW1kL3N5c3RlbS9tZHNkLnNlcnZpY2UuZApjYXQgPi9ldGMvc3lzdGVtZC9zeXN0ZW0vbWRzZC5zZXJ2aWNlLmQvb3ZlcnJpZGUuY29uZiA8PCdFT0YnCltVbml0XQpBZnRlcj1uZXR3b3JrLW9ubGluZS50YXJnZXQKRU9GCgpjYXQgPi9ldGMvZGVmYXVsdC9tZHNkIDw8RU9GCk1EU0RfUk9MRV9QUkVGSVg9L3Zhci9ydW4vbWRzZC9kZWZhdWx0Ck1EU0RfT1BUSU9OUz0iLUEgLWQgLXIgXCRNRFNEX1JPTEVfUFJFRklYIgoKZXhwb3J0IE1PTklUT1JJTkdfR0NTX0VOVklST05NRU5UPSckTURTREVOVklST05NRU5UJwpleHBvcnQgTU9OSVRPUklOR19HQ1NfQUNDT1VOVD0nJFJQTURTREFDQ09VTlQnCmV4cG9ydCBNT05JVE9SSU5HX0dDU19SRUdJT049JyRMT0NBVElPTicKZXhwb3J0IE1PTklUT1JJTkdfR0NTX0FVVEhfSURfVFlQRT1BdXRoS2V5VmF1bHQKZXhwb3J0IE1PTklUT1JJTkdfR0NTX0FVVEhfSUQ9JyRNRFNEQ0VSVElGSUNBVEVTQU4nCmV4cG9ydCBNT05JVE9SSU5HX0dDU19OQU1FU1BBQ0U9JyRSUE1EU0ROQU1FU1BBQ0UnCmV4cG9ydCBNT05JVE9SSU5HX0NPTkZJR19WRVJTSU9OPSckR0FURVdBWU1EU0RDT05GSUdWRVJTSU9OJwpleHBvcnQgTU9OSVRPUklOR19VU0VfR0VORVZBX0NPTkZJR19TRVJWSUNFPXRydWUKCmV4cG9ydCBNT05JVE9SSU5HX1RFTkFOVD0nJExPQ0FUSU9OJwpleHBvcnQgTU9OSVRPUklOR19ST0xFPWdhdGV3YXkKZXhwb3J0IE1PTklUT1JJTkdfUk9MRV9JTlNUQU5DRT0nJChob3N0bmFtZSknCgpleHBvcnQgTURTRF9NU0dQQUNLX1NPUlRfQ09MVU1OUz0xCkVPRgoKIyBzZXR0aW5nIE1PTklUT1JJTkdfR0NTX0FVVEhfSURfVFlQRT1BdXRoS2V5VmF1bHQgc2VlbXMgdG8gaGF2ZSBjYXVzZWQgbWRzZCBub3QKIyB0byBob25vdXIgU1NMX0NFUlRfRklMRSBhbnkgbW9yZSwgaGVhdmVuIG9ubHkga25vd3Mgd2h5Lgpta2RpciAtcCAvdXNyL2xpYi9zc2wvY2VydHMKY3NwbGl0IC1mIC91c3IvbGliL3NzbC9jZXJ0cy9jZXJ0LSAtYiAlMDNkLnBlbSAvZXRjL3BraS90bHMvY2VydHMvY2EtYnVuZGxlLmNydCAvXiQvMSB7Kn0gPi9kZXYvbnVsbApjX3JlaGFzaCAvdXNyL2xpYi9zc2wvY2VydHMKCiMgd2UgbGVhdmUgY2xpZW50SWQgYmxhbmsgYXMgbG9uZyBhcyBvbmx5IDEgbWFuYWdlZCBpZGVudGl0eSBhc3NpZ25lZCB0byB2bXNzCiMgaWYgd2UgaGF2ZSBtb3JlIHRoYW4gMSwgd2Ugd2lsbCBuZWVkIHRvIHBvcHVsYXRlIHdpdGggY2xpZW50SWQgdXNlZCBmb3Igb2ZmLW5vZGUgc2Nhbm5pbmcKY2F0ID4vZXRjL2RlZmF1bHQvdnNhLW5vZGVzY2FuLWFnZW50LmNvbmZpZyA8PEVPRgp7CiAgICAiTmljZSI6IDE5LAogICAgIlRpbWVvdXQiOiAxMDgwMCwKICAgICJDbGllbnRJZCI6ICIiLAogICAgIlRlbmFudElkIjogIiRBWlVSRVNFQ1BBQ0tWU0FURU5BTlRJRCIsCiAgICAiUXVhbHlzU3RvcmVCYXNlVXJsIjogIiRBWlVSRVNFQ1BBQ0tRVUFMWVNVUkwiLAogICAgIlByb2Nlc3NUaW1lb3V0IjogMzAwLAogICAgIkNvbW1hbmREZWxheSI6IDAKICB9CkVPRgoKIyB3ZSBzdGFydCBhIGNyb24gam9iIHRvIHJ1biBldmVyeSBob3VyIHRvIGVuc3VyZSB0aGUgc2FpZCBkaXJlY3RvcnkgaXMgYWNjZXNzaWJsZQojIGJ5IHRoZSBjb3JyZWN0IHVzZXIgYXMgaXQgZ2V0cyBjcmVhdGVkIGJ5IHJvb3QgYW5kIG1heSBjYXVzZSBhIHJhY2UgY29uZGl0aW9uCiMgd2hlcmUgcm9vdCBvd25zIHRoZSBkaXIgaW5zdGVhZCBvZiBzeXNsb2cKIyBUT0RPOiBodHRwczovL21zYXp1cmUudmlzdWFsc3R1ZGlvLmNvbS9BenVyZVJlZEhhdE9wZW5TaGlmdC9fd29ya2l0ZW1zL2VkaXQvMTI1OTEyMDcKY2F0ID4vZXRjL2Nyb24uZC9tZHNkLWNob3duLXdvcmthcm91bmQgPDxFT0YKU0hFTEw9L2Jpbi9iYXNoClBBVEg9L2JpbgowICogKiAqICogcm9vdCBjaG93biBzeXNsb2c6c3lzbG9nIC92YXIvb3B0L21pY3Jvc29mdC9saW51eG1vbmFnZW50L2VoL0V2ZW50Tm90aWNlL2Fyb3JwbG9ncyoKRU9GCgplY2hvICJlbmFibGluZyBhcm8gc2VydmljZXMiCmZvciBzZXJ2aWNlIGluIGFyby1nYXRld2F5IGF1b21zIGF6c2VjZCBhenNlY21vbmQgbWRzZCBtZG0gY2hyb255ZCBmbHVlbnRiaXQ7IGRvCiAgc3lzdGVtY3RsIGVuYWJsZSAkc2VydmljZS5zZXJ2aWNlCmRvbmUKCmZvciBzY2FuIGluIGJhc2VsaW5lIGNsYW1hdiBzb2Z0d2FyZTsgZG8KICAvdXNyL2xvY2FsL2Jpbi9henNlY2QgY29uZmlnIC1zICRzY2FuIC1kIFAxRApkb25lCgplY2hvICJyZWJvb3RpbmciCnJlc3RvcmVjb24gLVJGIC92YXIvbG9nLyoKKHNsZWVwIDMwOyByZWJvb3QpICYK')))]" } } } diff --git a/pkg/deploy/generator/scripts/gatewayVMSS.sh b/pkg/deploy/generator/scripts/gatewayVMSS.sh index 856a549a539..7e46aa7404e 100644 --- a/pkg/deploy/generator/scripts/gatewayVMSS.sh +++ b/pkg/deploy/generator/scripts/gatewayVMSS.sh @@ -14,8 +14,8 @@ yum -y -x WALinuxAgent -x WALinuxAgent-udev update --allowerasing echo "extending partition table" # Linux block devices are inconsistently named # it's difficult to tie the lvm pv to the physical disk using /dev/disk files, which is why lvs is used here -physicalDisk="$(lvs -o devices -a | head -n2 | tail -n1 | cut -d ' ' -f 3 | cut -d \( -f 1 | tr -d '[:digit:]')" -growpart "$physicalDisk" 2 +physical_disk="$(lvs -o devices -a | head -n2 | tail -n1 | cut -d ' ' -f 3 | cut -d \( -f 1 | tr -d '[:digit:]')" +growpart "$physical_disk" 2 echo "extending filesystems" lvextend -l +20%FREE /dev/rootvg/rootlv @@ -33,7 +33,12 @@ for attempt in {1..5}; do done echo "configuring logrotate" -cat >/etc/logrotate.conf <<'EOF' + +# gateway_logdir is a readonly variable that specifies the host path mount point for the gateway container log file +# for the purpose of rotating the gateway logs +declare -r gateway_logdir='/var/log/aro-gateway' + +cat >/etc/logrotate.conf </etc/systemd/system/aro-gateway.service <<'EOF' +cat >/etc/systemd/system/aro-gateway.service < 1 { + return nil, fmt.Errorf("found multiple etcd pods with conflicting IP addresses, only one degraded etcd is supported, unable to recover. Conflicting IPs found: %v", degradedEtcds) + // happens if the env variables are empty, check statuses next + } else if len(degradedEtcds) == 0 { + de = °radedEtcd{} + } else { + // array is no longer needed + de = °radedEtcds[0] + } + + return de, nil +} + +// comparePodEnvToIp compares the etcd container's environment variables to the pod's actual IP address +func findDegradedEtcd(log *logrus.Entry, pods *corev1.PodList) (*degradedEtcd, error) { + de, err := comparePodEnvToIp(log, pods) + if err != nil { + return °radedEtcd{}, err + } + + crashingPodSearchDe, err := findCrashloopingPods(log, pods) + log.Infof("Found degraded etcd while searching by Pod statuses: %v", crashingPodSearchDe) + if err != nil { + return °radedEtcd{}, err + } + + // Sanity check + // Since we are checking for both an etcd Pod with an IP mis match, and the statuses of all etcd pods, let's make sure the Pod's returned by both are the same + if de.Pod != crashingPodSearchDe.Pod && de.Pod != "" { + return de, fmt.Errorf("etcd Pod found in crashlooping state %s is not equal to etcd Pod with IP ENV mis match %s... failed sanity check", de.Pod, crashingPodSearchDe.Pod) + } + + // If no conflict is found a recent IP change may still be causing an issue + // Sometimes etcd can recovery the deployment itself, however there is still a data directory with the previous member's IP address present causing a failure + // This can still be remediated by relying on the pod statuses + if de.Node == "" { + log.Info("Unable to find an IP address conflict, using etcd Pod found during search by statuses") + return crashingPodSearchDe, nil + } + + return de, nil +} + +func ipFromEnv(containers []corev1.Container, podName string) string { + for _, c := range containers { + if c.Name == "etcd" { + for _, e := range c.Env { + // The environment variable that contains etcd's IP address has the following naming convention + // NODE_cluster_name_infra_ID_master_0_IP + // while the pod looks like this + // etcd-cluster-name-infra-id-master-0 + // To find the pod's IP address by variable name we use the pod's name + envName := strings.ReplaceAll(strings.ReplaceAll(podName, "-", "_"), "etcd_", "NODE_") + if e.Name == fmt.Sprintf("%s_IP", envName) { + return e.Value + } + } + } + } + + return "" +} + +func findCrashloopingPods(log *logrus.Entry, pods *corev1.PodList) (*degradedEtcd, error) { + // pods are collected in a list to check for multiple crashing etcd instances + // multiple etcd failures aren't supported so an error will be returned, rather than assuming the first found is the only one + crashingPods := &corev1.PodList{} + for _, p := range pods.Items { + for _, c := range p.Status.ContainerStatuses { + if !c.Ready && c.Name == "etcd" { + log.Infof("Found etcd container with status: %v", c) + crashingPods.Items = append(crashingPods.Items, p) + } + } + } + + if len(crashingPods.Items) > 1 { + // log multiple names in a readable way + names := []string{} + for _, c := range crashingPods.Items { + names = append(names, c.Name) + } + return nil, fmt.Errorf("only a single degraded etcd pod can can be recovered from, more than one NotReady etcd pods were found: %v", names) + } else if len(crashingPods.Items) == 0 { + return nil, errors.New("no etcd pod's were found in a CrashLoopBackOff state, unable to remediate etcd deployment") + } + crashingPod := &crashingPods.Items[0] + + return °radedEtcd{ + Node: strings.ReplaceAll(crashingPod.Name, "etcd-", ""), + Pod: crashingPod.Name, + OldIP: "unknown", + NewIP: "unknown", + }, nil +} diff --git a/pkg/frontend/fixetcd_test.go b/pkg/frontend/fixetcd_test.go new file mode 100644 index 00000000000..96d6bf28997 --- /dev/null +++ b/pkg/frontend/fixetcd_test.go @@ -0,0 +1,751 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + operatorv1fake "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1/fake" + "github.com/ugorji/go/codec" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/watch" + ktesting "k8s.io/client-go/testing" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/metrics/noop" + mock_adminactions "github.com/Azure/ARO-RP/pkg/util/mocks/adminactions" + testdatabase "github.com/Azure/ARO-RP/test/database" +) + +const degradedNode = "master-2" + +func TestFixEtcd(t *testing.T) { + // Context leak is intentional to make use of cancel function, and make it to our error check + ctx, ctxCancel := context.WithCancel(context.Background()) + const ( + mockSubID = "00000000-0000-0000-0000-000000000000" + mockTenantID = mockSubID + ) + resourceID := testdatabase.GetResourcePath(mockSubID, "cluster") + doc := &api.OpenShiftClusterDocument{ + Key: strings.ToLower(resourceID), + OpenShiftCluster: &api.OpenShiftCluster{ + Name: "cluster", + ID: resourceID, + Type: "Microsoft.RedHatOpenShift/openshiftClusters", + Properties: api.OpenShiftClusterProperties{ + InfraID: "zfsbk", + }, + }, + } + + type test struct { + name string + mocks func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) + wantErr string + pods *corev1.PodList + ctxCancel context.CancelFunc + cancel bool + } + + for _, tt := range []*test{ + { + name: "fail: list pods", + wantErr: "500: InternalServerError: : oh no, can't list pods", + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(nil, errors.New("oh no, can't list pods")) + }, + }, + { + name: "fail: invalid json, can't decode pods", + wantErr: "500: InternalServerError: : failed to decode pods, json decode error [pos 1]: only encoded map or array can decode into struct", + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(`{`) + if err != nil { + t.Fatalf("failed to encode pods, %s", err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + }, + }, + { + name: "pass: Expected degraded etcd scenario", + pods: newEtcdPods(t, doc, false, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil) + + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodSucceeded, false)() + + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).Times(1).Return([]byte("Backup job doing backup things..."), nil) + + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).Times(1).Return(nil) + + // fixPeers + // createPrivilegedServiceAccount + serviceAcc := newServiceAccount(serviceAccountName, doc.OpenShiftCluster.Name) + clusterRole := newClusterRole(kubeServiceAccount, doc.OpenShiftCluster.Name) + crb := newClusterRoleBinding(serviceAccountName, doc.OpenShiftCluster.Name) + scc := newSecurityContextConstraint(serviceAccountName, doc.OpenShiftCluster.Name, kubeServiceAccount) + + k.EXPECT().KubeCreateOrUpdate(ctx, serviceAcc).Times(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, clusterRole).Times(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, crb).Times(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, scc).Times(1).Return(nil) + + de, err := findDegradedEtcd(ti.log, pods) + if err != nil { + t.Fatal(err) + } + peerPods, err := getPeerPods(pods.Items, de, doc.OpenShiftCluster.Name) + if err != nil { + t.Fatal(err) + } + + jobFixPeers := newJobFixPeers(doc.OpenShiftCluster.Name, peerPods, de.Node) + k.EXPECT().KubeCreateOrUpdate(ctx, jobFixPeers).Times(1).Return(nil) + expectWatchEvent(gomock.Any(), jobFixPeers, k, "app", corev1.PodSucceeded, false)() + + k.EXPECT().KubeGetPodLogs(ctx, jobFixPeers.GetNamespace(), jobFixPeers.GetName(), jobFixPeers.GetName()).Times(1).Return([]byte("Fix peer job fixing peers..."), nil) + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobFixPeers.GetName(), true, &propPolicy).Times(1).Return(nil) + + // cleanup + k.EXPECT().KubeDelete(ctx, serviceAcc.GetKind(), serviceAcc.GetNamespace(), serviceAcc.GetName(), true, nil).Times(1).Return(nil) + k.EXPECT().KubeDelete(ctx, scc.GetKind(), scc.GetNamespace(), scc.GetName(), true, nil).Times(1).Return(nil) + k.EXPECT().KubeDelete(ctx, clusterRole.GetKind(), clusterRole.GetNamespace(), clusterRole.GetName(), true, nil).Times(1).Return(nil) + k.EXPECT().KubeDelete(ctx, crb.GetKind(), crb.GetNamespace(), crb.GetName(), true, nil).Times(1).Return(nil) + + err = codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(&operatorv1fake.FakeEtcds{}) + if err != nil { + t.Fatal(err) + } + k.EXPECT().KubeGet(ctx, "Etcd", "", doc.OpenShiftCluster.Name).MaxTimes(1).Return(buf.Bytes(), nil) + + // delete secrets + for _, prefix := range []string{"etcd-peer-", "etcd-serving-", "etcd-serving-metrics-"} { + k.EXPECT().KubeDelete(ctx, "Secret", namespaceEtcds, prefix+buildNodeName(doc, degradedNode), false, nil) + } + }, + }, + { + name: "pass: Empty env vars scenario", + pods: newEtcdPods(t, doc, false, false, true), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil) + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodSucceeded, false)() + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).MaxTimes(1).Return([]byte("Backup job doing backup things..."), nil) + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + + // fixPeers + // createPrivilegedServiceAccount + serviceAcc := newServiceAccount(serviceAccountName, doc.OpenShiftCluster.Name) + clusterRole := newClusterRole(kubeServiceAccount, doc.OpenShiftCluster.Name) + crb := newClusterRoleBinding(serviceAccountName, doc.OpenShiftCluster.Name) + scc := newSecurityContextConstraint(serviceAccountName, doc.OpenShiftCluster.Name, kubeServiceAccount) + + k.EXPECT().KubeCreateOrUpdate(ctx, serviceAcc).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, clusterRole).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, crb).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, scc).MaxTimes(1).Return(nil) + + de, err := findDegradedEtcd(ti.log, pods) + if err != nil { + t.Fatal(err) + } + peerPods, err := getPeerPods(pods.Items, de, doc.OpenShiftCluster.Name) + if err != nil { + t.Fatal(err) + } + + jobFixPeers := newJobFixPeers(doc.OpenShiftCluster.Name, peerPods, de.Node) + k.EXPECT().KubeCreateOrUpdate(ctx, jobFixPeers).MaxTimes(1).Return(nil) + expectWatchEvent(gomock.Any(), jobFixPeers, k, "app", corev1.PodSucceeded, false)() + k.EXPECT().KubeGetPodLogs(ctx, jobFixPeers.GetNamespace(), jobFixPeers.GetName(), jobFixPeers.GetName()).MaxTimes(1).Return([]byte("Fix peer job fixing peers..."), nil) + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobFixPeers.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + + // cleanup + k.EXPECT().KubeDelete(ctx, serviceAcc.GetKind(), serviceAcc.GetNamespace(), serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, scc.GetKind(), scc.GetNamespace(), scc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, clusterRole.GetKind(), clusterRole.GetNamespace(), clusterRole.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, crb.GetKind(), crb.GetNamespace(), crb.GetName(), true, nil).MaxTimes(1).Return(nil) + + err = codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(&operatorv1fake.FakeEtcds{}) + if err != nil { + t.Fatal(err) + } + k.EXPECT().KubeGet(ctx, "Etcd", "", doc.OpenShiftCluster.Name).MaxTimes(1).Return(buf.Bytes(), nil) + + // delete secrets + for _, prefix := range []string{"etcd-peer-", "etcd-serving-", "etcd-serving-metrics-"} { + k.EXPECT().KubeDelete(ctx, "Secret", namespaceEtcds, prefix+buildNodeName(doc, degradedNode), false, nil) + } + }, + }, + { + name: "fail: Multiple degraded etcd instances scenario", + wantErr: "500: InternalServerError: : only a single degraded etcd pod can can be recovered from, more than one NotReady etcd pods were found: [etcd-cluster-zfsbk-master-0 etcd-cluster-zfsbk-master-1 etcd-cluster-zfsbk-master-2]", + pods: newEtcdPods(t, doc, false, true, true), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + }, + }, + { + name: "fail: empty/correct pod env and no bad container statuses", + wantErr: "500: InternalServerError: : no etcd pod's were found in a CrashLoopBackOff state, unable to remediate etcd deployment", + pods: newEtcdPods(t, doc, true, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + }, + }, + { + name: "fail: create job data backup", + wantErr: "500: InternalServerError: : oh no, can't create job data backup", + pods: newEtcdPods(t, doc, false, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(errors.New("oh no, can't create job data backup")) + }, + }, + { + name: "fail: create job fix peers", + wantErr: "500: InternalServerError: : oh no, can't create job fix peers", + pods: newEtcdPods(t, doc, false, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil) + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodSucceeded, false)() + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).MaxTimes(1).Return([]byte("Backup job doing backup things..."), nil) + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + + // fixPeers + // createPrivilegedServiceAccount + serviceAcc := newServiceAccount(serviceAccountName, doc.OpenShiftCluster.Name) + clusterRole := newClusterRole(kubeServiceAccount, doc.OpenShiftCluster.Name) + crb := newClusterRoleBinding(serviceAccountName, doc.OpenShiftCluster.Name) + scc := newSecurityContextConstraint(serviceAccountName, doc.OpenShiftCluster.Name, kubeServiceAccount) + + k.EXPECT().KubeCreateOrUpdate(ctx, serviceAcc).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, clusterRole).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, crb).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, scc).MaxTimes(1).Return(nil) + + de, err := findDegradedEtcd(ti.log, pods) + if err != nil { + t.Fatal(err) + } + peerPods, err := getPeerPods(pods.Items, de, doc.OpenShiftCluster.Name) + if err != nil { + t.Fatal(err) + } + + jobFixPeers := newJobFixPeers(doc.OpenShiftCluster.Name, peerPods, de.Node) + k.EXPECT().KubeCreateOrUpdate(ctx, jobFixPeers).MaxTimes(1).Return(errors.New("oh no, can't create job fix peers")) + expectWatchEvent(gomock.Any(), jobFixPeers, k, "app", corev1.PodSucceeded, false)() + + // cleanup + k.EXPECT().KubeDelete(ctx, serviceAcc.GetKind(), serviceAcc.GetNamespace(), serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, scc.GetKind(), scc.GetNamespace(), scc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, clusterRole.GetKind(), clusterRole.GetNamespace(), clusterRole.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, crb.GetKind(), crb.GetNamespace(), crb.GetName(), true, nil).MaxTimes(1).Return(nil) + }, + }, + { + name: "fail: create service account", + wantErr: "500: InternalServerError: : oh no, can't create service account %!!(MISSING)s()", + pods: newEtcdPods(t, doc, false, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil) + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodSucceeded, false)() + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).MaxTimes(1).Return([]byte("Backup job doing backup things..."), nil) + + // fixPeers + serviceAcc := newServiceAccount(serviceAccountName, doc.OpenShiftCluster.Name) + + // k.EXPECT().KubeCreateOrUpdate(ctx, serviceAcc).MaxTimes(1).Return(errors.New(tt.wantErr)) + k.EXPECT().KubeCreateOrUpdate(ctx, serviceAcc).MaxTimes(1).Return(errors.New("oh no, can't create service account")) + + // nested cleanup + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "ServiceAccount", namespaceEtcds, serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "SecurityContextConstraints", "", serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "ClusterRole", "", "system:serviceaccountopenshift-etcd:etcd-recovery-privileged", true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "ClusterRoleBinding", "", serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + }, + }, + { + name: "fail: create cluster role", + wantErr: "500: InternalServerError: : oh no, can't create job fix peers %!!(MISSING)s()", + pods: newEtcdPods(t, doc, false, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil) + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodSucceeded, false)() + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).MaxTimes(1).Return([]byte("Backup job doing backup things..."), nil) + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + + // fixPeers + // createPrivilegedServiceAccount + serviceAcc := newServiceAccount(serviceAccountName, doc.OpenShiftCluster.Name) + clusterRole := newClusterRole(kubeServiceAccount, doc.OpenShiftCluster.Name) + + k.EXPECT().KubeCreateOrUpdate(ctx, serviceAcc).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, clusterRole).MaxTimes(1).Return(errors.New("oh no, can't create job fix peers")) + k.EXPECT().KubeDelete(ctx, "ServiceAccount", namespaceEtcds, serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "SecurityContextConstraints", "", serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "ClusterRole", "", "system:serviceaccountopenshift-etcd:etcd-recovery-privileged", true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "ClusterRoleBinding", "", serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + }, + }, + { + name: "fail: create cluster role binding", + wantErr: "500: InternalServerError: : oh no, can't create cluster role binding %!!(MISSING)s()", + pods: newEtcdPods(t, doc, false, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil) + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodSucceeded, false)() + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).MaxTimes(1).Return([]byte("Backup job doing backup things..."), nil) + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + + // fixPeers + // createPrivilegedServiceAccount + serviceAcc := newServiceAccount(serviceAccountName, doc.OpenShiftCluster.Name) + clusterRole := newClusterRole(kubeServiceAccount, doc.OpenShiftCluster.Name) + crb := newClusterRoleBinding(serviceAccountName, doc.OpenShiftCluster.Name) + + // cleanup + k.EXPECT().KubeCreateOrUpdate(ctx, serviceAcc).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, clusterRole).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, crb).MaxTimes(1).Return(errors.New("oh no, can't create cluster role binding")) + k.EXPECT().KubeDelete(ctx, "ServiceAccount", namespaceEtcds, serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "SecurityContextConstraints", "", serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "ClusterRole", "", "system:serviceaccountopenshift-etcd:etcd-recovery-privileged", true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "ClusterRoleBinding", "", serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + }, + }, + { + name: "fail: create security context constraint", + wantErr: "500: InternalServerError: : oh no, can't create security context constraint %!!(MISSING)s()", + pods: newEtcdPods(t, doc, false, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil) + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodSucceeded, false)() + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).MaxTimes(1).Return([]byte("Backup job doing backup things..."), nil) + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + + // fixPeers + // createPrivilegedServiceAccount + serviceAcc := newServiceAccount(serviceAccountName, doc.OpenShiftCluster.Name) + clusterRole := newClusterRole(kubeServiceAccount, doc.OpenShiftCluster.Name) + crb := newClusterRoleBinding(serviceAccountName, doc.OpenShiftCluster.Name) + scc := newSecurityContextConstraint(serviceAccountName, doc.OpenShiftCluster.Name, kubeServiceAccount) + + k.EXPECT().KubeCreateOrUpdate(ctx, serviceAcc).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, clusterRole).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, crb).MaxTimes(1).Return(nil) + k.EXPECT().KubeCreateOrUpdate(ctx, scc).MaxTimes(1).Return(errors.New("oh no, can't create security context constraint")) + + // cleanup + k.EXPECT().KubeDelete(ctx, "ServiceAccount", namespaceEtcds, serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "SecurityContextConstraints", "", serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "ClusterRole", "", "system:serviceaccountopenshift-etcd:etcd-recovery-privileged", true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "ClusterRoleBinding", "", serviceAcc.GetName(), true, nil).MaxTimes(1).Return(nil) + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + }, + }, + { + name: "fail: Backup job Pod failed", + wantErr: "500: InternalServerError: : pod etcd-recovery-data-backup event Failed received with message Pod Failed for reasons XYZ...", + pods: newEtcdPods(t, doc, false, false, false), + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil).MaxTimes(1) + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodFailed, false)() + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).MaxTimes(1).Return([]byte("oh no, Pod is in a failed state"), nil) + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + }, + }, + { + name: "fail: Context cancelled", + wantErr: "500: InternalServerError: : context was cancelled while waiting for etcd-recovery-data-backup because context canceled", + pods: newEtcdPods(t, doc, false, false, false), + cancel: true, + ctxCancel: ctxCancel, + mocks: func(tt *test, t *testing.T, ti *testInfra, k *mock_adminactions.MockKubeActions, pods *corev1.PodList, ctxCancel context.CancelFunc) { + buf := &bytes.Buffer{} + err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(pods) + if err != nil { + t.Fatalf("%s failed to encode pods, %s", t.Name(), err.Error()) + } + k.EXPECT().KubeList(ctx, "Pod", namespaceEtcds).MaxTimes(1).Return(buf.Bytes(), nil) + + // backupEtcd + jobBackupEtcd := createBackupEtcdDataJob(doc.OpenShiftCluster.Name, buildNodeName(doc, degradedNode)) + k.EXPECT().KubeCreateOrUpdate(ctx, jobBackupEtcd).MaxTimes(1).Return(nil).MaxTimes(1) + expectWatchEvent(gomock.Any(), jobBackupEtcd, k, "app", corev1.PodPending, true) + if tt.cancel { + tt.ctxCancel() + } + k.EXPECT().KubeGetPodLogs(ctx, jobBackupEtcd.GetNamespace(), jobBackupEtcd.GetName(), jobBackupEtcd.GetName()).MaxTimes(1).Return([]byte(tt.wantErr), nil) + propPolicy := metav1.DeletePropagationBackground + k.EXPECT().KubeDelete(ctx, "Job", namespaceEtcds, jobBackupEtcd.GetName(), true, &propPolicy).MaxTimes(1).Return(nil) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ti := newTestInfra(t).WithOpenShiftClusters().WithSubscriptions() + defer ti.done() + + k := mock_adminactions.NewMockKubeActions(ti.controller) + tt.mocks(tt, t, ti, k, tt.pods, ctxCancel) + + ti.fixture.AddOpenShiftClusterDocuments(doc) + ti.fixture.AddSubscriptionDocuments(&api.SubscriptionDocument{ + ID: mockSubID, + Subscription: &api.Subscription{ + State: api.SubscriptionStateRegistered, + Properties: &api.SubscriptionProperties{ + TenantID: mockTenantID, + }, + }, + }) + + f, err := NewFrontend(ctx, + ti.audit, + ti.log, + ti.env, + ti.asyncOperationsDatabase, + ti.clusterManagerDatabase, + ti.openShiftClustersDatabase, + ti.subscriptionsDatabase, + nil, + api.APIs, + &noop.Noop{}, + nil, + nil, + nil, + nil, + ti.enricher) + if err != nil { + t.Fatal(err) + } + + containerLogs, err := f.fixEtcd(ctx, ti.log, ti.env, doc, k, &operatorv1fake.FakeEtcds{ + Fake: &operatorv1fake.FakeOperatorV1{ + Fake: &ktesting.Fake{}, + }, + }) + ti.log.Infof("Container logs: \n%s", containerLogs) + if err != nil && err.Error() != tt.wantErr || + err == nil && tt.wantErr != "" { + t.Errorf("\n%s\n !=\n%s", err.Error(), tt.wantErr) + } + }) + } +} + +func expectWatchEvent(ctx gomock.Matcher, o *unstructured.Unstructured, k *mock_adminactions.MockKubeActions, labelKey string, podPhase corev1.PodPhase, noUpdates bool) func() { + message := "" + switch podPhase { + case corev1.PodSucceeded: + message = "Pod succeeded Successfully" + case corev1.PodFailed: + message = "Pod Failed for reasons XYZ..." + case corev1.PodPending: + message = "Pod is pending..." + case corev1.PodUnknown: + message = "Pod status is unknown..." + } + w := watch.NewFake() + k.EXPECT().KubeWatch(ctx, o, labelKey).MaxTimes(1).Return(watch.Interface(w), nil) + return func() { + go func() { + w.Add(&corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: o.GetName(), + Namespace: o.GetNamespace(), + }, + Status: corev1.PodStatus{ + Phase: podPhase, + Message: message, + }, + }) + w.Reset() + }() + } +} + +func buildClusterName(doc *api.OpenShiftClusterDocument) string { + return doc.OpenShiftCluster.Name + "-" + doc.OpenShiftCluster.Properties.InfraID +} + +func buildNodeName(doc *api.OpenShiftClusterDocument, node string) string { + c := buildClusterName(doc) + return c + "-" + node +} + +func newEtcdPods(t *testing.T, doc *api.OpenShiftClusterDocument, healthy, multiDegraded, emptyEnv bool) *corev1.PodList { + var ( + degradedNodeMaster2 = buildNodeName(doc, degradedNode) + nodeMaster0 = buildNodeName(doc, "master-0") + nodeMaster1 = buildNodeName(doc, "master-1") + ) + const ( + master0IP = "10.0.0.1" + master1IP = "10.0.0.2" + master2IP = "10.0.0.3" + master2ChangedIP = "10.0.0.9" + ) + + if healthy && multiDegraded { + t.Fatalf("TEST %s: healthy (value %t) and multiDegraded (value %t) cannot both be true, failed sanity check", t.Name(), healthy, multiDegraded) + } + + // Used to test scenario when etcd's env vars are empty, or there is no conflict found + // then statuses will be tests + envs := []corev1.EnvVar{ + { + Name: "NODE_" + doc.OpenShiftCluster.Name + "_" + doc.OpenShiftCluster.Properties.InfraID + "_master_0_IP", + Value: master0IP, + }, + { + Name: "NODE_ " + doc.OpenShiftCluster.Name + "_" + doc.OpenShiftCluster.Properties.InfraID + "_master_1_IP", + Value: master1IP, + }, + { + Name: "NODE_" + doc.OpenShiftCluster.Name + "_" + doc.OpenShiftCluster.Properties.InfraID + "_master_2_IP", + Value: master2IP, + }, + } + if emptyEnv { + envs = []corev1.EnvVar{} + } + containerID := "quay://etcd-container-id" + badStatus := []corev1.ContainerStatus{ + { + Name: "etcd", + Ready: false, + Started: to.BoolPtr(false), + RestartCount: 50, + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "Container is in a crashloop backoff", + Message: "Container crashloop backoff", + }, + }, + ContainerID: containerID, + }, + } + + statuses := []corev1.ContainerStatus{ + { + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + ContainerID: containerID, + }, + } + if multiDegraded { + statuses = badStatus + } + + pods := &corev1.PodList{ + TypeMeta: metav1.TypeMeta{ + Kind: "Etcd", + }, + Items: []corev1.Pod{ + // healthy pod + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-" + nodeMaster0, + Namespace: namespaceEtcds, + }, + Status: corev1.PodStatus{ + ContainerStatuses: statuses, + PodIPs: []corev1.PodIP{ + { + IP: master0IP, + }, + }, + }, + Spec: corev1.PodSpec{ + NodeName: nodeMaster0, + Containers: []corev1.Container{ + { + Name: "etcd", + Env: envs, + }, + }, + }, + }, + // healthy pod + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-" + nodeMaster1, + Namespace: namespaceEtcds, + }, + Status: corev1.PodStatus{ + ContainerStatuses: statuses, + PodIPs: []corev1.PodIP{ + { + IP: master1IP, + }, + }, + }, + Spec: corev1.PodSpec{ + NodeName: nodeMaster1, + Containers: []corev1.Container{ + { + Name: "etcd", + Env: envs, + }, + }, + }, + }, + // degraded pod + { + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-" + degradedNodeMaster2, + Namespace: namespaceEtcds, + }, + Status: corev1.PodStatus{ + ContainerStatuses: badStatus, + PodIPs: []corev1.PodIP{ + { + IP: master2ChangedIP, + }, + }, + }, + Spec: corev1.PodSpec{ + NodeName: degradedNodeMaster2, + Containers: []corev1.Container{ + { + Name: "etcd", + Env: envs, + }, + }, + }, + }, + }, + } + + if healthy { + pods.Items[len(pods.Items)-1].Status.ContainerStatuses = statuses + pods.Items[len(pods.Items)-1].Status.PodIPs = []corev1.PodIP{ + { + IP: master2IP, + }, + } + } + + return pods +} diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 24c461f8f57..5f7abe2f5ed 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -294,6 +294,11 @@ func (f *frontend) chiAuthenticatedRoutes(router chi.Router) { }) r.Get("/supportedvmsizes", f.supportedvmsizes) + r.Route("/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/etcdrecovery", + func(r chi.Router) { + r.Post("/", f.postAdminOpenShiftClusterEtcdRecovery) + }) + r.Route("/subscriptions/{subscriptionId}/resourcegroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}/kubernetesobjects", func(r chi.Router) { r.Get("/", f.getAdminKubernetesObjects) diff --git a/pkg/frontend/scripts.go b/pkg/frontend/scripts.go new file mode 100644 index 00000000000..d04efe91a34 --- /dev/null +++ b/pkg/frontend/scripts.go @@ -0,0 +1,9 @@ +package frontend + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import _ "embed" + +//go:embed scripts/backupandfixetcd.sh +var backupOrFixEtcd string diff --git a/pkg/frontend/scripts/backupandfixetcd.sh b/pkg/frontend/scripts/backupandfixetcd.sh new file mode 100755 index 00000000000..10520850fb7 --- /dev/null +++ b/pkg/frontend/scripts/backupandfixetcd.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# See for more information: https://docs.openshift.com/container-platform/4.10/backup_and_restore/control_plane_backup_and_restore/replacing-unhealthy-etcd-member.html + +remove_peer_members() { + echo "${PEER_PODS}" + for p in ${PEER_PODS}; do + echo "Attempting to get ID for pod/${p}" + members="$(oc rsh -n openshift-etcd -c etcdctl "pod/${p}" etcdctl member list -w json --hex true)" + id="$(jq -r --arg node "$DEGRADED_NODE" '.members[] | select( .name == $node).ID' <<< "$members")" + echo "id: ${id:-Not Found}" + if [[ -n $id ]]; then + echo "rshing into pod/${p} now to remove member id $id" + oc rsh \ + -n openshift-etcd \ + -c etcdctl \ + "pod/${p}" etcdctl member remove "$id" + else + echo "${DEGRADED_NODE} id not found in etcd member list for pod ${p}" + fi + done +} + +# jq expects it's required shared libraries to be present in /usr/lib64, not /host/usr/lib64. +# Because we are using jq mount under /host and haven't installed jq, those libraries exist under /host/usr/lib64 rather than /usr/lib64. +# Creating the symbolic links allows jq to resolve it's libraries without the need for installing. +create_sym_links() { + jq_lib1="/usr/lib64/libjq.so.1" + jq_lib2="/usr/lib64/libonig.so.5" + + if [[ ! -f $jq_lib1 ]]; then + ln -s "/host${jq_lib1}" "$jq_lib1" + fi + if [[ ! -f $jq_lib2 ]]; then + ln -s "/host${jq_lib2}" "$jq_lib2" + fi +} + +backup_etcd() { + local bdir etcd_yaml etcd_dir + bdir=/var/lib/etcd-backup + etcd_yaml=/etc/kubernetes/manifests/etcd-pod.yaml + etcd_dir=/var/lib/etcd + if [[ -d $etcd_dir ]] && [[ -f $etcd_yaml ]]; then + echo "Creating $bdir" + mkdir -p "$bdir" || abort "failed to make backup directory" + echo "Moving $etcd_yaml to $bdir" + mv "$etcd_yaml" "$bdir" || abort "failed to move $etcd_yaml to $bdir" + echo "Moving $etcd_dir to /host/tmp" + mv "$etcd_dir" /tmp || abort "failed to move $etcd_dir to /tmp" + else + echo "$etcd_dir doesn't exist or $etcd_yaml has already been moved" + echo "Not taking host etcd backup" + fi +} + +abort() { + echo "${1}, Aborting." + exit 1 +} + +if [[ -n $FIX_PEERS ]]; then + PATH+="${PATH}:/host/usr/bin" + create_sym_links + echo "Starting peer etcd member removal" + remove_peer_members +elif [[ -n $BACKUP ]]; then + echo "Starting etcd data backup" + backup_etcd +else + abort "BACKUP and FIX_PEERS are unset, no actions taken." +fi diff --git a/pkg/operator/controllers/clusteroperatoraro/clusteroperatoraro_controller_test.go b/pkg/operator/controllers/clusteroperatoraro/clusteroperatoraro_controller_test.go new file mode 100644 index 00000000000..b5102004f3f --- /dev/null +++ b/pkg/operator/controllers/clusteroperatoraro/clusteroperatoraro_controller_test.go @@ -0,0 +1,274 @@ +package clusteroperatoraro + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp/cmpopts" + configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/cmp" + _ "github.com/Azure/ARO-RP/pkg/util/scheme" + "github.com/Azure/ARO-RP/pkg/util/version" + utilconditions "github.com/Azure/ARO-RP/test/util/conditions" + utilerror "github.com/Azure/ARO-RP/test/util/error" +) + +func TestConditions(t *testing.T) { + tests := []struct { + name string + controllerConditions []operatorv1.OperatorCondition + wantConditions []configv1.ClusterOperatorStatusCondition + wantErr string + }{ + { + name: "no conditions sets defaults", + controllerConditions: []operatorv1.OperatorCondition{}, + wantConditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "NoData", + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "NoData", + }, + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "AsExpected", + }, + }, + }, + { + name: "All controllers available sets Available=True", + controllerConditions: []operatorv1.OperatorCondition{ + utilconditions.ControllerDefaultAvailable("ControllerA"), + utilconditions.ControllerDefaultAvailable("ControllerB"), + utilconditions.ControllerDefaultAvailable("ControllerC"), + }, + wantConditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "AsExpected", + Message: "All is well", + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "NoData", + }, + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "AsExpected", + }, + }, + }, + { + name: "Controller not available sets Available=False", + controllerConditions: []operatorv1.OperatorCondition{ + utilconditions.ControllerDefaultAvailable("ControllerA"), + { + Type: "ControllerBAvailable", + Status: operatorv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "SomeError", + Message: "An error occurred", + }, + utilconditions.ControllerDefaultAvailable("ControllerC"), + }, + wantConditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "ControllerB_SomeError", + Message: "ControllerBAvailable: An error occurred", + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "NoData", + }, + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "AsExpected", + }, + }, + }, + { + name: "All controllers not progressing sets Progressing=False", + controllerConditions: []operatorv1.OperatorCondition{ + utilconditions.ControllerDefaultProgressing("ControllerA"), + utilconditions.ControllerDefaultProgressing("ControllerB"), + utilconditions.ControllerDefaultProgressing("ControllerC"), + }, + wantConditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "NoData", + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "AsExpected", + Message: "All is well", + }, + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "AsExpected", + }, + }, + }, + { + name: "Controller progressing sets Progressing=True", + controllerConditions: []operatorv1.OperatorCondition{ + utilconditions.ControllerDefaultProgressing("ControllerA"), + { + Type: "ControllerBProgressing", + Status: operatorv1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "SomeProcess", + Message: "Something is happening", + }, + utilconditions.ControllerDefaultProgressing("ControllerC"), + }, + wantConditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "NoData", + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "ControllerB_SomeProcess", + Message: "ControllerBProgressing: Something is happening", + }, + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "AsExpected", + }, + }, + }, + { + name: "Controller degraded does NOT set Degraded=True", + controllerConditions: []operatorv1.OperatorCondition{ + utilconditions.ControllerDefaultDegraded("ControllerA"), + { + Type: "ControllerBDegraded", + Status: operatorv1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "SomeProcess", + Message: "Something bad is happening", + }, + utilconditions.ControllerDefaultDegraded("ControllerC"), + }, + wantConditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "NoData", + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "NoData", + }, + { + Type: configv1.OperatorDegraded, + Status: configv1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "AsExpected", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cluster := &arov1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: arov1alpha1.SingletonClusterName}, + Status: arov1alpha1.ClusterStatus{ + Conditions: tt.controllerConditions, + }, + } + clientFake := ctrlfake.NewClientBuilder(). + WithObjects(cluster). + Build() + + r := NewReconciler(logrus.NewEntry(logrus.StandardLogger()), clientFake) + + request := ctrl.Request{} + ctx := context.Background() + + _, err := r.Reconcile(ctx, request) + + utilerror.AssertErrorMessage(t, err, tt.wantErr) + + operator := &configv1.ClusterOperator{} + if err := clientFake.Get(ctx, types.NamespacedName{Name: clusterOperatorName}, operator); err != nil { + t.Error(err) + } + if diff := cmp.Diff(tt.wantConditions, operator.Status.Conditions, cmpopts.EquateApproxTime(time.Second)); diff != "" { + t.Error(diff) + } + + // static checks - these should always be set on the operator resource after every reconcile + wantVersion := []configv1.OperandVersion{{ + Name: "operator", + Version: version.GitCommit, + }} + if diff := cmp.Diff(wantVersion, operator.Status.Versions); diff != "" { + t.Error(diff) + } + + wantOwnerReference := []metav1.OwnerReference{{ + APIVersion: arov1alpha1.GroupVersion.Identifier(), + Kind: "Cluster", + Name: arov1alpha1.SingletonClusterName, + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }} + if diff := cmp.Diff(wantOwnerReference, operator.ObjectMeta.OwnerReferences); diff != "" { + t.Error(diff) + } + }) + } +} diff --git a/pkg/operator/controllers/dnsmasq/cluster_controller.go b/pkg/operator/controllers/dnsmasq/cluster_controller.go index b316b3e5709..d898e4558f3 100644 --- a/pkg/operator/controllers/dnsmasq/cluster_controller.go +++ b/pkg/operator/controllers/dnsmasq/cluster_controller.go @@ -9,7 +9,6 @@ import ( mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/sirupsen/logrus" kruntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -17,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/operator/controllers/base" "github.com/Azure/ARO-RP/pkg/util/dynamichelper" ) @@ -27,49 +27,51 @@ const ( ) type ClusterReconciler struct { - log *logrus.Entry - + base.AROController dh dynamichelper.Interface - - client client.Client } func NewClusterReconciler(log *logrus.Entry, client client.Client, dh dynamichelper.Interface) *ClusterReconciler { return &ClusterReconciler{ - log: log, - dh: dh, - client: client, + AROController: base.AROController{ + Log: log, + Client: client, + Name: ClusterControllerName, + }, + dh: dh, } } // Reconcile watches the ARO object, and if it changes, reconciles all the // 99-%s-aro-dns machineconfigs func (r *ClusterReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - instance := &arov1alpha1.Cluster{} - err := r.client.Get(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, instance) + instance, err := r.GetCluster(ctx) if err != nil { return reconcile.Result{}, err } if !instance.Spec.OperatorFlags.GetSimpleBoolean(controllerEnabled) { - r.log.Debug("controller is disabled") + r.Log.Debug("controller is disabled") return reconcile.Result{}, nil } - r.log.Debug("running") + r.Log.Debug("running") mcps := &mcv1.MachineConfigPoolList{} - err = r.client.List(ctx, mcps) + err = r.Client.List(ctx, mcps) if err != nil { - r.log.Error(err) + r.Log.Error(err) + r.SetDegraded(ctx, err) return reconcile.Result{}, err } err = reconcileMachineConfigs(ctx, instance, r.dh, mcps.Items...) if err != nil { - r.log.Error(err) + r.Log.Error(err) + r.SetDegraded(ctx, err) return reconcile.Result{}, err } + r.ClearConditions(ctx) return reconcile.Result{}, nil } diff --git a/pkg/operator/controllers/dnsmasq/cluster_controller_test.go b/pkg/operator/controllers/dnsmasq/cluster_controller_test.go index 55e39429b23..f2830ac5cb3 100644 --- a/pkg/operator/controllers/dnsmasq/cluster_controller_test.go +++ b/pkg/operator/controllers/dnsmasq/cluster_controller_test.go @@ -6,8 +6,10 @@ package dnsmasq import ( "context" "testing" + "time" "github.com/golang/mock/gomock" + operatorv1 "github.com/openshift/api/operator/v1" mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -17,20 +19,28 @@ import ( arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" mock_dynamichelper "github.com/Azure/ARO-RP/pkg/util/mocks/dynamichelper" + utilconditions "github.com/Azure/ARO-RP/test/util/conditions" utilerror "github.com/Azure/ARO-RP/test/util/error" ) func TestClusterReconciler(t *testing.T) { + transitionTime := metav1.Time{Time: time.Now()} + defaultAvailable := utilconditions.ControllerDefaultAvailable(ClusterControllerName) + defaultProgressing := utilconditions.ControllerDefaultProgressing(ClusterControllerName) + defaultDegraded := utilconditions.ControllerDefaultDegraded(ClusterControllerName) + defaultConditions := []operatorv1.OperatorCondition{defaultAvailable, defaultProgressing, defaultDegraded} + fakeDh := func(controller *gomock.Controller) *mock_dynamichelper.MockInterface { return mock_dynamichelper.NewMockInterface(controller) } tests := []struct { - name string - objects []client.Object - mocks func(mdh *mock_dynamichelper.MockInterface) - request ctrl.Request - wantErrMsg string + name string + objects []client.Object + mocks func(mdh *mock_dynamichelper.MockInterface) + request ctrl.Request + wantErrMsg string + wantConditions []operatorv1.OperatorCondition }{ { name: "no cluster", @@ -44,7 +54,9 @@ func TestClusterReconciler(t *testing.T) { objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: defaultConditions, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "false", @@ -52,16 +64,27 @@ func TestClusterReconciler(t *testing.T) { }, }, }, - mocks: func(mdh *mock_dynamichelper.MockInterface) {}, - request: ctrl.Request{}, - wantErrMsg: "", + mocks: func(mdh *mock_dynamichelper.MockInterface) {}, + request: ctrl.Request{}, + wantErrMsg: "", + wantConditions: defaultConditions, }, { name: "no MachineConfigPools does nothing", objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: []operatorv1.OperatorCondition{ + defaultAvailable, + defaultProgressing, + { + Type: ClusterControllerName + "Controller" + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionTrue, + LastTransitionTime: transitionTime, + }, + }, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "true", @@ -72,15 +95,18 @@ func TestClusterReconciler(t *testing.T) { mocks: func(mdh *mock_dynamichelper.MockInterface) { mdh.EXPECT().Ensure(gomock.Any()).Times(1) }, - request: ctrl.Request{}, - wantErrMsg: "", + request: ctrl.Request{}, + wantErrMsg: "", + wantConditions: defaultConditions, }, { name: "valid MachineConfigPool creates ARO DNS MachineConfig", objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: defaultConditions, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "true", @@ -96,8 +122,9 @@ func TestClusterReconciler(t *testing.T) { mocks: func(mdh *mock_dynamichelper.MockInterface) { mdh.EXPECT().Ensure(gomock.Any(), gomock.AssignableToTypeOf(&mcv1.MachineConfig{})).Times(1) }, - request: ctrl.Request{}, - wantErrMsg: "", + request: ctrl.Request{}, + wantErrMsg: "", + wantConditions: defaultConditions, }, } @@ -118,10 +145,11 @@ func TestClusterReconciler(t *testing.T) { client, dh, ) - - _, err := r.Reconcile(context.Background(), tt.request) + ctx := context.Background() + _, err := r.Reconcile(ctx, tt.request) utilerror.AssertErrorMessage(t, err, tt.wantErrMsg) + utilconditions.AssertControllerConditions(t, ctx, client, tt.wantConditions) }) } } diff --git a/pkg/operator/controllers/dnsmasq/machineconfig_controller.go b/pkg/operator/controllers/dnsmasq/machineconfig_controller.go index c93a9f7ad69..45c5c576d90 100644 --- a/pkg/operator/controllers/dnsmasq/machineconfig_controller.go +++ b/pkg/operator/controllers/dnsmasq/machineconfig_controller.go @@ -15,7 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/operator/controllers/base" "github.com/Azure/ARO-RP/pkg/util/dynamichelper" ) @@ -24,38 +24,38 @@ const ( ) type MachineConfigReconciler struct { - log *logrus.Entry + base.AROController dh dynamichelper.Interface - - client client.Client } var rxARODNS = regexp.MustCompile("^99-(.*)-aro-dns$") func NewMachineConfigReconciler(log *logrus.Entry, client client.Client, dh dynamichelper.Interface) *MachineConfigReconciler { return &MachineConfigReconciler{ - log: log, - dh: dh, - client: client, + AROController: base.AROController{ + Log: log, + Client: client, + Name: MachineConfigControllerName, + }, + dh: dh, } } // Reconcile watches ARO DNS MachineConfig objects, and if any changes, // reconciles it func (r *MachineConfigReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - instance := &arov1alpha1.Cluster{} - err := r.client.Get(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, instance) + instance, err := r.GetCluster(ctx) if err != nil { return reconcile.Result{}, err } if !instance.Spec.OperatorFlags.GetSimpleBoolean(controllerEnabled) { - r.log.Debug("controller is disabled") + r.Log.Debug("controller is disabled") return reconcile.Result{}, nil } - r.log.Debug("running") + r.Log.Debug("running") m := rxARODNS.FindStringSubmatch(request.Name) if m == nil { return reconcile.Result{}, nil @@ -63,12 +63,14 @@ func (r *MachineConfigReconciler) Reconcile(ctx context.Context, request ctrl.Re role := m[1] mcp := &mcv1.MachineConfigPool{} - err = r.client.Get(ctx, types.NamespacedName{Name: role}, mcp) + err = r.Client.Get(ctx, types.NamespacedName{Name: role}, mcp) if kerrors.IsNotFound(err) { + r.ClearDegraded(ctx) return reconcile.Result{}, nil } if err != nil { - r.log.Error(err) + r.Log.Error(err) + r.SetDegraded(ctx, err) return reconcile.Result{}, err } if mcp.GetDeletionTimestamp() != nil { @@ -77,10 +79,12 @@ func (r *MachineConfigReconciler) Reconcile(ctx context.Context, request ctrl.Re err = reconcileMachineConfigs(ctx, instance, r.dh, *mcp) if err != nil { - r.log.Error(err) + r.Log.Error(err) + r.SetDegraded(ctx, err) return reconcile.Result{}, err } + r.ClearConditions(ctx) return reconcile.Result{}, nil } diff --git a/pkg/operator/controllers/dnsmasq/machineconfig_controller_test.go b/pkg/operator/controllers/dnsmasq/machineconfig_controller_test.go index 6d45d5f203e..e47638d4f92 100644 --- a/pkg/operator/controllers/dnsmasq/machineconfig_controller_test.go +++ b/pkg/operator/controllers/dnsmasq/machineconfig_controller_test.go @@ -6,8 +6,10 @@ package dnsmasq import ( "context" "testing" + "time" "github.com/golang/mock/gomock" + operatorv1 "github.com/openshift/api/operator/v1" mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,20 +20,27 @@ import ( arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" mock_dynamichelper "github.com/Azure/ARO-RP/pkg/util/mocks/dynamichelper" + utilconditions "github.com/Azure/ARO-RP/test/util/conditions" utilerror "github.com/Azure/ARO-RP/test/util/error" ) func TestMachineConfigReconciler(t *testing.T) { + transitionTime := metav1.Time{Time: time.Now()} + defaultAvailable := utilconditions.ControllerDefaultAvailable(MachineConfigControllerName) + defaultProgressing := utilconditions.ControllerDefaultProgressing(MachineConfigControllerName) + defaultDegraded := utilconditions.ControllerDefaultDegraded(MachineConfigControllerName) + defaultConditions := []operatorv1.OperatorCondition{defaultAvailable, defaultProgressing, defaultDegraded} fakeDh := func(controller *gomock.Controller) *mock_dynamichelper.MockInterface { return mock_dynamichelper.NewMockInterface(controller) } tests := []struct { - name string - objects []client.Object - mocks func(mdh *mock_dynamichelper.MockInterface) - request ctrl.Request - wantErrMsg string + name string + objects []client.Object + mocks func(mdh *mock_dynamichelper.MockInterface) + request ctrl.Request + wantErrMsg string + wantConditions []operatorv1.OperatorCondition }{ { name: "no cluster", @@ -45,7 +54,9 @@ func TestMachineConfigReconciler(t *testing.T) { objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: defaultConditions, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "false", @@ -53,16 +64,27 @@ func TestMachineConfigReconciler(t *testing.T) { }, }, }, - mocks: func(mdh *mock_dynamichelper.MockInterface) {}, - request: ctrl.Request{}, - wantErrMsg: "", + mocks: func(mdh *mock_dynamichelper.MockInterface) {}, + request: ctrl.Request{}, + wantErrMsg: "", + wantConditions: defaultConditions, }, { name: "no MachineConfigPool for MachineConfig does nothing", objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: []operatorv1.OperatorCondition{ + defaultAvailable, + defaultProgressing, + { + Type: MachineConfigControllerName + "Controller" + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionTrue, + LastTransitionTime: transitionTime, + }, + }, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "true", @@ -77,14 +99,17 @@ func TestMachineConfigReconciler(t *testing.T) { Name: "99-custom-aro-dns", }, }, - wantErrMsg: "", + wantErrMsg: "", + wantConditions: defaultConditions, }, { name: "valid MachineConfigPool for MachineConfig reconciles MachineConfig", objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: defaultConditions, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "true", @@ -106,7 +131,8 @@ func TestMachineConfigReconciler(t *testing.T) { Name: "99-custom-aro-dns", }, }, - wantErrMsg: "", + wantErrMsg: "", + wantConditions: defaultConditions, }, } @@ -126,10 +152,11 @@ func TestMachineConfigReconciler(t *testing.T) { client, dh, ) - - _, err := r.Reconcile(context.Background(), tt.request) + ctx := context.Background() + _, err := r.Reconcile(ctx, tt.request) utilerror.AssertErrorMessage(t, err, tt.wantErrMsg) + utilconditions.AssertControllerConditions(t, ctx, client, tt.wantConditions) }) } } diff --git a/pkg/operator/controllers/dnsmasq/machineconfigpool_controller.go b/pkg/operator/controllers/dnsmasq/machineconfigpool_controller.go index 5be6a108cba..cb255db94f1 100644 --- a/pkg/operator/controllers/dnsmasq/machineconfigpool_controller.go +++ b/pkg/operator/controllers/dnsmasq/machineconfigpool_controller.go @@ -14,7 +14,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/operator/controllers/base" "github.com/Azure/ARO-RP/pkg/util/dynamichelper" ) @@ -23,52 +23,56 @@ const ( ) type MachineConfigPoolReconciler struct { - log *logrus.Entry + base.AROController dh dynamichelper.Interface - - client client.Client } func NewMachineConfigPoolReconciler(log *logrus.Entry, client client.Client, dh dynamichelper.Interface) *MachineConfigPoolReconciler { return &MachineConfigPoolReconciler{ - log: log, - dh: dh, - client: client, + AROController: base.AROController{ + Log: log, + Client: client, + Name: MachineConfigPoolControllerName, + }, + dh: dh, } } // Reconcile watches MachineConfigPool objects, and if any changes, // reconciles the associated ARO DNS MachineConfig object func (r *MachineConfigPoolReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { - instance := &arov1alpha1.Cluster{} - err := r.client.Get(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, instance) + instance, err := r.GetCluster(ctx) if err != nil { return reconcile.Result{}, err } if !instance.Spec.OperatorFlags.GetSimpleBoolean(controllerEnabled) { - r.log.Debug("controller is disabled") + r.Log.Debug("controller is disabled") return reconcile.Result{}, nil } - r.log.Debug("running") + r.Log.Debug("running") mcp := &mcv1.MachineConfigPool{} - err = r.client.Get(ctx, types.NamespacedName{Name: request.Name}, mcp) + err = r.Client.Get(ctx, types.NamespacedName{Name: request.Name}, mcp) if kerrors.IsNotFound(err) { + r.ClearDegraded(ctx) return reconcile.Result{}, nil } if err != nil { - r.log.Error(err) + r.Log.Error(err) + r.SetDegraded(ctx, err) return reconcile.Result{}, err } err = reconcileMachineConfigs(ctx, instance, r.dh, *mcp) if err != nil { - r.log.Error(err) + r.Log.Error(err) + r.SetDegraded(ctx, err) return reconcile.Result{}, err } + r.ClearConditions(ctx) return reconcile.Result{}, nil } diff --git a/pkg/operator/controllers/dnsmasq/machineconfigpool_controller_test.go b/pkg/operator/controllers/dnsmasq/machineconfigpool_controller_test.go index d88efa00b6e..f6d0ee32cbb 100644 --- a/pkg/operator/controllers/dnsmasq/machineconfigpool_controller_test.go +++ b/pkg/operator/controllers/dnsmasq/machineconfigpool_controller_test.go @@ -6,8 +6,10 @@ package dnsmasq import ( "context" "testing" + "time" "github.com/golang/mock/gomock" + operatorv1 "github.com/openshift/api/operator/v1" mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,20 +20,28 @@ import ( arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" mock_dynamichelper "github.com/Azure/ARO-RP/pkg/util/mocks/dynamichelper" + utilconditions "github.com/Azure/ARO-RP/test/util/conditions" utilerror "github.com/Azure/ARO-RP/test/util/error" ) func TestMachineConfigPoolReconciler(t *testing.T) { + transitionTime := metav1.Time{Time: time.Now()} + defaultAvailable := utilconditions.ControllerDefaultAvailable(MachineConfigPoolControllerName) + defaultProgressing := utilconditions.ControllerDefaultProgressing(MachineConfigPoolControllerName) + defaultDegraded := utilconditions.ControllerDefaultDegraded(MachineConfigPoolControllerName) + defaultConditions := []operatorv1.OperatorCondition{defaultAvailable, defaultProgressing, defaultDegraded} + fakeDh := func(controller *gomock.Controller) *mock_dynamichelper.MockInterface { return mock_dynamichelper.NewMockInterface(controller) } tests := []struct { - name string - objects []client.Object - mocks func(mdh *mock_dynamichelper.MockInterface) - request ctrl.Request - wantErrMsg string + name string + objects []client.Object + mocks func(mdh *mock_dynamichelper.MockInterface) + request ctrl.Request + wantErrMsg string + wantConditions []operatorv1.OperatorCondition }{ { name: "no cluster", @@ -45,7 +55,9 @@ func TestMachineConfigPoolReconciler(t *testing.T) { objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: defaultConditions, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "false", @@ -62,7 +74,17 @@ func TestMachineConfigPoolReconciler(t *testing.T) { objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: []operatorv1.OperatorCondition{ + defaultAvailable, + defaultProgressing, + { + Type: MachineConfigPoolControllerName + "Controller" + operatorv1.OperatorStatusTypeDegraded, + Status: operatorv1.ConditionTrue, + LastTransitionTime: transitionTime, + }, + }, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "true", @@ -77,14 +99,17 @@ func TestMachineConfigPoolReconciler(t *testing.T) { Name: "custom", }, }, - wantErrMsg: "", + wantErrMsg: "", + wantConditions: defaultConditions, }, { name: "MachineConfigPool reconciles ARO DNS MachineConfig", objects: []client.Object{ &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, - Status: arov1alpha1.ClusterStatus{}, + Status: arov1alpha1.ClusterStatus{ + Conditions: defaultConditions, + }, Spec: arov1alpha1.ClusterSpec{ OperatorFlags: arov1alpha1.OperatorFlags{ controllerEnabled: "true", @@ -109,7 +134,8 @@ func TestMachineConfigPoolReconciler(t *testing.T) { Name: "custom", }, }, - wantErrMsg: "", + wantErrMsg: "", + wantConditions: defaultConditions, }, } @@ -129,10 +155,11 @@ func TestMachineConfigPoolReconciler(t *testing.T) { client, dh, ) - - _, err := r.Reconcile(context.Background(), tt.request) + ctx := context.Background() + _, err := r.Reconcile(ctx, tt.request) utilerror.AssertErrorMessage(t, err, tt.wantErrMsg) + utilconditions.AssertControllerConditions(t, ctx, client, tt.wantConditions) }) } } diff --git a/pkg/operator/controllers/guardrails/staticresources/gk_audit_controller_deployment.yaml b/pkg/operator/controllers/guardrails/staticresources/gk_audit_controller_deployment.yaml index 01fa1aa0242..733afdd29c8 100644 --- a/pkg/operator/controllers/guardrails/staticresources/gk_audit_controller_deployment.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/gk_audit_controller_deployment.yaml @@ -100,11 +100,9 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsGroup: 999 runAsNonRoot: true - runAsUser: 1000 - # seccompProfile: - # type: RuntimeDefault + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /certs name: cert diff --git a/pkg/operator/controllers/guardrails/staticresources/gk_cluster_role.yaml b/pkg/operator/controllers/guardrails/staticresources/gk_cluster_role.yaml index 2c538e9dd13..5359564b5fd 100644 --- a/pkg/operator/controllers/guardrails/staticresources/gk_cluster_role.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/gk_cluster_role.yaml @@ -157,11 +157,3 @@ rules: - patch - update - watch -- apiGroups: # https://open-policy-agent.github.io/gatekeeper/website/docs/vendor-specific/#running-on-openshift-4x - - security.openshift.io - resourceNames: - - anyuid - resources: - - securitycontextconstraints - verbs: - - use diff --git a/pkg/operator/controllers/guardrails/staticresources/gk_controller_manager_deployment.yaml b/pkg/operator/controllers/guardrails/staticresources/gk_controller_manager_deployment.yaml index c46784988b7..f3bb7beaa00 100644 --- a/pkg/operator/controllers/guardrails/staticresources/gk_controller_manager_deployment.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/gk_controller_manager_deployment.yaml @@ -112,11 +112,9 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsGroup: 999 runAsNonRoot: true - runAsUser: 1000 - # seccompProfile: - # type: RuntimeDefault + seccompProfile: + type: RuntimeDefault volumeMounts: - mountPath: /certs name: cert diff --git a/pkg/operator/controllers/guardrails/staticresources/gk_role.yaml b/pkg/operator/controllers/guardrails/staticresources/gk_role.yaml index 41c2bd5476e..967a3bb43ef 100644 --- a/pkg/operator/controllers/guardrails/staticresources/gk_role.yaml +++ b/pkg/operator/controllers/guardrails/staticresources/gk_role.yaml @@ -26,11 +26,3 @@ rules: - patch - update - watch -- apiGroups: # https://open-policy-agent.github.io/gatekeeper/website/docs/vendor-specific/#running-on-openshift-4x - - security.openshift.io - resourceNames: - - anyuid - resources: - - securitycontextconstraints - verbs: - - use diff --git a/pkg/util/mocks/adminactions/adminactions.go b/pkg/util/mocks/adminactions/adminactions.go index 6dcd15f5751..e7366bc9ebc 100644 --- a/pkg/util/mocks/adminactions/adminactions.go +++ b/pkg/util/mocks/adminactions/adminactions.go @@ -14,8 +14,10 @@ import ( features "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-07-01/features" gomock "github.com/golang/mock/gomock" logrus "github.com/sirupsen/logrus" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" unstructured "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" schema "k8s.io/apimachinery/pkg/runtime/schema" + watch "k8s.io/apimachinery/pkg/watch" ) // MockKubeActions is a mock of KubeActions interface. @@ -112,17 +114,17 @@ func (mr *MockKubeActionsMockRecorder) KubeCreateOrUpdate(arg0, arg1 interface{} } // KubeDelete mocks base method. -func (m *MockKubeActions) KubeDelete(arg0 context.Context, arg1, arg2, arg3 string, arg4 bool) error { +func (m *MockKubeActions) KubeDelete(arg0 context.Context, arg1, arg2, arg3 string, arg4 bool, arg5 *v1.DeletionPropagation) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "KubeDelete", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "KubeDelete", arg0, arg1, arg2, arg3, arg4, arg5) ret0, _ := ret[0].(error) return ret0 } // KubeDelete indicates an expected call of KubeDelete. -func (mr *MockKubeActionsMockRecorder) KubeDelete(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { +func (mr *MockKubeActionsMockRecorder) KubeDelete(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KubeDelete", reflect.TypeOf((*MockKubeActions)(nil).KubeDelete), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KubeDelete", reflect.TypeOf((*MockKubeActions)(nil).KubeDelete), arg0, arg1, arg2, arg3, arg4, arg5) } // KubeGet mocks base method. @@ -170,6 +172,21 @@ func (mr *MockKubeActionsMockRecorder) KubeList(arg0, arg1, arg2 interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KubeList", reflect.TypeOf((*MockKubeActions)(nil).KubeList), arg0, arg1, arg2) } +// KubeWatch mocks base method. +func (m *MockKubeActions) KubeWatch(arg0 context.Context, arg1 *unstructured.Unstructured, arg2 string) (watch.Interface, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KubeWatch", arg0, arg1, arg2) + ret0, _ := ret[0].(watch.Interface) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// KubeWatch indicates an expected call of KubeWatch. +func (mr *MockKubeActionsMockRecorder) KubeWatch(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KubeWatch", reflect.TypeOf((*MockKubeActions)(nil).KubeWatch), arg0, arg1, arg2) +} + // ResolveGVR mocks base method. func (m *MockKubeActions) ResolveGVR(arg0 string) (*schema.GroupVersionResource, error) { m.ctrl.T.Helper() diff --git a/pkg/util/mocks/samples/samples.go b/pkg/util/mocks/samples/samples.go new file mode 100644 index 00000000000..c14a29c2b60 --- /dev/null +++ b/pkg/util/mocks/samples/samples.go @@ -0,0 +1,230 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/openshift/client-go/samples/clientset/versioned/typed/samples/v1 (interfaces: SamplesV1Interface,ConfigInterface) + +// Package mock_v1 is a generated GoMock package. +package mock_v1 + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/openshift/api/samples/v1" + v10 "github.com/openshift/client-go/samples/clientset/versioned/typed/samples/v1" + v11 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// MockSamplesV1Interface is a mock of SamplesV1Interface interface. +type MockSamplesV1Interface struct { + ctrl *gomock.Controller + recorder *MockSamplesV1InterfaceMockRecorder +} + +// MockSamplesV1InterfaceMockRecorder is the mock recorder for MockSamplesV1Interface. +type MockSamplesV1InterfaceMockRecorder struct { + mock *MockSamplesV1Interface +} + +// NewMockSamplesV1Interface creates a new mock instance. +func NewMockSamplesV1Interface(ctrl *gomock.Controller) *MockSamplesV1Interface { + mock := &MockSamplesV1Interface{ctrl: ctrl} + mock.recorder = &MockSamplesV1InterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSamplesV1Interface) EXPECT() *MockSamplesV1InterfaceMockRecorder { + return m.recorder +} + +// Configs mocks base method. +func (m *MockSamplesV1Interface) Configs() v10.ConfigInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Configs") + ret0, _ := ret[0].(v10.ConfigInterface) + return ret0 +} + +// Configs indicates an expected call of Configs. +func (mr *MockSamplesV1InterfaceMockRecorder) Configs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Configs", reflect.TypeOf((*MockSamplesV1Interface)(nil).Configs)) +} + +// RESTClient mocks base method. +func (m *MockSamplesV1Interface) RESTClient() rest.Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RESTClient") + ret0, _ := ret[0].(rest.Interface) + return ret0 +} + +// RESTClient indicates an expected call of RESTClient. +func (mr *MockSamplesV1InterfaceMockRecorder) RESTClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RESTClient", reflect.TypeOf((*MockSamplesV1Interface)(nil).RESTClient)) +} + +// MockConfigInterface is a mock of ConfigInterface interface. +type MockConfigInterface struct { + ctrl *gomock.Controller + recorder *MockConfigInterfaceMockRecorder +} + +// MockConfigInterfaceMockRecorder is the mock recorder for MockConfigInterface. +type MockConfigInterfaceMockRecorder struct { + mock *MockConfigInterface +} + +// NewMockConfigInterface creates a new mock instance. +func NewMockConfigInterface(ctrl *gomock.Controller) *MockConfigInterface { + mock := &MockConfigInterface{ctrl: ctrl} + mock.recorder = &MockConfigInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConfigInterface) EXPECT() *MockConfigInterfaceMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockConfigInterface) Create(arg0 context.Context, arg1 *v1.Config, arg2 v11.CreateOptions) (*v1.Config, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Config) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create. +func (mr *MockConfigInterfaceMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockConfigInterface)(nil).Create), arg0, arg1, arg2) +} + +// Delete mocks base method. +func (m *MockConfigInterface) Delete(arg0 context.Context, arg1 string, arg2 v11.DeleteOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockConfigInterfaceMockRecorder) Delete(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockConfigInterface)(nil).Delete), arg0, arg1, arg2) +} + +// DeleteCollection mocks base method. +func (m *MockConfigInterface) DeleteCollection(arg0 context.Context, arg1 v11.DeleteOptions, arg2 v11.ListOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteCollection", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteCollection indicates an expected call of DeleteCollection. +func (mr *MockConfigInterfaceMockRecorder) DeleteCollection(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCollection", reflect.TypeOf((*MockConfigInterface)(nil).DeleteCollection), arg0, arg1, arg2) +} + +// Get mocks base method. +func (m *MockConfigInterface) Get(arg0 context.Context, arg1 string, arg2 v11.GetOptions) (*v1.Config, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Config) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockConfigInterfaceMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockConfigInterface)(nil).Get), arg0, arg1, arg2) +} + +// List mocks base method. +func (m *MockConfigInterface) List(arg0 context.Context, arg1 v11.ListOptions) (*v1.ConfigList, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", arg0, arg1) + ret0, _ := ret[0].(*v1.ConfigList) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockConfigInterfaceMockRecorder) List(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockConfigInterface)(nil).List), arg0, arg1) +} + +// Patch mocks base method. +func (m *MockConfigInterface) Patch(arg0 context.Context, arg1 string, arg2 types.PatchType, arg3 []byte, arg4 v11.PatchOptions, arg5 ...string) (*v1.Config, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3, arg4} + for _, a := range arg5 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Patch", varargs...) + ret0, _ := ret[0].(*v1.Config) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Patch indicates an expected call of Patch. +func (mr *MockConfigInterfaceMockRecorder) Patch(arg0, arg1, arg2, arg3, arg4 interface{}, arg5 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3, arg4}, arg5...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockConfigInterface)(nil).Patch), varargs...) +} + +// Update mocks base method. +func (m *MockConfigInterface) Update(arg0 context.Context, arg1 *v1.Config, arg2 v11.UpdateOptions) (*v1.Config, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Config) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockConfigInterfaceMockRecorder) Update(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockConfigInterface)(nil).Update), arg0, arg1, arg2) +} + +// UpdateStatus mocks base method. +func (m *MockConfigInterface) UpdateStatus(arg0 context.Context, arg1 *v1.Config, arg2 v11.UpdateOptions) (*v1.Config, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateStatus", arg0, arg1, arg2) + ret0, _ := ret[0].(*v1.Config) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateStatus indicates an expected call of UpdateStatus. +func (mr *MockConfigInterfaceMockRecorder) UpdateStatus(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateStatus", reflect.TypeOf((*MockConfigInterface)(nil).UpdateStatus), arg0, arg1, arg2) +} + +// Watch mocks base method. +func (m *MockConfigInterface) Watch(arg0 context.Context, arg1 v11.ListOptions) (watch.Interface, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Watch", arg0, arg1) + ret0, _ := ret[0].(watch.Interface) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Watch indicates an expected call of Watch. +func (mr *MockConfigInterfaceMockRecorder) Watch(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Watch", reflect.TypeOf((*MockConfigInterface)(nil).Watch), arg0, arg1) +} diff --git a/pkg/util/mocks/samplesclient/versioned.go b/pkg/util/mocks/samplesclient/versioned.go new file mode 100644 index 00000000000..cd7a8c6f0dc --- /dev/null +++ b/pkg/util/mocks/samplesclient/versioned.go @@ -0,0 +1,64 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/openshift/client-go/samples/clientset/versioned (interfaces: Interface) + +// Package mock_versioned is a generated GoMock package. +package mock_versioned + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/openshift/client-go/samples/clientset/versioned/typed/samples/v1" + discovery "k8s.io/client-go/discovery" +) + +// MockInterface is a mock of Interface interface. +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface. +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance. +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// Discovery mocks base method. +func (m *MockInterface) Discovery() discovery.DiscoveryInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Discovery") + ret0, _ := ret[0].(discovery.DiscoveryInterface) + return ret0 +} + +// Discovery indicates an expected call of Discovery. +func (mr *MockInterfaceMockRecorder) Discovery() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Discovery", reflect.TypeOf((*MockInterface)(nil).Discovery)) +} + +// SamplesV1 mocks base method. +func (m *MockInterface) SamplesV1() v1.SamplesV1Interface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SamplesV1") + ret0, _ := ret[0].(v1.SamplesV1Interface) + return ret0 +} + +// SamplesV1 indicates an expected call of SamplesV1. +func (mr *MockInterfaceMockRecorder) SamplesV1() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SamplesV1", reflect.TypeOf((*MockInterface)(nil).SamplesV1)) +} diff --git a/pkg/util/steps/condition.go b/pkg/util/steps/condition.go index adcc953441f..3a6f8e8a517 100644 --- a/pkg/util/steps/condition.go +++ b/pkg/util/steps/condition.go @@ -5,13 +5,35 @@ package steps import ( "context" + "errors" "fmt" + "net/http" + "strings" "time" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" + + "github.com/Azure/ARO-RP/pkg/api" ) +// Functions that run as condition-steps return Error +// instead of InternalServerError +// Efforts are being made to not have generic Hive errors but specific, actionable failure cases. +// Instead of providing Hive-specific error messages to customers, the below will send a timeout error message. +var timeoutConditionErrors = map[string]string{ + "apiServersReady": "Kube API has not initialised successfully and is unavailable.", + "minimumWorkerNodesReady": "Minimum number of worker nodes have not been successfully created.", + "operatorConsoleExists": "Console Cluster Operator has failed to initialize successfully.", + "operatorConsoleReady": "Console Cluster Operator has not started successfully.", + "clusterVersionReady": "Cluster Verion is not reporting status as ready.", + "ingressControllerReady": "Ingress Cluster Operator has not started successfully.", + "aroDeploymentReady": "ARO Cluster Operator has failed to initialize successfully.", + "ensureAROOperatorRunningDesiredVersion": "ARO Cluster Operator is not running desired version.", + "hiveClusterDeploymentReady": "Timed out waiting for a condition, cluster Installation is unsuccessful.", + "hiveClusterInstallationComplete": "Timed out waiting for a condition, cluster Installation is unsuccessful.", +} + // conditionFunction is a function that takes a context and returns whether the // condition has been met and an error. // @@ -54,21 +76,49 @@ func (c conditionStep) run(ctx context.Context, log *logrus.Entry) error { // Run the condition function immediately, and then every // runner.pollInterval, until the condition returns true or timeoutCtx's - // timeout fires. Errors from `f` are returned directly. + // timeout fires. Errors from `f` are returned directly unless the error + // is ErrWaitTimeout. Internal ErrWaitTimeout errors are wrapped to avoid + // confusion with wait.PollImmediateUntil's own behavior of returning + // ErrWaitTimeout when the condition is not met. err := wait.PollImmediateUntil(pollInterval, func() (bool, error) { // We use the outer context, not the timeout context, as we do not want // to time out the condition function itself, only stop retrying once // timeoutCtx's timeout has fired. - return c.f(ctx) + cnd, cndErr := c.f(ctx) + if errors.Is(cndErr, wait.ErrWaitTimeout) { + return cnd, fmt.Errorf("condition encountered internal timeout: %w", cndErr) + } + + return cnd, cndErr }, timeoutCtx.Done()) if err != nil && !c.fail { log.Warnf("step %s failed but has configured 'fail=%t'. Continuing. Error: %s", c, c.fail, err.Error()) return nil } + if errors.Is(err, wait.ErrWaitTimeout) { + return enrichConditionTimeoutError(c.f) + } return err } +// Instead of giving Generic, timed out waiting for a condition, error +// returns enriched error messages mentioned in timeoutConditionErrors +func enrichConditionTimeoutError(f conditionFunction) error { + funcNameParts := strings.Split(FriendlyName(f), ".") + funcName := strings.TrimSuffix(funcNameParts[len(funcNameParts)-1], "-fm") + + message, exists := timeoutConditionErrors[funcName] + if !exists { + return errors.New("timed out waiting for the condition") + } + return api.NewCloudError( + http.StatusInternalServerError, + api.CloudErrorCodeDeploymentFailed, + "", message+"Please retry, if issue persists: raise azure support ticket", + ) +} + func (c conditionStep) String() string { return fmt.Sprintf("[Condition %s, timeout %s]", FriendlyName(c.f), c.timeout) } diff --git a/pkg/util/steps/condition_test.go b/pkg/util/steps/condition_test.go new file mode 100644 index 00000000000..cfecdaed7a1 --- /dev/null +++ b/pkg/util/steps/condition_test.go @@ -0,0 +1,95 @@ +package steps + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" +) + +// functionnames that will be used in the conditionFunction below +// All the keys of map timeoutConditionErrors +func apiServersReady(context.Context) (bool, error) { return false, nil } +func minimumWorkerNodesReady(context.Context) (bool, error) { return false, nil } +func operatorConsoleExists(context.Context) (bool, error) { return false, nil } +func operatorConsoleReady(context.Context) (bool, error) { return false, nil } +func clusterVersionReady(context.Context) (bool, error) { return false, nil } +func ingressControllerReady(context.Context) (bool, error) { return false, nil } +func aroDeploymentReady(context.Context) (bool, error) { return false, nil } +func ensureAROOperatorRunningDesiredVersion(context.Context) (bool, error) { return false, nil } +func hiveClusterDeploymentReady(context.Context) (bool, error) { return false, nil } +func hiveClusterInstallationComplete(context.Context) (bool, error) { return false, nil } + +func TestEnrichConditionTimeoutError(t *testing.T) { + for _, tt := range []struct { + desc string + function conditionFunction + wantErr string + }{ + // Verify response for func's mention in timeoutConditionErrors and + // Emit generic Error if an unknown func + { + // unknown function + desc: "test conditionfail for func - unknownFunc", + function: timingOutCondition, + wantErr: "timed out waiting for the condition", + }, + { + desc: "test conditionfail for func - apiServersReady", + function: apiServersReady, + wantErr: "500: DeploymentFailed: : Kube API has not initialised successfully and is unavailable.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - minimumWorkerNodesReady", + function: minimumWorkerNodesReady, + wantErr: "500: DeploymentFailed: : Minimum number of worker nodes have not been successfully created.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - operatorConsoleExists", + function: operatorConsoleExists, + wantErr: "500: DeploymentFailed: : Console Cluster Operator has failed to initialize successfully.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - operatorConsoleReady", + function: operatorConsoleReady, + wantErr: "500: DeploymentFailed: : Console Cluster Operator has not started successfully.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - clusterVersionReady", + function: clusterVersionReady, + wantErr: "500: DeploymentFailed: : Cluster Verion is not reporting status as ready.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - clusterVersionReady", + function: ingressControllerReady, + wantErr: "500: DeploymentFailed: : Ingress Cluster Operator has not started successfully.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - aroDeploymentReady", + function: aroDeploymentReady, + wantErr: "500: DeploymentFailed: : ARO Cluster Operator has failed to initialize successfully.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - ensureAROOperatorRunningDesiredVersion", + function: ensureAROOperatorRunningDesiredVersion, + wantErr: "500: DeploymentFailed: : ARO Cluster Operator is not running desired version.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - hiveClusterDeploymentReady", + function: hiveClusterDeploymentReady, + wantErr: "500: DeploymentFailed: : Timed out waiting for a condition, cluster Installation is unsuccessful.Please retry, if issue persists: raise azure support ticket", + }, + { + desc: "test conditionfail for func - hiveClusterInstallationComplete", + function: hiveClusterInstallationComplete, + wantErr: "500: DeploymentFailed: : Timed out waiting for a condition, cluster Installation is unsuccessful.Please retry, if issue persists: raise azure support ticket", + }, + } { + t.Run(tt.desc, func(t *testing.T) { + if got := enrichConditionTimeoutError(tt.function); got.Error() != tt.wantErr { + t.Errorf("invlaid enrichConditionTimeoutError: %s, got: %s", tt.wantErr, got) + } + }) + } +} diff --git a/pkg/util/steps/runner_test.go b/pkg/util/steps/runner_test.go index 69d6a9e76c9..4b3cf30ec3f 100644 --- a/pkg/util/steps/runner_test.go +++ b/pkg/util/steps/runner_test.go @@ -13,6 +13,7 @@ import ( "github.com/onsi/gomega" "github.com/onsi/gomega/types" "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" utilerror "github.com/Azure/ARO-RP/test/util/error" testlog "github.com/Azure/ARO-RP/test/util/log" @@ -26,6 +27,9 @@ func timingOutCondition(ctx context.Context) (bool, error) { time.Sleep(60 * time.Millisecond) return false, nil } +func internalTimeoutCondition(ctx context.Context) (bool, error) { + return false, wait.ErrWaitTimeout +} func currentTimeFunc() time.Time { return time.Now() @@ -169,6 +173,36 @@ func TestStepRunner(t *testing.T) { }, wantErr: "timed out waiting for the condition", }, + { + name: "A Condition that returns a timeout error causes a different failure from a timed out Condition", + steps: func(controller *gomock.Controller) []Step { + return []Step{ + Action(successfulFunc), + &conditionStep{ + f: internalTimeoutCondition, + fail: true, + pollInterval: 20 * time.Millisecond, + timeout: 50 * time.Millisecond, + }, + Action(successfulFunc), + } + }, + wantEntries: []map[string]types.GomegaMatcher{ + { + "msg": gomega.Equal("running step [Action github.com/Azure/ARO-RP/pkg/util/steps.successfulFunc]"), + "level": gomega.Equal(logrus.InfoLevel), + }, + { + "msg": gomega.Equal("running step [Condition github.com/Azure/ARO-RP/pkg/util/steps.internalTimeoutCondition, timeout 50ms]"), + "level": gomega.Equal(logrus.InfoLevel), + }, + { + "msg": gomega.Equal("step [Condition github.com/Azure/ARO-RP/pkg/util/steps.internalTimeoutCondition, timeout 50ms] encountered error: condition encountered internal timeout: timed out waiting for the condition"), + "level": gomega.Equal(logrus.ErrorLevel), + }, + }, + wantErr: "condition encountered internal timeout: timed out waiting for the condition", + }, { name: "A Condition that does not return true in the timeout time causes a failure", steps: func(controller *gomock.Controller) []Step { diff --git a/python/az/aro/azext_aro/_client_factory.py b/python/az/aro/azext_aro/_client_factory.py index c2402e2a9a2..775fba938d0 100644 --- a/python/az/aro/azext_aro/_client_factory.py +++ b/python/az/aro/azext_aro/_client_factory.py @@ -4,7 +4,7 @@ import urllib3 from azext_aro.custom import rp_mode_development -from azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2022_09_04 import AzureRedHatOpenShiftClient +from azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2023_04_01 import AzureRedHatOpenShiftClient from azure.cli.core.commands.client_factory import get_mgmt_service_client diff --git a/python/az/aro/azext_aro/_params.py b/python/az/aro/azext_aro/_params.py index a3e6a95a8c6..fcb56e536a0 100644 --- a/python/az/aro/azext_aro/_params.py +++ b/python/az/aro/azext_aro/_params.py @@ -16,6 +16,7 @@ from azext_aro._validators import validate_worker_vm_disk_size_gb from azext_aro._validators import validate_refresh_cluster_credentials from azext_aro._validators import validate_version_format +from azext_aro._validators import validate_outbound_type from azure.cli.core.commands.parameters import name_type from azure.cli.core.commands.parameters import get_enum_type, get_three_state_flag from azure.cli.core.commands.parameters import resource_group_name_type @@ -64,7 +65,9 @@ def load_arguments(self, _): c.argument('service_cidr', help='CIDR of service network. Must be a minimum of /18 or larger.', validator=validate_cidr('service_cidr')) - + c.argument('outbound_type', + help='Outbound type of cluster. Must be "Loadbalancer" (default) or "UserDefinedRouting".', + validator=validate_outbound_type) c.argument('disk_encryption_set', help='ResourceID of the DiskEncryptionSet to be used for master and worker VMs.', validator=validate_disk_encryption_set) diff --git a/python/az/aro/azext_aro/_validators.py b/python/az/aro/azext_aro/_validators.py index 5f18ed30e0c..06d9b5d9abb 100644 --- a/python/az/aro/azext_aro/_validators.py +++ b/python/az/aro/azext_aro/_validators.py @@ -118,6 +118,25 @@ def validate_pull_secret(namespace): raise InvalidArgumentValueError("Invalid --pull-secret.") from e +def validate_outbound_type(namespace): + outbound_type = getattr(namespace, 'outbound_type') + if outbound_type not in {'UserDefinedRouting', 'Loadbalancer', None}: + raise InvalidArgumentValueError('Invalid --outbound-type: must be "UserDefinedRouting" or "Loadbalancer"') + + ingress_visibility = getattr(namespace, 'ingress_visibility') + apiserver_visibility = getattr(namespace, 'apiserver_visibility') + + if (outbound_type == 'UserDefinedRouting' and + (is_visibility_public(ingress_visibility) or is_visibility_public(apiserver_visibility))): + raise InvalidArgumentValueError('Invalid --outbound-type: cannot use UserDefinedRouting when ' + + 'either --apiserver-visibility or --ingress-visibility is set ' + + 'to Public or not defined') + + +def is_visibility_public(visibility): + return visibility == 'Public' or visibility is None + + def validate_subnet(key): def _validate_subnet(cmd, namespace): subnet = getattr(namespace, key) diff --git a/python/az/aro/azext_aro/commands.py b/python/az/aro/azext_aro/commands.py index 247c62f8b34..e11171dcace 100644 --- a/python/az/aro/azext_aro/commands.py +++ b/python/az/aro/azext_aro/commands.py @@ -11,7 +11,7 @@ def load_command_table(self, _): aro_sdk = CliCommandType( - operations_tmpl='azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2022_09_04.operations#OpenShiftClustersOperations.{}', # pylint: disable=line-too-long + operations_tmpl='azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2023_04_01.operations#OpenShiftClustersOperations.{}', # pylint: disable=line-too-long client_factory=cf_aro) with self.command_group('aro', aro_sdk, client_factory=cf_aro) as g: diff --git a/python/az/aro/azext_aro/custom.py b/python/az/aro/azext_aro/custom.py index 6c2ec5520cc..459a5ec12ec 100644 --- a/python/az/aro/azext_aro/custom.py +++ b/python/az/aro/azext_aro/custom.py @@ -7,7 +7,7 @@ from base64 import b64decode import textwrap -import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2022_09_04.models as openshiftcluster +import azext_aro.vendored_sdks.azure.mgmt.redhatopenshift.v2023_04_01.models as openshiftcluster from azure.cli.command_modules.role import GraphError from azure.cli.core.commands.client_factory import get_mgmt_service_client @@ -53,6 +53,7 @@ def aro_create(cmd, # pylint: disable=too-many-locals client_secret=None, pod_cidr=None, service_cidr=None, + outbound_type=None, disk_encryption_set=None, master_encryption_at_host=False, master_vm_size=None, @@ -139,6 +140,7 @@ def aro_create(cmd, # pylint: disable=too-many-locals network_profile=openshiftcluster.NetworkProfile( pod_cidr=pod_cidr or '10.128.0.0/14', service_cidr=service_cidr or '172.30.0.0/16', + outbound_type=outbound_type or '', ), master_profile=openshiftcluster.MasterProfile( vm_size=master_vm_size or 'Standard_D8s_v3', diff --git a/python/az/aro/azext_aro/tests/latest/unit/test_validators.py b/python/az/aro/azext_aro/tests/latest/unit/test_validators.py index 4e3ce912416..bcae4a441af 100644 --- a/python/az/aro/azext_aro/tests/latest/unit/test_validators.py +++ b/python/az/aro/azext_aro/tests/latest/unit/test_validators.py @@ -3,7 +3,7 @@ from unittest.mock import Mock, patch from azext_aro._validators import ( - validate_cidr, validate_client_id, validate_client_secret, validate_cluster_resource_group, + validate_cidr, validate_client_id, validate_client_secret, validate_cluster_resource_group, validate_outbound_type, validate_disk_encryption_set, validate_domain, validate_pull_secret, validate_subnet, validate_subnets, validate_visibility, validate_vnet_resource_group_name, validate_worker_count, validate_worker_vm_disk_size_gb, validate_refresh_cluster_credentials ) @@ -775,3 +775,107 @@ def test_validate_refresh_cluster_credentials(test_description, namespace, expec else: with pytest.raises(expected_exception): validate_refresh_cluster_credentials(namespace) + + +test_validate_outbound_type_data = [ + ( + "Should not raise exception when key is Loadbalancer.", + Mock(outbound_type='Loadbalancer'), + None + ), + ( + "Should not raise exception when key is Loadbalancer and ingress visibility private", + Mock( + outbound_type='Loadbalancer', + apiserver_visibility="Public", + ingress_visibility="Private" + ), + None + ), + ( + "Should not raise exception when key is Loadbalancer and apiserver visibility private", + Mock( + outbound_type='Loadbalancer', + apiserver_visibility="Private", + ingress_visibility="Public" + ), + None + ), + ( + "Should not raise exception when key is Loadbalancer and ingress/apiserver visibility private", + Mock( + outbound_type='Loadbalancer', + apiserver_visibility="Private", + ingress_visibility="Private" + ), + None + ), + ( + "Should not raise exception when key is empty.", + Mock(outbound_type=None), + None + ), + ( + "Should not raise exception with UDR and ingress/apiserver visibility private", + Mock( + outbound_type="UserDefinedRouting", + apiserver_visibility="Private", + ingress_visibility="Private" + ), + None + ), + ( + "Should raise exception with UDR and ingress visibility is public", + Mock( + outbound_type="UserDefinedRouting", + apiserver_visibility="Private", + ingress_visibility="Public" + ), + InvalidArgumentValueError + ), + ( + "Should raise exception with UDR and apiserver visibility is public", + Mock( + outbound_type="UserDefinedRouting", + apiserver_visibility="Public", + ingress_visibility="Private" + ), + InvalidArgumentValueError + ), + ( + "Should raise exception with UDR and apiserver/ingress visibility is public", + Mock( + outbound_type="UserDefinedRouting", + apiserver_visibility="Public", + ingress_visibility="Public" + ), + InvalidArgumentValueError + ), + ( + "Should raise exception when key is UserDefinedRouting and apiserver/ingress visibilities are not defined.", + Mock( + outbound_type="UserDefinedRouting", + apiserver_visibility=None, + ingress_visibility=None + ), + InvalidArgumentValueError + ), + ( + "Should raise exception when key is a different value.", + Mock(outbound_type='testFail'), + InvalidArgumentValueError + ), +] + + +@pytest.mark.parametrize( + "test_description, namespace, expected_exception", + test_validate_outbound_type_data, + ids=[i[0] for i in test_validate_outbound_type_data] +) +def test_validate_outbound_type(test_description, namespace, expected_exception): + if expected_exception is None: + validate_outbound_type(namespace) + else: + with pytest.raises(expected_exception): + validate_outbound_type(namespace)