From f9c8a96b2292ac67ca07c183edf8ed0c7169ffb5 Mon Sep 17 00:00:00 2001 From: Tanmay Satam Date: Tue, 15 Aug 2023 14:47:29 -0400 Subject: [PATCH] Add hack script to generate hive install-log-regexes.yaml from ARO definition --- hack/genhiveconfig/genhiveconfig.go | 81 ++++++++++++++++ hack/genhiveconfig/genhiveconfig_test.go | 34 +++++++ .../hive-additional-install-log-regexes.yaml | 14 +++ pkg/hive/failure/reasons.go | 35 +++++++ pkg/hive/failure/reasons_test.go | 92 +++++++++++++++++++ 5 files changed, 256 insertions(+) create mode 100644 hack/genhiveconfig/genhiveconfig.go create mode 100644 hack/genhiveconfig/genhiveconfig_test.go create mode 100644 hack/hive-config/hive-additional-install-log-regexes.yaml create mode 100644 pkg/hive/failure/reasons.go create mode 100644 pkg/hive/failure/reasons_test.go diff --git a/hack/genhiveconfig/genhiveconfig.go b/hack/genhiveconfig/genhiveconfig.go new file mode 100644 index 00000000000..e8e26463b56 --- /dev/null +++ b/hack/genhiveconfig/genhiveconfig.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "os" + + "github.com/ghodss/yaml" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + failure "github.com/Azure/ARO-RP/pkg/hive/failure" + utillog "github.com/Azure/ARO-RP/pkg/util/log" +) + +const ( + hiveNamespaceName = "hive" + configMapName = "additional-install-log-regexes" + configMapPath = "hack/hive-config/hive-additional-install-log-regexes.yaml" + regexDataEntryName = "regexes" +) + +type installLogRegex struct { + Name string `json:"name"` + SearchRegexStrings []string `json:"searchRegexStrings"` + InstallFailingReason string `json:"installFailingReason"` + InstallFailingMessage string `json:"installFailingMessage"` +} + +func run(ctx context.Context) error { + ilrs := []installLogRegex{} + + for _, reason := range failure.Reasons { + ilrs = append(ilrs, failureReasonToInstallLogRegex(reason)) + } + + ilrsRaw, err := yaml.Marshal(ilrs) + if err != nil { + return err + } + + configmap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: hiveNamespaceName, + Name: configMapName, + }, + Data: map[string]string{ + regexDataEntryName: string(ilrsRaw), + }, + } + + configmapRaw, err := yaml.Marshal(configmap) + if err != nil { + return err + } + return os.WriteFile(configMapPath, configmapRaw, 0666) +} + +func failureReasonToInstallLogRegex(reason failure.InstallFailingReason) installLogRegex { + ilr := installLogRegex{ + Name: reason.Name, + InstallFailingReason: reason.Reason, + InstallFailingMessage: reason.Message, + SearchRegexStrings: []string{}, + } + for _, regex := range reason.SearchRegexes { + ilr.SearchRegexStrings = append(ilr.SearchRegexStrings, regex.String()) + } + return ilr +} + +func main() { + log := utillog.GetLogger() + + if err := run(context.Background()); err != nil { + log.Fatal(err) + } +} diff --git a/hack/genhiveconfig/genhiveconfig_test.go b/hack/genhiveconfig/genhiveconfig_test.go new file mode 100644 index 00000000000..48247c0fe64 --- /dev/null +++ b/hack/genhiveconfig/genhiveconfig_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "reflect" + "regexp" + "testing" + + "github.com/Azure/ARO-RP/pkg/hive/failure" +) + +func TestFailureReasonToInstallLogRegex(t *testing.T) { + input := failure.InstallFailingReason{ + Name: "TestReason", + Reason: "AzureTestReason", + Message: "This is a sentence.", + SearchRegexes: []*regexp.Regexp{ + regexp.MustCompile(".*"), + regexp.MustCompile("^$"), + }, + } + + want := installLogRegex{ + Name: "TestReason", + InstallFailingReason: "AzureTestReason", + InstallFailingMessage: "This is a sentence.", + SearchRegexStrings: []string{".*", "^$"}, + } + + got := failureReasonToInstallLogRegex(input) + + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} diff --git a/hack/hive-config/hive-additional-install-log-regexes.yaml b/hack/hive-config/hive-additional-install-log-regexes.yaml new file mode 100644 index 00000000000..a6d4d75db91 --- /dev/null +++ b/hack/hive-config/hive-additional-install-log-regexes.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + regexes: | + - installFailingMessage: The template deployment failed. Please see details for more + information. + installFailingReason: AzureInvalidTemplateDeployment + name: AzureInvalidTemplateDeployment + searchRegexStrings: + - '"code":\w?"InvalidTemplateDeployment"' +kind: ConfigMap +metadata: + creationTimestamp: null + name: additional-install-log-regexes + namespace: hive diff --git a/pkg/hive/failure/reasons.go b/pkg/hive/failure/reasons.go new file mode 100644 index 00000000000..f496c2068a3 --- /dev/null +++ b/pkg/hive/failure/reasons.go @@ -0,0 +1,35 @@ +package failure + +import "regexp" + +type InstallFailingReason struct { + Name string + Reason string + Message string + SearchRegexes []*regexp.Regexp +} + +var Reasons = []InstallFailingReason{ + // Order within this array determines precedence. Earlier entries will take + // priority over later ones. + AzureRequestDisallowedByPolicy, + AzureInvalidTemplateDeployment, +} + +var AzureRequestDisallowedByPolicy = InstallFailingReason{ + Name: "AzureRequestDisallowedByPolicy", + Reason: "AzureRequestDisallowedByPolicy", + Message: "Cluster Deployment was disallowed by policy. Please see install log for more information.", + SearchRegexes: []*regexp.Regexp{ + regexp.MustCompile(`"code":\w?"InvalidTemplateDeployment".*"code":\w?"RequestDisallowedByPolicy"`), + }, +} + +var AzureInvalidTemplateDeployment = InstallFailingReason{ + Name: "AzureInvalidTemplateDeployment", + Reason: "AzureInvalidTemplateDeployment", + Message: "The template deployment failed. Please see install log for more information.", + SearchRegexes: []*regexp.Regexp{ + regexp.MustCompile(`"code":\w?"InvalidTemplateDeployment"`), + }, +} diff --git a/pkg/hive/failure/reasons_test.go b/pkg/hive/failure/reasons_test.go new file mode 100644 index 00000000000..75e531dfb71 --- /dev/null +++ b/pkg/hive/failure/reasons_test.go @@ -0,0 +1,92 @@ +package failure + +import ( + "reflect" + "regexp" + "testing" +) + +func TestInstallFailingReasonRegexes(t *testing.T) { + for _, tt := range []struct { + name string + installLog string + want InstallFailingReason + }{ + { + name: "InvalidTemplateDeployment - no known errors", + installLog: ` +level=info msg=running in local development mode +level=info msg=creating development InstanceMetadata +level=info msg=InstanceMetadata: running on AzurePublicCloud +level=info msg=running step [Action github.com/openshift/ARO-Installer/pkg/installer.(*manager).Manifests.func1] +level=info msg=running step [Action github.com/openshift/ARO-Installer/pkg/installer.(*manager).Manifests.func2] +level=info msg=resolving graph +level=info msg=running step [Action github.com/openshift/ARO-Installer/pkg/installer.(*manager).Manifests.func3] +level=info msg=checking if graph exists +level=info msg=save graph +Generates the Ignition Config asset + +level=info msg=running in local development mode +level=info msg=creating development InstanceMetadata +level=info msg=InstanceMetadata: running on AzurePublicCloud +level=info msg=running step [AuthorizationRetryingAction github.com/openshift/ARO-Installer/pkg/installer.(*manager).deployResourceTemplate-fm] +level=info msg=load persisted graph +level=info msg=deploying resources template +level=error msg=step [AuthorizationRetryingAction github.com/openshift/ARO-Installer/pkg/installer.(*manager).deployResourceTemplate-fm] encountered error: 400: DeploymentFailed: : Deployment failed. Details: : : {"code":"InvalidTemplateDeployment","message":"The template deployment failed with multiple errors. Please see details for more information.","details":[]} +level=error msg=400: DeploymentFailed: : Deployment failed. Details: : : {"code":"InvalidTemplateDeployment","message":"The template deployment failed with multiple errors. Please see details for more information.","details":[]}`, + want: AzureInvalidTemplateDeployment, + }, + { + name: "InvalidTemplateDeployment - RequestDisallowedByPolicy", + installLog: ` +level=info msg=running in local development mode +level=info msg=creating development InstanceMetadata +level=info msg=InstanceMetadata: running on AzurePublicCloud +level=info msg=running step [Action github.com/openshift/ARO-Installer/pkg/installer.(*manager).Manifests.func1] +level=info msg=running step [Action github.com/openshift/ARO-Installer/pkg/installer.(*manager).Manifests.func2] +level=info msg=resolving graph +level=info msg=running step [Action github.com/openshift/ARO-Installer/pkg/installer.(*manager).Manifests.func3] +level=info msg=checking if graph exists +level=info msg=save graph +Generates the Ignition Config asset + +level=info msg=running in local development mode +level=info msg=creating development InstanceMetadata +level=info msg=InstanceMetadata: running on AzurePublicCloud +level=info msg=running step [AuthorizationRetryingAction github.com/openshift/ARO-Installer/pkg/installer.(*manager).deployResourceTemplate-fm] +level=info msg=load persisted graph +level=info msg=deploying resources template +level=error msg=step [AuthorizationRetryingAction github.com/openshift/ARO-Installer/pkg/installer.(*manager).deployResourceTemplate-fm] encountered error: 400: DeploymentFailed: : Deployment failed. Details: : : {"code":"InvalidTemplateDeployment","message":"The template deployment failed with multiple errors. Please see details for more information.","details":[{"additionalInfo":[],"code":"RequestDisallowedByPolicy","message":"Resource 'test-bootstrap' was disallowed by policy. Policy identifiers: ''.","target":"test-bootstrap"}]} +level=error msg=400: DeploymentFailed: : Deployment failed. Details: : : {"code":"InvalidTemplateDeployment","message":"The template deployment failed with multiple errors. Please see details for more information.","details":[{"additionalInfo":[],"code":"RequestDisallowedByPolicy","message":"Resource 'test-bootstrap' was disallowed by policy. Policy identifiers: ''.","target":"test-bootstrap"}]}`, + want: AzureRequestDisallowedByPolicy, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // This test uses a "mock" version of Hive's real implementation for matching install logs against regex patterns. + // https://github.com/bennerv/hive/blob/fec14dcf0-plus-base-image-update/pkg/controller/clusterprovision/installlogmonitor.go#L83 + // The purpose of this test is to test the regular expressions themselves, not the implementation. + got := mockHiveIdentifyReason(tt.installLog) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func mockHiveIdentifyReason(installLog string) InstallFailingReason { + for _, reason := range Reasons { + for _, regex := range reason.SearchRegexes { + if regex.MatchString(installLog) { + return reason + } + } + } + + return InstallFailingReason{ + Name: "UnknownError", + Reason: "UnknownError", + Message: installLog, + SearchRegexes: []*regexp.Regexp{}, + } +}