From dcd35a6c41fc45ddce354e3676aafaedb6eac5f5 Mon Sep 17 00:00:00 2001 From: Andrew Denton Date: Tue, 30 May 2023 10:45:45 -0700 Subject: [PATCH] wip --- cmd/aro/main.go | 3 + cmd/aro/operator.go | 8 + hack/db/db.go | 105 +++++- pkg/api/defaults.go | 3 +- pkg/database/database.go | 32 ++ pkg/database/openshiftclusters.go | 2 +- .../dnsv2/dnsv2_cluster_controller.go | 155 +++++++++ .../dnsv2/dnsv2_machineconfig_controller.go | 93 ++++++ .../dnsv2_machineconfigpool_controller.go | 81 +++++ .../installer/pkg/aro/dnsv2/resources.go | 302 ++++++++++++++++++ .../pkg/asset/ignition/bootstrap/common.go | 17 +- 11 files changed, 793 insertions(+), 8 deletions(-) create mode 100644 pkg/operator/controllers/dnsv2/dnsv2_cluster_controller.go create mode 100644 pkg/operator/controllers/dnsv2/dnsv2_machineconfig_controller.go create mode 100644 pkg/operator/controllers/dnsv2/dnsv2_machineconfigpool_controller.go create mode 100644 vendor/github.com/openshift/installer/pkg/aro/dnsv2/resources.go diff --git a/cmd/aro/main.go b/cmd/aro/main.go index 6d87f4c3080..691ea14d026 100644 --- a/cmd/aro/main.go +++ b/cmd/aro/main.go @@ -18,6 +18,7 @@ import ( utillog "github.com/Azure/ARO-RP/pkg/util/log" _ "github.com/Azure/ARO-RP/pkg/util/scheme" "github.com/Azure/ARO-RP/pkg/util/version" + "k8s.io/klog/v2" ) func usage() { @@ -36,9 +37,11 @@ func usage() { func main() { rand.Seed(time.Now().UnixNano()) + klog.InitFlags(nil) flag.Usage = usage flag.Parse() + klog.V(10).Info("Logging enabled.") ctx := context.Background() audit := utillog.GetAuditEntry() diff --git a/cmd/aro/operator.go b/cmd/aro/operator.go index 422e62db4e1..f1dd98c1f28 100644 --- a/cmd/aro/operator.go +++ b/cmd/aro/operator.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/ARO-RP/pkg/operator/controllers/checkers/serviceprincipalchecker" "github.com/Azure/ARO-RP/pkg/operator/controllers/clusteroperatoraro" "github.com/Azure/ARO-RP/pkg/operator/controllers/dnsmasq" + "github.com/Azure/ARO-RP/pkg/operator/controllers/dnsv2" "github.com/Azure/ARO-RP/pkg/operator/controllers/genevalogging" "github.com/Azure/ARO-RP/pkg/operator/controllers/guardrails" "github.com/Azure/ARO-RP/pkg/operator/controllers/imageconfig" @@ -126,6 +127,7 @@ func operator(ctx context.Context, log *logrus.Entry) error { client, dh)).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller %s: %v", rbac.ControllerName, err) } + // dnsmasq if err = (dnsmasq.NewClusterReconciler( log.WithField("controller", dnsmasq.ClusterControllerName), client, dh)).SetupWithManager(mgr); err != nil { @@ -141,6 +143,12 @@ func operator(ctx context.Context, log *logrus.Entry) error { client, dh)).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller %s: %v", dnsmasq.MachineConfigPoolControllerName, err) } + // dnsv2 + if err = (dnsmasq.NewClusterReconciler( + log.WithField("controller", dnsv2.ClusterControllerName), + client, dh)).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create controller %s: %v", dnsv2.ClusterControllerName, err) + } if err = (node.NewReconciler( log.WithField("controller", node.ControllerName), client, kubernetescli)).SetupWithManager(mgr); err != nil { diff --git a/hack/db/db.go b/hack/db/db.go index 9aad937f583..bed72941287 100644 --- a/hack/db/db.go +++ b/hack/db/db.go @@ -7,6 +7,8 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "net/http/httptrace" "os" "strings" @@ -27,19 +29,113 @@ const ( DatabaseAccountName = "DATABASE_ACCOUNT_NAME" KeyVaultPrefix = "KEYVAULT_PREFIX" ) +var logger = utillog.GetLogger() + +type transport struct { + current *http.Request +} + +func (t *transport) WroteRequest(wri httptrace.WroteRequestInfo) { + log := logger.WithField("wri", wri).WithError(wri.Err).WithField("t", t) + log.Infof("WroteRequest") +} + +func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { + log := logger.WithField("t", t) + t.current = req + log.Infof("RoundTrip start") + fmt.Fprintf(os.Stderr, "%s %s %s\n", req.Method, req.URL, req.Proto) + for k, v := range req.Header { + for x, vv := range v { + if x > 0 { + k = strings.Repeat(" ", len(k)) + } + fmt.Fprintf(os.Stderr, "%s: %s\n", k, vv) + } + } + fmt.Fprintf(os.Stderr, "---\n%s\n---\n", req.Body) + resp, err := http.DefaultTransport.RoundTrip(req) + log.WithError(err).Infof("RoundTrip stop") + if resp != nil { + fmt.Fprintf(os.Stderr, "%d.%d %s\n", resp.ProtoMajor, resp.ProtoMinor, resp.Status) + for k, v := range resp.Header { + for x, vv := range v { + if x > 0 { + k = strings.Repeat(" ", len(k)) + } + fmt.Fprintf(os.Stderr, "%s: %s\n", k, vv) + } + } + fmt.Fprintf(os.Stderr, "---\n%s\n---\n", resp.Body) + } + return resp, err +} + +// GotConn prints whether the connection has been used previously +// for the current request. +func (t *transport) GotConn(info httptrace.GotConnInfo) { + log := logger.WithFields(logrus.Fields{ + "idle": info.IdleTime, + "reused": info.Reused, + "wasidle": info.WasIdle, + "localaddr": info.Conn.LocalAddr(), + "remoteaddr": info.Conn.RemoteAddr(), + "transport": t, + }) + log.Infof("Connection reused") +} + +func newTrace(ctx context.Context) context.Context { + t := &transport{} + log := logger.WithContext(ctx) + + trace := &httptrace.ClientTrace{ + GotConn: t.GotConn, + WroteRequest: t.WroteRequest, + GotFirstResponseByte: func() { + log.Infoln("GotFirstResponseByte") + }, + DNSStart: func(di httptrace.DNSStartInfo) { + log.WithField("host", di.Host).Infof("DNSStart") + }, + DNSDone: func(di httptrace.DNSDoneInfo) { + log.WithError(di.Err).WithField("coalesced", di.Coalesced).WithField("addrs", di.Addrs).Infof("DNSDone") + }, + // WroteHeaderField: func(key string, value []string) { + // log.WithField("key", key).WithField("value", value).Infof("WroteHeaderField") + // }, + // WroteHeaders: func() { + // log.Info("WroteHeaders") + // }, + ConnectStart: func(network, addr string) { + log.WithField("network", network).WithField("addr", addr).Info("ConnectStart") + }, + ConnectDone: func(network, addr string, err error) { + log.WithField("network", network).WithField("addr", addr).WithError(err).Info("ConnectDone") + }, + } + ctx = httptrace.WithClientTrace(ctx, trace) + return ctx +} func run(ctx context.Context, log *logrus.Entry) error { if len(os.Args) != 2 { return fmt.Errorf("usage: %s resourceid", os.Args[0]) } + t := &transport{} + + ctx = newTrace(ctx) + _env, err := env.NewCore(ctx, log) if err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "env.NewCore").WithField("resourceid", os.Args[1]).Error("Failed to create environment") return err } tokenCredential, err := azidentity.NewAzureCLICredential(nil) if err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "azidentity.NewAzureCLICredential").WithField("resourceid", os.Args[1]).Error("Failed to create token credential") return err } @@ -48,10 +144,12 @@ func run(ctx context.Context, log *logrus.Entry) error { msiKVAuthorizer, err := _env.NewMSIAuthorizer(env.MSIContextRP, _env.Environment().KeyVaultScope) if err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "_env.NewMSIAuthorizer").WithField("resourceid", os.Args[1]).Error("Failed to create MSI authorizer") return err } if err := env.ValidateVars(KeyVaultPrefix); err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "keyvault.URI").WithField("resourceid", os.Args[1]).Error("Failed to create keyvault client") return err } keyVaultPrefix := os.Getenv(KeyVaultPrefix) @@ -60,6 +158,7 @@ func run(ctx context.Context, log *logrus.Entry) error { aead, err := encryption.NewMulti(ctx, serviceKeyvault, env.EncryptionSecretV2Name, env.EncryptionSecretName) if err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "encryption.NewMulti").WithField("resourceid", os.Args[1]).Error("Failed to create AEAD") return err } @@ -70,11 +169,13 @@ func run(ctx context.Context, log *logrus.Entry) error { dbAccountName := os.Getenv(DatabaseAccountName) dbAuthorizer, err := database.NewMasterKeyAuthorizer(ctx, _env, authorizer, dbAccountName) if err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "database.NewMasterKeyAuthorizer").WithField("resourceid", os.Args[1]).Error("Failed to create master key authorizer") return err } - dbc, err := database.NewDatabaseClient(log.WithField("component", "database"), _env, dbAuthorizer, &noop.Noop{}, aead, dbAccountName) + dbc, err := database.NewDatabaseClientWithTransport(log.WithField("component", "database"), _env, dbAuthorizer, &noop.Noop{}, aead, dbAccountName) if err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "database.NewDatabaseClient").WithField("resourceid", os.Args[1]).Error("Failed to create database client") return err } @@ -85,11 +186,13 @@ func run(ctx context.Context, log *logrus.Entry) error { openShiftClusters, err := database.NewOpenShiftClusters(ctx, dbc, dbName) if err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "database.NewOpenShiftClusters").WithField("resourceid", os.Args[1]).Error("Failed to connect to database") return err } doc, err := openShiftClusters.Get(ctx, strings.ToLower(os.Args[1])) if err != nil { + log.WithError(err).WithContext(ctx).WithField("fn", "openShiftClusters.Get").WithField("resourceid", os.Args[1]).Error("Failed to get cluster") return err } diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go index 355b691ab8f..92bbdb179e4 100644 --- a/pkg/api/defaults.go +++ b/pkg/api/defaults.go @@ -69,7 +69,8 @@ func DefaultOperatorFlags() OperatorFlags { "aro.azuresubnets.serviceendpoint.managed": flagTrue, "aro.banner.enabled": flagFalse, "aro.checker.enabled": flagTrue, - "aro.dnsmasq.enabled": flagTrue, + "aro.dnsmasq.enabled": flagFalse, + "aro.dnsv2.enabled": flagTrue, "aro.genevalogging.enabled": flagTrue, "aro.imageconfig.enabled": flagTrue, "aro.ingress.enabled": flagTrue, diff --git a/pkg/database/database.go b/pkg/database/database.go index 6233b7460c3..b9384b6606e 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -6,7 +6,9 @@ package database import ( "context" "crypto/tls" + "fmt" "net/http" + "os" "reflect" "time" @@ -53,7 +55,37 @@ func NewDatabaseClient(log *logrus.Entry, _env env.Core, authorizer cosmosdb.Aut return cosmosdb.NewDatabaseClient(log, c, h, databaseAccountName+"."+_env.Environment().CosmosDBDNSSuffix, authorizer), nil } +func NewDatabaseClientWithTransport(log *logrus.Entry, env env.Core, authorizer cosmosdb.Authorizer, m metrics.Emitter, aead encryption.AEAD, transport http.RoundTripper) (cosmosdb.DatabaseClient, error) { + for _, key := range []string{ + "DATABASE_ACCOUNT_NAME", + } { + if _, found := os.LookupEnv(key); !found { + return nil, fmt.Errorf("environment variable %q unset", key) + } + } + + h, err := NewJSONHandle(aead) + if err != nil { + return nil, err + } + + c := &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + } + + return cosmosdb.NewDatabaseClient(log, c, h, os.Getenv("DATABASE_ACCOUNT_NAME")+"."+env.Environment().CosmosDBDNSSuffix, authorizer), nil +} + func NewMasterKeyAuthorizer(ctx context.Context, _env env.Core, msiAuthorizer autorest.Authorizer, databaseAccountName string) (cosmosdb.Authorizer, error) { + for _, key := range []string{ + "DATABASE_ACCOUNT_NAME", + } { + if _, found := os.LookupEnv(key); !found { + return nil, fmt.Errorf("environment variable %q unset", key) + } + } + databaseaccounts := documentdb.NewDatabaseAccountsClient(_env.Environment(), _env.SubscriptionID(), msiAuthorizer) keys, err := databaseaccounts.ListKeys(ctx, _env.ResourceGroup(), databaseAccountName) diff --git a/pkg/database/openshiftclusters.go b/pkg/database/openshiftclusters.go index e3cf392e408..60eba10eb4a 100644 --- a/pkg/database/openshiftclusters.go +++ b/pkg/database/openshiftclusters.go @@ -148,7 +148,7 @@ func (c *openShiftClusters) Get(ctx context.Context, key string) (*api.OpenShift case len(docs.OpenShiftClusterDocuments) == 1: return docs.OpenShiftClusterDocuments[0], nil default: - return nil, &cosmosdb.Error{StatusCode: http.StatusNotFound} + return nil, &cosmosdb.Error{StatusCode: http.StatusNotFound, Message: "No cluster documents found"} } } diff --git a/pkg/operator/controllers/dnsv2/dnsv2_cluster_controller.go b/pkg/operator/controllers/dnsv2/dnsv2_cluster_controller.go new file mode 100644 index 00000000000..0564807b2b0 --- /dev/null +++ b/pkg/operator/controllers/dnsv2/dnsv2_cluster_controller.go @@ -0,0 +1,155 @@ +package dnsv2 + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "encoding/json" + "fmt" + + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/dynamichelper" + "github.com/openshift/installer/pkg/aro/dnsv2" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" +) + +const ( + ClusterControllerName = "Dnsv2Cluster" + controllerEnabled = "aro.dnsv2.enabled" +) + +type ClusterReconciler struct { + log *logrus.Entry + + dh dynamichelper.Interface + + client client.Client +} + +func NewClusterReconciler(log *logrus.Entry, client client.Client, dh dynamichelper.Interface) *ClusterReconciler { + return &ClusterReconciler{ + log: log, + dh: dh, + client: client, + } +} + +// Reconcile watches the ARO object, and if it changes, reconciles all the +// 99-%s-aro-dns machineconfigs +func (r *ClusterReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + instance := &arov1alpha1.Cluster{} + err := r.client.Get(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, instance) + if err != nil { + return reconcile.Result{}, err + } + + if !instance.Spec.OperatorFlags.GetSimpleBoolean(controllerEnabled) { + r.log.Debug("controller is disabled") + return reconcile.Result{}, nil + } + + r.log.Debug("running") + mcps := &mcv1.MachineConfigPoolList{} + err = r.client.List(ctx, mcps) + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + err = reconcileMachineConfigs(ctx, instance, r.dh, mcps.Items...) + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + aroClusterPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetName() == arov1alpha1.SingletonClusterName + }) + + return ctrl.NewControllerManagedBy(mgr). + For(&arov1alpha1.Cluster{}, builder.WithPredicates(aroClusterPredicate)). + Named(ClusterControllerName). + Complete(r) +} + +func reconcileMachineConfigs(ctx context.Context, instance *arov1alpha1.Cluster, dh dynamichelper.Interface, mcps ...mcv1.MachineConfigPool) error { + var resources []kruntime.Object + for _, mcp := range mcps { + resource, err := generateMachineConfig(instance.Spec.Domain, instance.Spec.APIIntIP, instance.Spec.IngressIP, mcp.Name, instance.Spec.GatewayDomains, instance.Spec.GatewayPrivateEndpointIP) + if err != nil { + return err + } + + err = dynamichelper.SetControllerReferences([]kruntime.Object{resource}, &mcp) + if err != nil { + return err + } + + resources = append(resources, resource) + } + + err := dynamichelper.Prepare(resources) + if err != nil { + return err + } + + return dh.Ensure(ctx, resources...) +} + +func generateMachineConfig(clusterDomain, apiIntIP, ingressIP, role string, gatewayDomains []string, gatewayPrivateEndpointIP string) (*mcfgv1.MachineConfig, error) { + ignConfig, err := dnsv2.Ignition2Config(clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + + b, err := json.Marshal(ignConfig) + if err != nil { + return nil, err + } + + // canonicalise the machineconfig payload the same way as MCO + var i interface{} + err = json.Unmarshal(b, &i) + if err != nil { + return nil, err + } + + rawExt := runtime.RawExtension{} + rawExt.Raw, err = json.Marshal(i) + if err != nil { + return nil, err + } + + return &mcfgv1.MachineConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: mcfgv1.SchemeGroupVersion.String(), + Kind: "MachineConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("99-%s-aro-dns", role), + Labels: map[string]string{ + "machineconfiguration.openshift.io/role": role, + }, + }, + Spec: mcfgv1.MachineConfigSpec{ + Config: rawExt, + }, + }, nil +} diff --git a/pkg/operator/controllers/dnsv2/dnsv2_machineconfig_controller.go b/pkg/operator/controllers/dnsv2/dnsv2_machineconfig_controller.go new file mode 100644 index 00000000000..527fb11a120 --- /dev/null +++ b/pkg/operator/controllers/dnsv2/dnsv2_machineconfig_controller.go @@ -0,0 +1,93 @@ +package dnsv2 + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "regexp" + + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "github.com/sirupsen/logrus" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/dynamichelper" +) + +const ( + MachineConfigControllerName = "Dnsv2MachineConfig" +) + +type MachineConfigReconciler struct { + log *logrus.Entry + + dh dynamichelper.Interface + + client client.Client +} + +var rxARODNS = regexp.MustCompile("^99-(.*)-aro-dns$") + +func NewMachineConfigReconciler(log *logrus.Entry, client client.Client, dh dynamichelper.Interface) *MachineConfigReconciler { + return &MachineConfigReconciler{ + log: log, + dh: dh, + client: client, + } +} + +// Reconcile watches ARO DNS MachineConfig objects, and if any changes, +// reconciles it +func (r *MachineConfigReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + instance := &arov1alpha1.Cluster{} + err := r.client.Get(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, instance) + if err != nil { + return reconcile.Result{}, err + } + + if !instance.Spec.OperatorFlags.GetSimpleBoolean(controllerEnabled) { + r.log.Debug("controller is disabled") + return reconcile.Result{}, nil + } + + r.log.Debug("running") + m := rxARODNS.FindStringSubmatch(request.Name) + if m == nil { + return reconcile.Result{}, nil + } + role := m[1] + + mcp := &mcv1.MachineConfigPool{} + err = r.client.Get(ctx, types.NamespacedName{Name: role}, mcp) + if kerrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + if mcp.GetDeletionTimestamp() != nil { + return reconcile.Result{}, nil + } + + err = reconcileMachineConfigs(ctx, instance, r.dh, *mcp) + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// SetupWithManager setup our mananger +func (r *MachineConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&mcv1.MachineConfig{}). + Named(MachineConfigControllerName). + Complete(r) +} diff --git a/pkg/operator/controllers/dnsv2/dnsv2_machineconfigpool_controller.go b/pkg/operator/controllers/dnsv2/dnsv2_machineconfigpool_controller.go new file mode 100644 index 00000000000..2789e447268 --- /dev/null +++ b/pkg/operator/controllers/dnsv2/dnsv2_machineconfigpool_controller.go @@ -0,0 +1,81 @@ +package dnsv2 + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "github.com/sirupsen/logrus" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + "github.com/Azure/ARO-RP/pkg/util/dynamichelper" +) + +const ( + MachineConfigPoolControllerName = "Dnsv2MachineConfigPool" +) + +type MachineConfigPoolReconciler struct { + log *logrus.Entry + + dh dynamichelper.Interface + + client client.Client +} + +func NewMachineConfigPoolReconciler(log *logrus.Entry, client client.Client, dh dynamichelper.Interface) *MachineConfigPoolReconciler { + return &MachineConfigPoolReconciler{ + log: log, + dh: dh, + client: client, + } +} + +// Reconcile watches MachineConfigPool objects, and if any changes, +// reconciles the associated ARO DNS MachineConfig object +func (r *MachineConfigPoolReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + instance := &arov1alpha1.Cluster{} + err := r.client.Get(ctx, types.NamespacedName{Name: arov1alpha1.SingletonClusterName}, instance) + if err != nil { + return reconcile.Result{}, err + } + + if !instance.Spec.OperatorFlags.GetSimpleBoolean(controllerEnabled) { + r.log.Debug("controller is disabled") + return reconcile.Result{}, nil + } + + r.log.Debug("running") + mcp := &mcv1.MachineConfigPool{} + err = r.client.Get(ctx, types.NamespacedName{Name: request.Name}, mcp) + if kerrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + err = reconcileMachineConfigs(ctx, instance, r.dh, *mcp) + if err != nil { + r.log.Error(err) + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// SetupWithManager setup our mananger +func (r *MachineConfigPoolReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&mcv1.MachineConfigPool{}). + Named(MachineConfigPoolControllerName). + Complete(r) +} diff --git a/vendor/github.com/openshift/installer/pkg/aro/dnsv2/resources.go b/vendor/github.com/openshift/installer/pkg/aro/dnsv2/resources.go new file mode 100644 index 00000000000..01364368f43 --- /dev/null +++ b/vendor/github.com/openshift/installer/pkg/aro/dnsv2/resources.go @@ -0,0 +1,302 @@ +package dnsv2 + +import ( + "bytes" + "text/template" + + "github.com/vincent-petithory/dataurl" + + ign2types "github.com/coreos/ignition/config/v2_2/types" + ignutil "github.com/coreos/ignition/v2/config/util" + ign3types "github.com/coreos/ignition/v2/config/v3_2/types" +) + +// Potentially add the IP of -n openshift-image-registry Service/image-registry, marked with # openshift-generated-node-resolver +var t = template.Must(template.New("").Parse(` + +{{ define "hosts" }} +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +XXX image-registry.openshift-image-registry.svc image-registry.openshift-image-registry.svc.cluster.local # openshift-generated-node-resolver +{{ .APIIntIP }} api.{{ .ClusterDomain }} +{{ .APIIntIP }} api-int.{{ .ClusterDomain }} +# {{ .IngressIP }} *.apps.{{ .ClusterDomain }}/ +{{- range $GatewayDomain := .GatewayDomains }} +{{ $.GatewayPrivateEndpointIP }} {{ $GatewayDomain }} +{{- end }} +{{ end }} + +{{ define "Corefile" }} +apps.{{ .ClusterDomain }}:53 { + root /etc/coredns/zones + file db.apps +} +.:53 { + bufsize 1232 # https://www.dnsflagday.net/2020/ + errors + log . { + class error + } + health { + lameduck 20s + } + ready + prometheus 127.0.0.1:9153 + hosts { + fallthrough + } + forward . /etc/resolv.conf + reload +} +hostname.bind:53 { + chaos +} +{{ end }} + +{{ define "db.apps" }} +$ORIGIN apps.{{ .ClusterDomain }}. +@ 3600 IN SOA ns1-09.azure-dns.com. azuredns-hostmaster.microsoft.com. 1 3600 300 2419200 300 + +* IN A {{ .IngressIP }} +{{ end }} + +{{ define "aro-coredns.service" }} +[Unit] +Description=CoreDNS +RequiredMountsFor=/etc/coredns +After=network-online.target +Before=bootkube.service + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +KillMode=mixed +ExecStartPre=cp /etc/hosts.aro /etc/hosts +ExecStart=/usr/bin/podman run --authfile=/var/lib/kubelet/config.json --rm --name aro-coredns -v /etc/coredns/:/etc/coredns/:Z --replace --cgroups=split --init --sdnotify=conmon --net=host --log-driver=journald -d quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:19829e1922bbbb02496d8647ba197a6db4aa58522aa9db588ff08897cd15ade8 -conf /etc/coredns/Corefile +ExecStop=/usr/bin/podman rm -f -i aro-coredns +ExecStopPost=-/usr/bin/podman rm -f -i aro-coredns +Delegate=yes +Type=notify +NotifyAccess=all +SyslogIdentifier=%N +Restart=always + +[Install] +WantedBy=multi-user.target +{{ end }} + +`)) + +func Ignition2Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) (*ign2types.Config, error) { + hosts, err := renderTemplateBytes("hosts", clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + corefile, err := renderTemplateBytes("Corefile", clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + dbApps, err := renderTemplateBytes("db.apps", clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + aroCorednsService, err := renderTemplateString("aro-coredns.service", clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + + return &ign2types.Config{ + Ignition: ign2types.Ignition{ + Version: ign2types.MaxVersion.String(), + }, + Storage: ign2types.Storage{ + Files: []ign2types.File{ + { + Node: ign2types.Node{ + Filesystem: "root", + Overwrite: ignutil.BoolToPtr(true), + Path: "/etc/hosts.aro", + User: &ign2types.NodeUser{ + Name: "root", + }, + }, + FileEmbedded1: ign2types.FileEmbedded1{ + Contents: ign2types.FileContents{ + Source: dataurl.EncodeBytes(hosts), + }, + Mode: ignutil.IntToPtr(0644), + }, + }, + { + Node: ign2types.Node{ + Filesystem: "root", + Overwrite: ignutil.BoolToPtr(true), + Path: "/etc/coredns/Corefile", + User: &ign2types.NodeUser{ + Name: "root", + }, + }, + FileEmbedded1: ign2types.FileEmbedded1{ + Contents: ign2types.FileContents{ + Source: dataurl.EncodeBytes(corefile), + }, + Mode: ignutil.IntToPtr(0644), + }, + }, + { + Node: ign2types.Node{ + Filesystem: "root", + Overwrite: ignutil.BoolToPtr(true), + Path: "/etc/coredns/zones/db.apps", + User: &ign2types.NodeUser{ + Name: "root", + }, + }, + FileEmbedded1: ign2types.FileEmbedded1{ + Contents: ign2types.FileContents{ + Source: dataurl.EncodeBytes(dbApps), + }, + Mode: ignutil.IntToPtr(0644), + }, + }, + }, + }, + Systemd: ign2types.Systemd{ + Units: []ign2types.Unit{ + { + Contents: aroCorednsService, + Enabled: ignutil.BoolToPtr(true), + Name: "aro-coredns.service", + }, + }, + }, + }, nil +} + +func Ignition3Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) (*ign3types.Config, error) { + hosts, err := renderTemplateBytes("hosts", clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + corefile, err := renderTemplateBytes("Corefile", clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + dbApps, err := renderTemplateBytes("db.apps", clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + aroCorednsService, err := renderTemplateString("aro-coredns.service", clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + + return &ign3types.Config{ + Ignition: ign3types.Ignition{ + Version: ign3types.MaxVersion.String(), + }, + Storage: ign3types.Storage{ + Files: []ign3types.File{ + { + Node: ign3types.Node{ + Overwrite: ignutil.BoolToPtr(true), + Path: "/etc/hosts.aro", + User: ign3types.NodeUser{ + Name: ignutil.StrToPtr("root"), + }, + }, + FileEmbedded1: ign3types.FileEmbedded1{ + Contents: ign3types.Resource{ + Source: ignutil.StrToPtr(dataurl.EncodeBytes(hosts)), + }, + Mode: ignutil.IntToPtr(0644), + }, + }, + { + Node: ign3types.Node{ + Overwrite: ignutil.BoolToPtr(true), + Path: "/etc/coredns/Corefile", + User: ign3types.NodeUser{ + Name: ignutil.StrToPtr("root"), + }, + }, + FileEmbedded1: ign3types.FileEmbedded1{ + Contents: ign3types.Resource{ + Source: ignutil.StrToPtr(dataurl.EncodeBytes(corefile)), + }, + Mode: ignutil.IntToPtr(0644), + }, + }, + { + Node: ign3types.Node{ + Overwrite: ignutil.BoolToPtr(true), + Path: "/etc/coredns/zones/db.apps", + User: ign3types.NodeUser{ + Name: ignutil.StrToPtr("root"), + }, + }, + FileEmbedded1: ign3types.FileEmbedded1{ + Contents: ign3types.Resource{ + Source: ignutil.StrToPtr(dataurl.EncodeBytes(dbApps)), + }, + Mode: ignutil.IntToPtr(0644), + }, + }, + }, + }, + Systemd: ign3types.Systemd{ + Units: []ign3types.Unit{ + { + Contents: &aroCorednsService, + Enabled: ignutil.BoolToPtr(true), + Name: "aro-coredns.service", + }, + }, + }, + }, nil +} + +func renderTemplateBytes(file, clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) ([]byte, error) { + buf := &bytes.Buffer{} + + err := t.ExecuteTemplate(buf, file, &struct { + ClusterDomain string + APIIntIP string + IngressIP string + GatewayDomains []string + GatewayPrivateEndpointIP string + }{ + ClusterDomain: clusterDomain, + APIIntIP: apiIntIP, + IngressIP: ingressIP, + GatewayDomains: gatewayDomains, + GatewayPrivateEndpointIP: gatewayPrivateEndpointIP, + }) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func renderTemplateString(file, clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) (string, error) { + buf := &bytes.Buffer{} + + err := t.ExecuteTemplate(buf, file, &struct { + ClusterDomain string + APIIntIP string + IngressIP string + GatewayDomains []string + GatewayPrivateEndpointIP string + }{ + ClusterDomain: clusterDomain, + APIIntIP: apiIntIP, + IngressIP: ingressIP, + GatewayDomains: gatewayDomains, + GatewayPrivateEndpointIP: gatewayPrivateEndpointIP, + }) + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/vendor/github.com/openshift/installer/pkg/asset/ignition/bootstrap/common.go b/vendor/github.com/openshift/installer/pkg/asset/ignition/bootstrap/common.go index e67966b3d1e..b7d1d610055 100644 --- a/vendor/github.com/openshift/installer/pkg/asset/ignition/bootstrap/common.go +++ b/vendor/github.com/openshift/installer/pkg/asset/ignition/bootstrap/common.go @@ -25,7 +25,7 @@ import ( "github.com/vincent-petithory/dataurl" "github.com/openshift/installer/data" - "github.com/openshift/installer/pkg/aro/dnsmasq" + "github.com/openshift/installer/pkg/aro/dnsv2" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/bootstraplogging" "github.com/openshift/installer/pkg/asset/ignition" @@ -209,13 +209,20 @@ func (a *Common) generateConfig(dependencies asset.Parents, templateData *bootst }}, ) - dnsmasqIgnConfig, err := dnsmasq.Ignition3Config(installConfig.Config.ClusterDomain(), aroDNSConfig.APIIntIP, aroDNSConfig.IngressIP, aroDNSConfig.GatewayDomains, aroDNSConfig.GatewayPrivateEndpointIP) + // dnsmasqIgnConfig, err := dnsmasq.Ignition3Config(installConfig.Config.ClusterDomain(), aroDNSConfig.APIIntIP, aroDNSConfig.IngressIP, aroDNSConfig.GatewayDomains, aroDNSConfig.GatewayPrivateEndpointIP) + // if err != nil { + // return err + // } + + // a.Config.Storage.Files = append(a.Config.Storage.Files, dnsmasqIgnConfig.Storage.Files...) + // a.Config.Systemd.Units = append(a.Config.Systemd.Units, dnsmasqIgnConfig.Systemd.Units...) + + aroDnsIgnConfig, err := dnsv2.Ignition3Config(installConfig.Config.ClusterDomain(), aroDNSConfig.APIIntIP, aroDNSConfig.IngressIP, aroDNSConfig.GatewayDomains, aroDNSConfig.GatewayPrivateEndpointIP) if err != nil { return err } - - a.Config.Storage.Files = append(a.Config.Storage.Files, dnsmasqIgnConfig.Storage.Files...) - a.Config.Systemd.Units = append(a.Config.Systemd.Units, dnsmasqIgnConfig.Systemd.Units...) + a.Config.Storage.Files = append(a.Config.Storage.Files, aroDnsIgnConfig.Storage.Files...) + a.Config.Systemd.Units = append(a.Config.Systemd.Units, aroDnsIgnConfig.Systemd.Units...) return nil }