[wip] Remove dnsmasq and replace with coredns

cmd/aro/main.go

@@ -14,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/klog/v2"
+
 	"github.com/Azure/ARO-RP/pkg/env"
 	utillog "github.com/Azure/ARO-RP/pkg/util/log"
 	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
@@ -36,9 +38,11 @@ func usage() {
 
 func main() {
 	rand.Seed(time.Now().UnixNano())
+	klog.InitFlags(nil)
 
 	flag.Usage = usage
 	flag.Parse()
+	klog.V(10).Info("Logging enabled.")
 
 	ctx := context.Background()
 	audit := utillog.GetAuditEntry()

cmd/aro/operator.go

@@ -24,6 +24,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/checkers/serviceprincipalchecker"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/clusteroperatoraro"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/dnsmasq"
+	"github.com/Azure/ARO-RP/pkg/operator/controllers/dnsv2"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/genevalogging"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/guardrails"
 	"github.com/Azure/ARO-RP/pkg/operator/controllers/imageconfig"
@@ -126,6 +127,8 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 			client, dh)).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to create controller %s: %v", rbac.ControllerName, err)
 		}
+
+		// dnsmasq
 		if err = (dnsmasq.NewClusterReconciler(
 			log.WithField("controller", dnsmasq.ClusterControllerName),
 			client, dh)).SetupWithManager(mgr); err != nil {
@@ -141,6 +144,23 @@ func operator(ctx context.Context, log *logrus.Entry) error {
 			client, dh)).SetupWithManager(mgr); err != nil {
 			return fmt.Errorf("unable to create controller %s: %v", dnsmasq.MachineConfigPoolControllerName, err)
 		}
+		// dnsv2
+		if err = (dnsv2.NewClusterReconciler(
+			log.WithField("controller", dnsv2.ClusterControllerName),
+			client, dh)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller %s: %v", dnsv2.ClusterControllerName, err)
+		}
+		if err = (dnsv2.NewMachineConfigReconciler(
+			log.WithField("controller", dnsv2.MachineConfigControllerName),
+			client, dh)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller %s: %v", dnsv2.MachineConfigControllerName, err)
+		}
+		if err = (dnsv2.NewMachineConfigPoolReconciler(
+			log.WithField("controller", dnsv2.MachineConfigPoolControllerName),
+			client, dh)).SetupWithManager(mgr); err != nil {
+			return fmt.Errorf("unable to create controller %s: %v", dnsv2.MachineConfigPoolControllerName, err)
+		}
+
 		if err = (node.NewReconciler(
 			log.WithField("controller", node.ControllerName),
 			client, kubernetescli)).SetupWithManager(mgr); err != nil {

go.mod

@@ -801,7 +801,7 @@ replace (
 	github.com/openshift/cluster-api-provider-libvirt => github.com/openshift/cluster-api-provider-libvirt v0.2.1-0.20191219173431-2336783d4603
 	github.com/openshift/cluster-api-provider-ovirt => github.com/openshift/cluster-api-provider-ovirt v0.1.1-0.20211215231458-35ce9aafee1f
 	github.com/openshift/console-operator => github.com/openshift/console-operator v0.0.0-20220318130441-e44516b9c315
-	github.com/openshift/installer => github.com/jewzaam/installer-aro v0.9.0-master.0.20220524230743-7e2aa7a0cc1a
+	github.com/openshift/installer => github.com/ventifus/installer-aro v0.0.0-20230705203259-26f39e3cef4e
 	github.com/openshift/library-go => github.com/openshift/library-go v0.0.0-20220303081124-fb4e7a2872f0
 	github.com/openshift/machine-api-operator => github.com/openshift/machine-api-operator v0.2.1-0.20220124104622-668c5b52b104
 	github.com/openshift/machine-config-operator => github.com/openshift/machine-config-operator v0.0.1-0.20220319215057-e6ba00b88555

hack/db/db.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/http/httptrace"
 	"os"
 	"strings"
 
@@ -28,18 +30,113 @@ const (
 	KeyVaultPrefix      = "KEYVAULT_PREFIX"
 )
 
+var logger = utillog.GetLogger()
+
+type transport struct {
+	current *http.Request
+}
+
+func (t *transport) WroteRequest(wri httptrace.WroteRequestInfo) {
+	log := logger.WithField("wri", wri).WithError(wri.Err).WithField("t", t)
+	log.Infof("WroteRequest")
+}
+
+func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	log := logger.WithField("t", t)
+	t.current = req
+	log.Infof("RoundTrip start")
+	fmt.Fprintf(os.Stderr, "%s %s %s\n", req.Method, req.URL, req.Proto)
+	for k, v := range req.Header {
+		for x, vv := range v {
+			if x > 0 {
+				k = strings.Repeat(" ", len(k))
+			}
+			fmt.Fprintf(os.Stderr, "%s: %s\n", k, vv)
+		}
+	}
+	fmt.Fprintf(os.Stderr, "---\n%s\n---\n", req.Body)
+	resp, err := http.DefaultTransport.RoundTrip(req)
+	log.WithError(err).Infof("RoundTrip stop")
+	if resp != nil {
+		fmt.Fprintf(os.Stderr, "%d.%d %s\n", resp.ProtoMajor, resp.ProtoMinor, resp.Status)
+		for k, v := range resp.Header {
+			for x, vv := range v {
+				if x > 0 {
+					k = strings.Repeat(" ", len(k))
+				}
+				fmt.Fprintf(os.Stderr, "%s: %s\n", k, vv)
+			}
+		}
+		fmt.Fprintf(os.Stderr, "---\n%s\n---\n", resp.Body)
+	}
+	return resp, err
+}
+
+// GotConn prints whether the connection has been used previously
+// for the current request.
+func (t *transport) GotConn(info httptrace.GotConnInfo) {
+	log := logger.WithFields(logrus.Fields{
+		"idle":       info.IdleTime,
+		"reused":     info.Reused,
+		"wasidle":    info.WasIdle,
+		"localaddr":  info.Conn.LocalAddr(),
+		"remoteaddr": info.Conn.RemoteAddr(),
+		"transport":  t,
+	})
+	log.Infof("Connection reused")
+}
+
+func newTrace(ctx context.Context) context.Context {
+	t := &transport{}
+	log := logger.WithContext(ctx)
+
+	trace := &httptrace.ClientTrace{
+		GotConn:      t.GotConn,
+		WroteRequest: t.WroteRequest,
+		GotFirstResponseByte: func() {
+			log.Infoln("GotFirstResponseByte")
+		},
+		DNSStart: func(di httptrace.DNSStartInfo) {
+			log.WithField("host", di.Host).Infof("DNSStart")
+		},
+		DNSDone: func(di httptrace.DNSDoneInfo) {
+			log.WithError(di.Err).WithField("coalesced", di.Coalesced).WithField("addrs", di.Addrs).Infof("DNSDone")
+		},
+		// WroteHeaderField: func(key string, value []string) {
+		// 	log.WithField("key", key).WithField("value", value).Infof("WroteHeaderField")
+		// },
+		// WroteHeaders: func() {
+		// 	log.Info("WroteHeaders")
+		// },
+		ConnectStart: func(network, addr string) {
+			log.WithField("network", network).WithField("addr", addr).Info("ConnectStart")
+		},
+		ConnectDone: func(network, addr string, err error) {
+			log.WithField("network", network).WithField("addr", addr).WithError(err).Info("ConnectDone")
+		},
+	}
+	ctx = httptrace.WithClientTrace(ctx, trace)
+	return ctx
+}
+
 func run(ctx context.Context, log *logrus.Entry) error {
 	if len(os.Args) != 2 {
 		return fmt.Errorf("usage: %s resourceid", os.Args[0])
 	}
 
+	t := &transport{}
+
+	ctx = newTrace(ctx)
+
 	_env, err := env.NewCore(ctx, log)
 	if err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "env.NewCore").WithField("resourceid", os.Args[1]).Error("Failed to create environment")
 		return err
 	}
 
 	tokenCredential, err := azidentity.NewAzureCLICredential(nil)
 	if err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "azidentity.NewAzureCLICredential").WithField("resourceid", os.Args[1]).Error("Failed to create token credential")
 		return err
 	}
 
@@ -48,10 +145,12 @@ func run(ctx context.Context, log *logrus.Entry) error {
 
 	msiKVAuthorizer, err := _env.NewMSIAuthorizer(env.MSIContextRP, _env.Environment().KeyVaultScope)
 	if err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "_env.NewMSIAuthorizer").WithField("resourceid", os.Args[1]).Error("Failed to create MSI authorizer")
 		return err
 	}
 
 	if err := env.ValidateVars(KeyVaultPrefix); err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "keyvault.URI").WithField("resourceid", os.Args[1]).Error("Failed to create keyvault client")
 		return err
 	}
 	keyVaultPrefix := os.Getenv(KeyVaultPrefix)
@@ -60,6 +159,7 @@ func run(ctx context.Context, log *logrus.Entry) error {
 
 	aead, err := encryption.NewMulti(ctx, serviceKeyvault, env.EncryptionSecretV2Name, env.EncryptionSecretName)
 	if err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "encryption.NewMulti").WithField("resourceid", os.Args[1]).Error("Failed to create AEAD")
 		return err
 	}
 
@@ -70,11 +170,13 @@ func run(ctx context.Context, log *logrus.Entry) error {
 	dbAccountName := os.Getenv(DatabaseAccountName)
 	dbAuthorizer, err := database.NewMasterKeyAuthorizer(ctx, _env, authorizer, dbAccountName)
 	if err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "database.NewMasterKeyAuthorizer").WithField("resourceid", os.Args[1]).Error("Failed to create master key authorizer")
 		return err
 	}
 
-	dbc, err := database.NewDatabaseClient(log.WithField("component", "database"), _env, dbAuthorizer, &noop.Noop{}, aead, dbAccountName)
+	dbc, err := database.NewDatabaseClientWithTransport(log.WithField("component", "database"), _env, dbAuthorizer, &noop.Noop{}, aead, dbAccountName, t)
 	if err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "database.NewDatabaseClient").WithField("resourceid", os.Args[1]).Error("Failed to create database client")
 		return err
 	}
 
@@ -85,11 +187,13 @@ func run(ctx context.Context, log *logrus.Entry) error {
 
 	openShiftClusters, err := database.NewOpenShiftClusters(ctx, dbc, dbName)
 	if err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "database.NewOpenShiftClusters").WithField("resourceid", os.Args[1]).Error("Failed to connect to database")
 		return err
 	}
 
 	doc, err := openShiftClusters.Get(ctx, strings.ToLower(os.Args[1]))
 	if err != nil {
+		log.WithError(err).WithContext(ctx).WithField("fn", "openShiftClusters.Get").WithField("resourceid", os.Args[1]).Error("Failed to get cluster")
 		return err
 	}
 

pkg/api/defaults.go

@@ -69,7 +69,8 @@ func DefaultOperatorFlags() OperatorFlags {
 		"aro.azuresubnets.serviceendpoint.managed": flagTrue,
 		"aro.banner.enabled":                       flagFalse,
 		"aro.checker.enabled":                      flagTrue,
-		"aro.dnsmasq.enabled":                      flagTrue,
+		"aro.dnsmasq.enabled":                      flagFalse,
+		"aro.dnsv2.enabled":                        flagTrue,
 		"aro.genevalogging.enabled":                flagTrue,
 		"aro.imageconfig.enabled":                  flagTrue,
 		"aro.ingress.enabled":                      flagTrue,

pkg/database/database.go

@@ -6,7 +6,9 @@ package database
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net/http"
+	"os"
 	"reflect"
 	"time"
 
@@ -53,7 +55,29 @@ func NewDatabaseClient(log *logrus.Entry, _env env.Core, authorizer cosmosdb.Aut
 	return cosmosdb.NewDatabaseClient(log, c, h, databaseAccountName+"."+_env.Environment().CosmosDBDNSSuffix, authorizer), nil
 }
 
+func NewDatabaseClientWithTransport(log *logrus.Entry, _env env.Core, authorizer cosmosdb.Authorizer, m metrics.Emitter, aead encryption.AEAD, databaseAccountName string, transport http.RoundTripper) (cosmosdb.DatabaseClient, error) {
+	h, err := NewJSONHandle(aead)
+	if err != nil {
+		return nil, err
+	}
+
+	c := &http.Client{
+		Transport: transport,
+		Timeout:   30 * time.Second,
+	}
+
+	return cosmosdb.NewDatabaseClient(log, c, h, databaseAccountName+"."+_env.Environment().CosmosDBDNSSuffix, authorizer), nil
+}
+
 func NewMasterKeyAuthorizer(ctx context.Context, _env env.Core, msiAuthorizer autorest.Authorizer, databaseAccountName string) (cosmosdb.Authorizer, error) {
+	for _, key := range []string{
+		"DATABASE_ACCOUNT_NAME",
+	} {
+		if _, found := os.LookupEnv(key); !found {
+			return nil, fmt.Errorf("environment variable %q unset", key)
+		}
+	}
+
 	databaseaccounts := documentdb.NewDatabaseAccountsClient(_env.Environment(), _env.SubscriptionID(), msiAuthorizer)
 
 	keys, err := databaseaccounts.ListKeys(ctx, _env.ResourceGroup(), databaseAccountName)

pkg/database/openshiftclusters.go

@@ -148,7 +148,7 @@ func (c *openShiftClusters) Get(ctx context.Context, key string) (*api.OpenShift
 	case len(docs.OpenShiftClusterDocuments) == 1:
 		return docs.OpenShiftClusterDocuments[0], nil
 	default:
-		return nil, &cosmosdb.Error{StatusCode: http.StatusNotFound}
+		return nil, &cosmosdb.Error{StatusCode: http.StatusNotFound, Message: "No cluster documents found"}
 	}
 }
 

pkg/monitor/cluster/clusterflagsandbanner_test.go

@@ -124,7 +124,7 @@ func TestEmitOperatorFlagsAndSupportBanner(t *testing.T) {
 			expectFlagsMetricsValue: 1,
 			expectFlagsMetricsDims: map[string]string{
 				"aro.imageconfig.enabled":    "false",
-				"aro.dnsmasq.enabled":        "false",
+				"aro.dnsmasq.enabled":        "true",
 				"aro.genevalogging.enabled":  "false",
 				"aro.autosizednodes.enabled": "true",
 			},
@@ -167,7 +167,7 @@ func TestEmitOperatorFlagsAndSupportBanner(t *testing.T) {
 			expectFlagsMetricsValue: 1,
 			expectFlagsMetricsDims: map[string]string{
 				"aro.imageconfig.enabled":    "false",
-				"aro.dnsmasq.enabled":        "false",
+				"aro.dnsmasq.enabled":        "true",
 				"aro.genevalogging.enabled":  "false",
 				"aro.autosizednodes.enabled": "true",
 			},