Skip to content

Commit

Permalink
Fix resolv.conf during Azure host servicing
Browse files Browse the repository at this point in the history
If the network interfaces are brought up/down without restarting NetworkManager, our resolv.conf will be overwritten by NetworkManager.
This can occur during [Azure Host Servicing](https://learn.microsoft.com/en-us/azure/developer/intro/hosting-apps-on-azure) events.

Addiontal information is written to the system logs to assist with SRE
troubleshooting.

By using NetworkManager dispatcher scripts, we can restart dnsmasq when
certian events happen.
  • Loading branch information
s-fairchild committed Jul 12, 2023
1 parent e34c6d6 commit 589e4ed
Showing 1 changed file with 72 additions and 0 deletions.
72 changes: 72 additions & 0 deletions pkg/operator/controllers/dnsmasq/dnsmasq.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,46 @@ chmod 0744 /etc/resolv.conf.dnsmasq
/bin/rm $TMPNETRESOLV
/bin/rm $TMPSELFRESOLV
{{ end }}
{{ define "99-dnsmasq-restart" }}
#!/bin/sh
# This is a NetworkManager dispatcher script to restart dnsmasq
# in the event of a network interface change (e. g. host servicing event https://learn.microsoft.com/en-us/azure/developer/intro/hosting-apps-on-azure)
# this will restart dnsmasq, reapplying our /etc/resolv.conf file and overwriting any modifications made by NetworkManager
interface=$1
action=$2
log() {
logger -i "$0" -t '99-DNSMASQ-RESTART SCRIPT' "$@"
}
if [[ $interface == eth* && $action == "up" ]] || [[ $interface == eth* && $action == "down" ]] || [ $interface == enP* && $action == "up" ] || [ $interface == enP* && $action == "down" ]; then
log "$action happened on $interface, connection state is now $CONNECTIVITY_STATE"
log "restarting dnsmasq now"
if systemctl restart dnsmasq; then
log "dnsmasq successfully restarted"
else
log "failed to restart dnsmasq"
fi
# log dns configuration information relevant to SRE while troubleshooting
# The line break used here is important for formatting
log "/etc/resolv.conf contents
$(cat /etc/resolv.conf)"
log "$(echo -n \"/etc/resolv.conf file metadata: \") $(ls -lZ /etc/resolv.conf)"
log "/etc/resolv.conf.dnsmasq contents
$(cat /etc/resolv.conf.dnsmasq)"
log "$(echo -n "/etc/resolv.conf.dnsmasq file metadata: ") $(ls -lZ /etc/resolv.conf.dnsmasq)"
fi
exit 0
{{ end }}
`))

func config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) ([]byte, error) {
Expand Down Expand Up @@ -157,6 +197,17 @@ func startpre() ([]byte, error) {
return buf.Bytes(), nil
}

func nmDispatcherRestartDnsmasq() ([]byte, error) {
buf := &bytes.Buffer{}

err := t.ExecuteTemplate(buf, "99-dnsmasq-restart", nil)
if err != nil {
return nil, err
}

return buf.Bytes(), nil
}

func ignition2Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) (*ign2types.Config, error) {
service, err := service()
if err != nil {
Expand All @@ -173,6 +224,11 @@ func ignition2Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains [
return nil, err
}

nmDispatcherRestartDnsmasq, err := nmDispatcherRestartDnsmasq()
if err != nil {
return nil, err
}

return &ign2types.Config{
Ignition: ign2types.Ignition{
Version: ign2types.MaxVersion.String(),
Expand Down Expand Up @@ -211,6 +267,22 @@ func ignition2Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains [
Mode: ignutil.IntToPtr(0744),
},
},
{
Node: ign2types.Node{
Filesystem: "root",
Overwrite: ignutil.BoolToPtr(true),
Path: "/etc/NetworkManager/dispatcher.d/99-dnsmasq-restart",
User: &ign2types.NodeUser{
Name: *ignutil.StrToPtr("root"),
},
},
FileEmbedded1: ign2types.FileEmbedded1{
Contents: ign2types.FileContents{
Source: *ignutil.StrToPtr(dataurl.EncodeBytes(nmDispatcherRestartDnsmasq)),
},
Mode: ignutil.IntToPtr(0744),
},
},
},
},
Systemd: ign2types.Systemd{
Expand Down

0 comments on commit 589e4ed

Please sign in to comment.