Put device in maintenance mode if no edge node certs

After a device has been reinstalled but reusing the the device
certificate the attempts to publish EdgeNodeCerts will be rejected
by the controller for security reasons since they have changed.
(They are stored in /persist/certs hence are lost and recreated on
a device reinstall.) This makes that unusual condition visible to the
user by putting the device in maintenance mode.

Signed-off-by: eriknordmark <[email protected]>

Author: eriknordmark

pkg/pillar/cmd/nodeagent/nodeagent.go

@@ -502,6 +502,17 @@ func handleZedAgentStatusImpl(ctxArg interface{}, key string,
 		handleDeviceCmd(ctxPtr, status, types.DeviceOperationPoweroff)
 	}
 	updateZedagentCloudConnectStatus(ctxPtr, status)
+	if status.EdgeNodeCertsRefused {
+		addMaintenanceModeReason(ctxPtr,
+			types.MaintenanceModeReasonEdgeNodeCertsRefused,
+			"handleZedAgentStatusImpl")
+		publishNodeAgentStatus(ctxPtr)
+	} else {
+		removeMaintenanceModeReason(ctxPtr,
+			types.MaintenanceModeReasonEdgeNodeCertsRefused,
+			"handleZedAgentStatusImpl")
+		publishNodeAgentStatus(ctxPtr)
+	}
 	log.Functionf("handleZedAgentStatusImpl(%s) done", key)
 }
 

pkg/pillar/cmd/zedagent/handlecertconfig.go

@@ -454,6 +454,10 @@ func edgeNodeCertsTask(ctx *zedagentContext, triggerEdgeNodeCerts chan struct{})
 }
 
 // prepare the edge node certs list proto message
+// Note that this can be rejected by the controller if a cert has changed
+// (which can happen if /persist, hence /persist/certs, have been destroyed
+// and recreated). In that case we set edgeNodeCertsRefused which will make
+// nodeagent put this device in maintenance mode.
 func publishEdgeNodeCertsToController(ctx *zedagentContext) {
 	var attestReq = &attest.ZAttestReq{}
 

pkg/pillar/cmd/zedagent/handleconfig.go

@@ -1107,6 +1107,7 @@ func publishZedAgentStatus(getconfigCtx *getconfigContext) {
 		RequestedBootReason:    ctx.requestedBootReason,
 		MaintenanceMode:        ctx.maintenanceMode,
 		MaintenanceModeReasons: ctx.maintModeReasons,
+		EdgeNodeCertsRefused:   ctx.edgeNodeCertsRefused,
 		ForceFallbackCounter:   ctx.forceFallbackCounter,
 		CurrentProfile:         getconfigCtx.localCmdAgent.GetCurrentProfile(),
 		RadioSilence:           getconfigCtx.localCmdAgent.GetRadioSilenceConfig(),

pkg/pillar/cmd/zedagent/zedagent.go

@@ -230,6 +230,7 @@ type zedagentContext struct {
 	apiMaintenanceMode    bool
 	localMaintenanceMode  bool                             //maintenance mode triggered by local failure
 	localMaintModeReasons types.MaintenanceModeMultiReason //local failure reason for maintenance mode
+	edgeNodeCertsRefused  bool                             // causes maintenance mode
 	devState              types.DeviceState
 	attestState           types.AttestState
 	attestError           string
@@ -2781,7 +2782,11 @@ func getDeferredSentHandlerFunction(ctx *zedagentContext) controllerconn.SentHan
 			}
 			if el, ok := itemType.(attest.ZAttestReqType); ok && el == attest.ZAttestReqType_ATTEST_REQ_CERT {
 				log.Noticef("sendAttestReqProtobuf: Sent EdgeNodeCerts")
-				ctx.publishedEdgeNodeCerts = true
+				if !ctx.publishedEdgeNodeCerts {
+					ctx.publishedEdgeNodeCerts = true
+					ctx.edgeNodeCertsRefused = false
+					publishZedAgentStatus(ctx.getconfigCtx)
+				}
 			}
 		} else {
 			if el, ok := itemType.(attest.ZAttestReqType); ok {
@@ -2801,9 +2806,26 @@ func getDeferredSentHandlerFunction(ctx *zedagentContext) controllerconn.SentHan
 				if el == attest.ZAttestReqType_ATTEST_REQ_CERT {
 					log.Warnf("sendAttestReqProtobuf: Failed to send EdgeNodeCerts: %s",
 						result.String())
-					// XXX should we declare maintenance mode?
 					// We get SenderStatusNotFound when a cert can
 					// not be replaced in the controller for security reasons.
+					if !ctx.publishedEdgeNodeCerts &&
+						ctx.getconfigCtx.configGetStatus == types.ConfigGetSuccess {
+						// Force maintenance mode
+						// Note: If the network is flaky we might set it early
+						// when we have ConfigGetSuccess and then clear it once the certs published
+						log.Errorf("Failed to send EdgeNodeCerts: %s, maint %t, %v",
+							result.String(),
+							ctx.maintenanceMode,
+							ctx.maintModeReasons)
+						ctx.edgeNodeCertsRefused = true
+						publishZedAgentStatus(ctx.getconfigCtx)
+					} else if ctx.getconfigCtx.configGetStatus != types.ConfigGetSuccess {
+						// Lost controller connectivity
+						// Currently no need for maintenance mode - it will
+						// be determined when connectivity is back
+						ctx.edgeNodeCertsRefused = false
+						publishZedAgentStatus(ctx.getconfigCtx)
+					}
 				}
 				if !ctx.publishedEdgeNodeCerts {
 					// Attestation request does not clog the send queue (issued

pkg/pillar/types/zedagenttypes.go

@@ -402,12 +402,13 @@ type MaintenanceModeMultiReason []MaintenanceModeReason
 // MaintenanceModeReason codes for storing reason for getting into maintenance mode,
 // this should match the values in api/proto/info/info.proto.MaintenanceModeReason
 const (
-	MaintenanceModeReasonNone            = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_NONE)
-	MaintenanceModeReasonUserRequested   = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_USER_REQUESTED)
-	MaintenanceModeReasonVaultLockedUp   = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_VAULT_LOCKED_UP)
-	MaintenanceModeReasonNoDiskSpace     = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_LOW_DISK_SPACE)
-	MaintenanceModeReasonTpmEncFailure   = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_TPM_ENCRYPTION_FAILURE)
-	MaintenanceModeReasonTpmQuoteFailure = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_TPM_QUOTE_FAILURE)
+	MaintenanceModeReasonNone                 = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_NONE)
+	MaintenanceModeReasonUserRequested        = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_USER_REQUESTED)
+	MaintenanceModeReasonVaultLockedUp        = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_VAULT_LOCKED_UP)
+	MaintenanceModeReasonNoDiskSpace          = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_LOW_DISK_SPACE)
+	MaintenanceModeReasonTpmEncFailure        = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_TPM_ENCRYPTION_FAILURE)
+	MaintenanceModeReasonTpmQuoteFailure      = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_TPM_QUOTE_FAILURE)
+	MaintenanceModeReasonEdgeNodeCertsRefused = MaintenanceModeReason(info.MaintenanceModeReason_MAINTENANCE_MODE_REASON_EDGE_NODE_CERTS_REFUSED)
 )
 
 // String returns the verbose equivalent of MaintenanceModeMultiReason code
@@ -441,6 +442,8 @@ func (mmr MaintenanceModeReason) String() string {
 		return "MaintenanceModeReasonNoDiskSpace"
 	case MaintenanceModeReasonTpmEncFailure:
 		return "MaintenanceModeReasonTpmEncFailure"
+	case MaintenanceModeReasonEdgeNodeCertsRefused:
+		return "MaintenanceModeReasonEdgeNodeCertsRefused"
 	default:
 		return "Unknown MaintenanceModeReason"
 	}
@@ -559,6 +562,7 @@ type ZedAgentStatus struct {
 	RequestedBootReason    BootReason // Why we will reboot
 	MaintenanceMode        bool       // Don't run apps etc
 	MaintenanceModeReasons MaintenanceModeMultiReason
+	EdgeNodeCertsRefused   bool         // Causes maintenance mode
 	ForceFallbackCounter   int          // Try image fallback when counter changes
 	CurrentProfile         string       // Current profile
 	RadioSilence           RadioSilence // Currently requested state of radio devices