From e4864f9f1d0f12a4a7d28514da43bcc75603a5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Wei=C3=9Fe?= <66256922+daniel-weisse@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:09:13 +0100 Subject: [PATCH] Merge commit from fork MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * verify recovery key signatures before comitting recovery Signed-off-by: Daniel Weiße * cli: fix private key loading function Signed-off-by: Daniel Weiße * coordinator: increase security version number to 2 Signed-off-by: Daniel Weiße * store: tighten up interface Signed-off-by: Daniel Weiße * stdstore: revert nil check Signed-off-by: Daniel Weiße * coordinator: bind sealed key to state Signed-off-by: Daniel Weiße * coordinator: split up setting and sealing of store key Signed-off-by: Daniel Weiße * coordinator: add context to BeginReadTransaction Signed-off-by: Daniel Weiße * coordinator: remove redundant key sealing Signed-off-by: Daniel Weiße * clientapi: fix incorrect secret when starting read transaction Signed-off-by: Daniel Weiße * cli: add test for pkcs11 rsa key loading Signed-off-by: Daniel Weiße * cli: update recover command documentation Signed-off-by: Daniel Weiße * sealer: fix doc comment Signed-off-by: Daniel Weiße * server: remove redundant return statement Signed-off-by: Daniel Weiße * stdstore: replace key backup with atomic file replace Signed-off-by: Daniel Weiße * add tests for security fix Signed-off-by: Daniel Weiße * stdstore: fix seal mode not being correctly set after recovery Signed-off-by: Daniel Weiße * add integration test to verify key binding Signed-off-by: Daniel Weiße * docs: update recovery workflows Signed-off-by: Daniel Weiße * fix linting issues Signed-off-by: Daniel Weiße --------- Signed-off-by: Daniel Weiße --- api/api.go | 66 ++++--- api/v1.go | 25 --- api/v2.go | 7 +- cli/internal/cmd/recover.go | 93 +++++++++- cli/internal/pkcs11/pkcs11.go | 44 ++++- .../pkcs11/pkcs11_integration_test.go | 86 ++++++++++ coordinator/clientapi/clientapi.go | 108 +++++++----- coordinator/clientapi/clientapi_test.go | 148 ++++++++++++---- coordinator/core/core.go | 8 +- coordinator/core/core_test.go | 20 ++- coordinator/core/metrics_test.go | 6 +- coordinator/recovery/recovery.go | 3 +- coordinator/recovery/single.go | 2 +- coordinator/seal/mocksealer.go | 16 +- coordinator/seal/noenclavesealer.go | 46 +++-- coordinator/seal/seal.go | 59 ++++--- coordinator/server/handler/handler.go | 2 +- coordinator/server/v1/v1.go | 40 +---- coordinator/server/v2/types.go | 2 + coordinator/server/v2/v2.go | 2 +- coordinator/store/stdstore/stdstore.go | 162 +++++++++++++----- coordinator/store/stdstore/stdstore_test.go | 10 +- coordinator/store/store.go | 20 ++- docs/docs/reference/cli.md | 12 +- docs/docs/reference/coordinator.md | 49 +----- docs/docs/workflows/define-manifest.md | 2 +- docs/docs/workflows/recover-coordinator.md | 43 ++--- enclave/coordinator.conf | 2 +- test/framework/framework.go | 15 +- test/integration_test.go | 101 ++++++++++- util/util.go | 14 ++ 31 files changed, 848 insertions(+), 365 deletions(-) diff --git a/api/api.go b/api/api.go index 22cd4cd1..053c0bed 100644 --- a/api/api.go +++ b/api/api.go @@ -9,7 +9,9 @@ package api import ( "bytes" "context" + "crypto" "crypto/ecdsa" + "crypto/rand" "crypto/rsa" "crypto/sha256" "crypto/tls" @@ -159,38 +161,33 @@ func VerifyMarbleRunDeployment(ctx context.Context, endpoint string, opts Verify } // Recover performs recovery on a Coordinator instance by setting the decrypted recoverySecret. +// The signer is used to generate a signature over the recoverySecret. +// The Coordinator will verify this signature matches one of the recovery public keys set in the manifest. // On success, it returns the number of remaining recovery secrets to be set, // as well as the verified SGX quote. // // If this function is called from inside an EGo enclave, the "marblerun_ego_enclave" build tag must be set when building the binary. -func Recover(ctx context.Context, endpoint string, opts VerifyOptions, recoverySecret []byte) (remaining int, sgxQuote []byte, err error) { - opts.setDefaults() - - rootCert, _, sgxQuote, err := VerifyCoordinator(ctx, endpoint, opts) +func Recover(ctx context.Context, endpoint string, opts VerifyOptions, recoverySecret []byte, signer crypto.Signer) (remaining int, sgxQuote []byte, err error) { + signature, err := util.SignPKCS1v15(signer, recoverySecret) if err != nil { return -1, nil, err } + return recoverCoordinator(ctx, endpoint, opts, recoverySecret, signature) +} - client, err := rest.NewClient(endpoint, rootCert, nil) - if err != nil { - return -1, nil, fmt.Errorf("setting up client: %w", err) - } - - // Attempt recovery using the v2 API first - remaining, err = recoverV2(ctx, client, recoverySecret) - if rest.IsNotAllowedErr(err) { - remaining, err = recoverV1(ctx, client, recoverySecret) - } - if err != nil { - return -1, nil, fmt.Errorf("sending recovery request: %w", err) - } - - return remaining, sgxQuote, err +// RecoverWithSignature performs recovery on a Coordinator instance by setting the decrypted recoverySecret. +// This is the same as [Recover], but allows passing in the recoverySecretSignature directly, +// instead of generating it using a [crypto.Signer]. +// The recoveryKeySignature must be a PKCS#1 v1.5 signature over the SHA-256 hash of recoverySecret. +// +// If this function is called from inside an EGo enclave, the "marblerun_ego_enclave" build tag must be set when building the binary. +func RecoverWithSignature(ctx context.Context, endpoint string, opts VerifyOptions, recoverySecret, recoverySecretSignature []byte) (remaining int, sgxQuote []byte, err error) { + return recoverCoordinator(ctx, endpoint, opts, recoverySecret, recoverySecretSignature) } // DecryptRecoveryData decrypts recovery data returned by a Coordinator during [ManifestSet] using a parties private recovery key. -func DecryptRecoveryData(recoveryData []byte, recoveryPrivateKey *rsa.PrivateKey) ([]byte, error) { - return util.DecryptOAEP(recoveryPrivateKey, recoveryData) +func DecryptRecoveryData(recoveryData []byte, recoveryPrivateKey crypto.Decrypter) ([]byte, error) { + return recoveryPrivateKey.Decrypt(rand.Reader, recoveryData, &rsa.OAEPOptions{Hash: crypto.SHA256}) } // GetStatus retrieves the status of a MarbleRun Coordinator instance. @@ -577,3 +574,30 @@ func getMarbleCredentialsFromEnv() (tls.Certificate, *x509.Certificate, error) { return tlsCert, coordinatorRoot, nil } + +// recoverCoordinator performs recovery on a Coordinator instance by setting the decrypted recoverySecret. +// The signer is used to generate a signature over the recoverySecret. +// The Coordinator will verify this signature matches one of the recovery public keys set in the manifest. +// On success, it returns the number of remaining recovery secrets to be set, +// as well as the verified SGX quote. +func recoverCoordinator(ctx context.Context, endpoint string, opts VerifyOptions, recoverySecret, recoverySecretSignature []byte) (remaining int, sgxQuote []byte, err error) { + opts.setDefaults() + + rootCert, _, sgxQuote, err := VerifyCoordinator(ctx, endpoint, opts) + if err != nil { + return -1, nil, err + } + + client, err := rest.NewClient(endpoint, rootCert, nil) + if err != nil { + return -1, nil, fmt.Errorf("setting up client: %w", err) + } + + // The v1 API does not support recovery, therefore only attempt the v2 API + remaining, err = recoverV2(ctx, client, recoverySecret, recoverySecretSignature) + if err != nil { + return -1, nil, fmt.Errorf("sending recovery request: %w", err) + } + + return remaining, sgxQuote, err +} diff --git a/api/v1.go b/api/v1.go index 75c1411d..d5652ad7 100644 --- a/api/v1.go +++ b/api/v1.go @@ -20,31 +20,6 @@ import ( apiv1 "github.com/edgelesssys/marblerun/coordinator/server/v1" ) -// recoverV1 performs recovery of the Coordinator using the legacy v1 API. -func recoverV1(ctx context.Context, client *rest.Client, recoverySecret []byte) (remaining int, err error) { - resp, err := client.Post(ctx, rest.RecoverEndpoint, rest.ContentPlain, bytes.NewReader(recoverySecret)) - if err != nil { - return -1, err - } - - var response apiv1.RecoveryStatusResponse - if err := json.Unmarshal(resp, &response); err != nil { - return -1, fmt.Errorf("unmarshalling Coordinator response: %w", err) - } - - if response.StatusMessage == "Recovery successful." { - return 0, nil - } - - remainingStr, _, _ := strings.Cut(response.StatusMessage, ": ") - remaining, err = strconv.Atoi(remainingStr) - if err != nil { - return -1, fmt.Errorf("parsing remaining recovery secrets: %w", err) - } - - return remaining, nil -} - // getStatusV1 retrieves the status of the Coordinator using the legacy v1 API. func getStatusV1(ctx context.Context, client *rest.Client) (int, string, error) { resp, err := client.Get(ctx, rest.StatusEndpoint, http.NoBody) diff --git a/api/v2.go b/api/v2.go index 80c17e30..acf4be06 100644 --- a/api/v2.go +++ b/api/v2.go @@ -19,8 +19,11 @@ import ( ) // recoverV2 performs recovery of the Coordinator using the v2 API. -func recoverV2(ctx context.Context, client *rest.Client, recoverySecret []byte) (remaining int, err error) { - recoverySecretJSON, err := json.Marshal(apiv2.RecoveryRequest{RecoverySecret: recoverySecret}) +func recoverV2(ctx context.Context, client *rest.Client, recoverySecret, recoverySecretSignature []byte) (remaining int, err error) { + recoverySecretJSON, err := json.Marshal(apiv2.RecoveryRequest{ + RecoverySecret: recoverySecret, + RecoverySecretSignature: recoverySecretSignature, + }) if err != nil { return -1, fmt.Errorf("marshalling request: %w", err) } diff --git a/cli/internal/cmd/recover.go b/cli/internal/cmd/recover.go index ec58cc36..b9443ff5 100644 --- a/cli/internal/cmd/recover.go +++ b/cli/internal/cmd/recover.go @@ -7,8 +7,18 @@ SPDX-License-Identifier: BUSL-1.1 package cmd import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "os" + "github.com/edgelesssys/marblerun/api" "github.com/edgelesssys/marblerun/cli/internal/file" + "github.com/edgelesssys/marblerun/cli/internal/pkcs11" "github.com/spf13/afero" "github.com/spf13/cobra" ) @@ -16,14 +26,24 @@ import ( // NewRecoverCmd returns the recover command. func NewRecoverCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "recover ", - Short: "Recover the MarbleRun Coordinator from a sealed state", - Long: "Recover the MarbleRun Coordinator from a sealed state", + Use: "recover ", + Short: "Recover the MarbleRun Coordinator from a sealed state", + Long: "Recover the MarbleRun Coordinator from a sealed state.\n" + + " may be either a decrypted recovery secret, or an encrypted recovery secret,\n" + + "in which case the private key is used to decrypt the secret.", Example: "marblerun recover recovery_key_decrypted $MARBLERUN", Args: cobra.ExactArgs(2), RunE: runRecover, } + cmd.Flags().StringP("key", "k", "", "Path to a the recovery private key to decrypt and/or sign the recovery key") + cmd.Flags().String("pkcs11-config", "", "Path to a PKCS#11 configuration file to load the recovery private key with") + cmd.Flags().String("pkcs11-key-id", "", "ID of the private key in the PKCS#11 token") + cmd.Flags().String("pkcs11-key-label", "", "Label of the private key in the PKCS#11 token") + must(cobra.MarkFlagFilename(cmd.Flags(), "pkcs11-config", "json")) + cmd.MarkFlagsOneRequired("pkcs11-key-id", "pkcs11-key-label", "key") + cmd.MarkFlagsMutuallyExclusive("pkcs11-config", "key") + return cmd } @@ -42,7 +62,22 @@ func runRecover(cmd *cobra.Command, args []string) error { return err } - remaining, sgxQuote, err := api.Recover(cmd.Context(), hostname, verifyOpts, recoveryKey) + keyHandle, cancel, err := getRecoveryKeySigner(cmd) + if err != nil { + return err + } + defer func() { + if err := cancel(); err != nil { + cmd.PrintErrf("Failed to close PKCS #11 session: %s\n", err) + } + }() + + recoveryKey, err = maybeDecryptRecoveryKey(recoveryKey, keyHandle) + if err != nil { + return err + } + + remaining, sgxQuote, err := api.Recover(cmd.Context(), hostname, verifyOpts, recoveryKey, keyHandle) if err != nil { return err } @@ -58,3 +93,53 @@ func runRecover(cmd *cobra.Command, args []string) error { } return nil } + +func getRecoveryKeySigner(cmd *cobra.Command) (pkcs11.SignerDecrypter, func() error, error) { + privKeyFile, err := cmd.Flags().GetString("key") + if err != nil { + return nil, nil, err + } + + if privKeyFile == "" { + pkcs11ConfigFile, err := cmd.Flags().GetString("pkcs11-config") + if err != nil { + return nil, nil, err + } + pkcs11KeyID, err := cmd.Flags().GetString("pkcs11-key-id") + if err != nil { + return nil, nil, err + } + pkcs11KeyLabel, err := cmd.Flags().GetString("pkcs11-key-label") + if err != nil { + return nil, nil, err + } + return pkcs11.LoadRSAPrivateKey(pkcs11ConfigFile, pkcs11KeyID, pkcs11KeyLabel) + } + + privKeyPEM, err := os.ReadFile(privKeyFile) + if err != nil { + return nil, nil, err + } + privateKeyBlock, _ := pem.Decode(privKeyPEM) + if privateKeyBlock == nil { + return nil, nil, fmt.Errorf("%q did not contain a valid PEM block", privKeyFile) + } + privK, err := x509.ParsePKCS8PrivateKey(privateKeyBlock.Bytes) + if err != nil { + return nil, nil, err + } + signer, ok := privK.(pkcs11.SignerDecrypter) + if !ok { + return nil, nil, errors.New("loaded private key does not fulfill required interface") + } + return signer, func() error { return nil }, nil +} + +// maybeDecryptRecoveryKey tries to decrypt the given recoveryKey using OAEP. +// If the recoveryKey is already 16 bytes long, it is returned as is. +func maybeDecryptRecoveryKey(recoveryKey []byte, decrypter crypto.Decrypter) ([]byte, error) { + if len(recoveryKey) != 16 { + return decrypter.Decrypt(rand.Reader, recoveryKey, &rsa.OAEPOptions{Hash: crypto.SHA256}) + } + return recoveryKey, nil +} diff --git a/cli/internal/pkcs11/pkcs11.go b/cli/internal/pkcs11/pkcs11.go index b997003c..0b07916c 100644 --- a/cli/internal/pkcs11/pkcs11.go +++ b/cli/internal/pkcs11/pkcs11.go @@ -16,8 +16,15 @@ import ( "github.com/ThalesGroup/crypto11" ) -// LoadX509KeyPair loads a [tls.Certificate] using the provided PKCS#11 configuration file. -// The returned cancel function must be called to release the PKCS#11 resources only after the certificate is no longer needed. +// SignerDecrypter is a combined interface for [crypto.Signer] and [crypto.Decrypter]. +// An RSA private key in a PKCS #11 token implements this interface. +type SignerDecrypter interface { + crypto.Signer + crypto.Decrypter +} + +// LoadX509KeyPair loads a [tls.Certificate] using the provided PKCS #11 configuration file. +// The returned cancel function must be called to release the PKCS #11 resources only after the certificate is no longer needed. func LoadX509KeyPair(pkcs11ConfigPath string, keyID, keyLabel, certID, certLabel string) (crt tls.Certificate, cancel func() error, err error) { pkcs11, err := crypto11.ConfigureFromFile(pkcs11ConfigPath) if err != nil { @@ -59,6 +66,39 @@ func LoadX509KeyPair(pkcs11ConfigPath string, keyID, keyLabel, certID, certLabel }, pkcs11.Close, nil } +// LoadRSAPrivateKey loads a [SignerDecrypter] using the provided PKCS #11 configuration file. +// The returned cancel function must be called to release the PKCS #11 resources only after the key is no longer needed. +func LoadRSAPrivateKey(pkcs11ConfigPath string, keyID, keyLabel string) (signer SignerDecrypter, cancel func() error, err error) { + pkcs11, err := crypto11.ConfigureFromFile(pkcs11ConfigPath) + if err != nil { + return nil, nil, err + } + defer func() { + if err != nil { + err = errors.Join(err, pkcs11.Close()) + } + }() + + var keyIDBytes, keyLabelBytes []byte + if keyID != "" { + keyIDBytes = []byte(keyID) + } + if keyLabel != "" { + keyLabelBytes = []byte(keyLabel) + } + + privK, err := loadPrivateKey(pkcs11, keyIDBytes, keyLabelBytes) + if err != nil { + return nil, nil, err + } + + signer, ok := privK.(crypto11.SignerDecrypter) + if !ok { + return nil, nil, errors.New("loaded private key does not support decryption") + } + return signer, pkcs11.Close, err +} + func loadPrivateKey(pkcs11 *crypto11.Context, id, label []byte) (crypto.Signer, error) { priv, err := pkcs11.FindKeyPair(id, label) if err != nil { diff --git a/cli/internal/pkcs11/pkcs11_integration_test.go b/cli/internal/pkcs11/pkcs11_integration_test.go index 2d8745c0..741940d6 100644 --- a/cli/internal/pkcs11/pkcs11_integration_test.go +++ b/cli/internal/pkcs11/pkcs11_integration_test.go @@ -12,6 +12,7 @@ import ( "crypto" "crypto/elliptic" "crypto/rand" + "crypto/sha256" "crypto/x509" "flag" "testing" @@ -156,6 +157,91 @@ func TestLoadX509KeyPair(t *testing.T) { } } +func TestLoadRSAPrivateKey(t *testing.T) { + // Ensure we can load the PKCS#11 configuration file + pkcs11, err := crypto11.ConfigureFromFile(*configPath) + require.NoError(t, err) + require.NoError(t, pkcs11.Close()) + + testCases := map[string]struct { + init func(t *testing.T) (keyID, keyLabel string) + wantErr bool + }{ + "identified by ID and label": { + init: func(t *testing.T) (keyID, keyLabel string) { + t.Helper() + require := require.New(t) + prefix := uuid.New().String() + keyID, keyLabel = prefix+"keyID", prefix+"keyLabel" + pkcs11, err := crypto11.ConfigureFromFile(*configPath) + require.NoError(err) + defer pkcs11.Close() + _, err = pkcs11.GenerateRSAKeyPairWithLabel([]byte(keyID), []byte(keyLabel), 2048) + require.NoError(err) + + return keyID, keyLabel + }, + }, + "identified by ID only": { + init: func(t *testing.T) (keyID, keyLabel string) { + t.Helper() + require := require.New(t) + prefix := uuid.New().String() + keyID, keyLabel = prefix+"keyID", prefix+"keyLabel" + pkcs11, err := crypto11.ConfigureFromFile(*configPath) + require.NoError(err) + defer pkcs11.Close() + _, err = pkcs11.GenerateRSAKeyPair([]byte(keyID), 2048) + require.NoError(err) + return keyID, "" + }, + }, + "identified by label only": { + init: func(t *testing.T) (keyID, keyLabel string) { + t.Helper() + require := require.New(t) + prefix := uuid.New().String() + keyID, keyLabel = prefix+"keyID", prefix+"keyLabel" + pkcs11, err := crypto11.ConfigureFromFile(*configPath) + require.NoError(err) + defer pkcs11.Close() + _, err = pkcs11.GenerateRSAKeyPairWithLabel([]byte(keyID), []byte(keyLabel), 2048) + require.NoError(err) + + return "", keyLabel + }, + }, + "key not found": { + init: func(_ *testing.T) (keyID, keyLabel string) { + return "not-found", "not-found" + }, + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + keyID, keyLabel := tc.init(t) + signer, cancel, err := LoadRSAPrivateKey(*configPath, keyID, keyLabel) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + defer func() { + _ = cancel() + }() + + data := sha256.Sum256([]byte("data")) + _, err = signer.Sign(rand.Reader, data[:], crypto.SHA256) + assert.NoError(err) + }) + } +} + func createSelfSignedCertificate(priv crypto.Signer) (*x509.Certificate, error) { serialNumber, err := util.GenerateCertificateSerialNumber() if err != nil { diff --git a/coordinator/clientapi/clientapi.go b/coordinator/clientapi/clientapi.go index 6535a230..2ce0388e 100644 --- a/coordinator/clientapi/clientapi.go +++ b/coordinator/clientapi/clientapi.go @@ -59,9 +59,11 @@ type core interface { type transactionHandle interface { BeginTransaction(context.Context) (store.Transaction, error) - SetEncryptionKey([]byte, seal.Mode) error + BeginReadTransaction(context.Context, []byte) (store.ReadTransaction, error) + SetEncryptionKey([]byte, seal.Mode) + SealEncryptionKey(additionalData []byte) error SetRecoveryData([]byte) - LoadState() ([]byte, error) + LoadState() (recoveryData, sealedState []byte, err error) } type updateLog interface { @@ -87,9 +89,10 @@ func (e QuoteVerifyError) Unwrap() error { // ClientAPI implements the client API. type ClientAPI struct { - core core - recovery recovery.Recovery - txHandle transactionHandle + core core + recovery recovery.Recovery + txHandle transactionHandle + recoverySignatureCache map[string][]byte updateLog updateLog log *zap.Logger @@ -275,7 +278,7 @@ func (a *ClientAPI) GetUpdateLog(ctx context.Context) ([]string, error) { } // Recover sets an encryption key (ideally decrypted from the recovery data) and tries to unseal and load a saved state of the Coordinator. -func (a *ClientAPI) Recover(ctx context.Context, encryptionKey []byte) (keysLeft int, err error) { +func (a *ClientAPI) Recover(ctx context.Context, encryptionKey, encryptionKeySignature []byte) (keysLeft int, retErr error) { a.log.Info("Recover called") defer a.core.Unlock() if err := a.core.RequireState(ctx, state.Recovery); err != nil { @@ -283,15 +286,19 @@ func (a *ClientAPI) Recover(ctx context.Context, encryptionKey []byte) (keysLeft return -1, err } defer func() { - if err != nil { - a.log.Error("Recover failed", zap.Error(err)) + if retErr != nil { + a.log.Error("Recover failed", zap.Error(retErr)) } }() + if a.recoverySignatureCache == nil { + a.recoverySignatureCache = make(map[string][]byte) + } remaining, secret, err := a.recovery.RecoverKey(encryptionKey) if err != nil { return -1, fmt.Errorf("setting recovery key: %w", err) } + a.recoverySignatureCache[string(encryptionKey)] = encryptionKeySignature // another key is needed to finish the recovery if remaining != 0 { @@ -299,46 +306,72 @@ func (a *ClientAPI) Recover(ctx context.Context, encryptionKey []byte) (keysLeft return remaining, nil } - // all keys are set, we can now load the state - if err := a.txHandle.SetEncryptionKey(secret, seal.ModeDisabled); err != nil { - return -1, fmt.Errorf("setting recovery key: %w", err) - } - - // load state - recoveryData, err := a.txHandle.LoadState() - if err != nil { - return -1, fmt.Errorf("loading state: %w", err) - } - - a.txHandle.SetRecoveryData(recoveryData) - if err := a.recovery.SetRecoveryData(recoveryData); err != nil { - a.log.Error("Could not retrieve recovery data from state. Recovery will be unavailable", zap.Error(err)) - } + // reset signature cache on return after this point + // the recovery module was already cleaned up if no more keys are missing + defer func() { + a.recoverySignatureCache = nil + }() - txdata, rollback, _, err := wrapper.WrapTransaction(ctx, a.txHandle) + // verify the recovery keys before properly loading the state and releasing recovery mode + sealedStore, err := a.txHandle.BeginReadTransaction(ctx, secret) if err != nil { - return -1, err + return -1, fmt.Errorf("loading sealed state: %w", err) } - defer rollback() - - // set seal mode defined in manifest - mnf, err := txdata.GetManifest() + readTx := wrapper.New(sealedStore) + mnf, err := readTx.GetManifest() if err != nil { return -1, fmt.Errorf("loading manifest from store: %w", err) } - if err := a.txHandle.SetEncryptionKey(secret, seal.ModeFromString(mnf.Config.SealMode)); err != nil { - return -1, fmt.Errorf("setting recovery key and seal mode: %w", err) + if len(mnf.RecoveryKeys) != len(a.recoverySignatureCache) { + return -1, fmt.Errorf("recovery keys in manifest do not match the keys used for recovery: expected %d, got %d", len(mnf.RecoveryKeys), len(a.recoverySignatureCache)) + } + for keyName, keyPEM := range mnf.RecoveryKeys { + pubKey, err := recovery.ParseRSAPublicKeyFromPEM(keyPEM) + if err != nil { + return -1, fmt.Errorf("parsing recovery public key %q: %w", keyName, err) + } + + found := false + for key, signature := range a.recoverySignatureCache { + if err := util.VerifyPKCS1v15(pubKey, []byte(key), signature); err == nil { + found = true + delete(a.recoverySignatureCache, key) + break + } + } + if !found { + return -1, fmt.Errorf("no matching recovery key found for recovery public key %q", keyName) + } } - rootCert, err := txdata.GetCertificate(constants.SKCoordinatorRootCert) + // cache SGX quote over the root certificate + rootCert, err := readTx.GetCertificate(constants.SKCoordinatorRootCert) if err != nil { return -1, fmt.Errorf("loading root certificate from store: %w", err) } - if err := a.core.GenerateQuote(rootCert.Raw); err != nil { return -1, fmt.Errorf("generating quote failed: %w", err) } + // load state and set seal mode defined in manifest + a.txHandle.SetEncryptionKey(secret, seal.ModeFromString(mnf.Config.SealMode)) + defer func() { + if retErr != nil { + a.txHandle.SetEncryptionKey(nil, seal.ModeDisabled) // reset encryption key in case of failure + } + }() + recoveryData, sealedState, err := a.txHandle.LoadState() + if err != nil { + return -1, fmt.Errorf("loading state: %w", err) + } + a.txHandle.SetRecoveryData(recoveryData) + if err := a.recovery.SetRecoveryData(recoveryData); err != nil { + a.log.Error("Could not retrieve recovery data from state. Recovery will be unavailable", zap.Error(err)) + } + if err := a.txHandle.SealEncryptionKey(sealedState); err != nil { + a.log.Error("Could not seal encryption key after recovery. Restart will require another recovery", zap.Error(err)) + } + a.log.Info("Recover successful") return 0, nil } @@ -403,18 +436,15 @@ func (a *ClientAPI) SetManifest(ctx context.Context, rawManifest []byte) (recove // Set encryption key & generate recovery data encryptionKey, err := a.recovery.GenerateEncryptionKey(mnf.RecoveryKeys) if err != nil { - a.log.Error("could not set up encryption key for sealing the state", zap.Error(err)) + a.log.Error("Could not set up encryption key for sealing the state", zap.Error(err)) return nil, fmt.Errorf("generating recovery encryption key: %w", err) } recoverySecretMap, recoveryData, err := a.recovery.GenerateRecoveryData(mnf.RecoveryKeys) if err != nil { - a.log.Error("could not generate recovery data", zap.Error(err)) + a.log.Error("Could not generate recovery data", zap.Error(err)) return nil, fmt.Errorf("generating recovery data: %w", err) } - if err := a.txHandle.SetEncryptionKey(encryptionKey, seal.ModeFromString(mnf.Config.SealMode)); err != nil { - a.log.Error("could not set encryption key to seal state", zap.Error(err)) - return nil, fmt.Errorf("setting encryption key: %w", err) - } + a.txHandle.SetEncryptionKey(encryptionKey, seal.ModeFromString(mnf.Config.SealMode)) // Parse X.509 user certificates and permissions from manifest users, err := mnf.GenerateUsers() diff --git a/coordinator/clientapi/clientapi_test.go b/coordinator/clientapi/clientapi_test.go index 0a0a05a0..c44c5e48 100644 --- a/coordinator/clientapi/clientapi_test.go +++ b/coordinator/clientapi/clientapi_test.go @@ -20,7 +20,9 @@ import ( "encoding/json" "encoding/pem" "errors" + "fmt" "math/big" + "strings" "testing" "time" @@ -40,6 +42,7 @@ import ( "github.com/edgelesssys/marblerun/coordinator/updatelog" "github.com/edgelesssys/marblerun/coordinator/user" "github.com/edgelesssys/marblerun/test" + "github.com/edgelesssys/marblerun/util" "github.com/google/uuid" "github.com/spf13/afero" "github.com/stretchr/testify/assert" @@ -53,7 +56,6 @@ func TestMain(m *testing.M) { } func TestGetCertQuote(t *testing.T) { - someErr := errors.New("failed") // these are not actually root and intermediate certs // but we don't care for this test rootCert, intermediateCert := test.MustSetupTestCerts(test.RecoveryPrivateKey) @@ -107,7 +109,7 @@ func TestGetCertQuote(t *testing.T) { "error getting state": { store: prepareDefaultStore(), core: &fakeCore{ - requireStateErr: someErr, + requireStateErr: assert.AnError, quote: []byte("quote"), }, wantErr: true, @@ -407,21 +409,28 @@ func TestGetUpdateLog(t *testing.T) { } func TestRecover(t *testing.T) { - someErr := errors.New("failed") _, rootCert := test.MustSetupTestCerts(test.RecoveryPrivateKey) defaultStore := func() store.Store { s := stdstore.New(&seal.MockSealer{}, afero.NewMemMapFs(), "", zaptest.NewLogger(t)) + s.SetEncryptionKey([]byte("key"), seal.ModeProductKey) // set encryption key to set seal mode wr := wrapper.New(s) require.NoError(t, wr.PutCertificate(constants.SKCoordinatorRootCert, rootCert)) - require.NoError(t, wr.PutRawManifest([]byte(`{}`))) + require.NoError(t, wr.PutRawManifest([]byte(test.ManifestJSONWithRecoveryKey))) return s } + signData := func(d []byte, k *rsa.PrivateKey) []byte { + sig, err := util.SignPKCS1v15(k, d) + require.NoError(t, err) + return sig + } testCases := map[string]struct { - store *fakeStore - recovery *stubRecovery - core *fakeCore - wantErr bool + store *fakeStore + recovery *stubRecovery + core *fakeCore + recoveryKey []byte + recoveryKeySig []byte + wantErr bool }{ "success": { store: &fakeStore{ @@ -431,6 +440,8 @@ func TestRecover(t *testing.T) { core: &fakeCore{ state: state.Recovery, }, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), }, "more than one key required": { store: &fakeStore{ @@ -442,17 +453,21 @@ func TestRecover(t *testing.T) { core: &fakeCore{ state: state.Recovery, }, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), }, "SetRecoveryData fails does not result in error": { store: &fakeStore{ store: defaultStore(), }, recovery: &stubRecovery{ - setRecoveryDataErr: someErr, + setRecoveryDataErr: assert.AnError, }, core: &fakeCore{ state: state.Recovery, }, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), }, "Coordinator not in recovery state": { store: &fakeStore{ @@ -462,51 +477,66 @@ func TestRecover(t *testing.T) { core: &fakeCore{ state: state.AcceptingManifest, }, - wantErr: true, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), + wantErr: true, }, "RecoverKey fails": { store: &fakeStore{ store: defaultStore(), }, recovery: &stubRecovery{ - recoverKeyErr: someErr, + recoverKeyErr: assert.AnError, }, core: &fakeCore{ state: state.Recovery, }, - wantErr: true, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), + wantErr: true, }, "LoadState fails": { store: &fakeStore{ store: defaultStore(), - loadStateErr: someErr, + loadStateErr: assert.AnError, }, recovery: &stubRecovery{}, core: &fakeCore{ state: state.Recovery, }, - wantErr: true, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), + wantErr: true, }, - "SetEncryptionKey fails": { + "SealEncryptionKey fails does return an error": { store: &fakeStore{ - store: defaultStore(), - setEncryptionKeyErr: someErr, + store: defaultStore(), + sealEncryptionKeyErr: assert.AnError, }, recovery: &stubRecovery{}, core: &fakeCore{ state: state.Recovery, }, - wantErr: true, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), }, "GetCertificate fails": { store: &fakeStore{ - store: stdstore.New(&seal.MockSealer{}, afero.NewMemMapFs(), "", zaptest.NewLogger(t)), + store: func() store.Store { + s := stdstore.New(&seal.MockSealer{}, afero.NewMemMapFs(), "", zaptest.NewLogger(t)) + s.SetEncryptionKey([]byte("key"), seal.ModeProductKey) // set encryption key to set seal mode + wr := wrapper.New(s) + require.NoError(t, wr.PutRawManifest([]byte(test.ManifestJSONWithRecoveryKey))) + return s + }(), }, recovery: &stubRecovery{}, core: &fakeCore{ state: state.Recovery, }, - wantErr: true, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), + wantErr: true, }, "GenerateQuote fails": { store: &fakeStore{ @@ -515,9 +545,44 @@ func TestRecover(t *testing.T) { recovery: &stubRecovery{}, core: &fakeCore{ state: state.Recovery, - generateQuoteErr: someErr, + generateQuoteErr: assert.AnError, }, - wantErr: true, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), + wantErr: true, + }, + "invalid recovery key signature": { + store: &fakeStore{ + store: defaultStore(), + }, + recovery: &stubRecovery{}, + core: &fakeCore{ + state: state.Recovery, + }, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0xFF}, 16), test.RecoveryPrivateKey), + wantErr: true, + }, + "manifest defines multiple recovery keys": { + store: &fakeStore{ + store: func() store.Store { + s := stdstore.New(&seal.MockSealer{}, afero.NewMemMapFs(), "", zaptest.NewLogger(t)) + s.SetEncryptionKey([]byte("key"), seal.ModeProductKey) // set encryption key to set seal mode + wr := wrapper.New(s) + require.NoError(t, wr.PutCertificate(constants.SKCoordinatorRootCert, rootCert)) + recoveryKey2Str := fmt.Sprintf("\"testRecKey2\": \"%s\",\"testRecKey1\":", strings.ReplaceAll(string(test.RecoveryPublicKey), "\n", "\\n")) + mnf := strings.Replace(test.ManifestJSONWithRecoveryKey, `"testRecKey1":`, recoveryKey2Str, 1) + require.NoError(t, wr.PutRawManifest([]byte(mnf))) + return s + }(), + }, + recovery: &stubRecovery{}, + core: &fakeCore{ + state: state.Recovery, + }, + recoveryKey: bytes.Repeat([]byte{0x01}, 16), + recoveryKeySig: signData(bytes.Repeat([]byte{0x01}, 16), test.RecoveryPrivateKey), + wantErr: true, }, } @@ -535,7 +600,7 @@ func TestRecover(t *testing.T) { log: log, } - keysLeft, err := api.Recover(context.Background(), []byte("recoveryKey")) + keysLeft, err := api.Recover(context.Background(), tc.recoveryKey, tc.recoveryKeySig) if tc.wantErr { assert.Error(err) return @@ -1231,10 +1296,10 @@ type fakeStore struct { setRecoveryDataCalled bool recoveryData []byte encryptionKey []byte - setEncryptionKeyErr error loadStateRes []byte loadStateErr error loadCalled bool + sealEncryptionKeyErr error } func (s *fakeStore) BeginTransaction(ctx context.Context) (store.Transaction, error) { @@ -1244,12 +1309,12 @@ func (s *fakeStore) BeginTransaction(ctx context.Context) (store.Transaction, er return s.store.BeginTransaction(ctx) } -func (s *fakeStore) SetEncryptionKey(key []byte, _ seal.Mode) error { - if s.setEncryptionKeyErr != nil { - return s.setEncryptionKeyErr - } +func (s *fakeStore) SetEncryptionKey(key []byte, _ seal.Mode) { s.encryptionKey = key - return nil +} + +func (s *fakeStore) SealEncryptionKey(_ []byte) error { + return s.sealEncryptionKeyErr } func (s *fakeStore) SetRecoveryData(recoveryData []byte) { @@ -1257,9 +1322,13 @@ func (s *fakeStore) SetRecoveryData(recoveryData []byte) { s.recoveryData = recoveryData } -func (s *fakeStore) LoadState() ([]byte, error) { +func (s *fakeStore) LoadState() ([]byte, []byte, error) { s.loadCalled = true - return s.loadStateRes, s.loadStateErr + return s.loadStateRes, nil, s.loadStateErr +} + +func (s *fakeStore) BeginReadTransaction(ctx context.Context, recoveryKey []byte) (store.ReadTransaction, error) { + return s.store.BeginReadTransaction(ctx, recoveryKey) } type stubRecovery struct { @@ -1309,11 +1378,11 @@ type fakeStoreTransaction struct { beginTransactionCalled bool beginTransactionErr error setEncryptionKeyCalled bool - setEncryptionKeyErr error sealMode seal.Mode loadStateCalled bool loadStateErr error setRecoveryDataCalled bool + sealEncryptionKeyErr error state map[string][]byte getErr error @@ -1330,19 +1399,26 @@ func (s *fakeStoreTransaction) BeginTransaction(_ context.Context) (store.Transa return s, s.beginTransactionErr } -func (s *fakeStoreTransaction) SetEncryptionKey(_ []byte, mode seal.Mode) error { +func (s *fakeStoreTransaction) SetEncryptionKey(_ []byte, mode seal.Mode) { s.setEncryptionKeyCalled = true s.sealMode = mode - return s.setEncryptionKeyErr +} + +func (s *fakeStoreTransaction) SealEncryptionKey(_ []byte) error { + return s.sealEncryptionKeyErr } func (s *fakeStoreTransaction) SetRecoveryData(_ []byte) { s.setRecoveryDataCalled = true } -func (s *fakeStoreTransaction) LoadState() ([]byte, error) { +func (s *fakeStoreTransaction) LoadState() ([]byte, []byte, error) { s.loadStateCalled = true - return nil, s.loadStateErr + return nil, nil, s.loadStateErr +} + +func (s *fakeStoreTransaction) BeginReadTransaction(_ context.Context, _ []byte) (store.ReadTransaction, error) { + return s, nil } func (s *fakeStoreTransaction) Get(key string) ([]byte, error) { diff --git a/coordinator/core/core.go b/coordinator/core/core.go index fc369478..7e9e0e8b 100644 --- a/coordinator/core/core.go +++ b/coordinator/core/core.go @@ -123,7 +123,7 @@ func NewCore( c.metrics = newCoreMetrics(promFactory, c, "coordinator") zapLogger.Info("Loading state") - recoveryData, loadErr := txHandle.LoadState() + recoveryData, _, loadErr := txHandle.LoadState() c.log.Debug("Loaded state", zap.Error(loadErr), zap.ByteString("recoveryData", recoveryData)) if err := c.recovery.SetRecoveryData(recoveryData); err != nil { c.log.Error("Could not retrieve recovery data from state. Recovery will be unavailable", zap.Error(err)) @@ -610,7 +610,9 @@ func (e QuoteError) Error() string { type transactionHandle interface { BeginTransaction(context.Context) (store.Transaction, error) - SetEncryptionKey([]byte, seal.Mode) error + BeginReadTransaction(context.Context, []byte) (store.ReadTransaction, error) + SetEncryptionKey([]byte, seal.Mode) + SealEncryptionKey([]byte) error SetRecoveryData([]byte) - LoadState() ([]byte, error) + LoadState() ([]byte, []byte, error) } diff --git a/coordinator/core/core_test.go b/coordinator/core/core_test.go index 53fbe2d2..43b13e3a 100644 --- a/coordinator/core/core_test.go +++ b/coordinator/core/core_test.go @@ -10,6 +10,7 @@ import ( "context" "crypto/ed25519" "crypto/rand" + "crypto/rsa" "crypto/tls" "crypto/x509" "crypto/x509/pkix" @@ -28,6 +29,7 @@ import ( "github.com/edgelesssys/marblerun/coordinator/store/stdstore" "github.com/edgelesssys/marblerun/coordinator/store/wrapper/testutil" "github.com/edgelesssys/marblerun/test" + "github.com/edgelesssys/marblerun/util" "github.com/google/uuid" "github.com/spf13/afero" "github.com/stretchr/testify/assert" @@ -131,16 +133,16 @@ func TestRecover(t *testing.T) { require.NoError(err) // new core does not allow recover - key := make([]byte, 16) - _, err = clientAPI.Recover(ctx, key) + key, sig := recoveryKeyWithSignature(t, test.RecoveryPrivateKey) + _, err = clientAPI.Recover(ctx, key, sig) assert.Error(err) // Set manifest. This will seal the state. - _, err = clientAPI.SetManifest(ctx, []byte(test.ManifestJSON)) + _, err = clientAPI.SetManifest(ctx, []byte(test.ManifestJSONWithRecoveryKey)) require.NoError(err) // core does not allow recover after manifest has been set - _, err = clientAPI.Recover(ctx, key) + _, err = clientAPI.Recover(ctx, key, sig) assert.Error(err) // Initialize new core and let unseal fail @@ -154,7 +156,7 @@ func TestRecover(t *testing.T) { require.Equal(state.Recovery, c2State) // recover - _, err = clientAPI.Recover(ctx, key) + _, err = clientAPI.Recover(ctx, key, sig) assert.NoError(err) c2State = testutil.GetState(t, c2.txHandle) assert.Equal(state.AcceptingMarbles, c2State) @@ -380,3 +382,11 @@ type stubIssuer struct { func (s *stubIssuer) Issue(message []byte) ([]byte, error) { return message, s.err } + +func recoveryKeyWithSignature(t *testing.T, priv *rsa.PrivateKey) ([]byte, []byte) { + t.Helper() + key := make([]byte, 16) + sig, err := util.SignPKCS1v15(priv, key) + require.NoError(t, err) + return key, sig +} diff --git a/coordinator/core/metrics_test.go b/coordinator/core/metrics_test.go index a52826ac..1a7c34f2 100644 --- a/coordinator/core/metrics_test.go +++ b/coordinator/core/metrics_test.go @@ -53,7 +53,7 @@ func TestStoreWrapperMetrics(t *testing.T) { clientAPI, err := clientapi.New(c.txHandle, c.recovery, c, zapLogger) require.NoError(err) - _, err = clientAPI.SetManifest(ctx, []byte(test.ManifestJSON)) + _, err = clientAPI.SetManifest(ctx, []byte(test.ManifestJSONWithRecoveryKey)) require.NoError(err) assert.Equal(1, promtest.CollectAndCount(c.metrics.coordinatorState)) assert.Equal(float64(state.AcceptingMarbles), promtest.ToFloat64(c.metrics.coordinatorState)) @@ -73,8 +73,8 @@ func TestStoreWrapperMetrics(t *testing.T) { clientAPI, err = clientapi.New(c.txHandle, c.recovery, c, zapLogger) require.NoError(err) - key := make([]byte, 16) - _, err = clientAPI.Recover(ctx, key) + key, sig := recoveryKeyWithSignature(t, test.RecoveryPrivateKey) + _, err = clientAPI.Recover(ctx, key, sig) require.NoError(err) state := testutil.GetState(t, c.txHandle) assert.Equal(1, promtest.CollectAndCount(c.metrics.coordinatorState)) diff --git a/coordinator/recovery/recovery.go b/coordinator/recovery/recovery.go index c696efce..b6a775c8 100644 --- a/coordinator/recovery/recovery.go +++ b/coordinator/recovery/recovery.go @@ -22,7 +22,8 @@ type Recovery interface { SetRecoveryData(data []byte) error } -func parseRSAPublicKeyFromPEM(pemContent string) (*rsa.PublicKey, error) { +// ParseRSAPublicKeyFromPEM parses a PEM encoded RSA public key to [*rsa.PublicKey]. +func ParseRSAPublicKeyFromPEM(pemContent string) (*rsa.PublicKey, error) { // Retrieve RSA public key for potential key recovery block, _ := pem.Decode([]byte(pemContent)) diff --git a/coordinator/recovery/single.go b/coordinator/recovery/single.go index 9ba370fc..8e4f0876 100644 --- a/coordinator/recovery/single.go +++ b/coordinator/recovery/single.go @@ -43,7 +43,7 @@ func (r *SinglePartyRecovery) GenerateRecoveryData(recoveryKeys map[string]strin secretMap := make(map[string][]byte, 1) for index, value := range recoveryKeys { // Parse RSA Public Key - recoveryk, err := parseRSAPublicKeyFromPEM(value) + recoveryk, err := ParseRSAPublicKeyFromPEM(value) if err != nil { return nil, nil, err } diff --git a/coordinator/seal/mocksealer.go b/coordinator/seal/mocksealer.go index 56635ff1..7a94baa3 100644 --- a/coordinator/seal/mocksealer.go +++ b/coordinator/seal/mocksealer.go @@ -10,6 +10,7 @@ package seal type MockSealer struct { data []byte unencryptedData []byte + key []byte // mock unseal error UnsealError error } @@ -19,6 +20,11 @@ func (s *MockSealer) Unseal(_ []byte) ([]byte, []byte, error) { return s.unencryptedData, s.data, s.UnsealError } +// UnsealWithKey implements the Sealer interface. +func (s *MockSealer) UnsealWithKey(_, _ []byte) ([]byte, []byte, error) { + return s.unencryptedData, s.data, s.UnsealError +} + // Seal implements the Sealer interface. func (s *MockSealer) Seal(unencryptedData []byte, toBeEncrypted []byte) ([]byte, error) { s.unencryptedData = unencryptedData @@ -28,17 +34,19 @@ func (s *MockSealer) Seal(unencryptedData []byte, toBeEncrypted []byte) ([]byte, // SealEncryptionKey implements the Sealer interface. // Since the MockSealer does not support sealing with an enclave key, it returns the key as is. -func (s *MockSealer) SealEncryptionKey(key []byte, mode Mode) ([]byte, error) { +func (s *MockSealer) SealEncryptionKey(_ []byte, mode Mode) ([]byte, error) { if mode == ModeProductKey || mode == ModeUniqueKey { - return key, nil + return s.key, nil } panic("invariant not met: unexpected mode") } // SetEncryptionKey implements the Sealer interface. -func (s *MockSealer) SetEncryptionKey(_ []byte) {} +func (s *MockSealer) SetEncryptionKey(key []byte) { + s.key = key +} // UnsealEncryptionKey implements the Sealer interface. -func (s *MockSealer) UnsealEncryptionKey(key []byte) ([]byte, error) { +func (s *MockSealer) UnsealEncryptionKey(key, _ []byte) ([]byte, error) { return key, nil } diff --git a/coordinator/seal/noenclavesealer.go b/coordinator/seal/noenclavesealer.go index 16aee2a5..5878de1a 100644 --- a/coordinator/seal/noenclavesealer.go +++ b/coordinator/seal/noenclavesealer.go @@ -29,22 +29,13 @@ func NewNoEnclaveSealer(log *zap.Logger) *NoEnclaveSealer { // Unseal decrypts sealedData and returns the decrypted data, // as well as the prefixed unencrypted metadata of the cipher text. func (s *NoEnclaveSealer) Unseal(sealedData []byte) ([]byte, []byte, error) { - unencryptedData, cipherText, err := prepareCipherText(sealedData, s.log) - if err != nil { - return unencryptedData, nil, err - } - - if s.encryptionKey == nil { - return unencryptedData, nil, fmt.Errorf("decrypting sealed data: %w", ErrMissingEncryptionKey) - } - - // Decrypt data with the unsealed encryption key and return it - decryptedData, err := ecrypto.Decrypt(cipherText, s.encryptionKey, nil) - if err != nil { - return unencryptedData, nil, fmt.Errorf("decrypting sealed data: %w", err) - } + return s.unseal(sealedData, s.encryptionKey) +} - return unencryptedData, decryptedData, nil +// UnsealWithKey decrypts sealedData using the given encryptionKey and returns the decrypted data, +// as well as the prefixed unencrypted metadata of the cipher text. +func (s *NoEnclaveSealer) UnsealWithKey(sealedData, encryptionKey []byte) ([]byte, []byte, error) { + return s.unseal(sealedData, encryptionKey) } // Seal encrypts the given data using the sealer's key. @@ -54,8 +45,8 @@ func (s *NoEnclaveSealer) Seal(unencryptedData []byte, toBeEncrypted []byte) ([] // SealEncryptionKey implements the Sealer interface. // Since the NoEnclaveSealer does not support sealing with an enclave key, it returns the key as is. -func (s *NoEnclaveSealer) SealEncryptionKey(key []byte, _ Mode) ([]byte, error) { - return key, nil +func (s *NoEnclaveSealer) SealEncryptionKey(_ []byte, _ Mode) ([]byte, error) { + return s.encryptionKey, nil } // SetEncryptionKey implements the Sealer interface. @@ -64,6 +55,25 @@ func (s *NoEnclaveSealer) SetEncryptionKey(key []byte) { } // UnsealEncryptionKey implements the Sealer interface. -func (s *NoEnclaveSealer) UnsealEncryptionKey(key []byte) ([]byte, error) { +func (s *NoEnclaveSealer) UnsealEncryptionKey(key, _ []byte) ([]byte, error) { return key, nil } + +func (s *NoEnclaveSealer) unseal(sealedData, encryptionKey []byte) ([]byte, []byte, error) { + unencryptedData, cipherText, err := prepareCipherText(sealedData, s.log) + if err != nil { + return unencryptedData, nil, err + } + + if encryptionKey == nil { + return unencryptedData, nil, fmt.Errorf("decrypting sealed data: %w", ErrMissingEncryptionKey) + } + + // Decrypt data with the unsealed encryption key and return it + decryptedData, err := ecrypto.Decrypt(cipherText, encryptionKey, nil) + if err != nil { + return unencryptedData, nil, fmt.Errorf("decrypting sealed data: %w", err) + } + + return unencryptedData, decryptedData, nil +} diff --git a/coordinator/seal/seal.go b/coordinator/seal/seal.go index b1e21ba8..064eee6f 100644 --- a/coordinator/seal/seal.go +++ b/coordinator/seal/seal.go @@ -39,12 +39,15 @@ type Sealer interface { Seal(unencryptedData []byte, toBeEncrypted []byte) (encryptedData []byte, err error) // Unseal decrypts the given data and returns the plain text, as well as the unencrypted metadata. Unseal(encryptedData []byte) (unencryptedData []byte, decryptedData []byte, err error) + // UnsealWithKey decrypts the given data with the provided encryption key and returns the plain text, + // as well as the unencrypted metadata. + UnsealWithKey(encryptedData, encryptionKey []byte) (unencryptedData []byte, decryptedData []byte, err error) // SealEncryptionKey seals an encryption key using the sealer. - SealEncryptionKey(key []byte, mode Mode) (encryptedKey []byte, err error) + SealEncryptionKey(additionalData []byte, mode Mode) (encryptedKey []byte, err error) // SetEncryptionKey sets the encryption key of the sealer. SetEncryptionKey(key []byte) // UnsealEncryptionKey decrypts an encrypted key. - UnsealEncryptionKey(encryptedKey []byte) ([]byte, error) + UnsealEncryptionKey(encryptedKey, additionalData []byte) ([]byte, error) } // AESGCMSealer implements the Sealer interface using AES-GCM for confidentiality and authentication. @@ -63,22 +66,13 @@ func NewAESGCMSealer(log *zap.Logger) *AESGCMSealer { // Unseal decrypts sealedData and returns the decrypted data, // as well as the prefixed unencrypted metadata of the cipher text. func (s *AESGCMSealer) Unseal(sealedData []byte) ([]byte, []byte, error) { - unencryptedData, cipherText, err := prepareCipherText(sealedData, s.log) - if err != nil { - return unencryptedData, nil, err - } - - if s.encryptionKey == nil { - return unencryptedData, nil, fmt.Errorf("decrypting sealed data: %w", ErrMissingEncryptionKey) - } - - // Decrypt data with the unsealed encryption key and return it - decryptedData, err := ecrypto.Decrypt(cipherText, s.encryptionKey, nil) - if err != nil { - return unencryptedData, nil, fmt.Errorf("decrypting sealed data: %w", err) - } + return s.unseal(sealedData, s.encryptionKey) +} - return unencryptedData, decryptedData, nil +// UnsealWithKey decrypts sealedData with the given encryption key and returns the decrypted data, +// as well as the prefixed unencrypted metadata of the cipher text. +func (s *AESGCMSealer) UnsealWithKey(sealedData, encryptionKey []byte) ([]byte, []byte, error) { + return s.unseal(sealedData, encryptionKey) } // Seal encrypts and stores information to the fs. @@ -86,15 +80,15 @@ func (s *AESGCMSealer) Seal(unencryptedData []byte, toBeEncrypted []byte) ([]byt return sealData(unencryptedData, toBeEncrypted, s.encryptionKey, s.log) } -// SealEncryptionKey seals an encryption key with the selected enclave key. -func (s *AESGCMSealer) SealEncryptionKey(encryptionKey []byte, mode Mode) ([]byte, error) { +// SealEncryptionKey seals the sealer's encryption key with the selected enclave key. +func (s *AESGCMSealer) SealEncryptionKey(additionalData []byte, mode Mode) ([]byte, error) { switch mode { case ModeProductKey: s.log.Debug("Sealing encryption key with product key") - return ecrypto.SealWithProductKey(encryptionKey, nil) + return ecrypto.SealWithProductKey(s.encryptionKey, additionalData) case ModeUniqueKey: s.log.Debug("Sealing encryption key with unique key") - return ecrypto.SealWithUniqueKey(encryptionKey, nil) + return ecrypto.SealWithUniqueKey(s.encryptionKey, additionalData) } return nil, errors.New("sealing is disabled") } @@ -105,9 +99,9 @@ func (s *AESGCMSealer) SetEncryptionKey(encryptionKey []byte) { } // UnsealEncryptionKey unseals the encryption key. -func (s *AESGCMSealer) UnsealEncryptionKey(encryptedKey []byte) ([]byte, error) { +func (s *AESGCMSealer) UnsealEncryptionKey(encryptedKey, additionalData []byte) ([]byte, error) { // Decrypt stored encryption key with seal key - encryptionKey, err := ecrypto.Unseal(encryptedKey, nil) + encryptionKey, err := ecrypto.Unseal(encryptedKey, additionalData) if err != nil { return nil, err } @@ -115,6 +109,25 @@ func (s *AESGCMSealer) UnsealEncryptionKey(encryptedKey []byte) ([]byte, error) return encryptionKey, nil } +func (s *AESGCMSealer) unseal(sealedData, encryptionKey []byte) ([]byte, []byte, error) { + unencryptedData, cipherText, err := prepareCipherText(sealedData, s.log) + if err != nil { + return unencryptedData, nil, err + } + + if encryptionKey == nil { + return unencryptedData, nil, fmt.Errorf("decrypting sealed data: %w", ErrMissingEncryptionKey) + } + + // Decrypt data with the unsealed encryption key and return it + decryptedData, err := ecrypto.Decrypt(cipherText, encryptionKey, nil) + if err != nil { + return unencryptedData, nil, fmt.Errorf("decrypting sealed data: %w", err) + } + + return unencryptedData, decryptedData, nil +} + // GenerateEncryptionKey generates a new random 16 byte encryption key. func GenerateEncryptionKey() ([]byte, error) { encryptionKey := make([]byte, 16) diff --git a/coordinator/server/handler/handler.go b/coordinator/server/handler/handler.go index 4f111f57..ae5ed3fa 100644 --- a/coordinator/server/handler/handler.go +++ b/coordinator/server/handler/handler.go @@ -28,7 +28,7 @@ type ClientAPI interface { GetSecrets(ctx context.Context, requestedSecrets []string, requestUser *user.User) (map[string]manifest.Secret, error) GetStatus(context.Context) (statusCode state.State, status string, err error) GetUpdateLog(context.Context) (updateLog []string, err error) - Recover(ctx context.Context, encryptionKey []byte) (int, error) + Recover(ctx context.Context, encryptionKey, encryptionKeySignature []byte) (int, error) SetMonotonicCounter(ctx context.Context, marbleType string, marbleUUID uuid.UUID, name string, value uint64) (uint64, error) SignQuote(ctx context.Context, quote []byte) (signature []byte, tcbStatus string, err error) VerifyMarble(ctx context.Context, clientCerts []*x509.Certificate) (string, uuid.UUID, error) diff --git a/coordinator/server/v1/v1.go b/coordinator/server/v1/v1.go index 7417cb6a..95f81bc9 100644 --- a/coordinator/server/v1/v1.go +++ b/coordinator/server/v1/v1.go @@ -10,7 +10,6 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" - "fmt" "io" "net/http" "strings" @@ -143,40 +142,11 @@ func (s *ClientAPIServer) QuoteGet(w http.ResponseWriter, r *http.Request) { handler.WriteJSON(w, CertQuoteResponse{cert, quote}) } -// RecoverPost recovers the Coordinator. -// -// Recover the Coordinator when unsealing of the existing state fails. -// -// This API endpoint is only available when the coordinator is in recovery mode. -// Before you can use the endpoint, you need to decrypt the recovery secret which you may have received when setting the manifest initially. -// See [Recovering the Coordinator](../#/workflows/recover-coordinator.md) to retrieve the recovery key needed to use this API endpoint correctly. -// -// Example for recovering the Coordinator with curl: -// -// curl -k -X POST --data-binary @recovery_key_decrypted "https://$MARBLERUN/recover" -func (s *ClientAPIServer) RecoverPost(w http.ResponseWriter, r *http.Request) { - key, err := io.ReadAll(r.Body) - if err != nil { - handler.WriteJSONError(w, err.Error(), http.StatusInternalServerError) - return - } - - // Perform recover and receive amount of remaining secrets (for multi-party recovery) - remaining, err := s.api.Recover(r.Context(), key) - if err != nil { - handler.WriteJSONError(w, err.Error(), http.StatusInternalServerError) - return - } - - // Construct status message based on remaining keys - var statusMessage string - if remaining != 0 { - statusMessage = fmt.Sprintf("Secret was processed successfully. Upload the next secret. Remaining secrets: %d", remaining) - } else { - statusMessage = "Recovery successful." - } - - handler.WriteJSON(w, RecoveryStatusResponse{statusMessage}) +// RecoverPost is a handler for the removed /recover endpoint. +// It only exists to inform users about using the new /api/v2/recover endpoint. +func (s *ClientAPIServer) RecoverPost(w http.ResponseWriter, _ *http.Request) { + errorMsg := "Recovering the Coordinator using the /recover API endpoint has been disabled. Use the /api/v2/recover endpoint instead." + handler.WriteJSONError(w, errorMsg, http.StatusGone) } // UpdateGet retrieves the update log. diff --git a/coordinator/server/v2/types.go b/coordinator/server/v2/types.go index a85f1ef1..86f11605 100644 --- a/coordinator/server/v2/types.go +++ b/coordinator/server/v2/types.go @@ -73,6 +73,8 @@ type QuoteSignResponse struct { type RecoveryRequest struct { // RecoverySecret is the decrypted secret (or secret share) to recover the Coordinator. RecoverySecret []byte `json:"recoverySecret"` + // RecoverySecretSignature is the RSA PKCS #1 v1.5 signature over the sha256 hash of the RecoverySecret. + RecoverySecretSignature []byte `json:"recoverySecretSignature"` } // RecoveryResponse contains the response for the recovery process. diff --git a/coordinator/server/v2/v2.go b/coordinator/server/v2/v2.go index a3aebf8d..bb246e31 100644 --- a/coordinator/server/v2/v2.go +++ b/coordinator/server/v2/v2.go @@ -129,7 +129,7 @@ func (s *ClientAPIServer) RecoverPost(w http.ResponseWriter, r *http.Request) { } // Perform recover and receive amount of remaining secrets (for multi-party recovery) - remaining, err := s.api.Recover(r.Context(), req.RecoverySecret) + remaining, err := s.api.Recover(r.Context(), req.RecoverySecret, req.RecoverySecretSignature) if err != nil { handler.WriteJSONError(w, err.Error(), http.StatusInternalServerError) return diff --git a/coordinator/store/stdstore/stdstore.go b/coordinator/store/stdstore/stdstore.go index 55454dce..b53c3ba5 100644 --- a/coordinator/store/stdstore/stdstore.go +++ b/coordinator/store/stdstore/stdstore.go @@ -8,16 +8,19 @@ package stdstore import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "path/filepath" "strings" "sync" - "time" + "github.com/edgelesssys/marblerun/coordinator/manifest" "github.com/edgelesssys/marblerun/coordinator/seal" "github.com/edgelesssys/marblerun/coordinator/store" + "github.com/edgelesssys/marblerun/coordinator/store/request" "github.com/spf13/afero" "go.uber.org/zap" ) @@ -113,21 +116,18 @@ func (s *StdStore) BeginTransaction(_ context.Context) (store.Transaction, error } // LoadState loads sealed data into StdStore's data. -func (s *StdStore) LoadState() ([]byte, error) { +func (s *StdStore) LoadState() (recoveryData, sealedData []byte, err error) { s.mux.Lock() defer s.mux.Unlock() // load from fs s.log.Debug("Loading sealed state from file system", zap.String("filename", filepath.Join(s.sealDir, SealedDataFname))) - sealedData, err := s.fs.ReadFile(filepath.Join(s.sealDir, SealedDataFname)) + sealedData, err = s.fs.ReadFile(filepath.Join(s.sealDir, SealedDataFname)) if errors.Is(err, afero.ErrFileNotFound) { - // No sealed data found, back up any existing seal keys - s.log.Debug("No sealed data found, backing up existing seal keys") - s.backupEncryptionKey() - return nil, nil + return nil, nil, nil } else if err != nil { s.log.Debug("Error reading sealed data", zap.Error(err)) - return nil, err + return nil, nil, err } s.log.Debug("Unsealing loaded state") @@ -137,15 +137,15 @@ func (s *StdStore) LoadState() ([]byte, error) { if !errors.Is(err, seal.ErrMissingEncryptionKey) { s.log.Debug("No encryption key found, entering recovery mode") s.recoveryMode = true - return encodedRecoveryData, fmt.Errorf("unsealing state: %w", err) + return encodedRecoveryData, sealedData, fmt.Errorf("unsealing state: %w", err) } // Try to unseal encryption key from disk using product key // And retry unsealing the sealed data s.log.Debug("Trying to unseal encryption key") - if err := s.unsealEncryptionKey(); err != nil { + if err := s.unsealEncryptionKey(sealedData); err != nil { s.log.Debug("Unsealing encryption key failed, entering recovery mode", zap.Error(err)) s.recoveryMode = true - return encodedRecoveryData, &seal.EncryptionKeyError{Err: err} + return encodedRecoveryData, sealedData, &seal.EncryptionKeyError{Err: err} } s.log.Debug("Retrying unsealing state") @@ -153,23 +153,26 @@ func (s *StdStore) LoadState() ([]byte, error) { if err != nil { s.log.Debug("Unsealing state failed, entering recovery mode", zap.Error(err)) s.recoveryMode = true - return encodedRecoveryData, fmt.Errorf("retry unsealing state with loaded key: %w", err) + return encodedRecoveryData, sealedData, fmt.Errorf("retry unsealing state with loaded key: %w", err) } } if len(stateRaw) == 0 { s.log.Debug("State is empty, nothing to do") - return encodedRecoveryData, nil + return encodedRecoveryData, sealedData, nil } // load state s.log.Debug("Loading state from unsealed JSON blob") var loadedData map[string][]byte if err := json.Unmarshal(stateRaw, &loadedData); err != nil { - return encodedRecoveryData, err + return encodedRecoveryData, sealedData, err + } + if err := s.reloadSealMode(loadedData); err != nil { + return encodedRecoveryData, sealedData, err } s.data = loadedData - return encodedRecoveryData, nil + return encodedRecoveryData, sealedData, nil } // SetRecoveryData sets the recovery data that is added to the sealed data. @@ -179,28 +182,62 @@ func (s *StdStore) SetRecoveryData(recoveryData []byte) { s.recoveryMode = false } -// SetEncryptionKey sets the encryption key for sealing and unsealing. -func (s *StdStore) SetEncryptionKey(encryptionKey []byte, mode seal.Mode) error { - s.log.Debug("Setting encryption key", zap.Int("mode", int(mode))) - if mode != seal.ModeDisabled { - // If there already is an existing key file stored on disk, save it - s.backupEncryptionKey() +// BeginReadTransaction loads the sealed state and returns a read-only transaction. +func (s *StdStore) BeginReadTransaction(_ context.Context, encryptionKey []byte) (store.ReadTransaction, error) { + s.log.Debug("Loading sealed state from file system", zap.String("filename", filepath.Join(s.sealDir, SealedDataFname))) + sealedData, err := s.fs.ReadFile(filepath.Join(s.sealDir, SealedDataFname)) + if err != nil { + return nil, fmt.Errorf("reading sealed data from disk: %w", err) + } - s.log.Debug("Sealing state encryption key") - encryptedKey, err := s.sealer.SealEncryptionKey(encryptionKey, mode) - if err != nil { - return fmt.Errorf("encrypting data key: %w", err) - } + s.log.Debug("Unsealing state") + _, data, err := s.sealer.UnsealWithKey(sealedData, encryptionKey) + if err != nil { + return nil, fmt.Errorf("unsealing state: %w", err) + } - // Write the sealed encryption key to disk - s.log.Debug("Writing sealed encryption key to disk", zap.String("filename", filepath.Join(s.sealDir, SealedKeyFname))) - if err = s.fs.WriteFile(filepath.Join(s.sealDir, SealedKeyFname), encryptedKey, 0o600); err != nil { - return fmt.Errorf("writing encrypted key to disk: %w", err) - } + s.log.Debug("Loading state from unsealed JSON blob") + var loadedData map[string][]byte + if err := json.Unmarshal(data, &loadedData); err != nil { + return nil, fmt.Errorf("unmarshalling state: %w", err) } + return &StdTransaction{ + // store is nil to prevent any writes or access to it + // callers of this function must not call Commit + store: nil, + data: loadedData, + log: s.log, + }, nil +} + +// SetEncryptionKey sets the encryption key for sealing and unsealing. +func (s *StdStore) SetEncryptionKey(encryptionKey []byte, mode seal.Mode) { + s.log.Debug("Setting encryption key", zap.Int("mode", int(mode))) s.sealer.SetEncryptionKey(encryptionKey) s.sealMode = mode +} + +// SealEncryptionKey seals the encryption key and writes it to disk. +func (s *StdStore) SealEncryptionKey(additionalData []byte) error { + s.log.Debug("Sealing state encryption key") + if s.sealMode == seal.ModeDisabled { + s.log.Debug("Sealing disabled, nothing to do") + return nil + } + + additionalDataHash := sha256.Sum256(additionalData) + s.log.Debug("Sealing state encryption key with additional data", zap.String("additionalData", hex.EncodeToString(additionalDataHash[:]))) + encryptedKey, err := s.sealer.SealEncryptionKey(additionalDataHash[:], s.sealMode) + if err != nil { + return fmt.Errorf("encrypting data key: %w", err) + } + + // Write the sealed encryption key to disk + s.log.Debug("Writing sealed encryption key to disk", zap.String("filename", filepath.Join(s.sealDir, SealedKeyFname))) + if err = s.atomicWriteFile(SealedKeyFname, encryptedKey); err != nil { + return fmt.Errorf("writing encrypted key to disk: %w", err) + } return nil } @@ -236,15 +273,23 @@ func (s *StdStore) commit(data map[string][]byte) error { return err } + additionalData := sha256.Sum256(sealedData) + s.log.Debug("Sealing encryption key", zap.String("additionalData", hex.EncodeToString(additionalData[:]))) + encryptedKey, err := s.sealer.SealEncryptionKey(additionalData[:], s.sealMode) + if err != nil { + return fmt.Errorf("sealing encryption key: %w", err) + } + // atomically replace the sealed data file s.log.Debug("Writing sealed transaction data to disk", zap.String("filename", filepath.Join(s.sealDir, SealedDataFname))) - sealedDataPath := filepath.Join(s.sealDir, SealedDataFname) - sealedDataPathTmp := sealedDataPath + ".tmp" - if err := s.fs.WriteFile(sealedDataPathTmp, sealedData, 0o600); err != nil { + if err := s.atomicWriteFile(SealedDataFname, sealedData); err != nil { return fmt.Errorf("writing sealed data file: %w", err) } - if err := s.fs.Rename(sealedDataPathTmp, sealedDataPath); err != nil { - return fmt.Errorf("renaming sealed data file: %w", err) + + // atomically replace the sealed key file + s.log.Debug("Writing sealed encryption key to disk", zap.String("filename", filepath.Join(s.sealDir, SealedKeyFname))) + if err := s.atomicWriteFile(SealedKeyFname, encryptedKey); err != nil { + return fmt.Errorf("writing encrypted key to disk: %w", err) } } @@ -256,13 +301,16 @@ func (s *StdStore) commit(data map[string][]byte) error { } // unsealEncryptionKey sets the seal key for the store's sealer by loading the encrypted key from disk. -func (s *StdStore) unsealEncryptionKey() error { +func (s *StdStore) unsealEncryptionKey(sealedData []byte) error { s.log.Debug("Loading sealed encryption key from disk", zap.String("filename", filepath.Join(s.sealDir, SealedKeyFname))) encryptedKey, err := s.fs.ReadFile(filepath.Join(s.sealDir, SealedKeyFname)) if err != nil { return fmt.Errorf("reading encrypted key from disk: %w", err) } - key, err := s.sealer.UnsealEncryptionKey(encryptedKey) + + additionalData := sha256.Sum256(sealedData) + s.log.Debug("Unsealing encryption key", zap.String("additionalData", hex.EncodeToString(additionalData[:]))) + key, err := s.sealer.UnsealEncryptionKey(encryptedKey, additionalData[:]) if err != nil { return fmt.Errorf("decrypting data key: %w", err) } @@ -270,14 +318,38 @@ func (s *StdStore) unsealEncryptionKey() error { return nil } -// backupEncryptionKey creates a backup of an existing seal key. -func (s *StdStore) backupEncryptionKey() { - if sealedKeyData, err := s.fs.ReadFile(filepath.Join(s.sealDir, SealedKeyFname)); err == nil { - t := time.Now() - newFileName := filepath.Join(s.sealDir, SealedKeyFname) + "_" + t.Format("20060102150405") + ".bak" - s.log.Debug("Creating backup of existing seal key", zap.String("filename", newFileName)) - _ = s.fs.WriteFile(newFileName, sealedKeyData, 0o600) +// atomicWriteFile writes data to a temporary file and then atomically replaces the target file. +func (s *StdStore) atomicWriteFile(fileName string, data []byte) error { + filePath := filepath.Join(s.sealDir, fileName) + filePathTmp := filePath + ".tmp" + filePathOld := filePath + ".old" + if err := s.fs.WriteFile(filePathTmp, data, 0o600); err != nil { + return fmt.Errorf("writing temporary file: %w", err) + } + if err := s.fs.Rename(filePath, filePathOld); err != nil && !errors.Is(err, afero.ErrFileNotFound) { + return fmt.Errorf("backing up old file: %w", err) + } + if err := s.fs.Rename(filePathTmp, filePath); err != nil { + return fmt.Errorf("replacing file: %w", err) } + return nil +} + +func (s *StdStore) reloadSealMode(rawState map[string][]byte) error { + s.log.Debug("Reloading seal mode") + rawMnf, ok := rawState[request.Manifest] + if !ok { + return nil // no manifest set + } + + var mnf manifest.Manifest + if err := json.Unmarshal(rawMnf, &mnf); err != nil { + return fmt.Errorf("unmarshaling manifest: %w", err) + } + + s.sealMode = seal.ModeFromString(mnf.Config.SealMode) + s.log.Debug("Seal mode set", zap.Int("sealMode", int(s.sealMode))) + return nil } // StdTransaction is a transaction for StdStore. diff --git a/coordinator/store/stdstore/stdstore_test.go b/coordinator/store/stdstore/stdstore_test.go index 1a9d3973..7b2e2232 100644 --- a/coordinator/store/stdstore/stdstore_test.go +++ b/coordinator/store/stdstore/stdstore_test.go @@ -23,7 +23,7 @@ func TestStdStore(t *testing.T) { ctx := context.Background() str := New(&seal.MockSealer{}, afero.NewMemMapFs(), "", zaptest.NewLogger(t)) - _, err := str.LoadState() + _, _, err := str.LoadState() assert.NoError(err) testData1 := []byte("test data") @@ -125,17 +125,17 @@ func TestStdStoreSealing(t *testing.T) { sealer := &seal.MockSealer{} store := New(sealer, fs, "", zaptest.NewLogger(t)) - _, err := store.LoadState() + _, _, err := store.LoadState() require.NoError(err) - require.NoError(store.SetEncryptionKey(nil, tc.mode)) + store.SetEncryptionKey(nil, tc.mode) testData1 := []byte("test data") require.NoError(store.Put("test:input", testData1)) // Check sealing with a new store initialized with the sealed state store2 := New(sealer, fs, "", zaptest.NewLogger(t)) - _, err = store2.LoadState() + _, _, err = store2.LoadState() require.NoError(err) val, err := store2.Get("test:input") @@ -154,7 +154,7 @@ func TestStdStoreRollback(t *testing.T) { ctx := context.Background() store := New(&seal.MockSealer{}, afero.NewMemMapFs(), "", zaptest.NewLogger(t)) - _, err := store.LoadState() + _, _, err := store.LoadState() assert.NoError(err) testData1 := []byte("test data") diff --git a/coordinator/store/store.go b/coordinator/store/store.go index 733b7d57..b81ee927 100644 --- a/coordinator/store/store.go +++ b/coordinator/store/store.go @@ -18,11 +18,16 @@ type Store interface { // BeginTransaction starts a new transaction. BeginTransaction(context.Context) (Transaction, error) // SetEncryptionKey sets the encryption key for the store. - SetEncryptionKey([]byte, seal.Mode) error + SetEncryptionKey([]byte, seal.Mode) + // SealEncryptionKey seals the encryption key for the store. + SealEncryptionKey(additionalData []byte) error // SetRecoveryData sets recovery data for the store. SetRecoveryData([]byte) // LoadState loads the sealed state of a store. - LoadState() ([]byte, error) + LoadState() (recoveryData, sealedData []byte, err error) + // BeginReadTransaction loads the store from a sealed state without committing any data to it, + // or modifying the underlying store in any way. + BeginReadTransaction(context.Context, []byte) (ReadTransaction, error) } // Transaction is a Store transaction. @@ -41,6 +46,17 @@ type Transaction interface { Rollback() } +// ReadTransaction is a read-only transaction on a [Store]. +// While data can be written to the transaction, it cannot be committed to the [Store]. +type ReadTransaction interface { + // Get returns a value from store by key + Get(string) ([]byte, error) + // Put saves a value to store by key + Put(string, []byte) error + // Iterator returns an Iterator for a given prefix + Iterator(string) (Iterator, error) +} + // Iterator is an iterator for the store. type Iterator interface { // Returns the next element of the iterator diff --git a/docs/docs/reference/cli.md b/docs/docs/reference/cli.md index a6577269..68ebcb62 100644 --- a/docs/docs/reference/cli.md +++ b/docs/docs/reference/cli.md @@ -932,10 +932,12 @@ Recover the MarbleRun Coordinator from a sealed state ### Synopsis -Recover the MarbleRun Coordinator from a sealed state +Recover the MarbleRun Coordinator from a sealed state. + may be either a decrypted recovery secret, or an encrypted recovery secret, +in which case the private key is used to decrypt the secret. ``` -marblerun recover [flags] +marblerun recover [flags] ``` ### Examples @@ -947,7 +949,11 @@ marblerun recover recovery_key_decrypted $MARBLERUN ### Options ``` - -h, --help help for recover + -h, --help help for recover + -k, --key string Path to a the recovery private key to decrypt and/or sign the recovery key + --pkcs11-config string Path to a PKCS#11 configuration file to load the recovery private key with + --pkcs11-key-id string ID of the private key in the PKCS#11 token + --pkcs11-key-label string Label of the private key in the PKCS#11 token ``` ### Options inherited from parent commands diff --git a/docs/docs/reference/coordinator.md b/docs/docs/reference/coordinator.md index f4bcd4fe..d2c3f63c 100644 --- a/docs/docs/reference/coordinator.md +++ b/docs/docs/reference/coordinator.md @@ -360,7 +360,7 @@ See [Recovering the Coordinator](../workflows/recover-coordinator.md) on how to Example for recovering the Coordinator with curl: ```bash -curl -k -X POST --data-binary @recovery_key.json "https://$MARBLERUN/recover" +curl -k -X POST --data-binary @recovery_key.json "https://$MARBLERUN/api/v2/recover" ``` #### Request body @@ -369,11 +369,16 @@ curl -k -X POST --data-binary @recovery_key.json "https://$MARBLERUN/recover" Base64 encoded recovery secret. +* `recoverySecretSignature` string + + Base64 encoded RSA PKCS #1 v1.5 signature over the sha256 hash of the RecoverySecret using the private key used to decrypt the recovery secret. + Example request body: ```JSON { - "recoverySecret": "AAECAwQFBgcICQoLDA0ODw==" + "recoverySecret": "AAECAwQFBgcICQoLDA0ODw==", + "recoverySecretSignature": "cmVjb3Zlcnkga2V5IHNpZ25hdHVyZQ==" } ``` @@ -399,46 +404,6 @@ Example response: } ``` - - - -```http -POST /recover -``` - -Recover the Coordinator using decrypted recovery secrets. - -This API endpoint is only available when the coordinator is in recovery mode. -Before you can use the endpoint, you need to decrypt the recovery secret which you may have received when setting the manifest initially. -See [Recovering the Coordinator](../workflows/recover-coordinator.md) on how to retrieve the recovery key needed to use this API endpoint correctly. - -Example for recovering the Coordinator with curl: - -```bash -curl -k -X POST --data-binary @recovery_key_decrypted "https://$MARBLERUN/recover" -``` - -#### Request body - -Raw binary encoded recovery secret. - -#### Returns - -* `statusMessage` string - - A human readable message indicating the success or progress of the recovery process. - -Example response: - -```JSON -{ - "status": "success", - "data": { - "statusMessage": "Secret was processed successfully. Upload the next secret. Remaining secrets: 2" - } -} -``` - diff --git a/docs/docs/workflows/define-manifest.md b/docs/docs/workflows/define-manifest.md index e11400db..3298df65 100644 --- a/docs/docs/workflows/define-manifest.md +++ b/docs/docs/workflows/define-manifest.md @@ -340,7 +340,7 @@ When verifying certificates in this context, MarbleRun ignores their `issuer`, ` Use OpenSSL to generate a compatible certificate. ```bash -openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout admin_private.key -out admin_certificate.crt +openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout admin_private.key -out admin_certificate.pem ``` Use the following command to preserve newlines correctly: diff --git a/docs/docs/workflows/recover-coordinator.md b/docs/docs/workflows/recover-coordinator.md index dadd5475..a357b1a3 100644 --- a/docs/docs/workflows/recover-coordinator.md +++ b/docs/docs/workflows/recover-coordinator.md @@ -14,40 +14,25 @@ You need the corresponding private key to the [`RecoveryKeys` defined in the man :::info If you don't want or can't recover the old state, you can also dismiss it by [uploading a new manifest](set-manifest.md). -The old state will be overwritten on disk, and the `/recover` endpoint won't be available anymore. +The old state will be overwritten on disk, and recovery won't be available anymore. ::: -You can upload the recovery secret through the `/recover` client API endpoint. To do so, you need to: +To recover the Coordinator, you need to decode your recovery secret and upload it using the MarbleRun CLI. -* Decode the Base64-encoded output returned to you during the initial upload of the manifest -* Decrypt the decoded output with the corresponding RSA private key of the key defined in the manifest -* Get the temporary root certificate (valid only during recovery mode) -* Upload the decrypted key to the `/recover` endpoint - -Assuming you saved the output from the manifest upload step in a file called `recovery_data` and the recovery private key in a file called `private_key.pem`, perform recovery like this: - -```bash -openssl base64 -d -in recovery_data \ - | openssl pkeyutl -inkey private_key.pem -decrypt \ - -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \ - -out recovery_key_decrypted -``` - -You can then upload the extracted secret using the MarbleRun CLI: +Assuming you named your recovery key `recoverKey1` in the manifest, and you saved the output from the manifest upload step in a file called `recovery_data`, decode your secret with the following command: ```bash -marblerun recover recovery_key_decrypted $MARBLERUN +jq -r '.RecoverySecrets.recoverKey1' recovery_data | openssl base64 -d > recovery_key_encrypted ``` -Alternatively, you can use `curl`: +Then decrypt and upload the extracted secret using the MarbleRun CLI: ```bash -era -c coordinator-era.json -h $MARBLERUN -output-root marblerun-temp.pem -curl --cacert marblerun-temp.pem --data-binary @recovery_key_decrypted https://$MARBLERUN/recover +marblerun recover recovery_key_encrypted $MARBLERUN --key private_key.pem ``` -On success, the Coordinator applies the sealed state again. If the Coordinator can't restore the state with the uploaded key, an error will be returned in the logs, and the `/recover` endpoint will stay open for further interaction. +On success, the Coordinator applies the sealed state again. If the Coordinator can't restore the state with the uploaded key, an error will be returned in the logs, and the recovery endpoint will stay open for further interaction. ## Multi-party recovery @@ -95,11 +80,8 @@ Assume the following `RecoveryKeys` were set in the manifest: $ marblerun status $MARBLERUN 1: Coordinator is in recovery mode. Either upload a key to unseal the saved state, or set a new manifest. For more information on how to proceed, consult the documentation. $ echo "EbkX/skIPrJISf8PiXdzRIKnwQyJ+VejtGzHGfES5NIPuCeEFedqgCVDk=" > recovery_data - $ openssl base64 -d -in recovery_data \ - | openssl pkeyutl -inkey private_key.pem -decrypt \ - -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \ - -out recovery_key_decrypted - $ marblerun recover recovery_key_decrypted $MARBLERUN + $ openssl base64 -d -in recovery_data > recovery_key_encrypted + $ marblerun recover recovery_key_encrypted $MARBLERUN --key private_key.pem Successfully verified Coordinator, now uploading key Secret was processed successfully. Upload the next secret. Remaining secrets: 1 ``` @@ -117,11 +99,8 @@ Assume the following `RecoveryKeys` were set in the manifest: $ marblerun status $MARBLERUN 1: Coordinator is in recovery mode. Either upload a key to unseal the saved state, or set a new manifest. For more information on how to proceed, consult the documentation. $ echo "bvPzio4A4SvzeHajsb+dFDpDarErcU9wMR0V9hyHtG2lC4ZfyrYjDBE7wtis3eOPgDaMG/HCt=" > recovery_data - $ openssl base64 -d -in recovery_data \ - | openssl pkeyutl -inkey private_key.pem -decrypt \ - -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \ - -out recovery_key_decrypted - $ marblerun recover recovery_key_decrypted $MARBLERUN + $ openssl base64 -d -in recovery_data > recovery_key_encrypted + $ marblerun recover recovery_key_encrypted $MARBLERUN --key private_key.pem Successfully verified Coordinator, now uploading key Recovery successful. $ marblerun status $MARBLERUN diff --git a/enclave/coordinator.conf b/enclave/coordinator.conf index cd69f34c..be2e0e7c 100644 --- a/enclave/coordinator.conf +++ b/enclave/coordinator.conf @@ -4,4 +4,4 @@ NumHeapPages=131072 NumStackPages=1024 NumTCS=32 ProductID=3 -SecurityVersion=1 +SecurityVersion=2 diff --git a/test/framework/framework.go b/test/framework/framework.go index 64404d51..34aa2ed9 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -10,6 +10,7 @@ package framework import ( "bufio" "context" + "crypto" "crypto/rsa" "crypto/tls" "crypto/x509" @@ -105,7 +106,7 @@ func (i IntegrationTest) UpdateManifest() { // CoordinatorConfig contains the configuration for the Coordinator. type CoordinatorConfig struct { dnsNames string - sealDir string + SealDir string extraEnv []string } @@ -117,14 +118,14 @@ func NewCoordinatorConfig(extraEnv ...string) CoordinatorConfig { } return CoordinatorConfig{ dnsNames: "localhost", - sealDir: sealDir, + SealDir: sealDir, extraEnv: extraEnv, } } // Cleanup removes the seal directory. func (c CoordinatorConfig) Cleanup() { - if err := os.RemoveAll(c.sealDir); err != nil { + if err := os.RemoveAll(c.SealDir); err != nil { panic(err) } } @@ -143,7 +144,7 @@ func (i IntegrationTest) StartCoordinator(ctx context.Context, cfg CoordinatorCo MakeEnv(constants.MeshAddr, i.MeshServerAddr), MakeEnv(constants.ClientAddr, i.ClientServerAddr), MakeEnv(constants.DNSNames, cfg.dnsNames), - MakeEnv(constants.SealDir, cfg.sealDir), + MakeEnv(constants.SealDir, cfg.SealDir), MakeEnv(constants.DebugLogging, "1"), i.SimulationFlag, } @@ -223,8 +224,8 @@ func (i IntegrationTest) SetUpdateManifest(manifest manifest.Manifest, certPEM [ } // SetRecover sets the recovery key of the Coordinator. -func (i IntegrationTest) SetRecover(recoveryKey []byte) error { - _, _, err := api.Recover(context.Background(), i.ClientServerAddr, api.VerifyOptions{InsecureSkipVerify: true}, recoveryKey) +func (i IntegrationTest) SetRecover(recoveryKey []byte, signer crypto.Signer) error { + _, _, err := api.Recover(context.Background(), i.ClientServerAddr, api.VerifyOptions{InsecureSkipVerify: true}, recoveryKey, signer) return err } @@ -345,7 +346,7 @@ func (i IntegrationTest) TriggerRecovery(coordinatorCfg CoordinatorConfig, cance // Remove sealed encryption key to trigger recovery state i.t.Log("Deleting sealed key to trigger recovery state...") - os.Remove(filepath.Join(coordinatorCfg.sealDir, stdstore.SealedKeyFname)) + os.Remove(filepath.Join(coordinatorCfg.SealDir, stdstore.SealedKeyFname)) // Restart server, we should be in recovery mode i.t.Log("Restarting the old instance") diff --git a/test/integration_test.go b/test/integration_test.go index 371b00a6..dda8668d 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -19,11 +19,14 @@ import ( "net" "net/http" "os" + "path/filepath" "testing" "github.com/edgelesssys/marblerun/api" corecrypto "github.com/edgelesssys/marblerun/coordinator/crypto" "github.com/edgelesssys/marblerun/coordinator/manifest" + "github.com/edgelesssys/marblerun/coordinator/state" + "github.com/edgelesssys/marblerun/coordinator/store/stdstore" "github.com/edgelesssys/marblerun/test/framework" "github.com/edgelesssys/marblerun/util" "github.com/stretchr/testify/assert" @@ -276,6 +279,14 @@ func TestRecoveryRestoreKey(t *testing.T) { defer serverCfg.Cleanup() f.StartMarbleServer(f.Ctx, serverCfg) + // Coordinator can restart automatically + cancelCoordinator() + cancelCoordinator = f.StartCoordinator(f.Ctx, cfg) + t.Log("Restarted Coordinator, checking status again...") + statusCode, err := f.GetStatus() + require.NoError(err) + assert.EqualValues(int(state.AcceptingMarbles), statusCode, "Server is in wrong status after restart.") + // Trigger recovery mode cancelCoordinator, cert := f.TriggerRecovery(cfg, cancelCoordinator) @@ -284,14 +295,98 @@ func TestRecoveryRestoreKey(t *testing.T) { require.NoError(err, "Failed to decrypt the recovery data.") // Perform recovery - require.NoError(f.SetRecover(recoveryKey)) + require.NoError(f.SetRecover(recoveryKey, RecoveryPrivateKey)) t.Log("Performed recovery, now checking status again...") + statusCode, err = f.GetStatus() + require.NoError(err) + assert.EqualValues(int(state.AcceptingMarbles), statusCode, "Server is in wrong status after recovery.") + + // Verify if old certificate is still valid + cancelCoordinator = f.VerifyCertAfterRecovery(cert, cancelCoordinator, cfg) + cancelCoordinator() + }) + } +} + +func TestRecoverySealedKeyStateBinding(t *testing.T) { + if *noenclave { + t.Skip("This test cannot be run in No Enclave mode.") + return + } + + for _, sealMode := range []string{"", "ProductKey", "UniqueKey"} { + t.Run("SealMode="+sealMode, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + f := newFramework(t) + + t.Log("Testing recovery...") + t.Log("Starting a coordinator enclave") + cfg := framework.NewCoordinatorConfig() + defer cfg.Cleanup() + cancelCoordinator := f.StartCoordinator(f.Ctx, cfg) + + // set Manifest + t.Log("Setting the Manifest") + f.TestManifest.Config.SealMode = sealMode + f.TestManifest.Config.FeatureGates = []string{"MonotonicCounter"} + recoveryData, err := f.SetManifest(f.TestManifest) + require.NoError(err, "failed to set Manifest") + + // start server + t.Log("Starting a Server-Marble") + serverCfg := framework.NewMarbleConfig(f.MeshServerAddr, "testMarbleServer", "server,backend,localhost") + defer serverCfg.Cleanup() + f.StartMarbleServer(f.Ctx, serverCfg) + + // Coordinator can restart automatically + cancelCoordinator() + cancelCoordinator = f.StartCoordinator(f.Ctx, cfg) + t.Log("Restarted Coordinator, checking status again...") statusCode, err := f.GetStatus() require.NoError(err) - assert.EqualValues(3, statusCode, "Server is in wrong status after recovery.") + assert.EqualValues(int(state.AcceptingMarbles), statusCode, "Server is in wrong status after restart.") + + // Get certificate before triggering recovery + cert, _, _, err := api.VerifyCoordinator(context.Background(), f.ClientServerAddr, api.VerifyOptions{InsecureSkipVerify: true}) + require.NoError(err) + + // Save backup of the encryption key + sealedEncryptionKey, err := os.ReadFile(filepath.Join(cfg.SealDir, stdstore.SealedKeyFname)) + require.NoError(err) + + // Start a marble which sets a monotonic counter + // This will update the state and re-seal the key + marbleCfg := framework.NewMarbleConfig(meshServerAddr, "testMarbleMonotonicCounter", "localhost") + defer marbleCfg.Cleanup() + assert.True(f.StartMarbleClient(f.Ctx, marbleCfg)) + + // Stop the Coordinator and replace the sealed key with the backup + cancelCoordinator() + require.NoError(os.WriteFile(filepath.Join(cfg.SealDir, stdstore.SealedKeyFname), sealedEncryptionKey, 0o600)) + + // Since the key backup is bound to a different state, the + // Coordinator should not be able to recovery automatically + cancelCoordinator = f.StartCoordinator(f.Ctx, cfg) + defer cancelCoordinator() + statusCode, err = f.GetStatus() + require.NoError(err) + assert.EqualValues(int(state.Recovery), statusCode, "Server is in wrong status after restart.") + + // Decrypt recovery data from when we set the manifest + recoveryKey, err := api.DecryptRecoveryData(recoveryData["testRecKey1"], RecoveryPrivateKey) + require.NoError(err, "Failed to decrypt the recovery data.") + + // Perform recovery + require.NoError(f.SetRecover(recoveryKey, RecoveryPrivateKey)) + t.Log("Performed recovery, now checking status again...") + statusCode, err = f.GetStatus() + require.NoError(err) + assert.EqualValues(int(state.AcceptingMarbles), statusCode, "Server is in wrong status after recovery.") // Verify if old certificate is still valid - f.VerifyCertAfterRecovery(cert, cancelCoordinator, cfg) + cancelCoordinator = f.VerifyCertAfterRecovery(cert, cancelCoordinator, cfg) + cancelCoordinator() }) } } diff --git a/util/util.go b/util/util.go index bf0df30c..d6ec0f2e 100644 --- a/util/util.go +++ b/util/util.go @@ -8,6 +8,7 @@ package util import ( "cmp" + "crypto" "crypto/rand" "crypto/rsa" "crypto/sha256" @@ -97,6 +98,19 @@ func DecryptOAEP(priv *rsa.PrivateKey, ciphertext []byte) ([]byte, error) { return rsa.DecryptOAEP(sha256.New(), rand.Reader, priv, ciphertext, nil) } +// SignPKCS1v15 signs the given data using the given RSA private key. +// The signature will be generated over the SHA-256 hash of the data. +func SignPKCS1v15(signer crypto.Signer, toSign []byte) ([]byte, error) { + hashed := sha256.Sum256(toSign) + return signer.Sign(rand.Reader, hashed[:], crypto.SHA256) +} + +// VerifyPKCS1v15 verifies a signature generated by [SignPKCS1v15] using the given RSA public key. +func VerifyPKCS1v15(pub *rsa.PublicKey, toVerify, signature []byte) error { + hashed := sha256.Sum256(toVerify) + return rsa.VerifyPKCS1v15(pub, crypto.SHA256, hashed[:], signature) +} + // MustGetwd returns the current working directory and panics if it cannot be dcetermined. func MustGetwd() string { // If MarbleRun runs in an enclave, EDG_CWD should be set.