From 02cd531c18dd48f21a5e6ba51f730a1f582e64cf Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Tue, 30 Jul 2024 14:41:03 +0300 Subject: [PATCH] WAL keyring improvements (#238) - Rename database key rotation functions to make room for the global space ones. - Now, during the first start, we would create a default temporary key provider for the global space. A user can (and should) create their own key provider afterwards. This allows use the same codepath and internal interfaces for the keyring management across databases and the global space. - Now need to cache the principal key for the global space as we use it only at the server start to decrypt internal key. Then internal key persists in the memory cache. Fixes https://perconadev.atlassian.net/browse/PG-835, https://perconadev.atlassian.net/browse/PG-833 --- Makefile.in | 2 +- documentation/docs/functions.md | 12 +- documentation/docs/test.md | 6 +- expected/pg_tde_is_encrypted.out | 1 + expected/test_issue_153_fix.out | 1 + meson.build | 2 +- pg_tde--1.0.sql | 97 ++++++++- src/access/pg_tde_tdemap.c | 18 +- src/access/pg_tde_xlog.c | 6 +- src/catalog/tde_global_catalog.c | 248 ----------------------- src/catalog/tde_global_space.c | 226 +++++++++++++++++++++ src/catalog/tde_keyring.c | 81 +++++--- src/catalog/tde_principal_key.c | 134 +++++++----- src/include/access/pg_tde_tdemap.h | 2 +- src/include/catalog/tde_global_catalog.h | 41 ---- src/include/catalog/tde_global_space.h | 36 ++++ src/include/catalog/tde_keyring.h | 13 +- src/include/catalog/tde_principal_key.h | 11 +- src/pg_tde.c | 6 +- src/smgr/pg_tde_smgr.c | 2 +- t/002_rotate_key.pl | 8 +- t/expected/002_rotate_key.out | 4 +- 22 files changed, 528 insertions(+), 429 deletions(-) delete mode 100644 src/catalog/tde_global_catalog.c create mode 100644 src/catalog/tde_global_space.c delete mode 100644 src/include/catalog/tde_global_catalog.h create mode 100644 src/include/catalog/tde_global_space.h diff --git a/Makefile.in b/Makefile.in index eb166667..5b4d98bf 100644 --- a/Makefile.in +++ b/Makefile.in @@ -42,7 +42,7 @@ src/keyring/keyring_curl.o \ src/keyring/keyring_file.o \ src/keyring/keyring_vault.o \ src/keyring/keyring_api.o \ -src/catalog/tde_global_catalog.o \ +src/catalog/tde_global_space.o \ src/catalog/tde_keyring.o \ src/catalog/tde_principal_key.o \ src/common/pg_tde_shmem.o \ diff --git a/documentation/docs/functions.md b/documentation/docs/functions.md index fa59cf38..62fba177 100644 --- a/documentation/docs/functions.md +++ b/documentation/docs/functions.md @@ -39,13 +39,13 @@ Sets the principal key for the database using the specified key provider. The principal key name is also used for constructing the name in the provider, for example on the remote Vault server. -You can use this function only to a principal key. For changes in the principal key, use the [`pg_tde_rotate_key`](#pg_tde_rotate_key) function. +You can use this function only to a principal key. For changes in the principal key, use the [`pg_tde_rotate_principal_key`](#pg_tde_rotate_principal_key) function. ```sql SELECT pg_tde_set_principal_key('name-of-the-principal-key', 'provider-name'); ``` -## pg_tde_rotate_key +## pg_tde_rotate_principal_key Creates a new version of the specified principal key and updates the database so that it uses the new principal key version. @@ -53,23 +53,23 @@ When used without any parameters, the function will just create a new version of principal key, using the same provider: ```sql -SELECT pg_tde_rotate_key(); +SELECT pg_tde_rotate_principal_key(); ``` Alternatively, you can pass two parameters to the function, specifying both a new key name and a new provider name: ```sql -SELECT pg_tde_rotate_key('name-of-the-new-principal-key', 'name-of-the-new-provider'); +SELECT pg_tde_rotate_principal_key('name-of-the-new-principal-key', 'name-of-the-new-provider'); ``` Both parameters support the `NULL` value, which means that the parameter won't be changed: ```sql -- creates new principal key on the same provider as before -SELECT pg_tde_rotate_key('name-of-the-new-principal-key', NULL); +SELECT pg_tde_rotate_principal_key('name-of-the-new-principal-key', NULL); -- copies the current principal key to a new provider -SELECT pg_tde_rotate_key(NULL, 'name-of-the-new-provider'); +SELECT pg_tde_rotate_principal_key(NULL, 'name-of-the-new-provider'); ``` ## pg_tde_is_encrypted diff --git a/documentation/docs/test.md b/documentation/docs/test.md index 917fa3ee..913c6cf3 100644 --- a/documentation/docs/test.md +++ b/documentation/docs/test.md @@ -27,9 +27,9 @@ To check if the data is encrypted, do the following: 3. Rotate the principal key when needed: ```sql - SELECT pg_tde_rotate_key(); -- uses automatic key versionin + SELECT pg_tde_rotate_principal_key(); -- uses automatic key versionin -- or - SELECT pg_tde_rotate_key('new-principal-key', NULL); -- specify new key name + SELECT pg_tde_rotate_principal_key('new-principal-key', NULL); -- specify new key name -- or - SELECT pg_tde_rotate_key('new-principal-key', 'new-provider'); -- change provider + SELECT pg_tde_rotate_principal_key('new-principal-key', 'new-provider'); -- change provider ``` \ No newline at end of file diff --git a/expected/pg_tde_is_encrypted.out b/expected/pg_tde_is_encrypted.out index e7f5774d..64467873 100644 --- a/expected/pg_tde_is_encrypted.out +++ b/expected/pg_tde_is_encrypted.out @@ -2,6 +2,7 @@ CREATE EXTENSION pg_tde; SELECT * FROM pg_tde_principal_key_info(); ERROR: Principal key does not exists for the database HINT: Use set_principal_key interface to set the principal key +CONTEXT: SQL function "pg_tde_principal_key_info" statement 1 SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per'); pg_tde_add_key_provider_file ------------------------------ diff --git a/expected/test_issue_153_fix.out b/expected/test_issue_153_fix.out index 2faf2074..1d3375fc 100644 --- a/expected/test_issue_153_fix.out +++ b/expected/test_issue_153_fix.out @@ -3,6 +3,7 @@ SET datestyle TO 'iso, dmy'; SELECT * FROM pg_tde_principal_key_info(); ERROR: Principal key does not exists for the database HINT: Use set_principal_key interface to set the principal key +CONTEXT: SQL function "pg_tde_principal_key_info" statement 1 SELECT pg_tde_add_key_provider_file('file-ring','/tmp/pg_tde_test_keyring.per'); pg_tde_add_key_provider_file ------------------------------ diff --git a/meson.build b/meson.build index 82410bb0..6445442d 100644 --- a/meson.build +++ b/meson.build @@ -41,7 +41,7 @@ pg_tde_sources = files( 'src/smgr/pg_tde_smgr.c', - 'src/catalog/tde_global_catalog.c', + 'src/catalog/tde_global_space.c', 'src/catalog/tde_keyring.c', 'src/catalog/tde_principal_key.c', 'src/common/pg_tde_shmem.c', diff --git a/pg_tde--1.0.sql b/pg_tde--1.0.sql index 7d98ca82..d9ecd7d9 100644 --- a/pg_tde--1.0.sql +++ b/pg_tde--1.0.sql @@ -3,8 +3,10 @@ -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION pg_tde" to load this file. \quit +CREATE type PG_TDE_GLOBAL AS ENUM('PG_TDE_GLOBAL'); + -- Key Provider Management -CREATE FUNCTION pg_tde_add_key_provider_internal(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON) +CREATE FUNCTION pg_tde_add_key_provider_internal(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON, is_global BOOLEAN) RETURNS INT AS 'MODULE_PATHNAME' LANGUAGE C; @@ -12,7 +14,7 @@ LANGUAGE C; CREATE OR REPLACE FUNCTION pg_tde_add_key_provider(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON) RETURNS INT AS $$ - SELECT pg_tde_add_key_provider_internal(provider_type, provider_name, options); + SELECT pg_tde_add_key_provider_internal(provider_type, provider_name, options, FALSE); $$ LANGUAGE SQL; @@ -65,6 +67,55 @@ RETURNS SETOF record AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; +-- Global Tblespace Key Provider Management +CREATE OR REPLACE FUNCTION pg_tde_add_key_provider(PG_TDE_GLOBAL, provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON) +RETURNS INT +AS $$ + SELECT pg_tde_add_key_provider_internal(provider_type, provider_name, options, TRUE); +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_file(PG_TDE_GLOBAL, provider_name VARCHAR(128), file_path TEXT) +RETURNS INT +AS $$ +-- JSON keys in the options must be matched to the keys in +-- load_file_keyring_provider_options function. + + SELECT pg_tde_add_key_provider('PG_TDE_GLOBAL', 'file', provider_name, + json_object('type' VALUE 'file', 'path' VALUE COALESCE(file_path, ''))); +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_file(PG_TDE_GLOBAL, provider_name VARCHAR(128), file_path JSON) +RETURNS INT +AS $$ +-- JSON keys in the options must be matched to the keys in +-- load_file_keyring_provider_options function. + + SELECT pg_tde_add_key_provider('PG_TDE_GLOBAL', 'file', provider_name, + json_object('type' VALUE 'file', 'path' VALUE file_path)); +$$ +LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_vault_v2(PG_TDE_GLOBAL, + provider_name VARCHAR(128), + vault_token TEXT, + vault_url TEXT, + vault_mount_path TEXT, + vault_ca_path TEXT) +RETURNS INT +AS $$ +-- JSON keys in the options must be matched to the keys in +-- load_vaultV2_keyring_provider_options function. + SELECT pg_tde_add_key_provider('PG_TDE_GLOBAL', 'vault-v2', provider_name, + json_object('type' VALUE 'vault-v2', + 'url' VALUE COALESCE(vault_url,''), + 'token' VALUE COALESCE(vault_token,''), + 'mountPath' VALUE COALESCE(vault_mount_path,''), + 'caPath' VALUE COALESCE(vault_ca_path,''))); +$$ +LANGUAGE SQL; + -- Table access method CREATE FUNCTION pg_tdeam_basic_handler(internal) RETURNS table_am_handler @@ -82,11 +133,25 @@ SELECT EXISTS ( )$$ LANGUAGE SQL; -CREATE FUNCTION pg_tde_rotate_key(new_principal_key_name VARCHAR(255) DEFAULT NULL, new_provider_name VARCHAR(255) DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT TRUE) +CREATE FUNCTION pg_tde_rotate_principal_key_internal(new_principal_key_name VARCHAR(255) DEFAULT NULL, new_provider_name VARCHAR(255) DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT TRUE, is_global BOOLEAN DEFAULT FALSE) RETURNS boolean AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION pg_tde_rotate_principal_key(new_principal_key_name VARCHAR(255) DEFAULT NULL, new_provider_name VARCHAR(255) DEFAULT NULL) +RETURNS boolean +AS $$ + SELECT pg_tde_rotate_principal_key_internal(new_principal_key_name, new_provider_name, TRUE, FALSE); +$$ +LANGUAGE SQL; + +CREATE FUNCTION pg_tde_rotate_principal_key(PG_TDE_GLOBAL, new_principal_key_name VARCHAR(255) DEFAULT NULL, new_provider_name VARCHAR(255) DEFAULT NULL) +RETURNS boolean +AS $$ + SELECT pg_tde_rotate_principal_key_internal(new_principal_key_name, new_provider_name, TRUE, TRUE); +$$ +LANGUAGE SQL; + CREATE FUNCTION pg_tde_set_principal_key(principal_key_name VARCHAR(255), provider_name VARCHAR(255), ensure_new_key BOOLEAN DEFAULT FALSE) RETURNS boolean AS 'MODULE_PATHNAME' @@ -97,7 +162,7 @@ RETURNS VOID AS 'MODULE_PATHNAME' LANGUAGE C; -CREATE FUNCTION pg_tde_principal_key_info() +CREATE FUNCTION pg_tde_principal_key_info_internal(is_global BOOLEAN) RETURNS TABLE ( principal_key_name text, key_provider_name text, key_provider_id integer, @@ -107,6 +172,30 @@ RETURNS TABLE ( principal_key_name text, AS 'MODULE_PATHNAME' LANGUAGE C; +CREATE FUNCTION pg_tde_principal_key_info() +RETURNS TABLE ( principal_key_name text, + key_provider_name text, + key_provider_id integer, + principal_key_internal_name text, + principal_key_version integer, + key_createion_time timestamp with time zone) +AS $$ + SELECT pg_tde_principal_key_info_internal(FALSE); +$$ +LANGUAGE SQL; + +CREATE FUNCTION pg_tde_principal_key_info(PG_TDE_GLOBAL) +RETURNS TABLE ( principal_key_name text, + key_provider_name text, + key_provider_id integer, + principal_key_internal_name text, + principal_key_version integer, + key_createion_time timestamp with time zone) +AS $$ + SELECT pg_tde_principal_key_info_internal(TRUE); +$$ +LANGUAGE SQL; + CREATE FUNCTION pg_tde_version() RETURNS TEXT AS 'MODULE_PATHNAME' LANGUAGE C; -- Access method diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index c274866c..efa3ba59 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -92,7 +92,6 @@ static bool pg_tde_read_one_map_entry(int fd, const RelFileLocator *rlocator, in static void pg_tde_write_keydata(char *db_keydata_path, TDEPrincipalKeyInfo *principal_key_info, int32 key_index, RelKeyData *enc_rel_key_data); static void pg_tde_write_one_keydata(int keydata_fd, int32 key_index, RelKeyData *enc_rel_key_data); -static RelKeyData* pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring); static RelKeyData* pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEPrincipalKey *principal_key); static RelKeyData* pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEPrincipalKey *principal_key); @@ -111,7 +110,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) TDEPrincipalKey *principal_key; XLogRelKey xlrec; - principal_key = GetPrincipalKey(newrlocator->dbOid, newrlocator->spcOid, NULL); + principal_key = GetPrincipalKey(newrlocator->dbOid, newrlocator->spcOid); if (principal_key == NULL) { ereport(ERROR, @@ -164,12 +163,6 @@ RelKey *tde_rel_key_map = NULL; */ RelKeyData * GetRelationKey(RelFileLocator rel) -{ - return GetRelationKeyWithKeyring(rel, NULL); -} - -RelKeyData * -GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring) { RelKey *curr; RelKeyData *key; @@ -183,7 +176,7 @@ GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring) } } - key = pg_tde_get_key_from_file(&rel, keyring); + key = pg_tde_get_key_from_file(&rel); if (key != NULL) { @@ -996,8 +989,8 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) * Reads the key of the required relation. It identifies its map entry and then simply * reads the key data from the keydata file. */ -static RelKeyData * -pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring) +RelKeyData * +pg_tde_get_key_from_file(const RelFileLocator *rlocator) { int32 key_index = 0; TDEPrincipalKey *principal_key; @@ -1013,7 +1006,7 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring LWLockAcquire(lock_files, LW_SHARED); /* Get/generate a principal key, create the key for relation and get the encrypted key with bytes to write */ - principal_key = GetPrincipalKey(rlocator->dbOid, rlocator->spcOid, keyring); + principal_key = GetPrincipalKey(rlocator->dbOid, rlocator->spcOid); if (principal_key == NULL) { LWLockRelease(lock_files); @@ -1179,6 +1172,7 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p xlrec->keydata_size = keydata_size; /* TODO: pgstat_report_wait_start / pgstat_report_wait_end */ + /* TODO: error handling */ pg_pread(m_fd[NEW_PRINCIPAL_KEY], xlrec->buff, xlrec->map_size, 0); pg_pread(k_fd[NEW_PRINCIPAL_KEY], &xlrec->buff[xlrec->map_size], xlrec->keydata_size, 0); diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index d02eb936..a19d50bf 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -28,7 +28,7 @@ #include "access/pg_tde_xlog.h" #include "encryption/enc_tde.h" #ifdef PERCONA_FORK -#include "catalog/tde_global_catalog.h" +#include "catalog/tde_global_space.h" static char *TDEXLogEncryptBuf = NULL; @@ -240,7 +240,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset) size_t data_size = 0; XLogPageHeader curr_page_hdr = &EncryptCurrentPageHrd; XLogPageHeader enc_buf_page; - RelKeyData *key = GetGlCatInternalKey(XLOG_TDE_OID); + RelKeyData *key = TDEGetGlobalInternalKey(XLOG_TDE_OID); off_t enc_off; size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; uint32 iv_ctr = 0; @@ -327,7 +327,7 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset) char iv_prefix[16] = {0,}; size_t data_size = 0; XLogPageHeader curr_page_hdr = &DecryptCurrentPageHrd; - RelKeyData *key = GetGlCatInternalKey(XLOG_TDE_OID); + RelKeyData *key = TDEGetGlobalInternalKey(XLOG_TDE_OID); size_t page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ; off_t dec_off; uint32 iv_ctr = 0; diff --git a/src/catalog/tde_global_catalog.c b/src/catalog/tde_global_catalog.c deleted file mode 100644 index 4124079b..00000000 --- a/src/catalog/tde_global_catalog.c +++ /dev/null @@ -1,248 +0,0 @@ -/*------------------------------------------------------------------------- - * - * tde_global_catalog.c - * Global catalog key management - * - * - * IDENTIFICATION - * src/catalog/tde_global_catalog.c - * - *------------------------------------------------------------------------- - */ - -#include "postgres.h" - -#ifdef PERCONA_FORK - -#include "storage/shmem.h" -#include "utils/guc.h" - -#include "access/pg_tde_tdemap.h" -#include "catalog/tde_global_catalog.h" -#include "catalog/tde_keyring.h" -#include "catalog/tde_principal_key.h" - -#include -#include -#include - -#define PRINCIPAL_KEY_DEFAULT_NAME "tde-global-catalog-key" - -/* TODO: not sure if we need an option of multiple principal keys for the global catalog */ -typedef enum -{ - TDE_GCAT_XLOG_KEY, - - /* must be last */ - TDE_GCAT_KEYS_COUNT -} GlobalCatalogKeyTypes; - -typedef struct EncryptionStateData -{ - GenericKeyring *keyring; - TDEPrincipalKey principal_keys[TDE_GCAT_KEYS_COUNT]; -} EncryptionStateData; - -static EncryptionStateData * EncryptionState = NULL; - -/* GUC */ -static char *KRingProviderType = NULL; -static char *KRingProviderFilePath = NULL; - -static void init_gl_catalog_keys(void); -static void init_keyring(void); -static TDEPrincipalKey * create_principal_key(const char *key_name, - GenericKeyring * keyring, Oid dbOid, Oid spcOid, - bool ensure_new_key); - -void -TDEGlCatInitGUC(void) -{ - DefineCustomStringVariable("pg_tde.global_keyring_type", - "Keyring type for global catalog", - NULL, - &KRingProviderType, - NULL, - PGC_POSTMASTER, - 0, /* no flags required */ - NULL, - NULL, - NULL - ); - DefineCustomStringVariable("pg_tde.global_keyring_file_path", - "Keyring file options for global catalog", - NULL, - &KRingProviderFilePath, - NULL, - PGC_POSTMASTER, - 0, /* no flags required */ - NULL, - NULL, - NULL - ); -} - - -Size -TDEGlCatEncStateSize(void) -{ - Size size; - - size = sizeof(EncryptionStateData); - size = add_size(size, sizeof(KeyringProviders)); - - return MAXALIGN(size); -} - -void -TDEGlCatShmemInit(void) -{ - bool foundBuf; - char *allocptr; - - EncryptionState = (EncryptionStateData *) - ShmemInitStruct("TDE XLog Encryption State", - TDEGlCatEncStateSize(), &foundBuf); - - allocptr = ((char *) EncryptionState) + MAXALIGN(sizeof(EncryptionStateData)); - EncryptionState->keyring = (GenericKeyring *) allocptr; - memset(EncryptionState->keyring, 0, sizeof(KeyringProviders)); - memset(EncryptionState->principal_keys, 0, sizeof(TDEPrincipalKey) * TDE_GCAT_KEYS_COUNT); -} - -void -TDEGlCatKeyInit(void) -{ - char db_map_path[MAXPGPATH] = {0}; - - init_keyring(); - - pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), - db_map_path, NULL); - if (access(db_map_path, F_OK) == -1) - { - init_gl_catalog_keys(); - } - else - { - /* put an internal key into the cache */ - GetGlCatInternalKey(XLOG_TDE_OID); - } -} - -TDEPrincipalKey * -TDEGetGlCatKeyFromCache(void) -{ - TDEPrincipalKey *mkey; - - mkey = &EncryptionState->principal_keys[TDE_GCAT_XLOG_KEY]; - if (mkey->keyLength == 0) - return NULL; - - return mkey; -} - -void -TDEPutGlCatKeyInCache(TDEPrincipalKey * mkey) -{ - memcpy(EncryptionState->principal_keys + TDE_GCAT_XLOG_KEY, mkey, sizeof(TDEPrincipalKey)); -} - -RelKeyData * -GetGlCatInternalKey(Oid obj_id) -{ - return GetRelationKeyWithKeyring(GLOBAL_SPACE_RLOCATOR(obj_id), EncryptionState->keyring); -} - -/* - * TODO: should be aligned with the rest of the keyring_provider code after its - * refactoring - * - * TODO: add Vault - */ -static void -init_keyring(void) -{ - EncryptionState->keyring->type = get_keyring_provider_from_typename(KRingProviderType); - switch (EncryptionState->keyring->type) - { - case FILE_KEY_PROVIDER: - FileKeyring * kring = (FileKeyring *) EncryptionState->keyring; - strncpy(kring->file_name, KRingProviderFilePath, sizeof(kring->file_name)); - break; - } -} - -/* - * Keys are created during the cluster start only, so no locks needed here. - */ -static void -init_gl_catalog_keys(void) -{ - InternalKey int_key; - RelKeyData *rel_key_data; - RelKeyData *enc_rel_key_data; - RelFileLocator *rlocator; - TDEPrincipalKey *mkey; - - mkey = create_principal_key(PRINCIPAL_KEY_DEFAULT_NAME, - EncryptionState->keyring, - GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, false); - - memset(&int_key, 0, sizeof(InternalKey)); - - /* Create and store an internal key for XLog */ - if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) - { - ereport(FATAL, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate internal key for \"WAL\": %s", - ERR_error_string(ERR_get_error(), NULL)))); - } - - rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); - rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &mkey->keyInfo); - enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); - pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); - - /* - * TODO: move global catalog internal keys into own cache. This cache should - * be in the TopMemmoryContext because of SSL contexts - * (see https://github.com/Percona-Lab/pg_tde/pull/214#discussion_r1648998317) - */ - pg_tde_put_key_into_map(rlocator->relNumber, rel_key_data); - TDEPutGlCatKeyInCache(mkey); -} - -static TDEPrincipalKey * -create_principal_key(const char *key_name, GenericKeyring * keyring, - Oid dbOid, Oid spcOid, bool ensure_new_key) -{ - TDEPrincipalKey *principalKey; - keyInfo *keyInfo = NULL; - - principalKey = palloc(sizeof(TDEPrincipalKey)); - principalKey->keyInfo.databaseId = dbOid; - principalKey->keyInfo.tablespaceId = spcOid; - principalKey->keyInfo.keyId.version = DEFAULT_PRINCIPAL_KEY_VERSION; - principalKey->keyInfo.keyringId = keyring->key_id; - strncpy(principalKey->keyInfo.keyId.name, key_name, TDE_KEY_NAME_LEN); - gettimeofday(&principalKey->keyInfo.creationTime, NULL); - - keyInfo = load_latest_versioned_key_name(&principalKey->keyInfo, keyring, ensure_new_key); - - if (keyInfo == NULL) - keyInfo = KeyringGenerateNewKeyAndStore(keyring, principalKey->keyInfo.keyId.versioned_name, INTERNAL_KEY_LEN, false); - - if (keyInfo == NULL) - { - ereport(ERROR, - (errmsg("failed to retrieve principal key"))); - } - - principalKey->keyLength = keyInfo->data.len; - memcpy(principalKey->keyData, keyInfo->data.data, keyInfo->data.len); - - return principalKey; -} -#endif /* PERCONA_FORK */ diff --git a/src/catalog/tde_global_space.c b/src/catalog/tde_global_space.c new file mode 100644 index 00000000..d45d24a8 --- /dev/null +++ b/src/catalog/tde_global_space.c @@ -0,0 +1,226 @@ +/*------------------------------------------------------------------------- + * + * tde_global_space.c + * Global catalog key management + * + * + * IDENTIFICATION + * src/catalog/tde_global_space.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#ifdef PERCONA_FORK + +#include "catalog/pg_tablespace_d.h" +#include "nodes/pg_list.h" +#include "storage/shmem.h" +#include "utils/guc.h" +#include "utils/memutils.h" + +#include "access/pg_tde_tdemap.h" +#include "catalog/tde_global_space.h" +#include "catalog/tde_keyring.h" +#include "catalog/tde_principal_key.h" + +#include +#include +#include + +#define PRINCIPAL_KEY_DEFAULT_NAME "tde-global-catalog-key" +#define KEYRING_DEFAULT_NAME "default_global_tablespace_keyring" + +#define DefaultKeyProvider GetKeyProviderByName(KEYRING_DEFAULT_NAME, \ + GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID) + +typedef enum +{ + TDE_INTERNAL_XLOG_KEY, + + /* must be last */ + TDE_INTERNAL_KEYS_COUNT +} InternalKeyType; + +/* Since for the global tablespace, we always keep the Internal key in the memory + * and read it from disk only once during the server start, we need no cache for + * the principal key. + */ +static RelKeyData * internal_keys_cache = NULL; + +static void init_keys(void); +static void init_default_keyring(void); +static TDEPrincipalKey * create_principal_key(const char *key_name, + GenericKeyring * keyring, Oid dbOid, + Oid spcOid); +static void cache_internal_key(RelKeyData * ikey, InternalKeyType type); + +void +TDEInitGlobalKeys(void) +{ + char db_map_path[MAXPGPATH] = {0}; + + pg_tde_set_db_file_paths(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID), + db_map_path, NULL); + if (access(db_map_path, F_OK) == -1) + { + init_default_keyring(); + init_keys(); + } + else + { + RelKeyData *ikey; + + ikey = pg_tde_get_key_from_file(&GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID)); + cache_internal_key(ikey, TDE_INTERNAL_XLOG_KEY); + } +} + +/* + * Internal Key should be in the TopMemmoryContext because of SSL contexts. This + * context is being initialized by OpenSSL with the pointer to the encryption + * context which is valid only for the current backend. So new backends have to + * inherit a cached key with NULL SSL connext and any changes to it have to remain + * local ot the backend. + * (see https://github.com/Percona-Lab/pg_tde/pull/214#discussion_r1648998317) + */ +static void +cache_internal_key(RelKeyData * ikey, InternalKeyType type) +{ + if (internal_keys_cache == NULL) + { + internal_keys_cache = + (RelKeyData *) MemoryContextAlloc(TopMemoryContext, + sizeof(RelKeyData) * TDE_INTERNAL_KEYS_COUNT); + } + memcpy(internal_keys_cache + type, ikey, sizeof(RelKeyData)); +} + + +RelKeyData * +TDEGetGlobalInternalKey(Oid obj_id) +{ + InternalKeyType ktype; + + Assert(internal_keys_cache != NULL); + switch (obj_id) + { + case XLOG_TDE_OID: + ktype = TDE_INTERNAL_XLOG_KEY; + break; + default: + elog(ERROR, "unknown internal key for Oid %u", obj_id); + } + return internal_keys_cache + ktype; +} + +static void +init_default_keyring(void) +{ + if (GetAllKeyringProviders(GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID) == NIL) + { + static KeyringProvideRecord provider = + { + .provider_name = KEYRING_DEFAULT_NAME, + .provider_type = FILE_KEY_PROVIDER, + .options = + "{" + "\"type\": \"file\"," + " \"path\": \"pg_tde_default_keyring_CHANGE_AND_REMOVE_IT\"" /* TODO: not sure about + * the location */ + "}" + }; + + /* + * TODO: should we remove it automaticaly on + * pg_tde_rotate_principal_key() ? + */ + save_new_key_provider_info(&provider, GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID, true); + elog(INFO, + "default keyring has been created for the global tablespace (WAL)." + " Change it with pg_tde_add_key_provider_* and run pg_tde_rotate_principal_key." + ); + } +} + +/* + * Create and store global space keys (principal and internal) and cache the + * internal key. + * + * This function has to be run during the cluster start only, so no locks here. + */ +static void +init_keys(void) +{ + InternalKey int_key; + RelKeyData *rel_key_data; + RelKeyData *enc_rel_key_data; + RelFileLocator *rlocator; + TDEPrincipalKey *mkey; + + mkey = create_principal_key(PRINCIPAL_KEY_DEFAULT_NAME, + DefaultKeyProvider, + GLOBAL_DATA_TDE_OID, GLOBALTABLESPACE_OID); + + memset(&int_key, 0, sizeof(InternalKey)); + + /* Create and store an internal key for XLog */ + if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) + { + ereport(FATAL, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate internal key for \"WAL\": %s", + ERR_error_string(ERR_get_error(), NULL)))); + } + + rlocator = &GLOBAL_SPACE_RLOCATOR(XLOG_TDE_OID); + rel_key_data = tde_create_rel_key(rlocator->relNumber, &int_key, &mkey->keyInfo); + enc_rel_key_data = tde_encrypt_rel_key(mkey, rel_key_data, rlocator); + pg_tde_write_key_map_entry(rlocator, enc_rel_key_data, &mkey->keyInfo); + + cache_internal_key(rel_key_data, TDE_INTERNAL_XLOG_KEY); + pfree(rel_key_data); + pfree(mkey); +} + +/* + * Substantially simplified version of set_principal_key_with_keyring() as during + * recovery (server start): + * - we can't insert XLog records; + * - no need for locks; + * - we run this func only once, during the first server start and always create + * a new key with the default keyring, hence no need to try to load the key + * first. + */ +static TDEPrincipalKey * +create_principal_key(const char *key_name, GenericKeyring * keyring, + Oid dbOid, Oid spcOid) +{ + TDEPrincipalKey *principalKey; + keyInfo *keyInfo = NULL; + + principalKey = palloc(sizeof(TDEPrincipalKey)); + principalKey->keyInfo.databaseId = dbOid; + principalKey->keyInfo.tablespaceId = spcOid; + principalKey->keyInfo.keyId.version = DEFAULT_PRINCIPAL_KEY_VERSION; + principalKey->keyInfo.keyringId = keyring->key_id; + strncpy(principalKey->keyInfo.keyId.name, key_name, TDE_KEY_NAME_LEN); + snprintf(principalKey->keyInfo.keyId.versioned_name, TDE_KEY_NAME_LEN, + "%s_%d", principalKey->keyInfo.keyId.name, principalKey->keyInfo.keyId.version); + gettimeofday(&principalKey->keyInfo.creationTime, NULL); + + keyInfo = KeyringGenerateNewKeyAndStore(keyring, principalKey->keyInfo.keyId.versioned_name, INTERNAL_KEY_LEN, false); + + if (keyInfo == NULL) + { + ereport(ERROR, + (errmsg("failed to generate principal key"))); + } + + principalKey->keyLength = keyInfo->data.len; + memcpy(principalKey->keyData, keyInfo->data.data, keyInfo->data.len); + + return principalKey; +} +#endif /* PERCONA_FORK */ diff --git a/src/catalog/tde_keyring.c b/src/catalog/tde_keyring.c index 6c5b4431..df3509a5 100644 --- a/src/catalog/tde_keyring.c +++ b/src/catalog/tde_keyring.c @@ -14,6 +14,7 @@ #include "access/xlog.h" #include "access/xloginsert.h" #include "access/pg_tde_xlog.h" +#include "catalog/tde_global_space.h" #include "catalog/tde_keyring.h" #include "catalog/tde_principal_key.h" #include "access/skey.h" @@ -64,7 +65,7 @@ typedef enum ProviderScanType PROVIDER_SCAN_ALL } ProviderScanType; -static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey); +static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid, Oid spcOid); static FileKeyring *load_file_keyring_provider_options(Datum keyring_options); static GenericKeyring *load_keyring_provider_options(ProviderType provider_type, Datum keyring_options); @@ -72,9 +73,11 @@ static VaultV2Keyring *load_vaultV2_keyring_provider_options(Datum keyring_optio static void debug_print_kerying(GenericKeyring *keyring); static char *get_keyring_infofile_path(char *resPath, Oid dbOid, Oid spcOid); static void key_provider_startup_cleanup(int tde_tbl_count, XLogExtensionInstall *ext_info, bool redo, void *arg); -static uint32 write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, Oid tablespace_id, off_t position, bool redo); -static uint32 save_new_key_provider_info(KeyringProvideRecord *provider); static const char *get_keyring_provider_typename(ProviderType p_type); +static uint32 write_key_provider_info(KeyringProvideRecord *provider, + Oid database_id, Oid tablespace_id, + off_t position, bool redo, bool recovery); + static Size initialize_shared_state(void *start_address); static Size required_shared_mem_size(void); @@ -180,16 +183,16 @@ static GenericKeyring *load_keyring_provider_from_record(KeyringProvideRecord *p } List * -GetAllKeyringProviders(void) +GetAllKeyringProviders(Oid dbOid, Oid spcOid) { - return scan_key_provider_file(PROVIDER_SCAN_ALL, NULL); + return scan_key_provider_file(PROVIDER_SCAN_ALL, NULL, dbOid, spcOid); } GenericKeyring * -GetKeyProviderByName(const char *provider_name) +GetKeyProviderByName(const char *provider_name, Oid dbOid, Oid spcOid) { GenericKeyring *keyring = NULL; - List *providers = scan_key_provider_file(PROVIDER_SCAN_BY_NAME, (void*)provider_name); + List *providers = scan_key_provider_file(PROVIDER_SCAN_BY_NAME, (void*)provider_name, dbOid, spcOid); if (providers != NIL) { keyring = (GenericKeyring *)linitial(providers); @@ -206,10 +209,10 @@ GetKeyProviderByName(const char *provider_name) } GenericKeyring * -GetKeyProviderByID(int provider_id) +GetKeyProviderByID(int provider_id, Oid dbOid, Oid spcOid) { GenericKeyring *keyring = NULL; - List *providers = scan_key_provider_file(PROVIDER_SCAN_BY_ID, &provider_id); + List *providers = scan_key_provider_file(PROVIDER_SCAN_BY_ID, &provider_id, dbOid, spcOid); if (providers != NIL) { keyring = (GenericKeyring *)linitial(providers); @@ -330,7 +333,8 @@ fetch_next_key_provider(int fd, off_t* curr_pos, KeyringProvideRecord *provider) } static uint32 -write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, Oid tablespace_id, off_t position, bool redo) +write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, + Oid tablespace_id, off_t position, bool redo, bool recovery) { off_t bytes_written = 0; off_t curr_pos = 0; @@ -355,7 +359,6 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, Oid tab } if (!redo) { - KeyringProviderXLRecord xlrec; /* we also need to verify the name conflict and generate the next provider ID */ while (fetch_next_key_provider(fd, &curr_pos, &existing_provider)) { @@ -372,15 +375,23 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, Oid tab } provider->provider_id = max_provider_id + 1; curr_pos = lseek(fd, 0, SEEK_END); - /* emit the xlog here. So that we can handle partial file write errors */ - xlrec.database_id = database_id; - xlrec.tablespace_id = tablespace_id; - xlrec.offset_in_file = curr_pos; - memcpy(&xlrec.provider, provider, sizeof(KeyringProvideRecord)); - - XLogBeginInsert(); - XLogRegisterData((char *)&xlrec, sizeof(KeyringProviderXLRecord)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_KEY_PROVIDER_KEY); + + /* emit the xlog here. So that we can handle partial file write errors + * but cannot make new WAL entries during recovery. + */ + if (!recovery) + { + KeyringProviderXLRecord xlrec; + + xlrec.database_id = database_id; + xlrec.tablespace_id = tablespace_id; + xlrec.offset_in_file = curr_pos; + memcpy(&xlrec.provider, provider, sizeof(KeyringProvideRecord)); + + XLogBeginInsert(); + XLogRegisterData((char *)&xlrec, sizeof(KeyringProviderXLRecord)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_ADD_KEY_PROVIDER_KEY); + } } else { @@ -420,22 +431,22 @@ write_key_provider_info(KeyringProvideRecord *provider, Oid database_id, Oid tab /* * Save the key provider info to the file */ -static uint32 -save_new_key_provider_info(KeyringProvideRecord* provider) +uint32 +save_new_key_provider_info(KeyringProvideRecord* provider, Oid databaseId, Oid tablespaceId, bool recovery) { - return write_key_provider_info(provider, MyDatabaseId, MyDatabaseTableSpace, 0, false); + return write_key_provider_info(provider, databaseId, tablespaceId, 0, false, recovery); } uint32 redo_key_provider_info(KeyringProviderXLRecord* xlrec) { - return write_key_provider_info(&xlrec->provider, xlrec->database_id, xlrec->tablespace_id, xlrec->offset_in_file, true); + return write_key_provider_info(&xlrec->provider, xlrec->database_id, xlrec->tablespace_id, xlrec->offset_in_file, true, false); } /* * Scan the key provider info file and can also apply filter based on scanType */ -static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey) +static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey, Oid dbOid, Oid spcOid) { off_t curr_pos = 0; int fd; @@ -446,7 +457,7 @@ static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey) if (scanType != PROVIDER_SCAN_ALL) Assert(scanKey != NULL); - get_keyring_infofile_path(kp_info_path, MyDatabaseId, MyDatabaseTableSpace); + get_keyring_infofile_path(kp_info_path, dbOid, spcOid); LWLockAcquire(tde_provider_info_lock(), LW_SHARED); @@ -500,10 +511,10 @@ static List *scan_key_provider_file(ProviderScanType scanType, void *scanKey) void cleanup_key_provider_info(Oid databaseId, Oid tablespaceId) { - /* Remove the key provider info fileß */ + /* Remove the key provider info file */ char kp_info_path[MAXPGPATH] = {0}; - get_keyring_infofile_path(kp_info_path, MyDatabaseId, MyDatabaseTableSpace); + get_keyring_infofile_path(kp_info_path, databaseId, tablespaceId); PathNameDeleteTemporaryFile(kp_info_path, false); } @@ -523,12 +534,21 @@ pg_tde_add_key_provider_internal(PG_FUNCTION_ARGS) char *provider_type = text_to_cstring(PG_GETARG_TEXT_PP(0)); char *provider_name = text_to_cstring(PG_GETARG_TEXT_PP(1)); char *options = text_to_cstring(PG_GETARG_TEXT_PP(2)); + bool is_global = PG_GETARG_BOOL(3); KeyringProvideRecord provider; + Oid dbOid = MyDatabaseId; + Oid spcOid = MyDatabaseTableSpace; + + if (is_global) + { + dbOid = GLOBAL_DATA_TDE_OID; + spcOid = GLOBALTABLESPACE_OID; + } strncpy(provider.options, options, sizeof(provider.options)); strncpy(provider.provider_name, provider_name, sizeof(provider.provider_name)); provider.provider_type = get_keyring_provider_from_typename(provider_type); - save_new_key_provider_info(&provider); + save_new_key_provider_info(&provider, dbOid, spcOid, false); PG_RETURN_INT32(provider.provider_id); } @@ -536,7 +556,7 @@ pg_tde_add_key_provider_internal(PG_FUNCTION_ARGS) Datum pg_tde_list_all_key_providers(PG_FUNCTION_ARGS) { - List* all_providers = GetAllKeyringProviders(); + List* all_providers = GetAllKeyringProviders(MyDatabaseId, MyDatabaseTableSpace); ListCell *lc; Tuplestorestate *tupstore; TupleDesc tupdesc; @@ -583,7 +603,6 @@ pg_tde_list_all_key_providers(PG_FUNCTION_ARGS) debug_print_kerying(keyring); } - tuplestore_donestoring(tupstore); list_free_deep(all_providers); return (Datum)0; } diff --git a/src/catalog/tde_principal_key.c b/src/catalog/tde_principal_key.c index 0571dc3a..09f28288 100644 --- a/src/catalog/tde_principal_key.c +++ b/src/catalog/tde_principal_key.c @@ -29,9 +29,7 @@ #include #include "access/pg_tde_tdemap.h" -#ifdef PERCONA_FORK -#include "catalog/tde_global_catalog.h" -#endif +#include "catalog/tde_global_space.h" typedef struct TdePrincipalKeySharedState { @@ -71,6 +69,14 @@ static void clear_principal_key_cache(Oid databaseId) ; static inline dshash_table *get_principal_key_Hash(void); static TDEPrincipalKey *get_principal_key_from_cache(Oid dbOid); static void push_principal_key_to_cache(TDEPrincipalKey *principalKey); +static Datum pg_tde_get_key_info(PG_FUNCTION_ARGS, Oid dbOid, Oid spcOid); +static keyInfo *load_latest_versioned_key_name(TDEPrincipalKeyInfo *principal_key_info, + GenericKeyring *keyring, + bool ensure_new_key); +static TDEPrincipalKey *set_principal_key_with_keyring(const char *key_name, + GenericKeyring *keyring, + Oid dbOid, Oid spcOid, + bool ensure_new_key); static const TDEShmemSetupRoutine principal_key_info_shmem_routine = { .init_shared_state = initialize_shared_state, @@ -205,8 +211,9 @@ save_principal_key_info(TDEPrincipalKeyInfo *principal_key_info) * throws an error. */ TDEPrincipalKey * -GetPrincipalKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) +GetPrincipalKey(Oid dbOid, Oid spcOid) { + GenericKeyring *keyring; TDEPrincipalKey *principalKey = NULL; TDEPrincipalKeyInfo *principalKeyInfo = NULL; const keyInfo *keyInfo = NULL; @@ -225,15 +232,13 @@ GetPrincipalKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) recursion++; - LWLockAcquire(lock_cache, LW_SHARED); -#ifdef PERCONA_FORK - /* Global catalog has its own cache */ - if (spcOid == GLOBALTABLESPACE_OID) - principalKey = TDEGetGlCatKeyFromCache(); - else -#endif + /* We don't store global space key in cache */ + if (spcOid != GLOBALTABLESPACE_OID) + { + LWLockAcquire(lock_cache, LW_SHARED); principalKey = get_principal_key_from_cache(dbOid); - LWLockRelease(lock_cache); + LWLockRelease(lock_cache); + } if (principalKey) { @@ -248,13 +253,11 @@ GetPrincipalKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) LWLockAcquire(lock_files, LW_SHARED); LWLockAcquire(lock_cache, LW_EXCLUSIVE); -#ifdef PERCONA_FORK - /* Global catalog has its own cache */ - if (spcOid == GLOBALTABLESPACE_OID) - principalKey = TDEGetGlCatKeyFromCache(); - else -#endif + /* We don't store global space key in cache */ + if (spcOid != GLOBALTABLESPACE_OID) + { principalKey = get_principal_key_from_cache(dbOid); + } if (principalKey) { @@ -275,17 +278,14 @@ GetPrincipalKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) return NULL; } + keyring = GetKeyProviderByID(principalKeyInfo->keyringId, dbOid, spcOid); if (keyring == NULL) { - keyring = GetKeyProviderByID(principalKeyInfo->keyringId); - if (keyring == NULL) - { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); + LWLockRelease(lock_cache); + LWLockRelease(lock_files); - recursion--; - return NULL; - } + recursion--; + return NULL; } keyInfo = KeyringGetKey(keyring, principalKeyInfo->keyId.versioned_name, false, &keyring_ret); @@ -305,12 +305,11 @@ GetPrincipalKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring) principalKey->keyLength = keyInfo->data.len; Assert(dbOid == principalKey->keyInfo.databaseId); -#ifdef PERCONA_FORK - if (spcOid == GLOBALTABLESPACE_OID) - TDEPutGlCatKeyInCache(principalKey); - else -#endif + /* We don't store global space key in cache */ + if (spcOid != GLOBALTABLESPACE_OID) + { push_principal_key_to_cache(principalKey); + } /* Release the exclusive locks here */ LWLockRelease(lock_cache); @@ -417,7 +416,7 @@ bool SetPrincipalKey(const char *key_name, const char *provider_name, bool ensure_new_key) { TDEPrincipalKey *principal_key = set_principal_key_with_keyring(key_name, - GetKeyProviderByName(provider_name), + GetKeyProviderByName(provider_name, MyDatabaseId, MyDatabaseTableSpace), MyDatabaseId, MyDatabaseTableSpace, ensure_new_key); @@ -425,19 +424,20 @@ SetPrincipalKey(const char *key_name, const char *provider_name, bool ensure_new } bool -RotatePrincipalKey(const char *new_key_name, const char *new_provider_name, bool ensure_new_key) +RotatePrincipalKey(TDEPrincipalKey *current_key, const char *new_key_name, const char *new_provider_name, bool ensure_new_key) { - TDEPrincipalKey *principal_key = GetPrincipalKey(MyDatabaseId, MyDatabaseTableSpace, NULL); TDEPrincipalKey new_principal_key; const keyInfo *keyInfo = NULL; GenericKeyring *keyring; bool is_rotated; + Assert(current_key != NULL); + /* * Let's set everything the same as the older principal key and * update only the required attributes. * */ - memcpy(&new_principal_key, principal_key, sizeof(TDEPrincipalKey)); + memcpy(&new_principal_key, current_key, sizeof(TDEPrincipalKey)); if (new_key_name == NULL) { @@ -450,12 +450,16 @@ RotatePrincipalKey(const char *new_key_name, const char *new_provider_name, bool if (new_provider_name != NULL) { - new_principal_key.keyInfo.keyringId = GetKeyProviderByName(new_provider_name)->key_id; + new_principal_key.keyInfo.keyringId = GetKeyProviderByName(new_provider_name, + new_principal_key.keyInfo.databaseId, + new_principal_key.keyInfo.tablespaceId)->key_id; } } /* We need a valid keyring structure */ - keyring = GetKeyProviderByID(new_principal_key.keyInfo.keyringId); + keyring = GetKeyProviderByID(new_principal_key.keyInfo.keyringId, + new_principal_key.keyInfo.databaseId, + new_principal_key.keyInfo.tablespaceId); keyInfo = load_latest_versioned_key_name(&new_principal_key.keyInfo, keyring, ensure_new_key); @@ -470,9 +474,9 @@ RotatePrincipalKey(const char *new_key_name, const char *new_provider_name, bool new_principal_key.keyLength = keyInfo->data.len; memcpy(new_principal_key.keyData, keyInfo->data.data, keyInfo->data.len); - is_rotated = pg_tde_perform_rotate_key(principal_key, &new_principal_key); - if (is_rotated) { - clear_principal_key_cache(principal_key->keyInfo.databaseId); + is_rotated = pg_tde_perform_rotate_key(current_key, &new_principal_key); + if (is_rotated && current_key->keyInfo.tablespaceId != GLOBALTABLESPACE_OID) { + clear_principal_key_cache(current_key->keyInfo.databaseId); push_principal_key_to_cache(&new_principal_key); } @@ -520,7 +524,7 @@ load_latest_versioned_key_name(TDEPrincipalKeyInfo *principal_key_info, GenericK /* vault-v2 returns 404 (KEYRING_CODE_RESOURCE_NOT_AVAILABLE) when key is not found */ if (kr_ret != KEYRING_CODE_SUCCESS && kr_ret != KEYRING_CODE_RESOURCE_NOT_AVAILABLE) { - ereport(ERROR, + ereport(FATAL, (errmsg("failed to retrieve principal key from keyring provider :\"%s\"", keyring->provider_name), errdetail("Error code: %d", kr_ret))); } @@ -718,29 +722,61 @@ Datum pg_tde_set_principal_key(PG_FUNCTION_ARGS) /* * SQL interface for key rotation */ -PG_FUNCTION_INFO_V1(pg_tde_rotate_key); +PG_FUNCTION_INFO_V1(pg_tde_rotate_principal_key_internal); Datum -pg_tde_rotate_key(PG_FUNCTION_ARGS) +pg_tde_rotate_principal_key_internal(PG_FUNCTION_ARGS) { char *new_principal_key_name = NULL; char *new_provider_name = NULL; bool ensure_new_key; + bool is_global; bool ret; + TDEPrincipalKey *current_key; + Oid dbOid = MyDatabaseId; + Oid spcOid = MyDatabaseTableSpace; if (!PG_ARGISNULL(0)) new_principal_key_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); if (!PG_ARGISNULL(1)) new_provider_name = text_to_cstring(PG_GETARG_TEXT_PP(1)); ensure_new_key = PG_GETARG_BOOL(2); + is_global = PG_GETARG_BOOL(3); +#ifdef PERCONA_FORK + if (is_global) + { + dbOid = GLOBAL_DATA_TDE_OID; + spcOid = GLOBALTABLESPACE_OID; + } +#endif - ereport(LOG, (errmsg("Rotating principal key to [%s : %s] for the database", new_principal_key_name, new_provider_name))); - ret = RotatePrincipalKey(new_principal_key_name, new_provider_name, ensure_new_key); + ereport(LOG, (errmsg("rotating principal key to [%s : %s] the for the %s", + new_principal_key_name, + new_provider_name, + is_global ? "cluster" : "database"))); + current_key = GetPrincipalKey(dbOid, spcOid); + ret = RotatePrincipalKey(current_key, new_principal_key_name, new_provider_name, ensure_new_key); PG_RETURN_BOOL(ret); } -PG_FUNCTION_INFO_V1(pg_tde_principal_key_info); -Datum pg_tde_principal_key_info(PG_FUNCTION_ARGS) +PG_FUNCTION_INFO_V1(pg_tde_principal_key_info_internal); +Datum pg_tde_principal_key_info_internal(PG_FUNCTION_ARGS) +{ + Oid dbOid = MyDatabaseId; + Oid spcOid = MyDatabaseTableSpace; + bool is_global = PG_GETARG_BOOL(0); + + if (is_global) + { + dbOid = GLOBAL_DATA_TDE_OID; + spcOid = GLOBALTABLESPACE_OID; + } + + return pg_tde_get_key_info(fcinfo, dbOid, spcOid); +} + +static Datum +pg_tde_get_key_info(PG_FUNCTION_ARGS, Oid dbOid, Oid spcOid) { TupleDesc tupdesc; Datum values[6]; @@ -757,7 +793,7 @@ Datum pg_tde_principal_key_info(PG_FUNCTION_ARGS) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context that cannot accept type record"))); - principal_key = GetPrincipalKey(MyDatabaseId, MyDatabaseTableSpace, NULL); + principal_key = GetPrincipalKey(dbOid, spcOid); if (principal_key == NULL) { ereport(ERROR, @@ -766,7 +802,7 @@ Datum pg_tde_principal_key_info(PG_FUNCTION_ARGS) PG_RETURN_NULL(); } - keyring = GetKeyProviderByID(principal_key->keyInfo.keyringId); + keyring = GetKeyProviderByID(principal_key->keyInfo.keyringId, dbOid, spcOid); /* Initialize the values and null flags */ diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 5c2cbee9..db629b18 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -53,7 +53,6 @@ extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator); extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); extern RelKeyData *GetRelationKey(RelFileLocator rel); -extern RelKeyData *GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring); extern void pg_tde_delete_tde_files(Oid dbOid, Oid spcOid); @@ -64,6 +63,7 @@ extern bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, of extern RelKeyData* tde_create_rel_key(Oid rel_id, InternalKey *key, TDEPrincipalKeyInfo *principal_key_info); extern RelKeyData *tde_encrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *rel_key_data, const RelFileLocator *rlocator); extern RelKeyData *tde_decrypt_rel_key(TDEPrincipalKey *principal_key, RelKeyData *enc_rel_key_data, const RelFileLocator *rlocator); +extern RelKeyData *pg_tde_get_key_from_file(const RelFileLocator *rlocator); extern void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *map_path, char *keydata_path); diff --git a/src/include/catalog/tde_global_catalog.h b/src/include/catalog/tde_global_catalog.h deleted file mode 100644 index 23d57380..00000000 --- a/src/include/catalog/tde_global_catalog.h +++ /dev/null @@ -1,41 +0,0 @@ -/*------------------------------------------------------------------------- - * - * tde_global_catalog.h - * Global catalog key management - * - * src/include/catalog/tde_global_catalog.h - * - *------------------------------------------------------------------------- - */ - -#ifndef TDE_GLOBAL_CATALOG_H -#define TDE_GLOBAL_CATALOG_H - -#include "postgres.h" - -#include "catalog/tde_principal_key.h" - -/* - * Needed for glogbal data (WAL etc) keys identification in caches and storage. - * We take IDs the oid type operators, so there is no overlap with the "real" - * catalog object possible. - */ -#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ -#define XLOG_TDE_OID 608 - -#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) { \ - GLOBALTABLESPACE_OID, \ - GLOBAL_DATA_TDE_OID, \ - _obj_oid \ -} - -extern void TDEGlCatInitGUC(void); -extern Size TDEGlCatEncStateSize(void); -extern void TDEGlCatShmemInit(void); -extern void TDEGlCatKeyInit(void); - -extern TDEPrincipalKey *TDEGetGlCatKeyFromCache(void); -extern void TDEPutGlCatKeyInCache(TDEPrincipalKey *mkey); -extern RelKeyData *GetGlCatInternalKey(Oid obj_id); - -#endif /*TDE_GLOBAL_CATALOG_H*/ diff --git a/src/include/catalog/tde_global_space.h b/src/include/catalog/tde_global_space.h new file mode 100644 index 00000000..6fe7d158 --- /dev/null +++ b/src/include/catalog/tde_global_space.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * tde_global_space.h + * Global catalog key management + * + * src/include/catalog/tde_global_space.h + * + *------------------------------------------------------------------------- + */ + +#ifndef TDE_GLOBAL_CATALOG_H +#define TDE_GLOBAL_CATALOG_H + +#include "postgres.h" + +#include "access/pg_tde_tdemap.h" +#include "catalog/tde_principal_key.h" + +/* + * Needed for global data (WAL etc) keys identification in caches and storage. + * We take Oids of the sql operators, so there is no overlap with the "real" + * catalog objects possible. + */ +#define GLOBAL_DATA_TDE_OID 607 /* Global objects fake "db" */ +#define XLOG_TDE_OID 608 + +#define GLOBAL_SPACE_RLOCATOR(_obj_oid) (RelFileLocator) { \ + GLOBALTABLESPACE_OID, \ + GLOBAL_DATA_TDE_OID, \ + _obj_oid \ +} + +extern void TDEInitGlobalKeys(void); +extern RelKeyData * TDEGetGlobalInternalKey(Oid obj_id); + +#endif /* TDE_GLOBAL_CATALOG_H */ diff --git a/src/include/catalog/tde_keyring.h b/src/include/catalog/tde_keyring.h index c6ec46f4..2256b495 100644 --- a/src/include/catalog/tde_keyring.h +++ b/src/include/catalog/tde_keyring.h @@ -56,12 +56,6 @@ typedef struct VaultV2Keyring char vault_mount_path[MAXPGPATH]; } VaultV2Keyring; -typedef union KeyringProviders -{ - FileKeyring file; - VaultV2Keyring vault; -} KeyringProviders; - /* This record goes into key provider info file */ typedef struct KeyringProvideRecord { @@ -78,11 +72,12 @@ typedef struct KeyringProviderXLRecord KeyringProvideRecord provider; } KeyringProviderXLRecord; -extern List *GetAllKeyringProviders(void); -extern GenericKeyring *GetKeyProviderByName(const char *provider_name); -extern GenericKeyring *GetKeyProviderByID(int provider_id); +extern List *GetAllKeyringProviders(Oid dbOid, Oid spcOid); +extern GenericKeyring *GetKeyProviderByName(const char *provider_name, Oid dbOid, Oid spcOid); +extern GenericKeyring *GetKeyProviderByID(int provider_id, Oid dbOid, Oid spcOid); extern ProviderType get_keyring_provider_from_typename(char *provider_type); extern void cleanup_key_provider_info(Oid databaseId, Oid tablespaceId); extern void InitializeKeyProviderInfo(void); +extern uint32 save_new_key_provider_info(KeyringProvideRecord *provider, Oid databaseId, Oid tablespaceId, bool recovery); extern uint32 redo_key_provider_info(KeyringProviderXLRecord *xlrec); #endif /*TDE_KEYRING_H*/ diff --git a/src/include/catalog/tde_principal_key.h b/src/include/catalog/tde_principal_key.h index 3cff9c26..0bebe59e 100644 --- a/src/include/catalog/tde_principal_key.h +++ b/src/include/catalog/tde_principal_key.h @@ -63,16 +63,9 @@ extern LWLock *tde_lwlock_mk_cache(void); extern bool save_principal_key_info(TDEPrincipalKeyInfo *principalKeyInfo); extern Oid GetPrincipalKeyProviderId(void); -extern TDEPrincipalKey* GetPrincipalKey(Oid dbOid, Oid spcOid, GenericKeyring *keyring); +extern TDEPrincipalKey* GetPrincipalKey(Oid dbOid, Oid spcOid); extern bool SetPrincipalKey(const char *key_name, const char *provider_name, bool ensure_new_key); -extern bool RotatePrincipalKey(const char *new_key_name, const char *new_provider_name, bool ensure_new_key); +extern bool RotatePrincipalKey(TDEPrincipalKey *current_key, const char *new_key_name, const char *new_provider_name, bool ensure_new_key); extern bool xl_tde_perform_rotate_key(XLogPrincipalKeyRotate *xlrec); -extern TDEPrincipalKey *set_principal_key_with_keyring(const char *key_name, - GenericKeyring *keyring, - Oid dbOid, Oid spcOid, - bool ensure_new_key); -extern keyInfo *load_latest_versioned_key_name(TDEPrincipalKeyInfo *principal_key_info, - GenericKeyring *keyring, - bool ensure_new_key); #endif /*PG_TDE_PRINCIPAL_KEY_H*/ diff --git a/src/pg_tde.c b/src/pg_tde.c index 056f0196..43d048c7 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -35,7 +35,7 @@ #include "pg_tde_defs.h" #include "smgr/pg_tde_smgr.h" #ifdef PERCONA_FORK -#include "catalog/tde_global_catalog.h" +#include "catalog/tde_global_space.h" #endif #define MAX_ON_INSTALLS 5 @@ -87,8 +87,7 @@ tde_shmem_startup(void) AesInit(); #ifdef PERCONA_FORK - TDEGlCatShmemInit(); - TDEGlCatKeyInit(); + TDEInitGlobalKeys(); TDEXLogShmemInit(); TDEXLogSmgrInit(); @@ -108,7 +107,6 @@ _PG_init(void) InitializeKeyProviderInfo(); #ifdef PERCONA_FORK XLogInitGUC(); - TDEGlCatInitGUC(); #endif prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = tde_shmem_request; diff --git a/src/smgr/pg_tde_smgr.c b/src/smgr/pg_tde_smgr.c index 0badb8fe..a50a2d24 100644 --- a/src/smgr/pg_tde_smgr.c +++ b/src/smgr/pg_tde_smgr.c @@ -35,7 +35,7 @@ tde_smgr_get_key(SMgrRelation reln) recursion++; - if(GetPrincipalKey(reln->smgr_rlocator.locator.relNumber, reln->smgr_rlocator.locator.spcOid, NULL)==NULL) + if(GetPrincipalKey(reln->smgr_rlocator.locator.relNumber, reln->smgr_rlocator.locator.spcOid)==NULL) { recursion--; return NULL; diff --git a/t/002_rotate_key.pl b/t/002_rotate_key.pl index 5139f017..07dca817 100644 --- a/t/002_rotate_key.pl +++ b/t/002_rotate_key.pl @@ -56,8 +56,8 @@ PGTDE::append_to_file($stdout); #rotate key -PGTDE::append_to_file("-- ROTATE KEY pg_tde_rotate_key('rotated-principal-key','file-2');"); -$rt_value = $node->psql('postgres', "SELECT pg_tde_rotate_key('rotated-principal-key','file-2');", extra_params => ['-a']); +PGTDE::append_to_file("-- ROTATE KEY pg_tde_rotate_principal_key('rotated-principal-key','file-2');"); +$rt_value = $node->psql('postgres', "SELECT pg_tde_rotate_principal_key('rotated-principal-key','file-2');", extra_params => ['-a']); $stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); PGTDE::append_to_file($stdout); @@ -70,8 +70,8 @@ PGTDE::append_to_file($stdout); #Again rotate key -PGTDE::append_to_file("-- ROTATE KEY pg_tde_rotate_key();"); -$rt_value = $node->psql('postgres', "SELECT pg_tde_rotate_key();", extra_params => ['-a']); +PGTDE::append_to_file("-- ROTATE KEY pg_tde_rotate_principal_key();"); +$rt_value = $node->psql('postgres', "SELECT pg_tde_rotate_principal_key();", extra_params => ['-a']); $stdout = $node->safe_psql('postgres', 'SELECT * FROM test_enc ORDER BY id ASC;', extra_params => ['-a']); PGTDE::append_to_file($stdout); diff --git a/t/expected/002_rotate_key.out b/t/expected/002_rotate_key.out index b99491b3..da00f28e 100644 --- a/t/expected/002_rotate_key.out +++ b/t/expected/002_rotate_key.out @@ -5,7 +5,7 @@ INSERT INTO test_enc (k) VALUES (5),(6); SELECT * FROM test_enc ORDER BY id ASC; 1|5 2|6 --- ROTATE KEY pg_tde_rotate_key('rotated-principal-key','file-2'); +-- ROTATE KEY pg_tde_rotate_principal_key('rotated-principal-key','file-2'); SELECT * FROM test_enc ORDER BY id ASC; 1|5 2|6 @@ -13,7 +13,7 @@ SELECT * FROM test_enc ORDER BY id ASC; SELECT * FROM test_enc ORDER BY id ASC; 1|5 2|6 --- ROTATE KEY pg_tde_rotate_key(); +-- ROTATE KEY pg_tde_rotate_principal_key(); SELECT * FROM test_enc ORDER BY id ASC; 1|5 2|6