From 077e1df5c78b7c10eb03f1a844a543f11dbe5d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 17 Jun 2025 18:10:14 +0200 Subject: [PATCH 01/27] Add `autoEncryption` configuration to the client (#889) --- config/command.php | 6 + config/schema/mongodb-1.0.xsd | 97 ++++++++ docs/config.rst | 59 ++++- docs/encryption.rst | 223 ++++++++++++++++++ phpstan-baseline.neon | 54 ++++- src/Command/ConnectionDiagnosticCommand.php | 120 ++++++++++ src/DataCollector/ConnectionDiagnostic.php | 109 +++++++++ src/DependencyInjection/Configuration.php | 138 +++++++++++ .../DoctrineMongoDBExtension.php | 57 ++++- .../DependencyInjection/ConfigurationTest.php | 212 ++++++++++++++++- .../DoctrineMongoDBExtensionTest.php | 108 +++++++++ .../Fixtures/config/xml/full.xml | 42 ++++ .../Fixtures/config/yml/full.yml | 37 +++ 13 files changed, 1230 insertions(+), 32 deletions(-) create mode 100644 docs/encryption.rst create mode 100644 src/Command/ConnectionDiagnosticCommand.php create mode 100644 src/DataCollector/ConnectionDiagnostic.php diff --git a/config/command.php b/config/command.php index 01e4e744..0a7e12d0 100644 --- a/config/command.php +++ b/config/command.php @@ -3,6 +3,7 @@ declare(strict_types=1); use Doctrine\Bundle\MongoDBBundle\Command\ClearMetadataCacheDoctrineODMCommand; +use Doctrine\Bundle\MongoDBBundle\Command\ConnectionDiagnosticCommand; use Doctrine\Bundle\MongoDBBundle\Command\CreateSchemaDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\DropSchemaDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\GenerateHydratorsDoctrineODMCommand; @@ -15,12 +16,17 @@ use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; use function Symfony\Component\DependencyInjection\Loader\Configurator\service; +use function Symfony\Component\DependencyInjection\Loader\Configurator\tagged_locator; return static function (ContainerConfigurator $containerConfigurator): void { $containerConfigurator->services() ->set('doctrine_mongodb.odm.command.clear_metadata_cache', ClearMetadataCacheDoctrineODMCommand::class) ->tag('console.command', ['command' => 'doctrine:mongodb:cache:clear-metadata']) + ->set('doctrine_mongodb.odm.command.connection_diagnostic', ConnectionDiagnosticCommand::class) + ->tag('console.command', ['command' => 'doctrine:mongodb:connection:diagnostic']) + ->args([tagged_locator('doctrine_mongodb.connection_diagnostic', 'name')]) + ->set('doctrine_mongodb.odm.command.create_schema', CreateSchemaDoctrineODMCommand::class) ->tag('console.command', ['command' => 'doctrine:mongodb:schema:create']) diff --git a/config/schema/mongodb-1.0.xsd b/config/schema/mongodb-1.0.xsd index 4a63e8be..026d714b 100644 --- a/config/schema/mongodb-1.0.xsd +++ b/config/schema/mongodb-1.0.xsd @@ -48,6 +48,7 @@ + @@ -84,6 +85,9 @@ + + + @@ -119,6 +123,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/config.rst b/docs/config.rst index d6aac188..dd4fa08d 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -624,6 +624,10 @@ Otherwise you will get a *auth failed* exception. ]); }; +Using Queryable Encryption +-------------------------- + +TODO: Add documentation for queryable encryption configuration. Full Default Configuration -------------------------- @@ -699,6 +703,33 @@ Full Default Configuration wTimeoutMS: ~ driver_options: context: ~ # stream context to use for connection + autoEncryption: # Options for client-side field-level encryption + keyVaultClient: null # Service ID of a MongoDB\Driver\Manager for the key vault + keyVaultNamespace: null # The namespace for the key vault collection (e.g., "encryption.__keyVault") + kmsProvider: {} # Configuration for Key Management System provider (see specific examples above) + # e.g., { type: "local", key: "YOUR_BASE64_KEY" } + # e.g., { type: "aws", accessKeyId: "...", secretAccessKey: "..." } + masterKey: ~ # Default master key to use when creating a new encrypted collection + schemaMap: [] # Document schemas for explicit encryption + encryptedFieldsMap: [] # Map of collections to their encrypted fields configuration + extraOptions: [] # Extra options for mongocryptd + # mongocryptdURI: "mongodb://localhost:27020" + # mongocryptdBypassSpawn: false + # mongocryptdSpawnPath: "/usr/local/bin/mongocryptd" + # mongocryptdSpawnArgs: ["--idleShutdownTimeoutSecs=60"] + # cryptSharedLibPath: null # Path to the crypt_shared library + # cryptSharedLibRequired: false # If true, fails if the crypt_shared library cannot be loaded + bypassQueryAnalysis: false # Disables automatic analysis of read and write operations for encryption + bypassAutoEncryption: false # Disables auto-encryption + tlsOptions: # TLS options for the Key Vault client (if keyVaultClient is not specified) + tlsCAFile: null # Path to CA file, e.g., /path/to/key-vault-ca.pem + tlsCertificateKeyFile: null # Path to client cert/key file, e.g., /path/to/key-vault-client.pem + tlsCertificateKeyFilePassword: null # Password for client cert/key file + tlsAllowInvalidCertificates: false # Bypass server certificate validation (use with caution) + tlsAllowInvalidHostnames: false # Bypass server hostname validation (use with caution) + tlsDisableCertificateRevocationCheck: false # Disable CRL checks + tlsDisableOCSPEndpointCheck: false # Disable OCSP checks + tlsInsecure: false # Allow invalid/no server cert (use with extreme caution) proxy_namespace: MongoDBODMProxies proxy_dir: "%kernel.cache_dir%/doctrine/odm/mongodb/Proxies" @@ -825,8 +856,32 @@ Full Default Configuration $config->connection('id') ->server('mongodb://localhost') - ->driverOptions([ - 'context' => null, // stream context to use for connection + ->driverOptions(['context' => null]), // stream context to use for connection + ->autoEncryption([ // Options for client-side field-level encryption + 'bypassAutoEncryption' => false, // Disables auto-encryption + 'keyVaultClient' => null, // Service ID of a MongoDB\Driver\Manager for the key vault + 'keyVaultNamespace' => null, // The namespace for the key vault collection (e.g., "encryption.__keyVault") + 'kmsProvider' => [ // Configuration for Key Management System provider + // e.g., ['type' => 'local', 'key' => 'YOUR_BASE64_KEY'] + // e.g., ['type' => 'aws', 'accessKeyId' => '...', 'secretAccessKey' => '...'] + ], + 'schemaMap' => [], // Document schemas for explicit encryption + 'encryptedFieldsMap' => [], // Map of collections to their encrypted fields configuration + 'extraOptions' => [ // Extra options for mongocryptd + // 'cryptSharedLibPath' => null, // Path to the crypt_shared library + // 'cryptSharedLibRequired' => false, // If true, fails if the crypt_shared library cannot be loaded + ], + 'bypassQueryAnalysis' => false, // Disables automatic analysis of read and write operations for encryption + 'tlsOptions' => [ // TLS options for the Key Vault client (if keyVaultClient is not specified) + // 'tlsCAFile' => null, // Path to CA file, e.g., /path/to/key-vault-ca.pem + // 'tlsCertificateKeyFile' => null, // Path to client cert/key file, e.g., /path/to/key-vault-client.pem + // 'tlsCertificateKeyFilePassword' => null, // Password for client cert/key file + // 'tlsAllowInvalidCertificates' => false, // Bypass server certificate validation (use with caution) + // 'tlsAllowInvalidHostnames' => false, // Bypass server hostname validation (use with caution) + // 'tlsDisableCertificateRevocation' => false, // Disable CRL checks + // 'tlsDisableOCSPEndpointCheck' => false, // Disable OCSP checks + // 'tlsInsecure' => false, // Allow invalid/no server cert (use with extreme caution) + ], ]) ->options([ 'authMechanism' => null, diff --git a/docs/encryption.rst b/docs/encryption.rst new file mode 100644 index 00000000..cbf4c47e --- /dev/null +++ b/docs/encryption.rst @@ -0,0 +1,223 @@ +Client-Side Field-Level Encryption (CSFLE) and Queryable Encryption (QE) +============================================================ + +This page documents how to configure and use MongoDB Client-Side Field-Level Encryption (CSFLE) and Queryable Encryption (QE) in DoctrineMongoDBBundle. + +.. note:: + + CSFLE and QE are advanced MongoDB features that allow you to encrypt specific fields in your documents, with optional support for searching encrypted data (Queryable Encryption). + +Configuration +------------- + +.. tip:: + + For a general overview of configuration options, see :doc:`config`. + +To enable CSFLE or QE, you need to configure the ``autoEncryption`` option under +your connection's configuration. At a minimum, you must specify the ``kmsProvider`` +and the ``masterKey`` for KMS provider other than "local". +Additional options are available for advanced use cases. + +.. configuration-block:: + + .. code-block:: yaml + + doctrine_mongodb: + connections: + default: + server: "mongodb://localhost:27017" + autoEncryption: + kmsProvider: + local: + key: "YOUR_BASE64_KEY" + # Optional: see below for more options + + .. code-block:: php + + use Symfony\Config\DoctrineMongodbConfig; + + return static function (DoctrineMongodbConfig $config): void { + $config->connection('default') + ->server('mongodb://localhost:27017') + ->autoEncryption([ + 'kmsProvider' => [ + 'type' => 'local', + 'key' => 'YOUR_BASE64_KEY', + ], + // ... other options ... + ]); + }; + +Supported KMS Providers +----------------------- + +The ``kmsProvider`` option specifies a single KMS provider that will be used for encryption. +The type of KMS provider is specified with the ``type`` property along with its options. + +The configuration for each KMS provider varies and is described in the +`MongoDB Manager constructor documentation `. + +Example of configuration for AWS + + +.. code-block:: yaml + + doctrine_mongodb: + connections: + default: + autoEncryption: + kmsProvider: + type: aws + accessKeyId: YOUR_AWS_ACCESS_KEY_ID + secretAccessKey: YOUR_AWS_SECRET_ACCESS_KEY + masterKey: + region: "eu-west-1" + key: "arn:aws:kms:eu-west-1:123456789012:key/abcd1234-12ab-34cd-56ef-1234567890ab" + + +Queryable Encryption (QE) +------------------------- + +Queryable Encryption (QE) allows you to run queries on encrypted fields. To use QE, you may need to provide an ``encryptedFieldsMap`` or use a schema map, depending on your driver and use case. + +.. tabs:: + + .. group-tab:: YAML + + .. code-block:: yaml + + doctrine_mongodb: + connections: + default: + autoEncryption: + encryptedFieldsMap: + "mydatabase.mycollection": + fields: + - path: "sensitive_field" + bsonType: "string" + + .. group-tab:: XML + + .. code-block:: xml + + + + + + + + + + + + .. group-tab:: PHP + + .. code-block:: php + + use Symfony\Config\DoctrineMongodbConfig; + + return static function (DoctrineMongodbConfig $config): void { + $config->connection('default') + ->autoEncryption([ + 'encryptedFieldsMap' => [ + 'mydatabase.mycollection' => [ + 'fields' => [ + [ + 'path' => 'sensitive_field', + 'bsonType' => 'string', + ], + ], + ], + ], + ]); + }; + +TLS Options +----------- + +If you are not specifying a custom ``keyVaultClient`` service, you can configure TLS settings for the internal key vault client using the ``tlsOptions`` key: + +.. tabs:: + + .. group-tab:: YAML + + .. code-block:: yaml + + doctrine_mongodb: + connections: + default: + autoEncryption: + tlsOptions: + tlsCAFile: "/path/to/key-vault-ca.pem" + tlsCertificateKeyFile: "/path/to/key-vault-client.pem" + tlsCertificateKeyFilePassword: "keyvaultclientpassword" + tlsAllowInvalidCertificates: false + tlsAllowInvalidHostnames: false + + .. group-tab:: XML + + .. code-block:: xml + + + + + /path/to/key-vault-ca.pem + /path/to/key-vault-client.pem + keyvaultclientpassword + false + false + + + + + .. group-tab:: PHP + + .. code-block:: php + + use Symfony\Config\DoctrineMongodbConfig; + + return static function (DoctrineMongodbConfig $config): void { + $config->connection('default') + ->autoEncryption([ + 'tlsOptions' => [ + 'tlsCAFile' => '/path/to/key-vault-ca.pem', + 'tlsCertificateKeyFile' => '/path/to/key-vault-client.pem', + 'tlsCertificateKeyFilePassword' => 'keyvaultclientpassword', + 'tlsAllowInvalidCertificates' => false, + 'tlsAllowInvalidHostnames' => false, + ], + ]); + }; + +Context Service for SSL +----------------------- + +You can use a Symfony service to provide a stream context for SSL options: + +.. code-block:: yaml + + services: + app.mongodb.context_service: + class: 'resource' + factory: 'stream_context_create' + arguments: + - { ssl: { verify_expiry: true } } + +Then reference this service in your connection configuration: + +.. code-block:: yaml + + doctrine_mongodb: + connections: + default: + server: "mongodb://localhost:27017" + driver_options: + context: "app.mongodb.context_service" + +Further Reading +--------------- + +- `MongoDB CSFLE documentation `_ +- `MongoDB PHP driver Manager::__construct `_ +- :doc:`config` diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index c09c6edb..53dd6849 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -102,6 +102,42 @@ parameters: count: 1 path: src/Command/UpdateSchemaDoctrineODMCommand.php + - + message: '#^Expression on left side of \?\? is not nullable\.$#' + identifier: nullCoalesce.expr + count: 1 + path: src/DataCollector/ConnectionDiagnostic.php + + - + message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DataCollector\\ConnectionDiagnostic\:\:__construct\(\) has parameter \$driverOptions with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/DataCollector/ConnectionDiagnostic.php + + - + message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DataCollector\\ConnectionDiagnostic\:\:getAutoEncryptionInfo\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/DataCollector/ConnectionDiagnostic.php + + - + message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DataCollector\\ConnectionDiagnostic\:\:getPhpExtensionInfo\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/DataCollector/ConnectionDiagnostic.php + + - + message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DataCollector\\ConnectionDiagnostic\:\:getServerInfo\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/DataCollector/ConnectionDiagnostic.php + + - + message: '#^Unreachable statement \- code above always terminates\.$#' + identifier: deadCode.unreachable + count: 1 + path: src/DataCollector/ConnectionDiagnostic.php + - message: '#^Cannot cast array\|bool\|float\|int\|string\|UnitEnum\|null to string\.$#' identifier: cast.string @@ -222,6 +258,12 @@ parameters: count: 1 path: src/DependencyInjection/DoctrineMongoDBExtension.php + - + message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DependencyInjection\\DoctrineMongoDBExtension\:\:loadConnections\(\) has parameter \$config with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/DependencyInjection/DoctrineMongoDBExtension.php + - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DependencyInjection\\DoctrineMongoDBExtension\:\:loadConnections\(\) has parameter \$connections with no value type specified in iterable type array\.$#' identifier: missingType.iterableValue @@ -252,12 +294,6 @@ parameters: count: 1 path: src/DependencyInjection/DoctrineMongoDBExtension.php - - - message: '#^PHPDoc tag @param references unknown parameter\: \$config$#' - identifier: parameter.notFound - count: 1 - path: src/DependencyInjection/DoctrineMongoDBExtension.php - - message: '#^Parameter \#2 \$bundles of method Symfony\\Bridge\\Doctrine\\DependencyInjection\\AbstractDoctrineExtension\:\:fixManagersAutoMappings\(\) expects array, array\|bool\|float\|int\|string\|UnitEnum\|null given\.$#' identifier: argument.type @@ -540,12 +576,6 @@ parameters: count: 1 path: tests/DependencyInjection/ConfigurationTest.php - - - message: '#^PHPDoc tag @param references unknown parameter\: \$configs$#' - identifier: parameter.notFound - count: 1 - path: tests/DependencyInjection/ConfigurationTest.php - - message: '#^Parameter \#1 \$input of static method Symfony\\Component\\Yaml\\Yaml\:\:parse\(\) expects string, string\|false given\.$#' identifier: argument.type diff --git a/src/Command/ConnectionDiagnosticCommand.php b/src/Command/ConnectionDiagnosticCommand.php new file mode 100644 index 00000000..0f6ed4c7 --- /dev/null +++ b/src/Command/ConnectionDiagnosticCommand.php @@ -0,0 +1,120 @@ + $diagnostics */ + public function __construct(private readonly ServiceProviderInterface $diagnostics) + { + parent::__construct(); + } + + protected function configure(): void + { + $this->addOption('connection', 'c', InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'The name of the connection to diagnose. If not specified, all connections will be diagnosed.', [], $this->getConnectionNames(...)); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + $io->title('MongoDB Encryption Diagnostics'); + + /** @var string[] $connectionNames */ + $connectionNames = $input->getOption('connection'); + if ($connectionNames) { + if (array_diff($connectionNames, $this->getConnectionNames())) { + $io->error('One or more specified connections do not exist. Available connections: ' . implode(', ', $this->getConnectionNames())); + + return Command::INVALID; + } + } else { + $connectionNames = $this->getConnectionNames(); + } + + foreach ($connectionNames as $name) { + $diagnostic = $this->diagnostics->get($name); + $io->section(sprintf('Connection: %s', $name)); + + $io->text('PHP Environment'); + try { + $phpInfo = $diagnostic->getPhpExtensionInfo(); + $io->listing([ + 'ext-mongodb loaded: ' . ($phpInfo['ext-mongodb loaded'] ? 'Yes' : 'No'), + 'ext-mongodb version: ' . ($phpInfo['ext-mongodb version'] ?: '[unknown]'), + 'library version: ' . ($phpInfo['library version'] ?: '[unknown]'), + ]); + } catch (Throwable $exception) { + $io->error('Could not retrieve PHP extension info: ' . $exception->getMessage()); + } + + $io->text('Server Information'); + try { + $serverInfo = $diagnostic->getServerInfo(); + $io->listing([ + 'MongoDB Version: ' . ($serverInfo['version'] ?? '[unknown]'), + 'Modules: ' . (isset($serverInfo['modules']) ? implode(', ', $serverInfo['modules']) : '[unknown]'), + 'crypt_shared version: ' . ($serverInfo['crypt_shared_version'] ?? '[unknown]'), + 'crypt_shared path: ' . ($serverInfo['crypt_shared_path'] ?? '[unknown]'), + 'Topology: ' . ($serverInfo['topology'] ?? '[unknown]'), + ]); + } catch (Throwable $exception) { + $io->error('Could not retrieve server info: ' . $exception->getMessage()); + } + + $io->text('Auto Encryption Configuration'); + try { + $autoEncryptionInfo = $diagnostic->getAutoEncryptionInfo(); + if ($autoEncryptionInfo) { + $io->listing([ + 'Auto Encryption Enabled: ' . ($autoEncryptionInfo['autoEncryption enabled'] ? 'Yes' : 'No'), + 'Key Vault Namespace: ' . $autoEncryptionInfo['keyVaultNamespace'], + 'Key Count: ' . $autoEncryptionInfo['keyCount'], + ]); + } else { + $io->text('No auto encryption configuration found for this connection.'); + } + } catch (Throwable $exception) { + $io->error('Could not retrieve auto encryption info: ' . $exception->getMessage()); + } + + $mongocryptdVersion = $diagnostic->getMongocryptdVersion(); + if ($mongocryptdVersion) { + $io->text('mongocryptd Version'); + $io->text($mongocryptdVersion); + } else { + $io->text('mongocryptd not found'); + } + } + + return Command::SUCCESS; + } + + /** @return list */ + private function getConnectionNames(): array + { + return array_keys($this->diagnostics->getProvidedServices()); + } +} diff --git a/src/DataCollector/ConnectionDiagnostic.php b/src/DataCollector/ConnectionDiagnostic.php new file mode 100644 index 00000000..9c6ae8bd --- /dev/null +++ b/src/DataCollector/ConnectionDiagnostic.php @@ -0,0 +1,109 @@ +client->getManager()->selectServer(new ReadPreference(ReadPreference::PRIMARY)); + $buildInfo = $server->executeCommand('admin', new Command(['buildInfo' => 1]))->toArray()[0] ?? null; + // command not supported for auto encryption: serverStatus + // $serverStatus = $server->executeCommand('admin', new Command(['serverStatus' => 1]))->toArray()[0] ?? null; + + return [ + 'version' => $buildInfo->version ?? null, + 'modules' => $buildInfo->modules ?? null, + //'crypt_shared_version' => $serverStatus->crypt_shared ?? null, + //'crypt_shared_path' => $serverStatus->crypt_shared_path ?? null, + 'topology' => $server->getType() ?? null, + ]; + } + + public function getPhpExtensionInfo(): array + { + return [ + 'ext-mongodb loaded' => extension_loaded('mongodb'), + 'ext-mongodb version' => phpversion('mongodb') ?: null, + 'library version' => InstalledVersions::getPrettyVersion('mongodb/mongodb'), + ]; + } + + /** + * Get the list of auto encryption providers configured for the MongoDB client + * and an indication of whether the configuration is valid. + */ + public function getAutoEncryptionInfo(): ?array + { + if (! isset($this->driverOptions['autoEncryption'])) { + return null; + } + + $autoEncryption = $this->driverOptions['autoEncryption']; + + // Check if the "keyVaultNamespace" collection exists and is properly formatted + $keyVaultNamespace = explode('.', $autoEncryption['keyVaultNamespace'], 2); + $keyCount = $this->client->getCollection($keyVaultNamespace[0], $keyVaultNamespace[1])->countDocuments(); + $clientEncryption = $this->client->createClientEncryption([]); + $clientEncryption->getKeys(); + dd($clientEncryption->getKeys()); + + return [ + 'autoEncryption enabled' => true, + 'keyVaultNamespace' => $autoEncryption['keyVaultNamespace'], + 'keyCount' => $keyCount, + ]; + } + + public function getMongocryptdVersion(): ?string + { + $mongocryptdPath = $this->findMongocryptdPath(); + if ($mongocryptdPath === null) { + return null; + } + + $output = []; + exec($mongocryptdPath . ' --version', $output); + + if (isset($output[0])) { + return trim($output[0]); + } + + return null; + } + + private function findMongocryptdPath(): ?string + { + $paths = explode(':', getenv('PATH') ?: ''); + + foreach ($paths as $path) { + if (file_exists($path . '/mongocryptd')) { + return $path . '/mongocryptd'; + } + } + + return null; + } +} diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 72511dad..17e483a3 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -11,7 +11,9 @@ use Symfony\Component\Config\Definition\Builder\TreeBuilder; use Symfony\Component\Config\Definition\ConfigurationInterface; +use function array_is_list; use function count; +use function in_array; use function is_array; use function is_string; use function json_decode; @@ -339,6 +341,142 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void ->end() ->end() ->end() + ->arrayNode('autoEncryption') + ->children() + ->booleanNode('bypassAutoEncryption')->end() + ->scalarNode('keyVaultClient')->end() + ->scalarNode('keyVaultNamespace') + ->validate() + ->ifTrue(static fn ($v) => ! preg_match('/^.+\..+$/', $v)) + ->thenInvalid('Invalid keyVaultNamespace format. It should be "database.collection".') + ->end() + ->end() + ->arrayNode('masterKey') + ->prototype('variable')->end() + ->end() + ->arrayNode('kmsProvider') + ->isRequired() + ->children() + ->scalarNode('type') + ->isRequired() + ->validate() + ->ifTrue(static fn ($v) => ! in_array($v, ['aws', 'azure', 'gcp', 'kmip', 'local'], true)) + ->thenInvalid('Invalid KMS provider type "%s". Valid values are "aws", "azure", "gcp", "kmip", or "local".') + ->end() + ->end() + // AWS + ->scalarNode('accessKeyId')->end() + ->scalarNode('secretAccessKey')->end() + ->scalarNode('sessionToken')->end() + // Azure + ->scalarNode('tenantId')->end() + ->scalarNode('clientId')->end() + ->scalarNode('clientSecret')->end() + ->scalarNode('keyVaultEndpoint')->end() + ->scalarNode('identityPlatformEndpoint')->end() + ->scalarNode('keyName')->end() + ->scalarNode('keyVersion')->end() + // GCP + ->scalarNode('email')->end() + ->scalarNode('privateKey')->end() + ->scalarNode('endpoint')->end() + ->scalarNode('projectId')->end() + ->scalarNode('location')->end() + ->scalarNode('keyRing')->end() + //->scalarNode('keyName')->end() + //->scalarNode('keyVersion')->end() + // KMIP + //->scalarNode('endpoint')->end() + ->scalarNode('tlsCAFile')->end() + ->scalarNode('tlsClientCertificateKeyFile')->end() + ->scalarNode('tlsClientCertificateKeyFilePassword')->end() + // Local + ->scalarNode('key')->end() + ->end() + ->end() + ->arrayNode('schemaMap') + ->prototype('variable')->end() + ->end() + ->arrayNode('encryptedFieldsMap') + ->useAttributeAsKey('name', false) + ->beforeNormalization() + ->always(static function ($v) { + if (isset($v['encryptedFields']) && is_array($v['encryptedFields'])) { + $encryptedFields = $v['encryptedFields']; + if (! array_is_list($encryptedFields)) { + $encryptedFields = [$encryptedFields]; + } + + $v = []; + foreach ($encryptedFields as $field) { + if (is_array($field['field'] ?? null) && ! array_is_list($field['field'])) { + $field['field'] = [$field['field']]; + } + + $v[$field['name'] ?? ''] = $field['field'] ?? []; + } + } + + return $v; + })->end() + ->prototype('array') + ->prototype('array') + ->children() + ->scalarNode('path')->isRequired()->cannotBeEmpty()->end() + ->scalarNode('bsonType')->isRequired()->cannotBeEmpty()->end() + ->arrayNode('queries') + ->children() + ->scalarNode('queryType')->isRequired()->cannotBeEmpty()->end() + ->integerNode('min')->end() + ->integerNode('max')->end() + ->integerNode('sparsity')->end() + ->integerNode('trimFactor')->end() + ->end() + ->end() + ->end() + ->end() + ->end() + ->end() + ->arrayNode('extraOptions') + ->prototype('variable')->end() + ->end() + ->booleanNode('bypassQueryAnalysis')->end() + ->arrayNode('tlsOptions') + ->children() + ->scalarNode('tlsCAFile')->end() + ->scalarNode('tlsCertificateKeyFile')->end() + ->scalarNode('tlsCertificateKeyFilePassword')->end() + ->booleanNode('tlsAllowInvalidCertificates')->end() + ->booleanNode('tlsAllowInvalidHostnames')->end() + ->booleanNode('tlsDisableCertificateRevocationCheck')->end() + ->booleanNode('tlsDisableOCSPEndpointCheck')->end() + ->booleanNode('tlsInsecure')->end() + ->end() + ->end() + ->end() + ->validate() + ->always(static function ($v) { + // Remove empty arrays for schemaMap, encryptedFieldsMap, extraOptions, tlsOptions + foreach ( + [ + 'masterKey', + 'schemaMap', + 'encryptedFieldsMap', + 'extraOptions', + 'tlsOptions', + ] as $key + ) { + if (! isset($v[$key]) || ! is_array($v[$key]) || count($v[$key]) !== 0) { + continue; + } + + unset($v[$key]); + } + + return $v; + }) + ->end() + ->end() ->end() ->end() ->end() diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index e45117af..cd3e06e6 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -7,6 +7,7 @@ use Composer\InstalledVersions; use Doctrine\Bundle\MongoDBBundle\Attribute\AsDocumentListener; use Doctrine\Bundle\MongoDBBundle\Attribute\MapDocument; +use Doctrine\Bundle\MongoDBBundle\DataCollector\ConnectionDiagnostic; use Doctrine\Bundle\MongoDBBundle\DependencyInjection\Compiler\FixturesCompilerPass; use Doctrine\Bundle\MongoDBBundle\DependencyInjection\Compiler\ServiceRepositoryCompilerPass; use Doctrine\Bundle\MongoDBBundle\Fixture\ODMFixtureInterface; @@ -46,6 +47,7 @@ use Symfony\Component\Messenger\MessageBusInterface; use Throwable; +use function array_diff_key; use function array_key_first; use function array_merge; use function class_exists; @@ -118,7 +120,7 @@ public function load(array $configs, ContainerBuilder $container): void ->setArgument(5, $config['enable_lazy_ghost_objects'] ? Proxy::class : LazyLoadingInterface::class); // load the connections - $this->loadConnections($config['connections'], $container); + $this->loadConnections($config['connections'], $container, $config); $config['document_managers'] = $this->fixManagersAutoMappings($config['document_managers'], $container->getParameter('kernel.bundles')); @@ -179,6 +181,22 @@ public function load(array $configs, ContainerBuilder $container): void $this->loadMessengerServices($container, $loader); $this->loadEntityValueResolverServices($container, $loader, $config); + + // Register EncryptionDiagnostics for each connection + $diagnosticsRefs = []; + foreach ($config['connections'] as $connName => $connConfig) { + $connService = sprintf('doctrine_mongodb.odm.%s_connection', $connName); + $driverOptions = $connConfig['driver_options'] ?? []; + $diagServiceId = sprintf('doctrine_mongodb.encryption_diagnostics.%s', $connName); + $container->setDefinition( + $diagServiceId, + new Definition(ConnectionDiagnostic::class, [ + new Reference($connService), // Use the connection service, which is a MongoDB\Client + $driverOptions, + ]), + ); + $diagnosticsRefs[$connName] = new Reference($diagServiceId); + } } /** @@ -382,9 +400,10 @@ protected function loadDocumentManager(array $documentManager, string|null $defa * @param array $config An array of connections configurations * @param ContainerBuilder $container A ContainerBuilder instance */ - protected function loadConnections(array $connections, ContainerBuilder $container): void + protected function loadConnections(array $connections, ContainerBuilder $container, array $config): void { - $cons = []; + $cons = []; + $diagnostics = []; foreach ($connections as $name => $connection) { // Define an event manager for this connection $eventManagerId = sprintf('doctrine_mongodb.odm.%s_connection.event_manager', $name); @@ -399,11 +418,12 @@ protected function loadConnections(array $connections, ContainerBuilder $contain new Definition(ODMConfiguration::class), ); - $odmConnArgs = [ + $driverOptions = $this->normalizeDriverOptions($connection, $config); + $odmConnArgs = [ $connection['server'] ?? null, /* phpcs:ignore Squiz.Arrays.ArrayDeclaration.ValueNoNewline */ $connection['options'] ?? [], - $this->normalizeDriverOptions($connection), + $driverOptions, ]; $odmConnDef = new Definition(Client::class, $odmConnArgs); @@ -411,6 +431,11 @@ protected function loadConnections(array $connections, ContainerBuilder $contain $id = sprintf('doctrine_mongodb.odm.%s_connection', $name); $container->setDefinition($id, $odmConnDef); $cons[$name] = $id; + + // Diagnostic service + $container->register(sprintf('doctrine_mongodb.odm.%s_connection_diagnostic', $name), ConnectionDiagnostic::class) + ->setArguments([new Reference($id), $driverOptions]) + ->addTag('doctrine_mongodb.connection_diagnostic', ['name' => $name]); } $container->setParameter('doctrine_mongodb.odm.connections', $cons); @@ -463,11 +488,12 @@ private function loadEntityValueResolverServices(ContainerBuilder $container, Fi /** * Normalizes the driver options array * - * @param array $connection + * @param array $connection Connection configuration + * @param array $config Full configuration * * @return array */ - private function normalizeDriverOptions(array $connection): array + private function normalizeDriverOptions(array $connection, array $config): array { $driverOptions = $connection['driver_options'] ?? []; $driverOptions['typeMap'] = DocumentManager::CLIENT_TYPEMAP; @@ -476,6 +502,23 @@ private function normalizeDriverOptions(array $connection): array $driverOptions['context'] = new Reference($driverOptions['context']); } + if (isset($connection['autoEncryption'])) { + $kmsProvider = $connection['autoEncryption']['kmsProvider']; + $driverOptions['autoEncryption'] = array_diff_key($connection['autoEncryption'], [ + 'kmsProvider' => false, + 'masterKey' => false, + ]); + + $driverOptions['autoEncryption']['keyVaultNamespace'] ??= $config['default_database'] . '.datakeys'; + if (isset($driverOptions['autoEncryption']['keyVaultClient'])) { + $driverOptions['autoEncryption']['keyVaultClient'] = new Reference($driverOptions['autoEncryption']['keyVaultClient']); + } + + $driverOptions['autoEncryption']['kmsProviders'] = [ + $kmsProvider['type'] => array_diff_key($kmsProvider, ['type' => true]), + ]; + } + $driverOptions['driver'] = [ 'name' => 'symfony-mongodb', 'version' => self::getODMVersion(), diff --git a/tests/DependencyInjection/ConfigurationTest.php b/tests/DependencyInjection/ConfigurationTest.php index b3be8720..755576c6 100644 --- a/tests/DependencyInjection/ConfigurationTest.php +++ b/tests/DependencyInjection/ConfigurationTest.php @@ -13,6 +13,7 @@ use Doctrine\ODM\MongoDB\Configuration as ODMConfiguration; use Doctrine\ODM\MongoDB\Repository\DefaultGridFSRepository; use Doctrine\ODM\MongoDB\Repository\DocumentRepository; +use Generator; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Group; use PHPUnit\Framework\TestCase; @@ -23,6 +24,7 @@ use Symfony\Component\Yaml\Yaml; use function array_key_exists; +use function array_merge; use function file_get_contents; use function method_exists; @@ -127,6 +129,56 @@ public function testFullConfiguration(array $config): void 'wTimeoutMS' => 1000, ], 'driver_options' => ['context' => 'conn1_context_service'], + 'autoEncryption' => [ + 'kmsProvider' => [ + 'type' => 'aws', + 'accessKeyId' => 'MONGODB_AWS_ACCESS_KEY_ID', + 'secretAccessKey' => 'MONGODB_AWS_SECRET_ACCESS_KEY', + 'sessionToken' => 'MONGODB_AWS_SESSION_TOKEN', + ], + 'masterKey' => ['key' => 'MONGODB_AWS_MASTER_KEY'], + 'keyVaultNamespace' => 'encryption.__keyVault', + 'tlsOptions' => [ + 'tlsCAFile' => '%kernel.project_dir%/config/certificates/mongodb-ca.pem', + 'tlsCertificateKeyFile' => '%kernel.project_dir%/config/certificates/mongodb-client.pem', + 'tlsCertificateKeyFilePassword' => 'MONGODB_TLS_CERTIFICATE_KEY_FILE_PASSWORD', + 'tlsDisableOCSPEndpointCheck' => false, + ], + 'bypassAutoEncryption' => true, + 'bypassQueryAnalysis' => true, + 'encryptedFieldsMap' => [ + 'encrypted.patients' => [ + [ + 'path' => 'patientRecord.ssn', + 'bsonType' => 'string', + 'queries' => ['queryType' => 'equality'], + ], + [ + 'path' => 'patientRecord.billing', + 'bsonType' => 'object', + ], + [ + 'path' => 'patientRecord.billingAmount', + 'bsonType' => 'int', + 'queries' => ['queryType' => 'range', 'min' => 100, 'max' => 2000, 'sparsity' => 1, 'trimFactor' => 4], + ], + ], + 'encrypted.users' => [ + [ + 'path' => 'email', + 'bsonType' => 'string', + 'queries' => ['queryType' => 'equality'], + ], + ], + ], + 'extraOptions' => [ + 'mongocryptdURI' => 'mongodb://localhost:27020', + 'mongocryptdBypassSpawn' => true, + 'mongocryptdSpawnPath' => '%kernel.project_dir%/bin/mongocryptd', + 'mongocryptdSpawnArgs' => '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60', + 'cryptSharedLibPath' => '%kernel.project_dir%/bin/libmongocrypt.so', + ], + ], ], 'conn2' => ['server' => 'mongodb://otherhost'], ], @@ -228,8 +280,8 @@ public static function provideFullConfiguration(): array $xml = XmlUtils::convertDomElementToArray($xml->getElementsByTagName('config')->item(0)); return [ - [$yaml], - [$xml], + 'yaml' => [$yaml], + 'xml' => [$xml], ]; } @@ -355,7 +407,7 @@ public static function provideMergeOptions(): array } /** - * @param array $configs A configuration array to process + * @param array $config A configuration array to process * @param array $expected Array of key/value options expected in the processed configuration */ #[DataProvider('provideNormalizeOptions')] @@ -370,13 +422,11 @@ public function testNormalizeOptions(array $config, array $expected): void } } - /** @return array */ - public static function provideNormalizeOptions(): array + /** @return Generator, 1: array}> */ + public static function provideNormalizeOptions(): Generator { - $cases = []; - // connection versus connections (id is the identifier) - $cases[] = [ + yield [ [ 'connection' => [ ['server' => 'mongodb://abc', 'id' => 'foo'], @@ -392,7 +442,7 @@ public static function provideNormalizeOptions(): array ]; // document_manager versus document_managers (id is the identifier) - $cases[] = [ + yield [ [ 'document_manager' => [ ['connection' => 'conn1', 'id' => 'foo'], @@ -408,7 +458,7 @@ public static function provideNormalizeOptions(): array ]; // mapping configuration that's beneath a specific document manager - $cases[] = [ + yield [ [ 'document_manager' => [ [ @@ -441,7 +491,88 @@ public static function provideNormalizeOptions(): array ], ]; - return $cases; + // Encrypted Field Map normalization from XML tags + yield [ + [ + 'connection' => [ + [ + 'server' => 'mongodb://abc', + 'id' => 'foo', + 'autoEncryption' => [ + 'kmsProvider' => ['type' => 'local', 'key' => '1234567890123456789012345678901234567890123456789012345678901234'], + 'encryptedFieldsMap' => [ + 'encryptedFields' => [ + [ + 'name' => 'encrypted.patients', + 'field' => [ + [ + 'path' => 'patientRecord.ssn', + 'bsonType' => 'string', + 'queries' => ['queryType' => 'equality'], + ], + [ + 'path' => 'patientRecord.billing', + 'bsonType' => 'object', + ], + [ + 'path' => 'patientRecord.billingAmount', + 'bsonType' => 'int', + 'queries' => ['queryType' => 'range', 'min' => 100, 'max' => 2000, 'sparsity' => 1, 'trimFactor' => 4], + ], + ], + ], + [ + 'name' => 'encrypted.users', + 'field' => + [ + 'path' => 'email', + 'bsonType' => 'string', + 'queries' => ['queryType' => 'equality'], + ], + ], + ], + ], + ], + ], + ], + ], + [ + 'connections' => [ + 'foo' => [ + 'server' => 'mongodb://abc', + 'autoEncryption' => [ + 'kmsProvider' => ['type' => 'local', 'key' => '1234567890123456789012345678901234567890123456789012345678901234'], + 'encryptedFieldsMap' => [ + 'encrypted.patients' => [ + [ + 'path' => 'patientRecord.ssn', + 'bsonType' => 'string', + 'queries' => ['queryType' => 'equality'], + ], + [ + 'path' => 'patientRecord.billing', + 'bsonType' => 'object', + ], + [ + 'path' => 'patientRecord.billingAmount', + 'bsonType' => 'int', + 'queries' => ['queryType' => 'range', 'min' => 100, 'max' => 2000, 'sparsity' => 1, 'trimFactor' => 4], + ], + ], + 'encrypted.users' => [ + [ + 'path' => 'email', + 'bsonType' => 'string', + 'queries' => ['queryType' => 'equality'], + ], + ], + ], + ], + + ], + ], + ], + ]; } public function testPasswordAndUsernameShouldBeUnsetIfNull(): void @@ -517,4 +648,63 @@ public function testNullReplicaSetValue(): void $processedConfig = $processor->processConfiguration($configuration, [$config]); $this->assertFalse(array_key_exists('replicaSet', $processedConfig['connections']['conn1']['options'])); } + + /** + * @param array $config + * + * @return array + */ + protected function processConfiguration(array $config): array + { + $processor = new Processor(); + $configuration = new Configuration(); + + return $processor->processConfiguration($configuration, [$this->getMinimalValidConfig($config)]); + } + + /** + * @param array $config + * + * @return array + */ + protected function getMinimalValidConfig(array $config = []): array + { + $baseConfig = [ + 'connections' => [ + 'default' => [ + 'driver_options' => [], // Placeholder for autoEncryption or other options + ], + ], + 'document_managers' => [ + 'default' => [], + ], + ]; + + // Deep merge config into baseConfig + if (isset($config['connections']['default']['driver_options'])) { + $baseConfig['connections']['default']['driver_options'] = array_merge( + $baseConfig['connections']['default']['driver_options'], + $config['connections']['default']['driver_options'], + ); + unset($config['connections']['default']['driver_options']); + } + + if (isset($config['connections']['default'])) { + $baseConfig['connections']['default'] = array_merge( + $baseConfig['connections']['default'], + $config['connections']['default'], + ); + unset($config['connections']['default']); + } + + if (isset($config['connections'])) { + $baseConfig['connections'] = array_merge( + $baseConfig['connections'], + $config['connections'], + ); + unset($config['connections']); + } + + return array_merge($baseConfig, $config); + } } diff --git a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php index e12925fb..435cd6cc 100644 --- a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php +++ b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php @@ -479,4 +479,112 @@ public function testUseTransactionalFlush(): void $configuration->getMethodCalls(), ); } + + public function testAutoEncryptionWithKeyVaultClientService(): void + { + $container = $this->buildMinimalContainer(); + $loader = new DoctrineMongoDBExtension(); + + // Define a dummy service for the keyVaultClient + $dummyServiceId = 'my_key_vault_client_service'; + $container->setDefinition($dummyServiceId, new Definition('stdClass')); + + $config = [ + 'connections' => [ + 'default' => [ + 'autoEncryption' => [ + 'keyVaultNamespace' => 'db.vault', + 'keyVaultClient' => $dummyServiceId, + 'kmsProvider' => ['type' => 'local', 'key' => 'cGFzc3dvcmQ='], + ], + ], + ], + 'document_managers' => ['default' => []], + ]; + + $loader->load([$config], $container); + + $clientDef = $container->getDefinition('doctrine_mongodb.odm.default_connection'); + $driverOptions = $clientDef->getArgument(2); + + $this->assertArrayHasKey('autoEncryption', $driverOptions); + $this->assertInstanceOf(Reference::class, $driverOptions['autoEncryption']['keyVaultClient']); + $this->assertEquals($dummyServiceId, (string) $driverOptions['autoEncryption']['keyVaultClient']); + $this->assertEquals('db.vault', $driverOptions['autoEncryption']['keyVaultNamespace']); + $this->assertEquals(['local' => ['key' => 'cGFzc3dvcmQ=']], $driverOptions['autoEncryption']['kmsProviders']); + } + + public function testAutoEncryptionWithComplexKmsAndSchemaMap(): void + { + $container = $this->buildMinimalContainer(); + $loader = new DoctrineMongoDBExtension(); + + $schemaMap = [ + 'db.coll.users' => [ + 'bsonType' => 'object', + 'encryptMetadata' => ['keyId' => '/dataKeyId'], + 'properties' => ['ssn' => ['encrypt' => ['bsonType' => 'string', 'algorithm' => 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic']]], + ], + ]; + $config = [ + 'connections' => [ + 'default' => [ + 'autoEncryption' => [ + 'keyVaultNamespace' => 'db.vault', + 'kmsProvider' => ['type' => 'aws', 'accessKeyId' => 'test', 'secretAccessKey' => 'secret'], + 'schemaMap' => $schemaMap, + ], + ], + ], + 'document_managers' => ['default' => []], + ]; + + $loader->load([$config], $container); + + $clientDef = $container->getDefinition('doctrine_mongodb.odm.default_connection'); + $driverOptions = $clientDef->getArgument(2); + + $this->assertArrayHasKey('autoEncryption', $driverOptions); + $this->assertEquals(['aws' => ['accessKeyId' => 'test', 'secretAccessKey' => 'secret']], $driverOptions['autoEncryption']['kmsProviders']); + $this->assertEquals($schemaMap, $driverOptions['autoEncryption']['schemaMap']); + $this->assertEquals('db.vault', $driverOptions['autoEncryption']['keyVaultNamespace']); + } + + public function testAutoEncryptionWithExtraOptions(): void + { + $container = $this->buildMinimalContainer(); + $loader = new DoctrineMongoDBExtension(); + + $config = [ + 'connections' => [ + 'default' => [ + 'autoEncryption' => [ + 'keyVaultNamespace' => 'db.vault', + 'kmsProvider' => ['type' => 'local', 'key' => 'cGFzc3dvcmQ='], + 'extraOptions' => [ + 'cryptSharedLibPath' => '/another/path.so', + 'cryptSharedLibRequired' => false, + 'mongocryptdSpawnPath' => '/custom/mongocryptd', + ], + ], + ], + ], + 'document_managers' => ['default' => []], + ]; + + $loader->load([$config], $container); + + $clientDef = $container->getDefinition('doctrine_mongodb.odm.default_connection'); + $driverOptions = $clientDef->getArgument(2); + + $this->assertArrayHasKey('autoEncryption', $driverOptions); + $this->assertEquals('/another/path.so', $driverOptions['autoEncryption']['extraOptions']['cryptSharedLibPath']); + $this->assertFalse($driverOptions['autoEncryption']['extraOptions']['cryptSharedLibRequired']); + $this->assertEquals('/custom/mongocryptd', $driverOptions['autoEncryption']['extraOptions']['mongocryptdSpawnPath']); + + $this->assertArrayHasKey('typeMap', $driverOptions); // Default option + $this->assertArrayHasKey('driver', $driverOptions); // Added by normalizeDriverOptions + $this->assertEquals('symfony-mongodb', $driverOptions['driver']['name']); + $this->assertArrayHasKey('version', $driverOptions['driver']); + } } diff --git a/tests/DependencyInjection/Fixtures/config/xml/full.xml b/tests/DependencyInjection/Fixtures/config/xml/full.xml index 363038c3..b6f6701d 100644 --- a/tests/DependencyInjection/Fixtures/config/xml/full.xml +++ b/tests/DependencyInjection/Fixtures/config/xml/full.xml @@ -67,6 +67,48 @@ + + + + encryption.__keyVault + + + + + + + + + + + + + + + + + + + diff --git a/tests/DependencyInjection/Fixtures/config/yml/full.yml b/tests/DependencyInjection/Fixtures/config/yml/full.yml index 5b3e8ef9..7b1614fb 100644 --- a/tests/DependencyInjection/Fixtures/config/yml/full.yml +++ b/tests/DependencyInjection/Fixtures/config/yml/full.yml @@ -55,6 +55,43 @@ doctrine_mongodb: wTimeoutMS: 1000 driver_options: context: conn1_context_service + autoEncryption: + kmsProvider: + type: 'aws' + accessKeyId: 'MONGODB_AWS_ACCESS_KEY_ID' + secretAccessKey: 'MONGODB_AWS_SECRET_ACCESS_KEY' + sessionToken: 'MONGODB_AWS_SESSION_TOKEN' + masterKey: + key: 'MONGODB_AWS_MASTER_KEY' + keyVaultNamespace: 'encryption.__keyVault' + tlsOptions: + tlsCAFile: '%kernel.project_dir%/config/certificates/mongodb-ca.pem' + tlsCertificateKeyFile: '%kernel.project_dir%/config/certificates/mongodb-client.pem' + tlsCertificateKeyFilePassword: 'MONGODB_TLS_CERTIFICATE_KEY_FILE_PASSWORD' + tlsDisableOCSPEndpointCheck: false + bypassAutoEncryption: true + bypassQueryAnalysis: true + encryptedFieldsMap: + encrypted.patients: + - path: patientRecord.ssn + bsonType: string + queries: { queryType: equality } + - path: patientRecord.billing + bsonType: object + - path: patientRecord.billingAmount + bsonType: int + queries: { queryType: range, min: 100, max: 2000, sparsity: 1, trimFactor: 4 } + encrypted.users: + - path: email + bsonType: string + queries: { queryType: equality } + extraOptions: + mongocryptdURI: 'mongodb://localhost:27020' + mongocryptdBypassSpawn: true + mongocryptdSpawnPath: '%kernel.project_dir%/bin/mongocryptd' + mongocryptdSpawnArgs: '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60' + cryptSharedLibPath: '%kernel.project_dir%/bin/libmongocrypt.so' + conn2: server: mongodb://otherhost From 9a43b4f5957b0703d4b17b5b4dc71cc532148226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 19 Jun 2025 22:14:39 +0200 Subject: [PATCH 02/27] [Encryption] Update required version of mongodb driver and odm (#899) * Update required version of mongodb driver and odm * Update minimum phpunit version to support readonly properties in nikic/php-parser --- .github/workflows/coding-standards.yml | 1 + .github/workflows/composer-lint.yml | 1 + .github/workflows/continuous-integration.yml | 4 ++-- .github/workflows/static-analysis.yml | 1 + composer.json | 4 ++-- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/coding-standards.yml b/.github/workflows/coding-standards.yml index e57b10f9..32444a03 100644 --- a/.github/workflows/coding-standards.yml +++ b/.github/workflows/coding-standards.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - "*.x" + - "feature/*" push: branches: - "*.x" diff --git a/.github/workflows/composer-lint.yml b/.github/workflows/composer-lint.yml index 24738f32..08bbce8a 100644 --- a/.github/workflows/composer-lint.yml +++ b/.github/workflows/composer-lint.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - "*.x" + - "feature/*" paths: - "composer.json" push: diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 2dd408eb..d48d65fc 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -4,7 +4,7 @@ on: pull_request: branches: - "*.x" - - "master" + - "feature/*" push: env: @@ -41,7 +41,7 @@ jobs: - dependencies: "lowest" os: "ubuntu-24.04" php-version: "8.1" - driver-version: "1.16.0" + driver-version: "1.21.0" stability: "stable" symfony-version: "6.4.*" - dependencies: "highest" diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 718750d9..49106d4d 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - "*.x" + - "feature/*" paths: - .github/workflows/static-analysis.yml - composer.* diff --git a/composer.json b/composer.json index dbd735bb..e63417fc 100644 --- a/composer.json +++ b/composer.json @@ -25,9 +25,9 @@ "homepage": "http://www.doctrine-project.org", "require": { "php": "^8.1", - "ext-mongodb": "^1.16 || ^2", + "ext-mongodb": "^1.21 || ^2", "composer-runtime-api": "^2.0", - "doctrine/mongodb-odm": "^2.6", + "doctrine/mongodb-odm": "dev-feature/queryable-encryption as 2.12.x-dev", "doctrine/persistence": "^3.0 || ^4.0", "psr/log": "^1.0 || ^2.0 || ^3.0", "symfony/config": "^6.4 || ^7.0", From 12b155924161be8ad55489ba4a52efa77fa5c4ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Mon, 23 Jun 2025 08:53:22 +0200 Subject: [PATCH 03/27] =?UTF-8?q?[Encryption]=C2=A0Inject=20auto=20encrypt?= =?UTF-8?q?ion=20options=20into=20ODM=20configuration=20(#901)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Inject auto encryption options into ODM configuration * Add tests on ODM configuration settings --- .../DoctrineMongoDBExtension.php | 78 +++++++++++------ .../DoctrineMongoDBExtensionTest.php | 86 +++++++++++++++---- 2 files changed, 121 insertions(+), 43 deletions(-) diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index cd3e06e6..c21bcabc 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -131,6 +131,7 @@ public function load(array $configs, ContainerBuilder $container): void $config['default_database'], $container, $config['enable_lazy_ghost_objects'], + $config['connections'], ); if ($config['resolve_target_documents']) { @@ -237,12 +238,13 @@ protected function overrideParameters(array $options, ContainerBuilder $containe /** * Loads the document managers configuration. * - * @param array $dmConfigs An array of document manager configs - * @param string|null $defaultDM The default document manager name - * @param string $defaultDB The default db name - * @param ContainerBuilder $container A ContainerBuilder instance + * @param array $dmConfigs An array of document manager configs + * @param string|null $defaultDM The default document manager name + * @param string $defaultDB The default db name + * @param ContainerBuilder $container A ContainerBuilder instance + * @param array $connections Configuration of connections */ - protected function loadDocumentManagers(array $dmConfigs, string|null $defaultDM, string $defaultDB, ContainerBuilder $container, bool $useLazyGhostObject = false): void + protected function loadDocumentManagers(array $dmConfigs, string|null $defaultDM, string $defaultDB, ContainerBuilder $container, bool $useLazyGhostObject = false, array $connections = []): void { $dms = []; foreach ($dmConfigs as $name => $documentManager) { @@ -253,6 +255,7 @@ protected function loadDocumentManagers(array $dmConfigs, string|null $defaultDM $defaultDB, $container, $useLazyGhostObject, + $connections, ); $dms[$name] = sprintf('doctrine_mongodb.odm.%s_document_manager', $name); } @@ -263,12 +266,13 @@ protected function loadDocumentManagers(array $dmConfigs, string|null $defaultDM /** * Loads a document manager configuration. * - * @param array $documentManager A document manager configuration array - * @param string|null $defaultDM The default document manager name - * @param string $defaultDB The default db name - * @param ContainerBuilder $container A ContainerBuilder instance + * @param array $documentManager A document manager configuration array + * @param string|null $defaultDM The default document manager name + * @param string $defaultDB The default db name + * @param ContainerBuilder $container A ContainerBuilder instance + * @param array $connections Configuration of connections */ - protected function loadDocumentManager(array $documentManager, string|null $defaultDM, string $defaultDB, ContainerBuilder $container, bool $useLazyGhostObject = false): void + protected function loadDocumentManager(array $documentManager, string|null $defaultDM, string $defaultDB, ContainerBuilder $container, bool $useLazyGhostObject = false, array $connections = []): void { $connectionName = $documentManager['connection'] ?? $documentManager['name']; $configurationId = sprintf('doctrine_mongodb.odm.%s_configuration', $documentManager['name']); @@ -302,6 +306,16 @@ protected function loadDocumentManager(array $documentManager, string|null $defa 'setAutoGeneratePersistentCollectionClasses' => '%doctrine_mongodb.odm.auto_generate_persistent_collection_classes%', ]; + if (isset($connections[$connectionName]['autoEncryption'])) { + $autoEncryption = $connections[$connectionName]['autoEncryption']; + $methods['setAutoEncryption'] = array_diff_key( + $this->normalizeAutoEncryption($autoEncryption, $defaultDB), + ['kmsProviders' => false], + ); + $methods['setKmsProvider'] = $autoEncryption['kmsProvider']; + $methods['setDefaultMasterKey'] = $autoEncryption['masterKey'] ?? null; + } + if ($useLazyGhostObject) { $methods['setUseLazyGhostObject'] = $useLazyGhostObject; } @@ -503,20 +517,7 @@ private function normalizeDriverOptions(array $connection, array $config): array } if (isset($connection['autoEncryption'])) { - $kmsProvider = $connection['autoEncryption']['kmsProvider']; - $driverOptions['autoEncryption'] = array_diff_key($connection['autoEncryption'], [ - 'kmsProvider' => false, - 'masterKey' => false, - ]); - - $driverOptions['autoEncryption']['keyVaultNamespace'] ??= $config['default_database'] . '.datakeys'; - if (isset($driverOptions['autoEncryption']['keyVaultClient'])) { - $driverOptions['autoEncryption']['keyVaultClient'] = new Reference($driverOptions['autoEncryption']['keyVaultClient']); - } - - $driverOptions['autoEncryption']['kmsProviders'] = [ - $kmsProvider['type'] => array_diff_key($kmsProvider, ['type' => true]), - ]; + $driverOptions['autoEncryption'] = $this->normalizeAutoEncryption($connection['autoEncryption'], $config['default_database']); } $driverOptions['driver'] = [ @@ -527,6 +528,35 @@ private function normalizeDriverOptions(array $connection, array $config): array return $driverOptions; } + /** + * Prepare the auto encryption configuration for the connection. + * + * @param array $autoEncryption The AutoEncryption configuration of a connection + * @param string $defaultDB The default database name + * + * @return array + */ + private function normalizeAutoEncryption(array $autoEncryption, string $defaultDB): array + { + if (! isset($autoEncryption['kmsProvider']['type'])) { + throw new InvalidArgumentException('The "kmsProvider" option must contain a "type" key.'); + } + + $autoEncryption['kmsProviders'] = [ + $autoEncryption['kmsProvider']['type'] => array_diff_key($autoEncryption['kmsProvider'], ['type' => true]), + ]; + unset($autoEncryption['kmsProvider']); + unset($autoEncryption['masterKey']); + + if (isset($autoEncryption['keyVaultClient'])) { + $autoEncryption['keyVaultClient'] = new Reference($autoEncryption['keyVaultClient']); + } + + $autoEncryption['keyVaultNamespace'] ??= $defaultDB . '.datakeys'; + + return $autoEncryption; + } + /** * Loads an ODM document managers bundle mapping information. * diff --git a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php index 435cd6cc..9a621bab 100644 --- a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php +++ b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php @@ -8,9 +8,12 @@ use Composer\InstalledVersions; use Composer\Semver\VersionParser; use Doctrine\Bundle\MongoDBBundle\Attribute\MapDocument; +use Doctrine\Bundle\MongoDBBundle\DependencyInjection\Compiler\ServiceRepositoryCompilerPass; use Doctrine\Bundle\MongoDBBundle\DependencyInjection\DoctrineMongoDBExtension; use Doctrine\Bundle\MongoDBBundle\Tests\DependencyInjection\Fixtures\Bundles\DocumentListenerBundle\EventListener\TestAttributeListener; +use Doctrine\ODM\MongoDB\Configuration; use Doctrine\ODM\MongoDB\Mapping\Annotations; +use MongoDB\Client; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Symfony\Bridge\Doctrine\Messenger\DoctrineClearEntityManagerWorkerSubscriber; @@ -23,6 +26,7 @@ use Symfony\Component\DependencyInjection\Reference; use Symfony\Component\Messenger\MessageBusInterface; +use function array_diff_key; use function array_merge; use function class_exists; use function interface_exists; @@ -51,6 +55,7 @@ public function buildMinimalContainer(): ContainerBuilder return new ContainerBuilder(new ParameterBag([ 'kernel.root_dir' => __DIR__, 'kernel.project_dir' => __DIR__, + 'kernel.cache_dir' => sys_get_temp_dir() . '/doctrine_mongodb_odm_bundle', 'kernel.name' => 'kernel', 'kernel.environment' => 'test', 'kernel.debug' => 'true', @@ -487,7 +492,7 @@ public function testAutoEncryptionWithKeyVaultClientService(): void // Define a dummy service for the keyVaultClient $dummyServiceId = 'my_key_vault_client_service'; - $container->setDefinition($dummyServiceId, new Definition('stdClass')); + $container->setDefinition($dummyServiceId, new Definition(Client::class)); $config = [ 'connections' => [ @@ -503,15 +508,31 @@ public function testAutoEncryptionWithKeyVaultClientService(): void ]; $loader->load([$config], $container); + (new ServiceRepositoryCompilerPass())->process($container); $clientDef = $container->getDefinition('doctrine_mongodb.odm.default_connection'); $driverOptions = $clientDef->getArgument(2); - $this->assertArrayHasKey('autoEncryption', $driverOptions); - $this->assertInstanceOf(Reference::class, $driverOptions['autoEncryption']['keyVaultClient']); - $this->assertEquals($dummyServiceId, (string) $driverOptions['autoEncryption']['keyVaultClient']); - $this->assertEquals('db.vault', $driverOptions['autoEncryption']['keyVaultNamespace']); - $this->assertEquals(['local' => ['key' => 'cGFzc3dvcmQ=']], $driverOptions['autoEncryption']['kmsProviders']); + self::assertArrayHasKey('autoEncryption', $driverOptions); + self::assertInstanceOf(Reference::class, $driverOptions['autoEncryption']['keyVaultClient']); + self::assertEquals($dummyServiceId, (string) $driverOptions['autoEncryption']['keyVaultClient']); + self::assertEquals('db.vault', $driverOptions['autoEncryption']['keyVaultNamespace']); + self::assertEquals(['local' => ['key' => 'cGFzc3dvcmQ=']], $driverOptions['autoEncryption']['kmsProviders']); + + // Auto encryption configuration should be set in the ODM configuration + $odmConfiguration = $container->get('doctrine_mongodb.odm.default_configuration'); + self::assertInstanceOf(Configuration::class, $odmConfiguration); + self::assertSame('local', $odmConfiguration->getDefaultKmsProvider()); + self::assertNull($odmConfiguration->getDefaultMasterKey()); + self::assertArrayHasKey('autoEncryption', $odmConfiguration->getDriverOptions()); + self::assertInstanceOf(Client::class, $odmConfiguration->getDriverOptions()['autoEncryption']['keyVaultClient']); + + // Ensure the driver option set in the client matches the ODM configuration + // except for the keyVaultClient, which is a service reference + self::assertEquals( + array_diff_key($driverOptions['autoEncryption'], ['keyVaultClient' => false]), + array_diff_key($odmConfiguration->getDriverOptions()['autoEncryption'], ['keyVaultClient' => false]), + ); } public function testAutoEncryptionWithComplexKmsAndSchemaMap(): void @@ -526,6 +547,10 @@ public function testAutoEncryptionWithComplexKmsAndSchemaMap(): void 'properties' => ['ssn' => ['encrypt' => ['bsonType' => 'string', 'algorithm' => 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic']]], ], ]; + $masterKey = [ + 'region' => 'eu-west-3', + 'key' => 'arn:aws:kms:eu-west-3:123456789012:key/abcd1234-a123-456a-a12b-a123b4cd56ef', + ]; $config = [ 'connections' => [ 'default' => [ @@ -533,6 +558,7 @@ public function testAutoEncryptionWithComplexKmsAndSchemaMap(): void 'keyVaultNamespace' => 'db.vault', 'kmsProvider' => ['type' => 'aws', 'accessKeyId' => 'test', 'secretAccessKey' => 'secret'], 'schemaMap' => $schemaMap, + 'masterKey' => $masterKey, ], ], ], @@ -540,14 +566,25 @@ public function testAutoEncryptionWithComplexKmsAndSchemaMap(): void ]; $loader->load([$config], $container); + (new ServiceRepositoryCompilerPass())->process($container); $clientDef = $container->getDefinition('doctrine_mongodb.odm.default_connection'); $driverOptions = $clientDef->getArgument(2); - $this->assertArrayHasKey('autoEncryption', $driverOptions); - $this->assertEquals(['aws' => ['accessKeyId' => 'test', 'secretAccessKey' => 'secret']], $driverOptions['autoEncryption']['kmsProviders']); - $this->assertEquals($schemaMap, $driverOptions['autoEncryption']['schemaMap']); - $this->assertEquals('db.vault', $driverOptions['autoEncryption']['keyVaultNamespace']); + self::assertArrayHasKey('autoEncryption', $driverOptions); + self::assertEquals(['aws' => ['accessKeyId' => 'test', 'secretAccessKey' => 'secret']], $driverOptions['autoEncryption']['kmsProviders']); + self::assertEquals($schemaMap, $driverOptions['autoEncryption']['schemaMap']); + self::assertEquals('db.vault', $driverOptions['autoEncryption']['keyVaultNamespace']); + + // Auto encryption configuration should be set in the ODM configuration + $odmConfiguration = $container->get('doctrine_mongodb.odm.default_configuration'); + self::assertInstanceOf(Configuration::class, $odmConfiguration); + self::assertSame('aws', $odmConfiguration->getDefaultKmsProvider()); + self::assertSame($masterKey, $odmConfiguration->getDefaultMasterKey()); + self::assertArrayHasKey('autoEncryption', $odmConfiguration->getDriverOptions()); + + // Ensure the driver option set in the client matches the ODM configuration + self::assertEquals($driverOptions['autoEncryption'], $odmConfiguration->getDriverOptions()['autoEncryption']); } public function testAutoEncryptionWithExtraOptions(): void @@ -573,18 +610,29 @@ public function testAutoEncryptionWithExtraOptions(): void ]; $loader->load([$config], $container); + (new ServiceRepositoryCompilerPass())->process($container); $clientDef = $container->getDefinition('doctrine_mongodb.odm.default_connection'); $driverOptions = $clientDef->getArgument(2); - $this->assertArrayHasKey('autoEncryption', $driverOptions); - $this->assertEquals('/another/path.so', $driverOptions['autoEncryption']['extraOptions']['cryptSharedLibPath']); - $this->assertFalse($driverOptions['autoEncryption']['extraOptions']['cryptSharedLibRequired']); - $this->assertEquals('/custom/mongocryptd', $driverOptions['autoEncryption']['extraOptions']['mongocryptdSpawnPath']); - - $this->assertArrayHasKey('typeMap', $driverOptions); // Default option - $this->assertArrayHasKey('driver', $driverOptions); // Added by normalizeDriverOptions - $this->assertEquals('symfony-mongodb', $driverOptions['driver']['name']); - $this->assertArrayHasKey('version', $driverOptions['driver']); + self::assertArrayHasKey('autoEncryption', $driverOptions); + self::assertEquals('/another/path.so', $driverOptions['autoEncryption']['extraOptions']['cryptSharedLibPath']); + self::assertFalse($driverOptions['autoEncryption']['extraOptions']['cryptSharedLibRequired']); + self::assertEquals('/custom/mongocryptd', $driverOptions['autoEncryption']['extraOptions']['mongocryptdSpawnPath']); + + self::assertArrayHasKey('typeMap', $driverOptions); // Default option + self::assertArrayHasKey('driver', $driverOptions); // Added by normalizeDriverOptions + self::assertEquals('symfony-mongodb', $driverOptions['driver']['name']); + self::assertArrayHasKey('version', $driverOptions['driver']); + + // Auto encryption configuration should be set in the ODM configuration + $odmConfiguration = $container->get('doctrine_mongodb.odm.default_configuration'); + self::assertInstanceOf(Configuration::class, $odmConfiguration); + self::assertSame('local', $odmConfiguration->getDefaultKmsProvider()); + self::assertNull($odmConfiguration->getDefaultMasterKey()); + self::assertArrayHasKey('autoEncryption', $odmConfiguration->getDriverOptions()); + + // Ensure the driver option set in the client matches the ODM configuration + self::assertEquals($driverOptions['autoEncryption'], $odmConfiguration->getDriverOptions()['autoEncryption']); } } From 76e96842a8f8e5c6c35b1200b96d3d178f4b3080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Wed, 2 Jul 2025 15:43:09 +0200 Subject: [PATCH 04/27] [Encryption] Improve diagnostic command (#898) Co-authored-by: Andreas Braun --- .github/workflows/composer-lint.yml | 1 + config/command.php | 5 + docs/encryption.rst | 66 ++++++- phpstan-baseline.neon | 48 ----- phpstan.neon.dist | 4 + src/Command/ConnectionDiagnosticCommand.php | 166 ++++++++++++------ src/Command/DumpEncryptedFieldsMapCommand.php | 116 ++++++++++++ src/DataCollector/ConnectionDiagnostic.php | 123 ++++++------- src/DataCollector/EncryptionDiagnostic.php | 105 +++++++++++ .../DoctrineMongoDBExtension.php | 16 -- .../ConnectionDiagnosticCommandTest.php | 28 +++ .../DependencyInjection/ConfigurationTest.php | 2 +- .../Fixtures/config/xml/full.xml | 2 +- .../Fixtures/config/yml/full.yml | 2 +- 14 files changed, 493 insertions(+), 191 deletions(-) create mode 100644 src/Command/DumpEncryptedFieldsMapCommand.php create mode 100644 src/DataCollector/EncryptionDiagnostic.php create mode 100644 tests/Command/ConnectionDiagnosticCommandTest.php diff --git a/.github/workflows/composer-lint.yml b/.github/workflows/composer-lint.yml index 08bbce8a..4c1dabd1 100644 --- a/.github/workflows/composer-lint.yml +++ b/.github/workflows/composer-lint.yml @@ -10,6 +10,7 @@ on: push: branches: - "*.x" + - "feature/*" paths: - "composer.json" diff --git a/config/command.php b/config/command.php index 0a7e12d0..9b064fa3 100644 --- a/config/command.php +++ b/config/command.php @@ -6,6 +6,7 @@ use Doctrine\Bundle\MongoDBBundle\Command\ConnectionDiagnosticCommand; use Doctrine\Bundle\MongoDBBundle\Command\CreateSchemaDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\DropSchemaDoctrineODMCommand; +use Doctrine\Bundle\MongoDBBundle\Command\DumpEncryptedFieldsMapCommand; use Doctrine\Bundle\MongoDBBundle\Command\GenerateHydratorsDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\GenerateProxiesDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\InfoDoctrineODMCommand; @@ -27,6 +28,10 @@ ->tag('console.command', ['command' => 'doctrine:mongodb:connection:diagnostic']) ->args([tagged_locator('doctrine_mongodb.connection_diagnostic', 'name')]) + ->set('doctrine_mongodb.odm.command.dump_encrypted_fields_map', DumpEncryptedFieldsMapCommand::class) + ->tag('console.command', ['command' => 'doctrine:mongodb:dump-encrypted-fields-map']) + ->args([tagged_locator('doctrine_mongodb.odm.document_manager', 'name')]) + ->set('doctrine_mongodb.odm.command.create_schema', CreateSchemaDoctrineODMCommand::class) ->tag('console.command', ['command' => 'doctrine:mongodb:schema:create']) diff --git a/docs/encryption.rst b/docs/encryption.rst index cbf4c47e..5be85afd 100644 --- a/docs/encryption.rst +++ b/docs/encryption.rst @@ -76,10 +76,23 @@ Example of configuration for AWS key: "arn:aws:kms:eu-west-1:123456789012:key/abcd1234-12ab-34cd-56ef-1234567890ab" -Queryable Encryption (QE) -------------------------- +Encrypted Fields Map +-------------------- -Queryable Encryption (QE) allows you to run queries on encrypted fields. To use QE, you may need to provide an ``encryptedFieldsMap`` or use a schema map, depending on your driver and use case. +You can configure which fields are encrypted in each collection by specifying the +``autoEncryption.encryptedFieldsMap`` option in the connection configuration. +This setting is **recommended** for improved security and performance. + +- If the connection ``encryptedFieldsMap`` object contains a key for the specified + collection, the client uses that object to perform automatic Queryable Encryption, + rather than using the remote schema. At minimum, the local rules must encrypt + all fields that the remote schema does. + +- If the connection ``encryptedFieldsMap`` object doesn't contain a key for the + specified collection, the client downloads the server-side remote schema for + the collection and uses it instead. + +For more details, see the official MongoDB documentation: `Encrypted Fields and Enabled Queries `_. .. tabs:: @@ -133,6 +146,51 @@ Queryable Encryption (QE) allows you to run queries on encrypted fields. To use ]); }; +Automatic Encryption Shared Library +----------------------------------- + +To use automatic encryption, the MongoDB PHP driver requires the `Automatic Encryption Shared Library`_. + +If the driver is not able to find the library, you can specify its path using the ``cryptSharedLibPath`` extra option in your connection configuration. + +.. tabs:: + + .. group-tab:: YAML + + .. code-block:: yaml + + doctrine_mongodb: + connections: + default: + autoEncryption: + extraOptions: + cryptSharedLibPath: '%kernel.project_dir%/bin/mongo_crypt_v1.so' + + .. group-tab:: XML + + .. code-block:: xml + + + + + + + + .. group-tab:: PHP + + .. code-block:: php + + use Symfony\Config\DoctrineMongodbConfig; + + return static function (DoctrineMongodbConfig $config): void { + $config->connection('default') + ->autoEncryption([ + 'extraOptions' => [ + 'cryptSharedLibPath' => '%kernel.project_dir%/bin/mongo_crypt_v1.so', + ], + ]); + }; + TLS Options ----------- @@ -221,3 +279,5 @@ Further Reading - `MongoDB CSFLE documentation `_ - `MongoDB PHP driver Manager::__construct `_ - :doc:`config` + +.. _`Automatic Encryption Shared Library`: https://www.mongodb.com/docs/manual/core/queryable-encryption/install-library/ diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 53dd6849..cb21d61c 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -36,12 +36,6 @@ parameters: count: 1 path: src/CacheWarmer/ProxyCacheWarmer.php - - - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\CacheWarmer\\ProxyCacheWarmer\:\:getClassesForProxyGeneration\(\) return type with generic class Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata does not specify its types\: T$#' - identifier: missingType.generics - count: 1 - path: src/CacheWarmer/ProxyCacheWarmer.php - - message: '#^Parameter \#1 \$application of static method Doctrine\\Bundle\\MongoDBBundle\\Command\\DoctrineODMCommand\:\:setApplicationDocumentManager\(\) expects Symfony\\Bundle\\FrameworkBundle\\Console\\Application, Symfony\\Component\\Console\\Application\|null given\.$#' identifier: argument.type @@ -102,42 +96,12 @@ parameters: count: 1 path: src/Command/UpdateSchemaDoctrineODMCommand.php - - - message: '#^Expression on left side of \?\? is not nullable\.$#' - identifier: nullCoalesce.expr - count: 1 - path: src/DataCollector/ConnectionDiagnostic.php - - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DataCollector\\ConnectionDiagnostic\:\:__construct\(\) has parameter \$driverOptions with no value type specified in iterable type array\.$#' identifier: missingType.iterableValue count: 1 path: src/DataCollector/ConnectionDiagnostic.php - - - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DataCollector\\ConnectionDiagnostic\:\:getAutoEncryptionInfo\(\) return type has no value type specified in iterable type array\.$#' - identifier: missingType.iterableValue - count: 1 - path: src/DataCollector/ConnectionDiagnostic.php - - - - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DataCollector\\ConnectionDiagnostic\:\:getPhpExtensionInfo\(\) return type has no value type specified in iterable type array\.$#' - identifier: missingType.iterableValue - count: 1 - path: src/DataCollector/ConnectionDiagnostic.php - - - - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\DataCollector\\ConnectionDiagnostic\:\:getServerInfo\(\) return type has no value type specified in iterable type array\.$#' - identifier: missingType.iterableValue - count: 1 - path: src/DataCollector/ConnectionDiagnostic.php - - - - message: '#^Unreachable statement \- code above always terminates\.$#' - identifier: deadCode.unreachable - count: 1 - path: src/DataCollector/ConnectionDiagnostic.php - - message: '#^Cannot cast array\|bool\|float\|int\|string\|UnitEnum\|null to string\.$#' identifier: cast.string @@ -354,12 +318,6 @@ parameters: count: 1 path: src/Form/ChoiceList/MongoDBQueryBuilderLoader.php - - - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\Form\\DoctrineMongoDBTypeGuesser\:\:getMetadata\(\) return type with generic class Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata does not specify its types\: T$#' - identifier: missingType.generics - count: 1 - path: src/Form/DoctrineMongoDBTypeGuesser.php - - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\Form\\DoctrineMongoDBTypeGuesser\:\:getMetadata\(\) should return array\{Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata, string\}\|null but returns array\{Doctrine\\Persistence\\Mapping\\ClassMetadata\, string\}\.$#' identifier: return.type @@ -402,12 +360,6 @@ parameters: count: 1 path: src/Form/DoctrineMongoDBTypeGuesser.php - - - message: '#^Property Doctrine\\Bundle\\MongoDBBundle\\Form\\DoctrineMongoDBTypeGuesser\:\:\$cache with generic class Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata does not specify its types\: T$#' - identifier: missingType.generics - count: 1 - path: src/Form/DoctrineMongoDBTypeGuesser.php - - message: '#^Unable to resolve the template type T in call to method Doctrine\\Persistence\\ObjectManager\:\:getClassMetadata\(\)$#' identifier: argument.templateType diff --git a/phpstan.neon.dist b/phpstan.neon.dist index bb852bd2..03fa1fc3 100644 --- a/phpstan.neon.dist +++ b/phpstan.neon.dist @@ -8,3 +8,7 @@ parameters: - config - src - tests + + ignoreErrors: + - message: '# with generic class Doctrine\\ODM\\MongoDB\\Mapping\\ClassMetadata#' + identifier: missingType.generics diff --git a/src/Command/ConnectionDiagnosticCommand.php b/src/Command/ConnectionDiagnosticCommand.php index 0f6ed4c7..7dce530c 100644 --- a/src/Command/ConnectionDiagnosticCommand.php +++ b/src/Command/ConnectionDiagnosticCommand.php @@ -5,6 +5,8 @@ namespace Doctrine\Bundle\MongoDBBundle\Command; use Doctrine\Bundle\MongoDBBundle\DataCollector\ConnectionDiagnostic; +use Doctrine\Bundle\MongoDBBundle\DataCollector\EncryptionDiagnostic; +use MongoDB\Driver\Exception\RuntimeException; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -12,7 +14,6 @@ use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; use Symfony\Contracts\Service\ServiceProviderInterface; -use Throwable; use function array_diff; use function array_keys; @@ -27,8 +28,10 @@ final class ConnectionDiagnosticCommand extends Command { /** @param ServiceProviderInterface $diagnostics */ - public function __construct(private readonly ServiceProviderInterface $diagnostics) - { + public function __construct( + private readonly ServiceProviderInterface $diagnostics, + private readonly EncryptionDiagnostic $encryptionDiagnostic = new EncryptionDiagnostic(), + ) { parent::__construct(); } @@ -54,59 +57,18 @@ protected function execute(InputInterface $input, OutputInterface $output): int $connectionNames = $this->getConnectionNames(); } + $configOk = $this->printAndCheckExtensionInfo($io); + $this->printMongocryptdInfo($io); + foreach ($connectionNames as $name) { $diagnostic = $this->diagnostics->get($name); - $io->section(sprintf('Connection: %s', $name)); - - $io->text('PHP Environment'); - try { - $phpInfo = $diagnostic->getPhpExtensionInfo(); - $io->listing([ - 'ext-mongodb loaded: ' . ($phpInfo['ext-mongodb loaded'] ? 'Yes' : 'No'), - 'ext-mongodb version: ' . ($phpInfo['ext-mongodb version'] ?: '[unknown]'), - 'library version: ' . ($phpInfo['library version'] ?: '[unknown]'), - ]); - } catch (Throwable $exception) { - $io->error('Could not retrieve PHP extension info: ' . $exception->getMessage()); - } - - $io->text('Server Information'); - try { - $serverInfo = $diagnostic->getServerInfo(); - $io->listing([ - 'MongoDB Version: ' . ($serverInfo['version'] ?? '[unknown]'), - 'Modules: ' . (isset($serverInfo['modules']) ? implode(', ', $serverInfo['modules']) : '[unknown]'), - 'crypt_shared version: ' . ($serverInfo['crypt_shared_version'] ?? '[unknown]'), - 'crypt_shared path: ' . ($serverInfo['crypt_shared_path'] ?? '[unknown]'), - 'Topology: ' . ($serverInfo['topology'] ?? '[unknown]'), - ]); - } catch (Throwable $exception) { - $io->error('Could not retrieve server info: ' . $exception->getMessage()); - } - - $io->text('Auto Encryption Configuration'); - try { - $autoEncryptionInfo = $diagnostic->getAutoEncryptionInfo(); - if ($autoEncryptionInfo) { - $io->listing([ - 'Auto Encryption Enabled: ' . ($autoEncryptionInfo['autoEncryption enabled'] ? 'Yes' : 'No'), - 'Key Vault Namespace: ' . $autoEncryptionInfo['keyVaultNamespace'], - 'Key Count: ' . $autoEncryptionInfo['keyCount'], - ]); - } else { - $io->text('No auto encryption configuration found for this connection.'); - } - } catch (Throwable $exception) { - $io->error('Could not retrieve auto encryption info: ' . $exception->getMessage()); - } + $configOk = $this->printAndCheckConnectionDiagnostic($name, $diagnostic, $io) && $configOk; + } - $mongocryptdVersion = $diagnostic->getMongocryptdVersion(); - if ($mongocryptdVersion) { - $io->text('mongocryptd Version'); - $io->text($mongocryptdVersion); - } else { - $io->text('mongocryptd not found'); - } + if ($configOk) { + $io->success('System looks ok for encryption support.'); + } else { + $io->warning('Not all requirements for encryption support are met. Please check the diagnostics above.'); } return Command::SUCCESS; @@ -117,4 +79,102 @@ private function getConnectionNames(): array { return array_keys($this->diagnostics->getProvidedServices()); } + + /** @return bool True if the server is compatible with auto-encryption configuration, false otherwise. */ + private function printAndCheckConnectionDiagnostic(string $name, ConnectionDiagnostic $diagnostic, SymfonyStyle $io): bool + { + $io->section(sprintf('Connection: %s', $name)); + + $autoEncryptionEnabled = $this->printAutoEncryptionConfiguration($io, $diagnostic); + + if (! $autoEncryptionEnabled) { + return true; + } + + return $this->printAndCheckServerInfo($io, $diagnostic); + } + + /** @return bool True if the driver supports auto-encryption, false otherwise */ + private function printAndCheckExtensionInfo(SymfonyStyle $io): bool + { + $io->text('PHP Environment'); + $phpInfo = $this->encryptionDiagnostic->getPhpExtensionInfo(); + $io->listing([ + 'MongoDB extension loaded: ' . ($phpInfo['extensionLoaded'] ? 'Yes' : 'No'), + 'MongoDB extension version: ' . ($phpInfo['extensionVersion'] ?: '[unknown]'), + 'MongoDB extension supports libmongocrypt: ' . ($phpInfo['extensionSupportsLibmongocrypt'] ? 'Yes' : 'No'), + 'MongoDB library version: ' . ($phpInfo['libraryVersion'] ?: '[unknown]'), + ]); + + $extensionOk = $phpInfo['extensionLoaded'] && $phpInfo['extensionSupportsLibmongocrypt']; + + if (! $extensionOk) { + $io->warning('At least one extension requirement is not met. Encryption may not work.'); + } + + return $extensionOk; + } + + private function printMongocryptdInfo(SymfonyStyle $io): void + { + $io->text('mongocryptd information'); + $mongocryptdInfo = $this->encryptionDiagnostic->getMongocryptdInfo(); + + if ($mongocryptdInfo['mongocryptdPath'] === null) { + $io->listing(['mongocryptd: not found']); + } else { + $io->listing([ + 'mongocryptd path: ' . $mongocryptdInfo['mongocryptdPath'], + 'mongocryptd version: ' . ($mongocryptdInfo['mongocryptdVersion'] ?: '[unknown]'), + ]); + } + } + + /** @return bool True if the server supports auto-encryption, false otherwise */ + private function printAndCheckServerInfo(SymfonyStyle $io, ConnectionDiagnostic $diagnostic): bool + { + $io->text('Server Information'); + $serverInfo = $diagnostic->getServerInfo(); + + $io->listing([ + 'Server Version: ' . ($serverInfo['version'] ?? '[unknown]'), + 'Topology: ' . $serverInfo['topologyName'], + ]); + + if (! $serverInfo['versionSupported']) { + $io->warning('This server version does not support encryption.'); + } + + if (! $serverInfo['topologySupported']) { + $io->warning('This topology does not support encryption.'); + } + + return $serverInfo['versionSupported'] && $serverInfo['topologySupported']; + } + + /** @return bool True if the connection uses auto encryption, false otherwise. */ + private function printAutoEncryptionConfiguration(SymfonyStyle $io, ConnectionDiagnostic $diagnostic): bool + { + $io->text('Auto Encryption Configuration'); + if (! $diagnostic->usesAutoEncryption()) { + $io->text('Auto encryption is not enabled for this connection.'); + + return false; + } + + try { + $autoEncryptionInfo = $diagnostic->getAutoEncryptionInfo(); + + $io->listing([ + 'Auto Encryption Enabled: ' . ($autoEncryptionInfo['autoEncryptionEnabled'] ? 'Yes' : 'No'), + 'Key Vault Namespace: ' . $autoEncryptionInfo['keyVaultNamespace'], + 'Key Count: ' . $autoEncryptionInfo['keyCount'], + ]); + } catch (RuntimeException $e) { + // We typically get an error when mongocryptd is not running or not reachable. + $io->error('Failed to retrieve auto encryption information: ' . $e->getMessage()); + } + + return true; + } } diff --git a/src/Command/DumpEncryptedFieldsMapCommand.php b/src/Command/DumpEncryptedFieldsMapCommand.php new file mode 100644 index 00000000..f2c999dc --- /dev/null +++ b/src/Command/DumpEncryptedFieldsMapCommand.php @@ -0,0 +1,116 @@ + $documentManagers */ + public function __construct(private readonly ServiceCollectionInterface $documentManagers) + { + parent::__construct(); + } + + protected function configure(): void + { + $this->addOption( + 'format', + 'f', + InputOption::VALUE_REQUIRED, + 'The output format for the encrypted fields map (yaml, php)', + 'yaml', + ['yaml', 'php', 'json'] + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + + $format = $input->getOption('format'); + + $dumper = new Dumper(); + + foreach ($this->documentManagers as $name => $documentManager) { + $generator = new EncryptedFieldsMapGenerator($documentManager->getMetadataFactory()); + $encryptedFieldsMap = $generator->getEncryptedFieldsMap(); + + if (empty($encryptedFieldsMap)) { + continue; + } + + $encryptedFieldsMap = array_combine( + // Convert class names in keys to their full namespaces + array_map( + fn (string $fqcn): string => $this->getDocumentNamespace( + $documentManager->getClassMetadata($fqcn), + $documentManager->getConfiguration()->getDefaultDB(), + ), + array_keys($encryptedFieldsMap), + ), + array_values($encryptedFieldsMap), + ); + + $io->section(sprintf('Dumping encrypted fields map for document manager "%s"', $name)); + switch ($format) { + case 'yaml': + $outputContent = $dumper->dump($encryptedFieldsMap, 3); + break; + case 'php': + $outputContent = var_export($encryptedFieldsMap, true); + break; + case 'json': + $outputContent = json_encode($encryptedFieldsMap, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_THROW_ON_ERROR); + break; + default: + $io->error(sprintf('Unknown format "%s"', $format)); + + return Command::FAILURE; + } + + $io->block($outputContent); + } + + return Command::SUCCESS; + } + + private function getDocumentNamespace(ClassMetadata $metadata, string $defaultDb): string + { + $db = $metadata->getDatabase(); + $db = $db ?: $defaultDb; + $db = $db ?: 'doctrine'; + + return $db . '.' . $metadata->getCollection(); + } +} diff --git a/src/DataCollector/ConnectionDiagnostic.php b/src/DataCollector/ConnectionDiagnostic.php index 9c6ae8bd..59bda32c 100644 --- a/src/DataCollector/ConnectionDiagnostic.php +++ b/src/DataCollector/ConnectionDiagnostic.php @@ -4,56 +4,44 @@ namespace Doctrine\Bundle\MongoDBBundle\DataCollector; -use Composer\InstalledVersions; use MongoDB\Client; use MongoDB\Driver\Command; use MongoDB\Driver\ReadPreference; +use MongoDB\Driver\Server; -use function dd; -use function exec; -use function explode; -use function extension_loaded; -use function file_exists; -use function getenv; -use function phpversion; -use function trim; - -// Check if mongocryptd (enterprise installed locally) or crypt_shared is available -// Install openssl +use function array_flip; +use function array_intersect_key; +use function in_array; +use function iterator_count; +use function version_compare; + +/** @internal */ class ConnectionDiagnostic { - public function __construct(private Client $client, private array $driverOptions) - { - } - - public function getServerInfo(): array - { - $server = $this->client->getManager()->selectServer(new ReadPreference(ReadPreference::PRIMARY)); - $buildInfo = $server->executeCommand('admin', new Command(['buildInfo' => 1]))->toArray()[0] ?? null; - // command not supported for auto encryption: serverStatus - // $serverStatus = $server->executeCommand('admin', new Command(['serverStatus' => 1]))->toArray()[0] ?? null; - - return [ - 'version' => $buildInfo->version ?? null, - 'modules' => $buildInfo->modules ?? null, - //'crypt_shared_version' => $serverStatus->crypt_shared ?? null, - //'crypt_shared_path' => $serverStatus->crypt_shared_path ?? null, - 'topology' => $server->getType() ?? null, - ]; - } - - public function getPhpExtensionInfo(): array - { - return [ - 'ext-mongodb loaded' => extension_loaded('mongodb'), - 'ext-mongodb version' => phpversion('mongodb') ?: null, - 'library version' => InstalledVersions::getPrettyVersion('mongodb/mongodb'), - ]; + private const CLIENT_ENCRYPTION_OPTION_NAMES = [ + 'keyVaultClient', + 'keyVaultNamespace', + 'kmsProviders', + 'tlsOptions', + ]; + + private const SUPPORTED_SERVER_TYPES = [ + Server::TYPE_MONGOS, + Server::TYPE_RS_PRIMARY, + Server::TYPE_RS_SECONDARY, + ]; + + public function __construct( + private readonly Client $client, + private readonly array $driverOptions, + ) { } /** * Get the list of auto encryption providers configured for the MongoDB client * and an indication of whether the configuration is valid. + * + * @return array{autoEncryptionEnabled: bool, keyVaultNamespace: string, keyCount: int}|null */ public function getAutoEncryptionInfo(): ?array { @@ -63,47 +51,46 @@ public function getAutoEncryptionInfo(): ?array $autoEncryption = $this->driverOptions['autoEncryption']; - // Check if the "keyVaultNamespace" collection exists and is properly formatted - $keyVaultNamespace = explode('.', $autoEncryption['keyVaultNamespace'], 2); - $keyCount = $this->client->getCollection($keyVaultNamespace[0], $keyVaultNamespace[1])->countDocuments(); - $clientEncryption = $this->client->createClientEncryption([]); - $clientEncryption->getKeys(); - dd($clientEncryption->getKeys()); + $clientEncryptionOpts = array_intersect_key($autoEncryption, array_flip(self::CLIENT_ENCRYPTION_OPTION_NAMES)); + $clientEncryption = $this->client->createClientEncryption($clientEncryptionOpts); return [ - 'autoEncryption enabled' => true, + 'autoEncryptionEnabled' => true, 'keyVaultNamespace' => $autoEncryption['keyVaultNamespace'], - 'keyCount' => $keyCount, + 'keyCount' => iterator_count($clientEncryption->getKeys()), ]; } - public function getMongocryptdVersion(): ?string + /** @return array{topologyName: string, topologySupported: bool, version: ?string, versionSupported: bool} */ + public function getServerInfo(): array { - $mongocryptdPath = $this->findMongocryptdPath(); - if ($mongocryptdPath === null) { - return null; - } - - $output = []; - exec($mongocryptdPath . ' --version', $output); + $server = $this->client->getManager()->selectServer(new ReadPreference(ReadPreference::PRIMARY_PREFERRED)); + $buildInfo = $server->executeCommand('admin', new Command(['buildInfo' => 1]))->toArray()[0] ?? null; - if (isset($output[0])) { - return trim($output[0]); - } + $version = $buildInfo->version ?? null; - return null; + return [ + 'topologyName' => $this->getTopologyType($server), + 'topologySupported' => in_array($server->getType(), self::SUPPORTED_SERVER_TYPES), + 'version' => $version, + 'versionSupported' => $version ? version_compare($version, '8.0.0', '>=') : false, + ]; } - private function findMongocryptdPath(): ?string + public function usesAutoEncryption(): bool { - $paths = explode(':', getenv('PATH') ?: ''); - - foreach ($paths as $path) { - if (file_exists($path . '/mongocryptd')) { - return $path . '/mongocryptd'; - } - } + return isset($this->driverOptions['autoEncryption']); + } - return null; + private function getTopologyType(Server $server): string + { + return match ($server->getType()) { + Server::TYPE_STANDALONE => 'Standalone', + Server::TYPE_MONGOS => 'Sharded Cluster', + Server::TYPE_RS_PRIMARY, + Server::TYPE_RS_SECONDARY, + Server::TYPE_RS_ARBITER => 'Replica Set', + default => 'Unknown', + }; } } diff --git a/src/DataCollector/EncryptionDiagnostic.php b/src/DataCollector/EncryptionDiagnostic.php new file mode 100644 index 00000000..1d55bcea --- /dev/null +++ b/src/DataCollector/EncryptionDiagnostic.php @@ -0,0 +1,105 @@ +getExtensionInfoRow('libmongocrypt') !== 'disabled'; + + return [ + 'extensionLoaded' => extension_loaded('mongodb'), + 'extensionVersion' => phpversion('mongodb') ?: null, + 'extensionSupportsLibmongocrypt' => $libmongocryptAvailable, + 'libraryVersion' => InstalledVersions::getPrettyVersion('mongodb/mongodb'), + ]; + } + + /** @return array{mongocryptdPath: ?string, mongocryptdVersion: ?string} */ + public function getMongocryptdInfo(): array + { + $mongocryptdPath = $this->findMongocryptdPath(); + + return [ + 'mongocryptdPath' => $mongocryptdPath, + 'mongocryptdVersion' => $this->getMongocryptdVersion($mongocryptdPath), + ]; + } + + private function findMongocryptdPath(): ?string + { + $paths = explode(':', getenv('PATH') ?: ''); + + foreach ($paths as $path) { + if (file_exists($path . '/mongocryptd')) { + return $path . '/mongocryptd'; + } + } + + return null; + } + + private function getMongocryptdVersion(?string $mongocryptdPath): ?string + { + if ($mongocryptdPath === null) { + return null; + } + + $output = []; + exec($mongocryptdPath . ' --version', $output); + + if (isset($output[0])) { + return trim($output[0]); + } + + return null; + } + + private function getExtensionInfo(): string + { + $extension = new ReflectionExtension('mongodb'); + + ob_start(); + $extension->info(); + $info = ob_get_contents(); + ob_end_clean(); + + return (string) $info; + } + + private function getExtensionInfoRow(string $row): ?string + { + $pattern = sprintf('/^%s(.*)$/m', preg_quote($row . ' => ')); + + if (preg_match($pattern, $this->getExtensionInfo(), $matches) !== 1) { + return null; + } + + return $matches[1]; + } +} diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index c21bcabc..595e5147 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -182,22 +182,6 @@ public function load(array $configs, ContainerBuilder $container): void $this->loadMessengerServices($container, $loader); $this->loadEntityValueResolverServices($container, $loader, $config); - - // Register EncryptionDiagnostics for each connection - $diagnosticsRefs = []; - foreach ($config['connections'] as $connName => $connConfig) { - $connService = sprintf('doctrine_mongodb.odm.%s_connection', $connName); - $driverOptions = $connConfig['driver_options'] ?? []; - $diagServiceId = sprintf('doctrine_mongodb.encryption_diagnostics.%s', $connName); - $container->setDefinition( - $diagServiceId, - new Definition(ConnectionDiagnostic::class, [ - new Reference($connService), // Use the connection service, which is a MongoDB\Client - $driverOptions, - ]), - ); - $diagnosticsRefs[$connName] = new Reference($diagServiceId); - } } /** diff --git a/tests/Command/ConnectionDiagnosticCommandTest.php b/tests/Command/ConnectionDiagnosticCommandTest.php new file mode 100644 index 00000000..0948236c --- /dev/null +++ b/tests/Command/ConnectionDiagnosticCommandTest.php @@ -0,0 +1,28 @@ +find('doctrine:mongodb:connection:diagnostic'); + $commandTester = new CommandTester($command); + $commandTester->execute([]); + + $output = $commandTester->getDisplay(); + $this->assertStringContainsString('MongoDB extension loaded', $output); + $this->assertStringContainsString('mongocryptd', $output); + $this->assertStringContainsString('Connection: default', $output); + $this->assertStringContainsString('Auto encryption is not enabled for this connection.', $output); + } +} diff --git a/tests/DependencyInjection/ConfigurationTest.php b/tests/DependencyInjection/ConfigurationTest.php index 755576c6..47613d78 100644 --- a/tests/DependencyInjection/ConfigurationTest.php +++ b/tests/DependencyInjection/ConfigurationTest.php @@ -176,7 +176,7 @@ public function testFullConfiguration(array $config): void 'mongocryptdBypassSpawn' => true, 'mongocryptdSpawnPath' => '%kernel.project_dir%/bin/mongocryptd', 'mongocryptdSpawnArgs' => '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60', - 'cryptSharedLibPath' => '%kernel.project_dir%/bin/libmongocrypt.so', + 'cryptSharedLibPath' => '%kernel.project_dir%/bin/mongo_crypt_v1.dylib', ], ], ], diff --git a/tests/DependencyInjection/Fixtures/config/xml/full.xml b/tests/DependencyInjection/Fixtures/config/xml/full.xml index b6f6701d..65cdbdb6 100644 --- a/tests/DependencyInjection/Fixtures/config/xml/full.xml +++ b/tests/DependencyInjection/Fixtures/config/xml/full.xml @@ -106,7 +106,7 @@ mongocryptdBypassSpawn="true" mongocryptdSpawnPath="%kernel.project_dir%/bin/mongocryptd" mongocryptdSpawnArgs="--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60" - cryptSharedLibPath="%kernel.project_dir%/bin/libmongocrypt.so" + cryptSharedLibPath="%kernel.project_dir%/bin/mongo_crypt_v1.dylib" /> diff --git a/tests/DependencyInjection/Fixtures/config/yml/full.yml b/tests/DependencyInjection/Fixtures/config/yml/full.yml index 7b1614fb..e2f4215c 100644 --- a/tests/DependencyInjection/Fixtures/config/yml/full.yml +++ b/tests/DependencyInjection/Fixtures/config/yml/full.yml @@ -90,7 +90,7 @@ doctrine_mongodb: mongocryptdBypassSpawn: true mongocryptdSpawnPath: '%kernel.project_dir%/bin/mongocryptd' mongocryptdSpawnArgs: '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60' - cryptSharedLibPath: '%kernel.project_dir%/bin/libmongocrypt.so' + cryptSharedLibPath: '%kernel.project_dir%/bin/mongo_crypt_v1.dylib' conn2: server: mongodb://otherhost From af57fa042a2a8449c1feee585778f938ef9dea54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Fri, 1 Aug 2025 10:30:33 +0200 Subject: [PATCH 05/27] [Encryption] Fix format of `encryptedFieldsMaps` in the `autoEncryption` configuration (#905) * Fix format of encryptedFieldsMaps in the configuration * EncryptedFieldsMaps loaded from a JSON string from XML configuration * Enable client configuration for tests - partial --- config/schema/mongodb-1.0.xsd | 23 +-- docs/encryption.rst | 34 ++-- phpunit.xml.dist | 1 + src/Command/DumpEncryptedFieldsMapCommand.php | 34 ++-- src/DependencyInjection/Configuration.php | 26 +-- .../DoctrineMongoDBExtension.php | 8 + .../DependencyInjection/ConfigurationTest.php | 189 +++++++++++++----- .../Fixtures/config/xml/full.xml | 108 ++++++++-- .../Fixtures/config/yml/full.yml | 58 +++++- tests/TestCase.php | 5 +- 10 files changed, 347 insertions(+), 139 deletions(-) diff --git a/config/schema/mongodb-1.0.xsd b/config/schema/mongodb-1.0.xsd index 026d714b..3d336ff6 100644 --- a/config/schema/mongodb-1.0.xsd +++ b/config/schema/mongodb-1.0.xsd @@ -179,26 +179,9 @@ - - - - - - - - - - - - - - - - - - - - + + + diff --git a/docs/encryption.rst b/docs/encryption.rst index 5be85afd..13c6bc7b 100644 --- a/docs/encryption.rst +++ b/docs/encryption.rst @@ -92,7 +92,12 @@ This setting is **recommended** for improved security and performance. specified collection, the client downloads the server-side remote schema for the collection and uses it instead. -For more details, see the official MongoDB documentation: `Encrypted Fields and Enabled Queries `_. +For more details, see the official MongoDB documentation: +`Encrypted Fields and Enabled Queries `_. + +Note that there is no ``fields`` key in the configuration of each collection +for the bundle configuration. Instead, you directly specify the list of +encrypted fields as an array under the collection namespace. .. tabs:: @@ -106,9 +111,9 @@ For more details, see the official MongoDB documentation: `Encrypted Fields and autoEncryption: encryptedFieldsMap: "mydatabase.mycollection": - fields: - - path: "sensitive_field" - bsonType: "string" + - keyId: { $binary: { base64: 2CSosXLSTEKaYphcSnUuCw==, subType: '04' } } + path: "sensitive_field" + bsonType: "string" .. group-tab:: XML @@ -117,9 +122,15 @@ For more details, see the official MongoDB documentation: `Encrypted Fields and - - - + @@ -135,11 +146,10 @@ For more details, see the official MongoDB documentation: `Encrypted Fields and ->autoEncryption([ 'encryptedFieldsMap' => [ 'mydatabase.mycollection' => [ - 'fields' => [ - [ - 'path' => 'sensitive_field', - 'bsonType' => 'string', - ], + [ + 'path' => 'sensitive_field', + 'keyId' => ['$binary' => ['base64' => '2CSosXLSTEKaYphcSnUuCw==', 'subType' => '04' ] ], + 'bsonType' => 'string', ], ], ], diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 50eaaae1..2ef3bdec 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -20,6 +20,7 @@ + diff --git a/src/Command/DumpEncryptedFieldsMapCommand.php b/src/Command/DumpEncryptedFieldsMapCommand.php index f2c999dc..25b0d630 100644 --- a/src/Command/DumpEncryptedFieldsMapCommand.php +++ b/src/Command/DumpEncryptedFieldsMapCommand.php @@ -6,7 +6,7 @@ use Doctrine\ODM\MongoDB\DocumentManager; use Doctrine\ODM\MongoDB\Mapping\ClassMetadata; -use Doctrine\ODM\MongoDB\Utility\EncryptedFieldsMapGenerator; +use MongoDB\BSON\PackedArray; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -16,10 +16,7 @@ use Symfony\Component\Yaml\Dumper; use Symfony\Contracts\Service\ServiceCollectionInterface; -use function array_combine; -use function array_keys; -use function array_map; -use function array_values; +use function json_decode; use function json_encode; use function sprintf; use function var_export; @@ -63,24 +60,25 @@ protected function execute(InputInterface $input, OutputInterface $output): int $dumper = new Dumper(); foreach ($this->documentManagers as $name => $documentManager) { - $generator = new EncryptedFieldsMapGenerator($documentManager->getMetadataFactory()); - $encryptedFieldsMap = $generator->getEncryptedFieldsMap(); + $encryptedFieldsMap = []; + foreach ($documentManager->getMetadataFactory()->getAllMetadata() as $metadata) { + $database = $documentManager->getDocumentDatabase($metadata->getName()); + $collectionInfoIterator = $database->listCollections(['filter' => ['name' => $metadata->getCollection()]]); + + foreach ($collectionInfoIterator as $collectionInfo) { + if ($collectionInfo['options']['encryptedFields'] ?? null) { + $encryptedFieldsMap[$this->getDocumentNamespace($metadata, $database->getDatabaseName())] = $collectionInfo['options']['encryptedFields']; + } + } + } if (empty($encryptedFieldsMap)) { continue; } - $encryptedFieldsMap = array_combine( - // Convert class names in keys to their full namespaces - array_map( - fn (string $fqcn): string => $this->getDocumentNamespace( - $documentManager->getClassMetadata($fqcn), - $documentManager->getConfiguration()->getDefaultDB(), - ), - array_keys($encryptedFieldsMap), - ), - array_values($encryptedFieldsMap), - ); + foreach ($encryptedFieldsMap as $ns => $encryptedFields) { + $encryptedFieldsMap[$ns] = json_decode(PackedArray::fromPHP($encryptedFields['fields'])->toRelaxedExtendedJSON(), true); + } $io->section(sprintf('Dumping encrypted fields map for document manager "%s"', $name)); switch ($format) { diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 17e483a3..b4b6a2b7 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -11,7 +11,6 @@ use Symfony\Component\Config\Definition\Builder\TreeBuilder; use Symfony\Component\Config\Definition\ConfigurationInterface; -use function array_is_list; use function count; use function in_array; use function is_array; @@ -20,6 +19,8 @@ use function method_exists; use function preg_match; +use const JSON_THROW_ON_ERROR; + /** * FrameworkExtension configuration structure. */ @@ -401,20 +402,8 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void ->useAttributeAsKey('name', false) ->beforeNormalization() ->always(static function ($v) { - if (isset($v['encryptedFields']) && is_array($v['encryptedFields'])) { - $encryptedFields = $v['encryptedFields']; - if (! array_is_list($encryptedFields)) { - $encryptedFields = [$encryptedFields]; - } - - $v = []; - foreach ($encryptedFields as $field) { - if (is_array($field['field'] ?? null) && ! array_is_list($field['field'])) { - $field['field'] = [$field['field']]; - } - - $v[$field['name'] ?? ''] = $field['field'] ?? []; - } + if (is_string($v)) { + return json_decode($v, true, 512, JSON_THROW_ON_ERROR); } return $v; @@ -424,13 +413,16 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void ->children() ->scalarNode('path')->isRequired()->cannotBeEmpty()->end() ->scalarNode('bsonType')->isRequired()->cannotBeEmpty()->end() + ->variableNode('keyId')->isRequired()->cannotBeEmpty()->end() ->arrayNode('queries') ->children() ->scalarNode('queryType')->isRequired()->cannotBeEmpty()->end() - ->integerNode('min')->end() - ->integerNode('max')->end() + ->variableNode('min')->end() + ->variableNode('max')->end() ->integerNode('sparsity')->end() + ->integerNode('precision')->end() ->integerNode('trimFactor')->end() + ->integerNode('contention')->end() ->end() ->end() ->end() diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index 595e5147..66b6ca75 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -27,6 +27,7 @@ use Doctrine\Persistence\Mapping\Driver\MappingDriverChain; use Doctrine\Persistence\Proxy; use InvalidArgumentException; +use MongoDB\BSON\Document as BsonDocument; use MongoDB\Client; use ProxyManager\Proxy\LazyLoadingInterface; use Symfony\Bridge\Doctrine\DependencyInjection\AbstractDoctrineExtension; @@ -55,6 +56,7 @@ use function in_array; use function interface_exists; use function is_dir; +use function json_encode; use function method_exists; use function sprintf; @@ -538,6 +540,12 @@ private function normalizeAutoEncryption(array $autoEncryption, string $defaultD $autoEncryption['keyVaultNamespace'] ??= $defaultDB . '.datakeys'; + if (isset($autoEncryption['encryptedFieldsMap'])) { + foreach ($autoEncryption['encryptedFieldsMap'] as &$value) { + $value = (new Definition(BsonDocument::class))->setFactory([BsonDocument::class, 'fromJSON'])->setArguments([json_encode(['fields' => $value])]); + } + } + return $autoEncryption; } diff --git a/tests/DependencyInjection/ConfigurationTest.php b/tests/DependencyInjection/ConfigurationTest.php index 47613d78..dae2c3a0 100644 --- a/tests/DependencyInjection/ConfigurationTest.php +++ b/tests/DependencyInjection/ConfigurationTest.php @@ -147,27 +147,108 @@ public function testFullConfiguration(array $config): void 'bypassAutoEncryption' => true, 'bypassQueryAnalysis' => true, 'encryptedFieldsMap' => [ + 'encrypted.RangeTypes' => [ + [ + 'keyId' => ['$binary' => ['base64' => 'lhZHItpvRkqXevh4Wtqg/g==', 'subType' => '04']], + 'path' => 'intField', + 'bsonType' => 'int', + 'queries' => ['queryType' => 'range', 'contention' => 8, 'min' => 5, 'max' => 10], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'qd9PEKIPTE2J30ev29lMpQ==', 'subType' => '04']], + 'path' => 'floatField', + 'bsonType' => 'double', + 'queries' => ['queryType' => 'range', 'contention' => 8, 'min' => 5.5, 'max' => 10.5, 'precision' => 1], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'zVLg8CF4RSSu4xn7x7dOyQ==', 'subType' => '04']], + 'path' => 'decimalField', + 'bsonType' => 'decimal', + 'queries' => [ + 'queryType' => 'range', + 'contention' => 8, + 'min' => ['$numberDecimal' => '0.1'], + 'max' => ['$numberDecimal' => '0.2'], + 'precision' => 2, + ], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'ySdd8lZ2QBqnwKPJTp/yLA==', 'subType' => '04']], + 'path' => 'immutableDateField', + 'bsonType' => 'date', + 'queries' => [ + 'queryType' => 'range', + 'contention' => 8, + 'min' => ['$date' => '2000-01-01T00:00:00Z'], + 'max' => ['$date' => '2100-01-01T00:00:00Z'], + ], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'NWKI+DyES/OlNkUbJbWJ9w==', 'subType' => '04']], + 'path' => 'dateField', + 'bsonType' => 'date', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'wiiv+0K/QAquyEq3HDxRKw==', 'subType' => '04']], + 'path' => 'binField', + 'bsonType' => 'binData', + ], + [ + 'keyId' => ['$binary' => ['base64' => '2CSosXLSTEKaYphcSnUuCw==', 'subType' => '04']], + 'path' => 'timestampField', + 'bsonType' => 'timestamp', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'h3H6HdG3T5CK+Z2yQ4Ho+Q==', 'subType' => '04']], + 'path' => 'hashField', + 'bsonType' => 'object', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'X78UZZ/HTX2wLw4K3uG42w==', 'subType' => '04']], + 'path' => 'collectionField', + 'bsonType' => 'objectId', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'LugQL/ZXTJOl856Yacmkwg==', 'subType' => '04']], + 'path' => 'boolField', + 'bsonType' => 'bool', + ], + ], 'encrypted.patients' => [ [ - 'path' => 'patientRecord.ssn', - 'bsonType' => 'string', - 'queries' => ['queryType' => 'equality'], + 'keyId' => ['$binary' => ['base64' => 'GH25/XvYSaCgTUQLAo1hQw==', 'subType' => '04']], + 'path' => 'pathologies', + 'bsonType' => 'array', ], [ + 'keyId' => ['$binary' => ['base64' => 'krVWyFlNTUOaGFMfk+s7UA==', 'subType' => '04']], 'path' => 'patientRecord.billing', 'bsonType' => 'object', ], [ + 'keyId' => ['$binary' => ['base64' => 'X1ZaSI1GSAKnZ+sPGcmYBA==', 'subType' => '04']], 'path' => 'patientRecord.billingAmount', 'bsonType' => 'int', - 'queries' => ['queryType' => 'range', 'min' => 100, 'max' => 2000, 'sparsity' => 1, 'trimFactor' => 4], + 'queries' => [ + 'queryType' => 'range', + 'contention' => 8, + 'min' => 100, + 'max' => 2000, + 'sparsity' => 1, + 'trimFactor' => 4, + ], ], ], - 'encrypted.users' => [ + 'encrypted.client' => [ [ - 'path' => 'email', + 'keyId' => ['$binary' => ['base64' => 'I0Aw18vnRGWzVS1t3uejpQ==', 'subType' => '04']], + 'path' => 'name', 'bsonType' => 'string', - 'queries' => ['queryType' => 'equality'], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'XSPRK3vaTLmMZr9IEj/qwQ==', 'subType' => '04']], + 'path' => 'clientCards', + 'bsonType' => 'array', ], ], ], @@ -491,7 +572,7 @@ public static function provideNormalizeOptions(): Generator ], ]; - // Encrypted Field Map normalization from XML tags + // Encrypted Field Map can be a JSON string in a yield [ [ 'connection' => [ @@ -500,38 +581,40 @@ public static function provideNormalizeOptions(): Generator 'id' => 'foo', 'autoEncryption' => [ 'kmsProvider' => ['type' => 'local', 'key' => '1234567890123456789012345678901234567890123456789012345678901234'], - 'encryptedFieldsMap' => [ - 'encryptedFields' => [ - [ - 'name' => 'encrypted.patients', - 'field' => [ - [ - 'path' => 'patientRecord.ssn', - 'bsonType' => 'string', - 'queries' => ['queryType' => 'equality'], - ], - [ - 'path' => 'patientRecord.billing', - 'bsonType' => 'object', - ], - [ - 'path' => 'patientRecord.billingAmount', - 'bsonType' => 'int', - 'queries' => ['queryType' => 'range', 'min' => 100, 'max' => 2000, 'sparsity' => 1, 'trimFactor' => 4], - ], - ], - ], - [ - 'name' => 'encrypted.users', - 'field' => - [ - 'path' => 'email', - 'bsonType' => 'string', - 'queries' => ['queryType' => 'equality'], - ], - ], + 'encryptedFieldsMap' => <<<'JSON' + { + "encrypted.patients": [ + { + "keyId": { "$binary": { "base64": "GH25/XvYSaCgTUQLAo1hQw==", "subType": "04" } }, + "path": "pathologies", + "bsonType": "array" + }, + { + "keyId": { "$binary": { "base64": "krVWyFlNTUOaGFMfk+s7UA==", "subType": "04" } }, + "path": "patientRecord.billing", + "bsonType": "object" + }, + { + "keyId": { "$binary": { "base64": "X1ZaSI1GSAKnZ+sPGcmYBA==", "subType": "04" } }, + "path": "patientRecord.billingAmount", + "bsonType": "int", + "queries": { "queryType": "range", "contention": 8, "min": 100, "max": 2000, "sparsity": 1, "trimFactor": 4 } + } ], - ], + "encrypted.client": [ + { + "keyId": { "$binary": { "base64": "I0Aw18vnRGWzVS1t3uejpQ==", "subType": "04" } }, + "path": "name", + "bsonType": "string" + }, + { + "keyId": { "$binary": { "base64": "XSPRK3vaTLmMZr9IEj/qwQ==", "subType": "04" } }, + "path": "clientCards", + "bsonType": "array" + } + ] + } + JSON, ], ], ], @@ -545,25 +628,39 @@ public static function provideNormalizeOptions(): Generator 'encryptedFieldsMap' => [ 'encrypted.patients' => [ [ - 'path' => 'patientRecord.ssn', - 'bsonType' => 'string', - 'queries' => ['queryType' => 'equality'], + 'keyId' => ['$binary' => ['base64' => 'GH25/XvYSaCgTUQLAo1hQw==', 'subType' => '04']], + 'path' => 'pathologies', + 'bsonType' => 'array', ], [ + 'keyId' => ['$binary' => ['base64' => 'krVWyFlNTUOaGFMfk+s7UA==', 'subType' => '04']], 'path' => 'patientRecord.billing', 'bsonType' => 'object', ], [ + 'keyId' => ['$binary' => ['base64' => 'X1ZaSI1GSAKnZ+sPGcmYBA==', 'subType' => '04']], 'path' => 'patientRecord.billingAmount', 'bsonType' => 'int', - 'queries' => ['queryType' => 'range', 'min' => 100, 'max' => 2000, 'sparsity' => 1, 'trimFactor' => 4], + 'queries' => [ + 'queryType' => 'range', + 'contention' => 8, + 'min' => 100, + 'max' => 2000, + 'sparsity' => 1, + 'trimFactor' => 4, + ], ], ], - 'encrypted.users' => [ + 'encrypted.client' => [ [ - 'path' => 'email', + 'keyId' => ['$binary' => ['base64' => 'I0Aw18vnRGWzVS1t3uejpQ==', 'subType' => '04']], + 'path' => 'name', 'bsonType' => 'string', - 'queries' => ['queryType' => 'equality'], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'XSPRK3vaTLmMZr9IEj/qwQ==', 'subType' => '04']], + 'path' => 'clientCards', + 'bsonType' => 'array', ], ], ], diff --git a/tests/DependencyInjection/Fixtures/config/xml/full.xml b/tests/DependencyInjection/Fixtures/config/xml/full.xml index 65cdbdb6..c4a19374 100644 --- a/tests/DependencyInjection/Fixtures/config/xml/full.xml +++ b/tests/DependencyInjection/Fixtures/config/xml/full.xml @@ -7,8 +7,8 @@ http://symfony.com/schema/dic/doctrine/odm/mongodb http://symfony.com/schema/dic/doctrine/odm/mongodb/mongodb-1.0.xsd"> - - - - - - - - - - - - - - + setHydratorNamespace('SymfonyTests\Doctrine'); $config->setMetadataDriverImpl(new AttributeDriver($paths)); $config->setMetadataCache(new ArrayAdapter()); + $uri = getenv('DOCTRINE_MONGODB_SERVER') ?: 'mongodb://localhost:27017'; - return DocumentManager::create(null, $config); + return DocumentManager::create(new Client($uri), $config); } } From 8a8bdd45a8d40962e9f340f2e54e49b5b4b4696b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Fri, 8 Aug 2025 20:05:49 +0200 Subject: [PATCH 06/27] Add link to encryption page from config docs --- docs/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.rst b/docs/config.rst index dd4fa08d..8a4a4b97 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -627,7 +627,7 @@ Otherwise you will get a *auth failed* exception. Using Queryable Encryption -------------------------- -TODO: Add documentation for queryable encryption configuration. +For details on configuring Queryable Encryption (QE) and Client-Side Field-Level Encryption (CSFLE), see :doc:`encryption`. Full Default Configuration -------------------------- From 61a81187ccfb4b081466922d617a628c64a248d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Mon, 11 Aug 2025 18:12:51 +0200 Subject: [PATCH 07/27] Escape mongocryptd path before calling it --- src/DataCollector/EncryptionDiagnostic.php | 3 ++- src/DependencyInjection/DoctrineMongoDBExtension.php | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/DataCollector/EncryptionDiagnostic.php b/src/DataCollector/EncryptionDiagnostic.php index 1d55bcea..3a02b64a 100644 --- a/src/DataCollector/EncryptionDiagnostic.php +++ b/src/DataCollector/EncryptionDiagnostic.php @@ -7,6 +7,7 @@ use Composer\InstalledVersions; use ReflectionExtension; +use function escapeshellarg; use function exec; use function explode; use function extension_loaded; @@ -71,7 +72,7 @@ private function getMongocryptdVersion(?string $mongocryptdPath): ?string } $output = []; - exec($mongocryptdPath . ' --version', $output); + exec(escapeshellarg($mongocryptdPath) . ' --version', $output); if (isset($output[0])) { return trim($output[0]); diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index 66b6ca75..a8f779ff 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -542,6 +542,8 @@ private function normalizeAutoEncryption(array $autoEncryption, string $defaultD if (isset($autoEncryption['encryptedFieldsMap'])) { foreach ($autoEncryption['encryptedFieldsMap'] as &$value) { + // Wrap the encrypted fields in a 'fields' key as required the encryptedFieldsMap structure. + // Some values can be BSON binary, date or numbers, the extended JSON format is used to convert them BSON document. $value = (new Definition(BsonDocument::class))->setFactory([BsonDocument::class, 'fromJSON'])->setArguments([json_encode(['fields' => $value])]); } } From 823dd72787a24fd47310c33ba4f648602b923457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Wed, 13 Aug 2025 21:50:51 +0200 Subject: [PATCH 08/27] [Encryption] Fix XML configuration (#910) * Add keyVaultClient to XML configuration * Require 1 kms provider in the XML config * Remove tls attributes from kms-provider node * Add cryptSharedLibRequired to XSD and test * Clean tlsOptions and wrap into an array with the kms provider name as key * Add comment for duplicate options --- config/schema/mongodb-1.0.xsd | 8 ++++---- src/DependencyInjection/Configuration.php | 8 +------- src/DependencyInjection/DoctrineMongoDBExtension.php | 7 ++++++- tests/DependencyInjection/ConfigurationTest.php | 2 ++ tests/DependencyInjection/Fixtures/config/xml/full.xml | 2 ++ tests/DependencyInjection/Fixtures/config/yml/full.yml | 2 ++ 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/config/schema/mongodb-1.0.xsd b/config/schema/mongodb-1.0.xsd index 3d336ff6..8e4641cd 100644 --- a/config/schema/mongodb-1.0.xsd +++ b/config/schema/mongodb-1.0.xsd @@ -125,8 +125,9 @@ - + + @@ -157,13 +158,11 @@ + - - - @@ -197,6 +196,7 @@ + diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index b4b6a2b7..db190e3c 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -384,13 +384,11 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void ->scalarNode('projectId')->end() ->scalarNode('location')->end() ->scalarNode('keyRing')->end() + // Attribute already present for another KMS type //->scalarNode('keyName')->end() //->scalarNode('keyVersion')->end() // KMIP //->scalarNode('endpoint')->end() - ->scalarNode('tlsCAFile')->end() - ->scalarNode('tlsClientCertificateKeyFile')->end() - ->scalarNode('tlsClientCertificateKeyFilePassword')->end() // Local ->scalarNode('key')->end() ->end() @@ -438,11 +436,7 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void ->scalarNode('tlsCAFile')->end() ->scalarNode('tlsCertificateKeyFile')->end() ->scalarNode('tlsCertificateKeyFilePassword')->end() - ->booleanNode('tlsAllowInvalidCertificates')->end() - ->booleanNode('tlsAllowInvalidHostnames')->end() - ->booleanNode('tlsDisableCertificateRevocationCheck')->end() ->booleanNode('tlsDisableOCSPEndpointCheck')->end() - ->booleanNode('tlsInsecure')->end() ->end() ->end() ->end() diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index a8f779ff..bbfaae34 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -528,9 +528,14 @@ private function normalizeAutoEncryption(array $autoEncryption, string $defaultD throw new InvalidArgumentException('The "kmsProvider" option must contain a "type" key.'); } + $provider = $autoEncryption['kmsProvider']['type']; $autoEncryption['kmsProviders'] = [ - $autoEncryption['kmsProvider']['type'] => array_diff_key($autoEncryption['kmsProvider'], ['type' => true]), + $provider => array_diff_key($autoEncryption['kmsProvider'], ['type' => true]), ]; + if (isset($autoEncryption['tlsOptions'])) { + $autoEncryption['tlsOptions'] = [$provider => $autoEncryption['tlsOptions']]; + } + unset($autoEncryption['kmsProvider']); unset($autoEncryption['masterKey']); diff --git a/tests/DependencyInjection/ConfigurationTest.php b/tests/DependencyInjection/ConfigurationTest.php index dae2c3a0..42a66d78 100644 --- a/tests/DependencyInjection/ConfigurationTest.php +++ b/tests/DependencyInjection/ConfigurationTest.php @@ -137,6 +137,7 @@ public function testFullConfiguration(array $config): void 'sessionToken' => 'MONGODB_AWS_SESSION_TOKEN', ], 'masterKey' => ['key' => 'MONGODB_AWS_MASTER_KEY'], + 'keyVaultClient' => 'my_key_vault_client_service', 'keyVaultNamespace' => 'encryption.__keyVault', 'tlsOptions' => [ 'tlsCAFile' => '%kernel.project_dir%/config/certificates/mongodb-ca.pem', @@ -258,6 +259,7 @@ public function testFullConfiguration(array $config): void 'mongocryptdSpawnPath' => '%kernel.project_dir%/bin/mongocryptd', 'mongocryptdSpawnArgs' => '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60', 'cryptSharedLibPath' => '%kernel.project_dir%/bin/mongo_crypt_v1.dylib', + 'cryptSharedLibRequired' => true, ], ], ], diff --git a/tests/DependencyInjection/Fixtures/config/xml/full.xml b/tests/DependencyInjection/Fixtures/config/xml/full.xml index c4a19374..69dd638b 100644 --- a/tests/DependencyInjection/Fixtures/config/xml/full.xml +++ b/tests/DependencyInjection/Fixtures/config/xml/full.xml @@ -78,6 +78,7 @@ sessionToken="MONGODB_AWS_SESSION_TOKEN" /> + my_key_vault_client_service encryption.__keyVault diff --git a/tests/DependencyInjection/Fixtures/config/yml/full.yml b/tests/DependencyInjection/Fixtures/config/yml/full.yml index 2e9bfb9f..4b334a38 100644 --- a/tests/DependencyInjection/Fixtures/config/yml/full.yml +++ b/tests/DependencyInjection/Fixtures/config/yml/full.yml @@ -64,6 +64,7 @@ doctrine_mongodb: masterKey: key: 'MONGODB_AWS_MASTER_KEY' keyVaultNamespace: 'encryption.__keyVault' + keyVaultClient: 'my_key_vault_client_service' tlsOptions: tlsCAFile: '%kernel.project_dir%/config/certificates/mongodb-ca.pem' tlsCertificateKeyFile: '%kernel.project_dir%/config/certificates/mongodb-client.pem' @@ -131,6 +132,7 @@ doctrine_mongodb: mongocryptdSpawnPath: '%kernel.project_dir%/bin/mongocryptd' mongocryptdSpawnArgs: '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60' cryptSharedLibPath: '%kernel.project_dir%/bin/mongo_crypt_v1.dylib' + cryptSharedLibRequired: true conn2: server: mongodb://otherhost From f21d4ee7a84d3cfd62d2e42e49423ca9825404a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Mon, 18 Aug 2025 20:07:27 +0200 Subject: [PATCH 09/27] Remove invalid tlsOptions from the config doc --- docs/config.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 8a4a4b97..db08a37b 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -725,11 +725,7 @@ Full Default Configuration tlsCAFile: null # Path to CA file, e.g., /path/to/key-vault-ca.pem tlsCertificateKeyFile: null # Path to client cert/key file, e.g., /path/to/key-vault-client.pem tlsCertificateKeyFilePassword: null # Password for client cert/key file - tlsAllowInvalidCertificates: false # Bypass server certificate validation (use with caution) - tlsAllowInvalidHostnames: false # Bypass server hostname validation (use with caution) - tlsDisableCertificateRevocationCheck: false # Disable CRL checks tlsDisableOCSPEndpointCheck: false # Disable OCSP checks - tlsInsecure: false # Allow invalid/no server cert (use with extreme caution) proxy_namespace: MongoDBODMProxies proxy_dir: "%kernel.cache_dir%/doctrine/odm/mongodb/Proxies" From 23737b390cf1bd9ad5160eed06e6ccb674ff3ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Mon, 18 Aug 2025 20:12:37 +0200 Subject: [PATCH 10/27] Remove unecessary attributes from the XSD (default values) --- config/schema/mongodb-1.0.xsd | 106 +++++++++++++++++----------------- docs/encryption.rst | 12 ++-- 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/config/schema/mongodb-1.0.xsd b/config/schema/mongodb-1.0.xsd index 8e4641cd..457987d7 100644 --- a/config/schema/mongodb-1.0.xsd +++ b/config/schema/mongodb-1.0.xsd @@ -9,7 +9,7 @@ - + @@ -46,9 +46,9 @@ - - - + + + @@ -125,46 +125,46 @@ - - - - - - - + + + + + + + - - + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + - - + + - + - + @@ -172,10 +172,10 @@ - - - - + + + + @@ -184,27 +184,27 @@ - - - - + + + + - - - - - - + + + + + + - - + + @@ -247,10 +247,10 @@ - - - - + + + + diff --git a/docs/encryption.rst b/docs/encryption.rst index 13c6bc7b..2ae90df3 100644 --- a/docs/encryption.rst +++ b/docs/encryption.rst @@ -204,7 +204,8 @@ If the driver is not able to find the library, you can specify its path using th TLS Options ----------- -If you are not specifying a custom ``keyVaultClient`` service, you can configure TLS settings for the internal key vault client using the ``tlsOptions`` key: +If you are not specifying a custom ``keyVaultClient`` service, you can configure +TLS settings for the internal key vault client using the ``tlsOptions`` key: .. tabs:: @@ -220,8 +221,7 @@ If you are not specifying a custom ``keyVaultClient`` service, you can configure tlsCAFile: "/path/to/key-vault-ca.pem" tlsCertificateKeyFile: "/path/to/key-vault-client.pem" tlsCertificateKeyFilePassword: "keyvaultclientpassword" - tlsAllowInvalidCertificates: false - tlsAllowInvalidHostnames: false + tlsDisableOCSPEndpointCheck: false .. group-tab:: XML @@ -233,8 +233,7 @@ If you are not specifying a custom ``keyVaultClient`` service, you can configure /path/to/key-vault-ca.pem /path/to/key-vault-client.pem keyvaultclientpassword - false - false + false @@ -252,8 +251,7 @@ If you are not specifying a custom ``keyVaultClient`` service, you can configure 'tlsCAFile' => '/path/to/key-vault-ca.pem', 'tlsCertificateKeyFile' => '/path/to/key-vault-client.pem', 'tlsCertificateKeyFilePassword' => 'keyvaultclientpassword', - 'tlsAllowInvalidCertificates' => false, - 'tlsAllowInvalidHostnames' => false, + 'tlsDisableOCSPEndpointCheck' => false, ], ]); }; From 953613d40fbcccc43871aa419c8243615f54aba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Mon, 18 Aug 2025 21:55:51 +0200 Subject: [PATCH 11/27] Cast KMS provider to object to support AWS empty config https://github.com/doctrine/mongodb-odm/pull/2801 --- .../DoctrineMongoDBExtension.php | 14 +++++++--- .../DoctrineMongoDBExtensionTest.php | 28 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index bbfaae34..09ad0d41 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -528,10 +528,16 @@ private function normalizeAutoEncryption(array $autoEncryption, string $defaultD throw new InvalidArgumentException('The "kmsProvider" option must contain a "type" key.'); } - $provider = $autoEncryption['kmsProvider']['type']; - $autoEncryption['kmsProviders'] = [ - $provider => array_diff_key($autoEncryption['kmsProvider'], ['type' => true]), - ]; + $provider = $autoEncryption['kmsProvider']['type']; + $providerOpts = array_diff_key($autoEncryption['kmsProvider'], ['type' => true]); + // To use "Automatic Credentials", the provider options must be an empty document. + // Fix the empty array to an empty stdClass object, as the driver expects it. + if ($providerOpts === []) { + $providerOpts = new Definition('stdClass'); + } + + $autoEncryption['kmsProviders'] = [$provider => $providerOpts]; + if (isset($autoEncryption['tlsOptions'])) { $autoEncryption['tlsOptions'] = [$provider => $autoEncryption['tlsOptions']]; } diff --git a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php index 9a621bab..c8b462d0 100644 --- a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php +++ b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php @@ -16,6 +16,7 @@ use MongoDB\Client; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use stdClass; use Symfony\Bridge\Doctrine\Messenger\DoctrineClearEntityManagerWorkerSubscriber; use Symfony\Component\DependencyInjection\Alias; use Symfony\Component\DependencyInjection\ChildDefinition; @@ -635,4 +636,31 @@ public function testAutoEncryptionWithExtraOptions(): void // Ensure the driver option set in the client matches the ODM configuration self::assertEquals($driverOptions['autoEncryption'], $odmConfiguration->getDriverOptions()['autoEncryption']); } + + public function testAutoEncryptionWithEmptyKmsProvider(): void + { + $container = $this->buildMinimalContainer(); + $loader = new DoctrineMongoDBExtension(); + + $config = [ + 'connections' => [ + 'default' => [ + 'autoEncryption' => [ + 'keyVaultNamespace' => 'db.vault', + 'kmsProvider' => ['type' => 'aws'], + ], + ], + ], + 'document_managers' => ['default' => []], + ]; + + $loader->load([$config], $container); + (new ServiceRepositoryCompilerPass())->process($container); + + $clientDef = $container->getDefinition('doctrine_mongodb.odm.default_connection'); + $driverOptions = $clientDef->getArgument(2); + + self::assertArrayHasKey('autoEncryption', $driverOptions); + self::assertEquals(['aws' => new Definition(stdClass::class)], $driverOptions['autoEncryption']['kmsProviders']); + } } From ea8ac92c9d1ef2cdeaf42a7f1f9b6a56e567880a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Wed, 20 Aug 2025 14:13:25 +0200 Subject: [PATCH 12/27] Add "fields" property to encryptedFieldsMap config (#912) --- docs/encryption.rst | 62 ++-- src/Command/DumpEncryptedFieldsMapCommand.php | 7 +- src/DependencyInjection/Configuration.php | 30 +- .../DoctrineMongoDBExtension.php | 2 +- .../DependencyInjection/ConfigurationTest.php | 318 +++++++++--------- .../Fixtures/config/xml/full.xml | 178 +++++----- .../Fixtures/config/yml/full.yml | 103 +++--- 7 files changed, 371 insertions(+), 329 deletions(-) diff --git a/docs/encryption.rst b/docs/encryption.rst index 2ae90df3..3ba920bf 100644 --- a/docs/encryption.rst +++ b/docs/encryption.rst @@ -79,26 +79,37 @@ Example of configuration for AWS Encrypted Fields Map -------------------- -You can configure which fields are encrypted in each collection by specifying the +The encrypted fields are set to the collection when you create it, and the MongoDB +client will query the server for the collection schema before performing any +operations. For additional security, you can also specify the encrypted fields +in the connection configuration, which allows the client to use local rules +instead of downloading the remote schema from the server, that could potentially +be tampered with if an attacker compromises the server. + +The Encrypted Fields Maps is a list of all encrypted fields associated with all +the collection namespaces that has encryption enabled. To configure it, you +can run a command that extract the encrypted fields from the server and generate +the ``encryptedFieldsMap`` configuration. + +.. code-block:: console + + php bin/console doctrine:mongodb:dump-encrypted-fields-map --format yaml + +The output of the command will be a YAML configuration for the ``autoEncryption.encryptedFieldsMap`` option in the connection configuration. -This setting is **recommended** for improved security and performance. - If the connection ``encryptedFieldsMap`` object contains a key for the specified - collection, the client uses that object to perform automatic Queryable Encryption, - rather than using the remote schema. At minimum, the local rules must encrypt - all fields that the remote schema does. + collection namespace, the client uses that object to perform automatic + Queryable Encryption, rather than using the remote schema. At minimum, the + local rules must encrypt all fields that the remote schema does. - If the connection ``encryptedFieldsMap`` object doesn't contain a key for the - specified collection, the client downloads the server-side remote schema for - the collection and uses it instead. + specified collection namespace, the client downloads the server-side remote + schema for the collection and uses it instead. For more details, see the official MongoDB documentation: `Encrypted Fields and Enabled Queries `_. -Note that there is no ``fields`` key in the configuration of each collection -for the bundle configuration. Instead, you directly specify the list of -encrypted fields as an array under the collection namespace. - .. tabs:: .. group-tab:: YAML @@ -111,9 +122,10 @@ encrypted fields as an array under the collection namespace. autoEncryption: encryptedFieldsMap: "mydatabase.mycollection": - - keyId: { $binary: { base64: 2CSosXLSTEKaYphcSnUuCw==, subType: '04' } } - path: "sensitive_field" - bsonType: "string" + fields: + - keyId: { $binary: { base64: 2CSosXLSTEKaYphcSnUuCw==, subType: '04' } } + path: "sensitive_field" + bsonType: "string" .. group-tab:: XML @@ -124,11 +136,13 @@ encrypted fields as an array under the collection namespace. @@ -146,10 +160,12 @@ encrypted fields as an array under the collection namespace. ->autoEncryption([ 'encryptedFieldsMap' => [ 'mydatabase.mycollection' => [ - [ - 'path' => 'sensitive_field', - 'keyId' => ['$binary' => ['base64' => '2CSosXLSTEKaYphcSnUuCw==', 'subType' => '04' ] ], - 'bsonType' => 'string', + 'fields' => [ + [ + 'path' => 'sensitive_field', + 'keyId' => ['$binary' => ['base64' => '2CSosXLSTEKaYphcSnUuCw==', 'subType' => '04' ] ], + 'bsonType' => 'string', + ], ], ], ], diff --git a/src/Command/DumpEncryptedFieldsMapCommand.php b/src/Command/DumpEncryptedFieldsMapCommand.php index 25b0d630..f57fb88e 100644 --- a/src/Command/DumpEncryptedFieldsMapCommand.php +++ b/src/Command/DumpEncryptedFieldsMapCommand.php @@ -77,7 +77,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int } foreach ($encryptedFieldsMap as $ns => $encryptedFields) { - $encryptedFieldsMap[$ns] = json_decode(PackedArray::fromPHP($encryptedFields['fields'])->toRelaxedExtendedJSON(), true); + // Keep only the "fields" key and ignore "escCollection" and "ecocCollection" + $encryptedFieldsMap[$ns] = ['fields' => json_decode(PackedArray::fromPHP($encryptedFields['fields'])->toRelaxedExtendedJSON(), true)]; } $io->section(sprintf('Dumping encrypted fields map for document manager "%s"', $name)); @@ -105,9 +106,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int private function getDocumentNamespace(ClassMetadata $metadata, string $defaultDb): string { - $db = $metadata->getDatabase(); - $db = $db ?: $defaultDb; - $db = $db ?: 'doctrine'; + $db = $metadata->getDatabase() ?: $defaultDb ?: 'doctrine'; return $db . '.' . $metadata->getCollection(); } diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index db190e3c..a39f1eff 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -407,20 +407,24 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void return $v; })->end() ->prototype('array') - ->prototype('array') - ->children() - ->scalarNode('path')->isRequired()->cannotBeEmpty()->end() - ->scalarNode('bsonType')->isRequired()->cannotBeEmpty()->end() - ->variableNode('keyId')->isRequired()->cannotBeEmpty()->end() - ->arrayNode('queries') + ->children() + ->arrayNode('fields') + ->prototype('array') ->children() - ->scalarNode('queryType')->isRequired()->cannotBeEmpty()->end() - ->variableNode('min')->end() - ->variableNode('max')->end() - ->integerNode('sparsity')->end() - ->integerNode('precision')->end() - ->integerNode('trimFactor')->end() - ->integerNode('contention')->end() + ->scalarNode('path')->isRequired()->cannotBeEmpty()->end() + ->scalarNode('bsonType')->isRequired()->cannotBeEmpty()->end() + ->variableNode('keyId')->isRequired()->cannotBeEmpty()->end() + ->arrayNode('queries') + ->children() + ->scalarNode('queryType')->isRequired()->cannotBeEmpty()->end() + ->variableNode('min')->end() + ->variableNode('max')->end() + ->integerNode('sparsity')->end() + ->integerNode('precision')->end() + ->integerNode('trimFactor')->end() + ->integerNode('contention')->end() + ->end() + ->end() ->end() ->end() ->end() diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index 09ad0d41..241b5634 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -555,7 +555,7 @@ private function normalizeAutoEncryption(array $autoEncryption, string $defaultD foreach ($autoEncryption['encryptedFieldsMap'] as &$value) { // Wrap the encrypted fields in a 'fields' key as required the encryptedFieldsMap structure. // Some values can be BSON binary, date or numbers, the extended JSON format is used to convert them BSON document. - $value = (new Definition(BsonDocument::class))->setFactory([BsonDocument::class, 'fromJSON'])->setArguments([json_encode(['fields' => $value])]); + $value = (new Definition(BsonDocument::class))->setFactory([BsonDocument::class, 'fromJSON'])->setArguments([json_encode($value)]); } } diff --git a/tests/DependencyInjection/ConfigurationTest.php b/tests/DependencyInjection/ConfigurationTest.php index 42a66d78..d0ae535f 100644 --- a/tests/DependencyInjection/ConfigurationTest.php +++ b/tests/DependencyInjection/ConfigurationTest.php @@ -149,107 +149,113 @@ public function testFullConfiguration(array $config): void 'bypassQueryAnalysis' => true, 'encryptedFieldsMap' => [ 'encrypted.RangeTypes' => [ - [ - 'keyId' => ['$binary' => ['base64' => 'lhZHItpvRkqXevh4Wtqg/g==', 'subType' => '04']], - 'path' => 'intField', - 'bsonType' => 'int', - 'queries' => ['queryType' => 'range', 'contention' => 8, 'min' => 5, 'max' => 10], - ], - [ - 'keyId' => ['$binary' => ['base64' => 'qd9PEKIPTE2J30ev29lMpQ==', 'subType' => '04']], - 'path' => 'floatField', - 'bsonType' => 'double', - 'queries' => ['queryType' => 'range', 'contention' => 8, 'min' => 5.5, 'max' => 10.5, 'precision' => 1], - ], - [ - 'keyId' => ['$binary' => ['base64' => 'zVLg8CF4RSSu4xn7x7dOyQ==', 'subType' => '04']], - 'path' => 'decimalField', - 'bsonType' => 'decimal', - 'queries' => [ - 'queryType' => 'range', - 'contention' => 8, - 'min' => ['$numberDecimal' => '0.1'], - 'max' => ['$numberDecimal' => '0.2'], - 'precision' => 2, + 'fields' => [ + [ + 'keyId' => ['$binary' => ['base64' => 'lhZHItpvRkqXevh4Wtqg/g==', 'subType' => '04']], + 'path' => 'intField', + 'bsonType' => 'int', + 'queries' => ['queryType' => 'range', 'contention' => 8, 'min' => 5, 'max' => 10], ], - ], - [ - 'keyId' => ['$binary' => ['base64' => 'ySdd8lZ2QBqnwKPJTp/yLA==', 'subType' => '04']], - 'path' => 'immutableDateField', - 'bsonType' => 'date', - 'queries' => [ - 'queryType' => 'range', - 'contention' => 8, - 'min' => ['$date' => '2000-01-01T00:00:00Z'], - 'max' => ['$date' => '2100-01-01T00:00:00Z'], + [ + 'keyId' => ['$binary' => ['base64' => 'qd9PEKIPTE2J30ev29lMpQ==', 'subType' => '04']], + 'path' => 'floatField', + 'bsonType' => 'double', + 'queries' => ['queryType' => 'range', 'contention' => 8, 'min' => 5.5, 'max' => 10.5, 'precision' => 1], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'zVLg8CF4RSSu4xn7x7dOyQ==', 'subType' => '04']], + 'path' => 'decimalField', + 'bsonType' => 'decimal', + 'queries' => [ + 'queryType' => 'range', + 'contention' => 8, + 'min' => ['$numberDecimal' => '0.1'], + 'max' => ['$numberDecimal' => '0.2'], + 'precision' => 2, + ], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'ySdd8lZ2QBqnwKPJTp/yLA==', 'subType' => '04']], + 'path' => 'immutableDateField', + 'bsonType' => 'date', + 'queries' => [ + 'queryType' => 'range', + 'contention' => 8, + 'min' => ['$date' => '2000-01-01T00:00:00Z'], + 'max' => ['$date' => '2100-01-01T00:00:00Z'], + ], + ], + [ + 'keyId' => ['$binary' => ['base64' => 'NWKI+DyES/OlNkUbJbWJ9w==', 'subType' => '04']], + 'path' => 'dateField', + 'bsonType' => 'date', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'wiiv+0K/QAquyEq3HDxRKw==', 'subType' => '04']], + 'path' => 'binField', + 'bsonType' => 'binData', + ], + [ + 'keyId' => ['$binary' => ['base64' => '2CSosXLSTEKaYphcSnUuCw==', 'subType' => '04']], + 'path' => 'timestampField', + 'bsonType' => 'timestamp', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'h3H6HdG3T5CK+Z2yQ4Ho+Q==', 'subType' => '04']], + 'path' => 'hashField', + 'bsonType' => 'object', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'X78UZZ/HTX2wLw4K3uG42w==', 'subType' => '04']], + 'path' => 'collectionField', + 'bsonType' => 'objectId', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'LugQL/ZXTJOl856Yacmkwg==', 'subType' => '04']], + 'path' => 'boolField', + 'bsonType' => 'bool', ], - ], - [ - 'keyId' => ['$binary' => ['base64' => 'NWKI+DyES/OlNkUbJbWJ9w==', 'subType' => '04']], - 'path' => 'dateField', - 'bsonType' => 'date', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'wiiv+0K/QAquyEq3HDxRKw==', 'subType' => '04']], - 'path' => 'binField', - 'bsonType' => 'binData', - ], - [ - 'keyId' => ['$binary' => ['base64' => '2CSosXLSTEKaYphcSnUuCw==', 'subType' => '04']], - 'path' => 'timestampField', - 'bsonType' => 'timestamp', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'h3H6HdG3T5CK+Z2yQ4Ho+Q==', 'subType' => '04']], - 'path' => 'hashField', - 'bsonType' => 'object', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'X78UZZ/HTX2wLw4K3uG42w==', 'subType' => '04']], - 'path' => 'collectionField', - 'bsonType' => 'objectId', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'LugQL/ZXTJOl856Yacmkwg==', 'subType' => '04']], - 'path' => 'boolField', - 'bsonType' => 'bool', ], ], 'encrypted.patients' => [ - [ - 'keyId' => ['$binary' => ['base64' => 'GH25/XvYSaCgTUQLAo1hQw==', 'subType' => '04']], - 'path' => 'pathologies', - 'bsonType' => 'array', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'krVWyFlNTUOaGFMfk+s7UA==', 'subType' => '04']], - 'path' => 'patientRecord.billing', - 'bsonType' => 'object', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'X1ZaSI1GSAKnZ+sPGcmYBA==', 'subType' => '04']], - 'path' => 'patientRecord.billingAmount', - 'bsonType' => 'int', - 'queries' => [ - 'queryType' => 'range', - 'contention' => 8, - 'min' => 100, - 'max' => 2000, - 'sparsity' => 1, - 'trimFactor' => 4, + 'fields' => [ + [ + 'keyId' => ['$binary' => ['base64' => 'GH25/XvYSaCgTUQLAo1hQw==', 'subType' => '04']], + 'path' => 'pathologies', + 'bsonType' => 'array', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'krVWyFlNTUOaGFMfk+s7UA==', 'subType' => '04']], + 'path' => 'patientRecord.billing', + 'bsonType' => 'object', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'X1ZaSI1GSAKnZ+sPGcmYBA==', 'subType' => '04']], + 'path' => 'patientRecord.billingAmount', + 'bsonType' => 'int', + 'queries' => [ + 'queryType' => 'range', + 'contention' => 8, + 'min' => 100, + 'max' => 2000, + 'sparsity' => 1, + 'trimFactor' => 4, + ], ], ], ], 'encrypted.client' => [ - [ - 'keyId' => ['$binary' => ['base64' => 'I0Aw18vnRGWzVS1t3uejpQ==', 'subType' => '04']], - 'path' => 'name', - 'bsonType' => 'string', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'XSPRK3vaTLmMZr9IEj/qwQ==', 'subType' => '04']], - 'path' => 'clientCards', - 'bsonType' => 'array', + 'fields' => [ + [ + 'keyId' => ['$binary' => ['base64' => 'I0Aw18vnRGWzVS1t3uejpQ==', 'subType' => '04']], + 'path' => 'name', + 'bsonType' => 'string', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'XSPRK3vaTLmMZr9IEj/qwQ==', 'subType' => '04']], + 'path' => 'clientCards', + 'bsonType' => 'array', + ], ], ], ], @@ -585,36 +591,40 @@ public static function provideNormalizeOptions(): Generator 'kmsProvider' => ['type' => 'local', 'key' => '1234567890123456789012345678901234567890123456789012345678901234'], 'encryptedFieldsMap' => <<<'JSON' { - "encrypted.patients": [ - { - "keyId": { "$binary": { "base64": "GH25/XvYSaCgTUQLAo1hQw==", "subType": "04" } }, - "path": "pathologies", - "bsonType": "array" - }, - { - "keyId": { "$binary": { "base64": "krVWyFlNTUOaGFMfk+s7UA==", "subType": "04" } }, - "path": "patientRecord.billing", - "bsonType": "object" - }, - { - "keyId": { "$binary": { "base64": "X1ZaSI1GSAKnZ+sPGcmYBA==", "subType": "04" } }, - "path": "patientRecord.billingAmount", - "bsonType": "int", - "queries": { "queryType": "range", "contention": 8, "min": 100, "max": 2000, "sparsity": 1, "trimFactor": 4 } - } - ], - "encrypted.client": [ - { - "keyId": { "$binary": { "base64": "I0Aw18vnRGWzVS1t3uejpQ==", "subType": "04" } }, - "path": "name", - "bsonType": "string" - }, - { - "keyId": { "$binary": { "base64": "XSPRK3vaTLmMZr9IEj/qwQ==", "subType": "04" } }, - "path": "clientCards", - "bsonType": "array" - } - ] + "encrypted.patients": { + "fields": [ + { + "keyId": { "$binary": { "base64": "GH25/XvYSaCgTUQLAo1hQw==", "subType": "04" } }, + "path": "pathologies", + "bsonType": "array" + }, + { + "keyId": { "$binary": { "base64": "krVWyFlNTUOaGFMfk+s7UA==", "subType": "04" } }, + "path": "patientRecord.billing", + "bsonType": "object" + }, + { + "keyId": { "$binary": { "base64": "X1ZaSI1GSAKnZ+sPGcmYBA==", "subType": "04" } }, + "path": "patientRecord.billingAmount", + "bsonType": "int", + "queries": { "queryType": "range", "contention": 8, "min": 100, "max": 2000, "sparsity": 1, "trimFactor": 4 } + } + ] + }, + "encrypted.client": { + "fields": [ + { + "keyId": { "$binary": { "base64": "I0Aw18vnRGWzVS1t3uejpQ==", "subType": "04" } }, + "path": "name", + "bsonType": "string" + }, + { + "keyId": { "$binary": { "base64": "XSPRK3vaTLmMZr9IEj/qwQ==", "subType": "04" } }, + "path": "clientCards", + "bsonType": "array" + } + ] + } } JSON, ], @@ -629,40 +639,44 @@ public static function provideNormalizeOptions(): Generator 'kmsProvider' => ['type' => 'local', 'key' => '1234567890123456789012345678901234567890123456789012345678901234'], 'encryptedFieldsMap' => [ 'encrypted.patients' => [ - [ - 'keyId' => ['$binary' => ['base64' => 'GH25/XvYSaCgTUQLAo1hQw==', 'subType' => '04']], - 'path' => 'pathologies', - 'bsonType' => 'array', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'krVWyFlNTUOaGFMfk+s7UA==', 'subType' => '04']], - 'path' => 'patientRecord.billing', - 'bsonType' => 'object', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'X1ZaSI1GSAKnZ+sPGcmYBA==', 'subType' => '04']], - 'path' => 'patientRecord.billingAmount', - 'bsonType' => 'int', - 'queries' => [ - 'queryType' => 'range', - 'contention' => 8, - 'min' => 100, - 'max' => 2000, - 'sparsity' => 1, - 'trimFactor' => 4, + 'fields' => [ + [ + 'keyId' => ['$binary' => ['base64' => 'GH25/XvYSaCgTUQLAo1hQw==', 'subType' => '04']], + 'path' => 'pathologies', + 'bsonType' => 'array', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'krVWyFlNTUOaGFMfk+s7UA==', 'subType' => '04']], + 'path' => 'patientRecord.billing', + 'bsonType' => 'object', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'X1ZaSI1GSAKnZ+sPGcmYBA==', 'subType' => '04']], + 'path' => 'patientRecord.billingAmount', + 'bsonType' => 'int', + 'queries' => [ + 'queryType' => 'range', + 'contention' => 8, + 'min' => 100, + 'max' => 2000, + 'sparsity' => 1, + 'trimFactor' => 4, + ], ], ], ], 'encrypted.client' => [ - [ - 'keyId' => ['$binary' => ['base64' => 'I0Aw18vnRGWzVS1t3uejpQ==', 'subType' => '04']], - 'path' => 'name', - 'bsonType' => 'string', - ], - [ - 'keyId' => ['$binary' => ['base64' => 'XSPRK3vaTLmMZr9IEj/qwQ==', 'subType' => '04']], - 'path' => 'clientCards', - 'bsonType' => 'array', + 'fields' => [ + [ + 'keyId' => ['$binary' => ['base64' => 'I0Aw18vnRGWzVS1t3uejpQ==', 'subType' => '04']], + 'path' => 'name', + 'bsonType' => 'string', + ], + [ + 'keyId' => ['$binary' => ['base64' => 'XSPRK3vaTLmMZr9IEj/qwQ==', 'subType' => '04']], + 'path' => 'clientCards', + 'bsonType' => 'array', + ], ], ], ], diff --git a/tests/DependencyInjection/Fixtures/config/xml/full.xml b/tests/DependencyInjection/Fixtures/config/xml/full.xml index 69dd638b..a5c55a7c 100644 --- a/tests/DependencyInjection/Fixtures/config/xml/full.xml +++ b/tests/DependencyInjection/Fixtures/config/xml/full.xml @@ -89,92 +89,98 @@ diff --git a/tests/DependencyInjection/Fixtures/config/yml/full.yml b/tests/DependencyInjection/Fixtures/config/yml/full.yml index 4b334a38..bd8b8ce5 100644 --- a/tests/DependencyInjection/Fixtures/config/yml/full.yml +++ b/tests/DependencyInjection/Fixtures/config/yml/full.yml @@ -74,58 +74,61 @@ doctrine_mongodb: bypassQueryAnalysis: true encryptedFieldsMap: encrypted.RangeTypes: - - keyId: { $binary: { base64: lhZHItpvRkqXevh4Wtqg/g==, subType: '04' } } - path: intField - bsonType: int - queries: { queryType: range, contention: 8, min: 5, max: 10 } - - keyId: { $binary: { base64: qd9PEKIPTE2J30ev29lMpQ==, subType: '04' } } - path: floatField - bsonType: double - queries: { queryType: range, contention: 8, min: 5.5, max: 10.5, precision: 1 } - - keyId: { $binary: { base64: zVLg8CF4RSSu4xn7x7dOyQ==, subType: '04' } } - path: decimalField - bsonType: decimal - queries: { queryType: range, contention: 8, min: { $numberDecimal: '0.1' }, max: { $numberDecimal: '0.2' }, precision: 2 } - - keyId: { $binary: { base64: ySdd8lZ2QBqnwKPJTp/yLA==, subType: '04' } } - path: immutableDateField - bsonType: date - queries: { queryType: range, contention: 8, min: { $date: '2000-01-01T00:00:00Z' }, max: { $date: '2100-01-01T00:00:00Z' } } - - keyId: { $binary: { base64: NWKI+DyES/OlNkUbJbWJ9w==, subType: '04' } } - path: dateField - bsonType: date - - keyId: { $binary: { base64: wiiv+0K/QAquyEq3HDxRKw==, subType: '04' } } - path: binField - bsonType: binData - - keyId: { $binary: { base64: 2CSosXLSTEKaYphcSnUuCw==, subType: '04' } } - path: timestampField - bsonType: timestamp - - keyId: { $binary: { base64: h3H6HdG3T5CK+Z2yQ4Ho+Q==, subType: '04' } } - path: hashField - bsonType: object - - keyId: { $binary: { base64: X78UZZ/HTX2wLw4K3uG42w==, subType: '04' } } - path: collectionField - bsonType: objectId - - keyId: { $binary: { base64: LugQL/ZXTJOl856Yacmkwg==, subType: '04' } } - path: boolField - bsonType: bool + fields: + - keyId: { $binary: { base64: lhZHItpvRkqXevh4Wtqg/g==, subType: '04' } } + path: intField + bsonType: int + queries: { queryType: range, contention: 8, min: 5, max: 10 } + - keyId: { $binary: { base64: qd9PEKIPTE2J30ev29lMpQ==, subType: '04' } } + path: floatField + bsonType: double + queries: { queryType: range, contention: 8, min: 5.5, max: 10.5, precision: 1 } + - keyId: { $binary: { base64: zVLg8CF4RSSu4xn7x7dOyQ==, subType: '04' } } + path: decimalField + bsonType: decimal + queries: { queryType: range, contention: 8, min: { $numberDecimal: '0.1' }, max: { $numberDecimal: '0.2' }, precision: 2 } + - keyId: { $binary: { base64: ySdd8lZ2QBqnwKPJTp/yLA==, subType: '04' } } + path: immutableDateField + bsonType: date + queries: { queryType: range, contention: 8, min: { $date: '2000-01-01T00:00:00Z' }, max: { $date: '2100-01-01T00:00:00Z' } } + - keyId: { $binary: { base64: NWKI+DyES/OlNkUbJbWJ9w==, subType: '04' } } + path: dateField + bsonType: date + - keyId: { $binary: { base64: wiiv+0K/QAquyEq3HDxRKw==, subType: '04' } } + path: binField + bsonType: binData + - keyId: { $binary: { base64: 2CSosXLSTEKaYphcSnUuCw==, subType: '04' } } + path: timestampField + bsonType: timestamp + - keyId: { $binary: { base64: h3H6HdG3T5CK+Z2yQ4Ho+Q==, subType: '04' } } + path: hashField + bsonType: object + - keyId: { $binary: { base64: X78UZZ/HTX2wLw4K3uG42w==, subType: '04' } } + path: collectionField + bsonType: objectId + - keyId: { $binary: { base64: LugQL/ZXTJOl856Yacmkwg==, subType: '04' } } + path: boolField + bsonType: bool encrypted.patients: - - keyId: { $binary: { base64: GH25/XvYSaCgTUQLAo1hQw==, subType: '04' } } - path: pathologies - bsonType: array - - keyId: { $binary: { base64: krVWyFlNTUOaGFMfk+s7UA==, subType: '04' } } - path: patientRecord.billing - bsonType: object - - keyId: { $binary: { base64: X1ZaSI1GSAKnZ+sPGcmYBA==, subType: '04' } } - path: patientRecord.billingAmount - bsonType: int - queries: { queryType: range, contention: 8, min: 100, max: 2000, sparsity: 1, trimFactor: 4 } + fields: + - keyId: { $binary: { base64: GH25/XvYSaCgTUQLAo1hQw==, subType: '04' } } + path: pathologies + bsonType: array + - keyId: { $binary: { base64: krVWyFlNTUOaGFMfk+s7UA==, subType: '04' } } + path: patientRecord.billing + bsonType: object + - keyId: { $binary: { base64: X1ZaSI1GSAKnZ+sPGcmYBA==, subType: '04' } } + path: patientRecord.billingAmount + bsonType: int + queries: { queryType: range, contention: 8, min: 100, max: 2000, sparsity: 1, trimFactor: 4 } encrypted.client: - - keyId: { $binary: { base64: I0Aw18vnRGWzVS1t3uejpQ==, subType: '04' } } - path: name - bsonType: string - - keyId: { $binary: { base64: XSPRK3vaTLmMZr9IEj/qwQ==, subType: '04' } } - path: clientCards - bsonType: array + fields: + - keyId: { $binary: { base64: I0Aw18vnRGWzVS1t3uejpQ==, subType: '04' } } + path: name + bsonType: string + - keyId: { $binary: { base64: XSPRK3vaTLmMZr9IEj/qwQ==, subType: '04' } } + path: clientCards + bsonType: array extraOptions: mongocryptdURI: 'mongodb://localhost:27020' mongocryptdBypassSpawn: true From 59bea27d909869f94e2f09a28767affb0b68533b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Wed, 20 Aug 2025 14:23:06 +0200 Subject: [PATCH 13/27] Validate extraOptions configuration (#913) --- src/DependencyInjection/Configuration.php | 16 +++++++++++++++- tests/DependencyInjection/ConfigurationTest.php | 2 +- .../Fixtures/config/yml/full.yml | 4 +++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index a39f1eff..14f1b202 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -12,6 +12,7 @@ use Symfony\Component\Config\Definition\ConfigurationInterface; use function count; +use function explode; use function in_array; use function is_array; use function is_string; @@ -432,7 +433,20 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void ->end() ->end() ->arrayNode('extraOptions') - ->prototype('variable')->end() + ->children() + ->scalarNode('mongocryptdURI')->end() + ->booleanNode('mongocryptdBypassSpawn')->end() + ->scalarNode('mongocryptdSpawnPath')->end() + ->arrayNode('mongocryptdSpawnArgs') + ->beforeNormalization() + ->ifString() + ->then(static fn ($v) => explode(' ', $v)) + ->end() + ->prototype('scalar')->cannotBeEmpty()->end() + ->end() + ->scalarNode('cryptSharedLibPath')->end() + ->booleanNode('cryptSharedLibRequired')->end() + ->end() ->end() ->booleanNode('bypassQueryAnalysis')->end() ->arrayNode('tlsOptions') diff --git a/tests/DependencyInjection/ConfigurationTest.php b/tests/DependencyInjection/ConfigurationTest.php index d0ae535f..a0d8df49 100644 --- a/tests/DependencyInjection/ConfigurationTest.php +++ b/tests/DependencyInjection/ConfigurationTest.php @@ -263,7 +263,7 @@ public function testFullConfiguration(array $config): void 'mongocryptdURI' => 'mongodb://localhost:27020', 'mongocryptdBypassSpawn' => true, 'mongocryptdSpawnPath' => '%kernel.project_dir%/bin/mongocryptd', - 'mongocryptdSpawnArgs' => '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60', + 'mongocryptdSpawnArgs' => ['--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid', '--idleShutdownTimeoutSecs=60'], 'cryptSharedLibPath' => '%kernel.project_dir%/bin/mongo_crypt_v1.dylib', 'cryptSharedLibRequired' => true, ], diff --git a/tests/DependencyInjection/Fixtures/config/yml/full.yml b/tests/DependencyInjection/Fixtures/config/yml/full.yml index bd8b8ce5..d8bc2a29 100644 --- a/tests/DependencyInjection/Fixtures/config/yml/full.yml +++ b/tests/DependencyInjection/Fixtures/config/yml/full.yml @@ -133,7 +133,9 @@ doctrine_mongodb: mongocryptdURI: 'mongodb://localhost:27020' mongocryptdBypassSpawn: true mongocryptdSpawnPath: '%kernel.project_dir%/bin/mongocryptd' - mongocryptdSpawnArgs: '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60' + mongocryptdSpawnArgs: + - '--pidfilepath=%kernel.project_dir%/var/mongocryptd.pid' + - '--idleShutdownTimeoutSecs=60' cryptSharedLibPath: '%kernel.project_dir%/bin/mongo_crypt_v1.dylib' cryptSharedLibRequired: true From 7028b0ea1decf173259490970fce7aa9d2923d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 21 Aug 2025 11:16:31 +0200 Subject: [PATCH 14/27] Convert extra-options to sub-elements to allow multiple mongocryptdSpawnArgs --- config/schema/mongodb-1.0.xsd | 14 ++++++++------ src/DependencyInjection/Configuration.php | 2 +- .../Fixtures/config/xml/full.xml | 17 +++++++++-------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/config/schema/mongodb-1.0.xsd b/config/schema/mongodb-1.0.xsd index 457987d7..dfeae1dd 100644 --- a/config/schema/mongodb-1.0.xsd +++ b/config/schema/mongodb-1.0.xsd @@ -191,12 +191,14 @@ - - - - - - + + + + + + + + diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 14f1b202..46d7c937 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -440,7 +440,7 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void ->arrayNode('mongocryptdSpawnArgs') ->beforeNormalization() ->ifString() - ->then(static fn ($v) => explode(' ', $v)) + ->then(static fn ($v) => [$v]) ->end() ->prototype('scalar')->cannotBeEmpty()->end() ->end() diff --git a/tests/DependencyInjection/Fixtures/config/xml/full.xml b/tests/DependencyInjection/Fixtures/config/xml/full.xml index a5c55a7c..c0193ffc 100644 --- a/tests/DependencyInjection/Fixtures/config/xml/full.xml +++ b/tests/DependencyInjection/Fixtures/config/xml/full.xml @@ -184,14 +184,15 @@ } ]]> - + + mongodb://localhost:27020 + true + %kernel.project_dir%/bin/mongocryptd + --pidfilepath=%kernel.project_dir%/var/mongocryptd.pid + --idleShutdownTimeoutSecs=60 + %kernel.project_dir%/bin/mongo_crypt_v1.dylib + true + From cf11f812824ef7eb63d99a9db923a8e0078508f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 21 Aug 2025 17:54:47 +0200 Subject: [PATCH 15/27] Revert extraOptions config to attributes for string and boolean options --- config/schema/mongodb-1.0.xsd | 10 +++++----- src/DependencyInjection/Configuration.php | 1 - .../Fixtures/config/xml/full.xml | 13 +++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config/schema/mongodb-1.0.xsd b/config/schema/mongodb-1.0.xsd index dfeae1dd..d4f695a4 100644 --- a/config/schema/mongodb-1.0.xsd +++ b/config/schema/mongodb-1.0.xsd @@ -192,13 +192,13 @@ - - - - - + + + + + diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 46d7c937..030d5ed6 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -12,7 +12,6 @@ use Symfony\Component\Config\Definition\ConfigurationInterface; use function count; -use function explode; use function in_array; use function is_array; use function is_string; diff --git a/tests/DependencyInjection/Fixtures/config/xml/full.xml b/tests/DependencyInjection/Fixtures/config/xml/full.xml index c0193ffc..f76ee3ed 100644 --- a/tests/DependencyInjection/Fixtures/config/xml/full.xml +++ b/tests/DependencyInjection/Fixtures/config/xml/full.xml @@ -184,14 +184,15 @@ } ]]> - - mongodb://localhost:27020 - true - %kernel.project_dir%/bin/mongocryptd + --pidfilepath=%kernel.project_dir%/var/mongocryptd.pid --idleShutdownTimeoutSecs=60 - %kernel.project_dir%/bin/mongo_crypt_v1.dylib - true From c49d20dc889d33acc2202578bc3131e3e8767c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 21 Aug 2025 18:37:20 +0200 Subject: [PATCH 16/27] Update diagnostic criteria --- src/DataCollector/ConnectionDiagnostic.php | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/DataCollector/ConnectionDiagnostic.php b/src/DataCollector/ConnectionDiagnostic.php index 59bda32c..86a9326a 100644 --- a/src/DataCollector/ConnectionDiagnostic.php +++ b/src/DataCollector/ConnectionDiagnostic.php @@ -11,7 +11,6 @@ use function array_flip; use function array_intersect_key; -use function in_array; use function iterator_count; use function version_compare; @@ -25,12 +24,6 @@ class ConnectionDiagnostic 'tlsOptions', ]; - private const SUPPORTED_SERVER_TYPES = [ - Server::TYPE_MONGOS, - Server::TYPE_RS_PRIMARY, - Server::TYPE_RS_SECONDARY, - ]; - public function __construct( private readonly Client $client, private readonly array $driverOptions, @@ -71,9 +64,9 @@ public function getServerInfo(): array return [ 'topologyName' => $this->getTopologyType($server), - 'topologySupported' => in_array($server->getType(), self::SUPPORTED_SERVER_TYPES), + 'topologySupported' => $server->getType() !== Server::TYPE_STANDALONE, 'version' => $version, - 'versionSupported' => $version ? version_compare($version, '8.0.0', '>=') : false, + 'versionSupported' => $version ? version_compare($version, '7.0.0', '>=') : false, ]; } From e28616aa5a216c6c2b8e031426a4ee742e9ca63d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 21 Aug 2025 18:47:12 +0200 Subject: [PATCH 17/27] Add comment on encryptedFieldsMap JSON decoding --- src/DependencyInjection/Configuration.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/DependencyInjection/Configuration.php b/src/DependencyInjection/Configuration.php index 030d5ed6..c4fec903 100644 --- a/src/DependencyInjection/Configuration.php +++ b/src/DependencyInjection/Configuration.php @@ -400,6 +400,9 @@ private function addConnectionsSection(ArrayNodeDefinition $rootNode): void ->useAttributeAsKey('name', false) ->beforeNormalization() ->always(static function ($v) { + // Create a PHP array representation of the Extended BSON that is later + // converted to JSON string to create a BSON document from this JSON. + // This lets the DI dumper transform the parameters in the string and dump it. if (is_string($v)) { return json_decode($v, true, 512, JSON_THROW_ON_ERROR); } From 887b3195e2a8d70cc58bee5787ffabd3fbc64b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Fri, 22 Aug 2025 19:25:32 +0200 Subject: [PATCH 18/27] Rename encryption commands (#914) --- config/command.php | 12 ++++++------ docs/encryption.rst | 2 +- ...icCommand.php => EncryptionDiagnosticCommand.php} | 4 ++-- ...ommand.php => EncryptionDumpFieldsMapCommand.php} | 4 ++-- tests/Command/ConnectionDiagnosticCommandTest.php | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) rename src/Command/{ConnectionDiagnosticCommand.php => EncryptionDiagnosticCommand.php} (98%) rename src/Command/{DumpEncryptedFieldsMapCommand.php => EncryptionDumpFieldsMapCommand.php} (97%) diff --git a/config/command.php b/config/command.php index 9b064fa3..2d86b888 100644 --- a/config/command.php +++ b/config/command.php @@ -3,10 +3,10 @@ declare(strict_types=1); use Doctrine\Bundle\MongoDBBundle\Command\ClearMetadataCacheDoctrineODMCommand; -use Doctrine\Bundle\MongoDBBundle\Command\ConnectionDiagnosticCommand; use Doctrine\Bundle\MongoDBBundle\Command\CreateSchemaDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\DropSchemaDoctrineODMCommand; -use Doctrine\Bundle\MongoDBBundle\Command\DumpEncryptedFieldsMapCommand; +use Doctrine\Bundle\MongoDBBundle\Command\EncryptionDiagnosticCommand; +use Doctrine\Bundle\MongoDBBundle\Command\EncryptionDumpFieldsMapCommand; use Doctrine\Bundle\MongoDBBundle\Command\GenerateHydratorsDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\GenerateProxiesDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\InfoDoctrineODMCommand; @@ -24,12 +24,12 @@ ->set('doctrine_mongodb.odm.command.clear_metadata_cache', ClearMetadataCacheDoctrineODMCommand::class) ->tag('console.command', ['command' => 'doctrine:mongodb:cache:clear-metadata']) - ->set('doctrine_mongodb.odm.command.connection_diagnostic', ConnectionDiagnosticCommand::class) - ->tag('console.command', ['command' => 'doctrine:mongodb:connection:diagnostic']) + ->set('doctrine_mongodb.odm.command.connection_diagnostic', EncryptionDiagnosticCommand::class) + ->tag('console.command', ['command' => 'doctrine:mongodb:encryption:diagnostic']) ->args([tagged_locator('doctrine_mongodb.connection_diagnostic', 'name')]) - ->set('doctrine_mongodb.odm.command.dump_encrypted_fields_map', DumpEncryptedFieldsMapCommand::class) - ->tag('console.command', ['command' => 'doctrine:mongodb:dump-encrypted-fields-map']) + ->set('doctrine_mongodb.odm.command.dump_encrypted_fields_map', EncryptionDumpFieldsMapCommand::class) + ->tag('console.command', ['command' => 'doctrine:mongodb:encryption:dump-fields-map']) ->args([tagged_locator('doctrine_mongodb.odm.document_manager', 'name')]) ->set('doctrine_mongodb.odm.command.create_schema', CreateSchemaDoctrineODMCommand::class) diff --git a/docs/encryption.rst b/docs/encryption.rst index 3ba920bf..29f7c59a 100644 --- a/docs/encryption.rst +++ b/docs/encryption.rst @@ -93,7 +93,7 @@ the ``encryptedFieldsMap`` configuration. .. code-block:: console - php bin/console doctrine:mongodb:dump-encrypted-fields-map --format yaml + php bin/console doctrine:mongodb:encryption:dump-fields-map --format yaml The output of the command will be a YAML configuration for the ``autoEncryption.encryptedFieldsMap`` option in the connection configuration. diff --git a/src/Command/ConnectionDiagnosticCommand.php b/src/Command/EncryptionDiagnosticCommand.php similarity index 98% rename from src/Command/ConnectionDiagnosticCommand.php rename to src/Command/EncryptionDiagnosticCommand.php index 7dce530c..7761568d 100644 --- a/src/Command/ConnectionDiagnosticCommand.php +++ b/src/Command/EncryptionDiagnosticCommand.php @@ -22,10 +22,10 @@ /** @internal */ #[AsCommand( - name: 'doctrine:mongodb:connection:diagnostic', + name: 'doctrine:mongodb:encryption:diagnostic', description: 'Diagnose MongoDB configuration and server capabilities for each connection.', )] -final class ConnectionDiagnosticCommand extends Command +final class EncryptionDiagnosticCommand extends Command { /** @param ServiceProviderInterface $diagnostics */ public function __construct( diff --git a/src/Command/DumpEncryptedFieldsMapCommand.php b/src/Command/EncryptionDumpFieldsMapCommand.php similarity index 97% rename from src/Command/DumpEncryptedFieldsMapCommand.php rename to src/Command/EncryptionDumpFieldsMapCommand.php index f57fb88e..17e0c895 100644 --- a/src/Command/DumpEncryptedFieldsMapCommand.php +++ b/src/Command/EncryptionDumpFieldsMapCommand.php @@ -28,10 +28,10 @@ /** @internal */ #[AsCommand( - name: 'doctrine:mongodb:dump-encrypted-fields-map', + name: 'doctrine:mongodb:encryption:dump-fields-map', description: 'Dumps the encrypted fields map for all documents in the configured connections.', )] -final class DumpEncryptedFieldsMapCommand extends Command +final class EncryptionDumpFieldsMapCommand extends Command { /** @param ServiceCollectionInterface $documentManagers */ public function __construct(private readonly ServiceCollectionInterface $documentManagers) diff --git a/tests/Command/ConnectionDiagnosticCommandTest.php b/tests/Command/ConnectionDiagnosticCommandTest.php index 0948236c..e9f3328b 100644 --- a/tests/Command/ConnectionDiagnosticCommandTest.php +++ b/tests/Command/ConnectionDiagnosticCommandTest.php @@ -15,7 +15,7 @@ public function testExecute(): void $kernel = new CommandTestKernel('test', false); $application = new Application($kernel); - $command = $application->find('doctrine:mongodb:connection:diagnostic'); + $command = $application->find('doctrine:mongodb:encryption:diagnostic'); $commandTester = new CommandTester($command); $commandTester->execute([]); From 37c0f036e1b2ea59c105c805c7dac1b3fbc9513f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Fri, 22 Aug 2025 19:44:19 +0200 Subject: [PATCH 19/27] Update docs --- docs/config.rst | 5 --- docs/encryption.rst | 41 ++++--------------- .../EncryptionDumpFieldsMapCommand.php | 2 +- .../DoctrineMongoDBExtension.php | 3 +- 4 files changed, 11 insertions(+), 40 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index db08a37b..59978e45 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -852,7 +852,6 @@ Full Default Configuration $config->connection('id') ->server('mongodb://localhost') - ->driverOptions(['context' => null]), // stream context to use for connection ->autoEncryption([ // Options for client-side field-level encryption 'bypassAutoEncryption' => false, // Disables auto-encryption 'keyVaultClient' => null, // Service ID of a MongoDB\Driver\Manager for the key vault @@ -872,11 +871,7 @@ Full Default Configuration // 'tlsCAFile' => null, // Path to CA file, e.g., /path/to/key-vault-ca.pem // 'tlsCertificateKeyFile' => null, // Path to client cert/key file, e.g., /path/to/key-vault-client.pem // 'tlsCertificateKeyFilePassword' => null, // Password for client cert/key file - // 'tlsAllowInvalidCertificates' => false, // Bypass server certificate validation (use with caution) - // 'tlsAllowInvalidHostnames' => false, // Bypass server hostname validation (use with caution) - // 'tlsDisableCertificateRevocation' => false, // Disable CRL checks // 'tlsDisableOCSPEndpointCheck' => false, // Disable OCSP checks - // 'tlsInsecure' => false, // Allow invalid/no server cert (use with extreme caution) ], ]) ->options([ diff --git a/docs/encryption.rst b/docs/encryption.rst index 29f7c59a..3384db79 100644 --- a/docs/encryption.rst +++ b/docs/encryption.rst @@ -86,7 +86,7 @@ in the connection configuration, which allows the client to use local rules instead of downloading the remote schema from the server, that could potentially be tampered with if an attacker compromises the server. -The Encrypted Fields Maps is a list of all encrypted fields associated with all +The Encrypted Fields Map is a list of all encrypted fields associated with all the collection namespaces that has encryption enabled. To configure it, you can run a command that extract the encrypted fields from the server and generate the ``encryptedFieldsMap`` configuration. @@ -163,6 +163,8 @@ For more details, see the official MongoDB documentation: 'fields' => [ [ 'path' => 'sensitive_field', + // Extended JSON representation of a BSON binary type + // The MongoDB\BSON\Binary class cannot be used here 'keyId' => ['$binary' => ['base64' => '2CSosXLSTEKaYphcSnUuCw==', 'subType' => '04' ] ], 'bsonType' => 'string', ], @@ -245,12 +247,12 @@ TLS settings for the internal key vault client using the ``tlsOptions`` key: - - /path/to/key-vault-ca.pem - /path/to/key-vault-client.pem - keyvaultclientpassword - false - + @@ -272,31 +274,6 @@ TLS settings for the internal key vault client using the ``tlsOptions`` key: ]); }; -Context Service for SSL ------------------------ - -You can use a Symfony service to provide a stream context for SSL options: - -.. code-block:: yaml - - services: - app.mongodb.context_service: - class: 'resource' - factory: 'stream_context_create' - arguments: - - { ssl: { verify_expiry: true } } - -Then reference this service in your connection configuration: - -.. code-block:: yaml - - doctrine_mongodb: - connections: - default: - server: "mongodb://localhost:27017" - driver_options: - context: "app.mongodb.context_service" - Further Reading --------------- diff --git a/src/Command/EncryptionDumpFieldsMapCommand.php b/src/Command/EncryptionDumpFieldsMapCommand.php index 17e0c895..2fae6d34 100644 --- a/src/Command/EncryptionDumpFieldsMapCommand.php +++ b/src/Command/EncryptionDumpFieldsMapCommand.php @@ -45,7 +45,7 @@ protected function configure(): void 'format', 'f', InputOption::VALUE_REQUIRED, - 'The output format for the encrypted fields map (yaml, php)', + 'The output format for the encrypted fields map (yaml, json, php)', 'yaml', ['yaml', 'php', 'json'] ); diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index 241b5634..7c881b6c 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -402,8 +402,7 @@ protected function loadDocumentManager(array $documentManager, string|null $defa */ protected function loadConnections(array $connections, ContainerBuilder $container, array $config): void { - $cons = []; - $diagnostics = []; + $cons = []; foreach ($connections as $name => $connection) { // Define an event manager for this connection $eventManagerId = sprintf('doctrine_mongodb.odm.%s_connection.event_manager', $name); From 28b8e68e16d472f8ac0d9acdcefc1970b29277af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 26 Aug 2025 13:52:48 +0200 Subject: [PATCH 20/27] Return dumped encrypted fields map even in quiet mode --- src/Command/EncryptionDumpFieldsMapCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Command/EncryptionDumpFieldsMapCommand.php b/src/Command/EncryptionDumpFieldsMapCommand.php index 2fae6d34..1833018d 100644 --- a/src/Command/EncryptionDumpFieldsMapCommand.php +++ b/src/Command/EncryptionDumpFieldsMapCommand.php @@ -98,7 +98,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::FAILURE; } - $io->block($outputContent); + $output->writeln($outputContent, OutputInterface::VERBOSITY_QUIET); } return Command::SUCCESS; From 235b64df99c52980a5e7a5826b584532f48de812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 26 Aug 2025 15:53:17 +0200 Subject: [PATCH 21/27] =?UTF-8?q?[Encryption]=C2=A0Fix=20query=20option=20?= =?UTF-8?q?min/max=20type=20for=20exported=20encrypted=20fields=20map=20(#?= =?UTF-8?q?915)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix query option min/max type for exported encrypted fields map * Type 'decimal' is not a supported equality indexed type --- src/Command/EncryptionDumpFieldsMapCommand.php | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Command/EncryptionDumpFieldsMapCommand.php b/src/Command/EncryptionDumpFieldsMapCommand.php index 1833018d..a4d3ac65 100644 --- a/src/Command/EncryptionDumpFieldsMapCommand.php +++ b/src/Command/EncryptionDumpFieldsMapCommand.php @@ -76,9 +76,24 @@ protected function execute(InputInterface $input, OutputInterface $output): int continue; } + // The min/max query options must have the same type as the field. + // But the PHP driver always convert to "int" or "float" when the value fit in the range foreach ($encryptedFieldsMap as $ns => $encryptedFields) { + $fields = json_decode(PackedArray::fromPHP($encryptedFields['fields'])->toCanonicalExtendedJSON(), true); + foreach ($fields as &$field) { + if ($field['bsonType'] === 'long') { + if (isset($field['queries']['min']['$numberInt'])) { + $field['queries']['min'] = ['$numberLong' => $field['queries']['min']['$numberInt']]; + } + + if (isset($field['queries']['max']['$numberInt'])) { + $field['queries']['max'] = ['$numberLong' => $field['queries']['max']['$numberInt']]; + } + } + } + // Keep only the "fields" key and ignore "escCollection" and "ecocCollection" - $encryptedFieldsMap[$ns] = ['fields' => json_decode(PackedArray::fromPHP($encryptedFields['fields'])->toRelaxedExtendedJSON(), true)]; + $encryptedFieldsMap[$ns] = ['fields' => $fields]; } $io->section(sprintf('Dumping encrypted fields map for document manager "%s"', $name)); From 6d74e7990c8e0b852397fafa4840ae007b7ed5f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 28 Aug 2025 14:28:41 +0200 Subject: [PATCH 22/27] Revert mongodb-odm branch in composer.json --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index e63417fc..8d1c4036 100644 --- a/composer.json +++ b/composer.json @@ -27,7 +27,7 @@ "php": "^8.1", "ext-mongodb": "^1.21 || ^2", "composer-runtime-api": "^2.0", - "doctrine/mongodb-odm": "dev-feature/queryable-encryption as 2.12.x-dev", + "doctrine/mongodb-odm": "^2.6", "doctrine/persistence": "^3.0 || ^4.0", "psr/log": "^1.0 || ^2.0 || ^3.0", "symfony/config": "^6.4 || ^7.0", From e1e82e65007e9f2e77175a988fdbfa71aa03e3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 28 Aug 2025 15:33:52 +0200 Subject: [PATCH 23/27] Add attributes for all master key formats --- config/schema/mongodb-1.0.xsd | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/config/schema/mongodb-1.0.xsd b/config/schema/mongodb-1.0.xsd index d4f695a4..90588f99 100644 --- a/config/schema/mongodb-1.0.xsd +++ b/config/schema/mongodb-1.0.xsd @@ -168,7 +168,26 @@ - + + + + + + + + + + + + + + + + + + + + From 90ad0b1dbf3ff75f0d0bffe5ea3242c8421a8a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 28 Aug 2025 15:36:56 +0200 Subject: [PATCH 24/27] Change random string --- tests/DependencyInjection/DoctrineMongoDBExtensionTest.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php index c8b462d0..b6560ee9 100644 --- a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php +++ b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php @@ -501,7 +501,7 @@ public function testAutoEncryptionWithKeyVaultClientService(): void 'autoEncryption' => [ 'keyVaultNamespace' => 'db.vault', 'keyVaultClient' => $dummyServiceId, - 'kmsProvider' => ['type' => 'local', 'key' => 'cGFzc3dvcmQ='], + 'kmsProvider' => ['type' => 'local', 'key' => 'base64_encoded_key'], ], ], ], @@ -518,7 +518,7 @@ public function testAutoEncryptionWithKeyVaultClientService(): void self::assertInstanceOf(Reference::class, $driverOptions['autoEncryption']['keyVaultClient']); self::assertEquals($dummyServiceId, (string) $driverOptions['autoEncryption']['keyVaultClient']); self::assertEquals('db.vault', $driverOptions['autoEncryption']['keyVaultNamespace']); - self::assertEquals(['local' => ['key' => 'cGFzc3dvcmQ=']], $driverOptions['autoEncryption']['kmsProviders']); + self::assertEquals(['local' => ['key' => 'base64_encoded_key']], $driverOptions['autoEncryption']['kmsProviders']); // Auto encryption configuration should be set in the ODM configuration $odmConfiguration = $container->get('doctrine_mongodb.odm.default_configuration'); @@ -598,7 +598,7 @@ public function testAutoEncryptionWithExtraOptions(): void 'default' => [ 'autoEncryption' => [ 'keyVaultNamespace' => 'db.vault', - 'kmsProvider' => ['type' => 'local', 'key' => 'cGFzc3dvcmQ='], + 'kmsProvider' => ['type' => 'local', 'key' => 'base64_encoded_key'], 'extraOptions' => [ 'cryptSharedLibPath' => '/another/path.so', 'cryptSharedLibRequired' => false, From 93ddd8acde3b8057a973aec293cf690575204b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 28 Aug 2025 15:44:53 +0200 Subject: [PATCH 25/27] Throw exception in JSON decode from toCanonicalExtendedJSON + support decimal range min/max --- src/Command/EncryptionDumpFieldsMapCommand.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Command/EncryptionDumpFieldsMapCommand.php b/src/Command/EncryptionDumpFieldsMapCommand.php index a4d3ac65..dae1d643 100644 --- a/src/Command/EncryptionDumpFieldsMapCommand.php +++ b/src/Command/EncryptionDumpFieldsMapCommand.php @@ -21,6 +21,7 @@ use function sprintf; use function var_export; +use const JSON_BIGINT_AS_STRING; use const JSON_PRETTY_PRINT; use const JSON_THROW_ON_ERROR; use const JSON_UNESCAPED_SLASHES; @@ -79,7 +80,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int // The min/max query options must have the same type as the field. // But the PHP driver always convert to "int" or "float" when the value fit in the range foreach ($encryptedFieldsMap as $ns => $encryptedFields) { - $fields = json_decode(PackedArray::fromPHP($encryptedFields['fields'])->toCanonicalExtendedJSON(), true); + $fields = json_decode(PackedArray::fromPHP($encryptedFields['fields'])->toCanonicalExtendedJSON(), true, flags: JSON_BIGINT_AS_STRING | JSON_THROW_ON_ERROR); foreach ($fields as &$field) { if ($field['bsonType'] === 'long') { if (isset($field['queries']['min']['$numberInt'])) { @@ -89,6 +90,14 @@ protected function execute(InputInterface $input, OutputInterface $output): int if (isset($field['queries']['max']['$numberInt'])) { $field['queries']['max'] = ['$numberLong' => $field['queries']['max']['$numberInt']]; } + } elseif ($field['bsonType'] === 'decimal') { + if (isset($field['queries']['min']['$numberDouble'])) { + $field['queries']['min'] = ['$numberDecimal' => $field['queries']['min']['$numberDouble']]; + } + + if (isset($field['queries']['max']['$numberDouble'])) { + $field['queries']['max'] = ['$numberDecimal' => $field['queries']['max']['$numberDouble']]; + } } } From 7183f33d081cd102e35315790c2dbb9edc177b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 28 Aug 2025 16:44:29 +0200 Subject: [PATCH 26/27] Move the commands to the Encryption namespace --- config/command.php | 8 ++++---- .../DiagnosticCommand.php} | 4 ++-- .../DumpFieldsMapCommand.php} | 4 ++-- .../DiagnosticCommandTest.php} | 5 +++-- 4 files changed, 11 insertions(+), 10 deletions(-) rename src/Command/{EncryptionDiagnosticCommand.php => Encryption/DiagnosticCommand.php} (98%) rename src/Command/{EncryptionDumpFieldsMapCommand.php => Encryption/DumpFieldsMapCommand.php} (98%) rename tests/Command/{ConnectionDiagnosticCommandTest.php => Encryption/DiagnosticCommandTest.php} (82%) diff --git a/config/command.php b/config/command.php index 2d86b888..6b5af282 100644 --- a/config/command.php +++ b/config/command.php @@ -5,8 +5,8 @@ use Doctrine\Bundle\MongoDBBundle\Command\ClearMetadataCacheDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\CreateSchemaDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\DropSchemaDoctrineODMCommand; -use Doctrine\Bundle\MongoDBBundle\Command\EncryptionDiagnosticCommand; -use Doctrine\Bundle\MongoDBBundle\Command\EncryptionDumpFieldsMapCommand; +use Doctrine\Bundle\MongoDBBundle\Command\Encryption\DiagnosticCommand; +use Doctrine\Bundle\MongoDBBundle\Command\Encryption\DumpFieldsMapCommand; use Doctrine\Bundle\MongoDBBundle\Command\GenerateHydratorsDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\GenerateProxiesDoctrineODMCommand; use Doctrine\Bundle\MongoDBBundle\Command\InfoDoctrineODMCommand; @@ -24,11 +24,11 @@ ->set('doctrine_mongodb.odm.command.clear_metadata_cache', ClearMetadataCacheDoctrineODMCommand::class) ->tag('console.command', ['command' => 'doctrine:mongodb:cache:clear-metadata']) - ->set('doctrine_mongodb.odm.command.connection_diagnostic', EncryptionDiagnosticCommand::class) + ->set('doctrine_mongodb.odm.command.encryption_diagnostic', DiagnosticCommand::class) ->tag('console.command', ['command' => 'doctrine:mongodb:encryption:diagnostic']) ->args([tagged_locator('doctrine_mongodb.connection_diagnostic', 'name')]) - ->set('doctrine_mongodb.odm.command.dump_encrypted_fields_map', EncryptionDumpFieldsMapCommand::class) + ->set('doctrine_mongodb.odm.command.encryption_dump_fields_map', DumpFieldsMapCommand::class) ->tag('console.command', ['command' => 'doctrine:mongodb:encryption:dump-fields-map']) ->args([tagged_locator('doctrine_mongodb.odm.document_manager', 'name')]) diff --git a/src/Command/EncryptionDiagnosticCommand.php b/src/Command/Encryption/DiagnosticCommand.php similarity index 98% rename from src/Command/EncryptionDiagnosticCommand.php rename to src/Command/Encryption/DiagnosticCommand.php index 7761568d..0c2290f9 100644 --- a/src/Command/EncryptionDiagnosticCommand.php +++ b/src/Command/Encryption/DiagnosticCommand.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Doctrine\Bundle\MongoDBBundle\Command; +namespace Doctrine\Bundle\MongoDBBundle\Command\Encryption; use Doctrine\Bundle\MongoDBBundle\DataCollector\ConnectionDiagnostic; use Doctrine\Bundle\MongoDBBundle\DataCollector\EncryptionDiagnostic; @@ -25,7 +25,7 @@ name: 'doctrine:mongodb:encryption:diagnostic', description: 'Diagnose MongoDB configuration and server capabilities for each connection.', )] -final class EncryptionDiagnosticCommand extends Command +final class DiagnosticCommand extends Command { /** @param ServiceProviderInterface $diagnostics */ public function __construct( diff --git a/src/Command/EncryptionDumpFieldsMapCommand.php b/src/Command/Encryption/DumpFieldsMapCommand.php similarity index 98% rename from src/Command/EncryptionDumpFieldsMapCommand.php rename to src/Command/Encryption/DumpFieldsMapCommand.php index dae1d643..4df6cbe9 100644 --- a/src/Command/EncryptionDumpFieldsMapCommand.php +++ b/src/Command/Encryption/DumpFieldsMapCommand.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Doctrine\Bundle\MongoDBBundle\Command; +namespace Doctrine\Bundle\MongoDBBundle\Command\Encryption; use Doctrine\ODM\MongoDB\DocumentManager; use Doctrine\ODM\MongoDB\Mapping\ClassMetadata; @@ -32,7 +32,7 @@ name: 'doctrine:mongodb:encryption:dump-fields-map', description: 'Dumps the encrypted fields map for all documents in the configured connections.', )] -final class EncryptionDumpFieldsMapCommand extends Command +final class DumpFieldsMapCommand extends Command { /** @param ServiceCollectionInterface $documentManagers */ public function __construct(private readonly ServiceCollectionInterface $documentManagers) diff --git a/tests/Command/ConnectionDiagnosticCommandTest.php b/tests/Command/Encryption/DiagnosticCommandTest.php similarity index 82% rename from tests/Command/ConnectionDiagnosticCommandTest.php rename to tests/Command/Encryption/DiagnosticCommandTest.php index e9f3328b..ac3a6e77 100644 --- a/tests/Command/ConnectionDiagnosticCommandTest.php +++ b/tests/Command/Encryption/DiagnosticCommandTest.php @@ -2,13 +2,14 @@ declare(strict_types=1); -namespace Doctrine\Bundle\MongoDBBundle\Tests\Command; +namespace Doctrine\Bundle\MongoDBBundle\Tests\Command\Encryption; +use Doctrine\Bundle\MongoDBBundle\Tests\Command\CommandTestKernel; use Symfony\Bundle\FrameworkBundle\Console\Application; use Symfony\Bundle\FrameworkBundle\Test\KernelTestCase; use Symfony\Component\Console\Tester\CommandTester; -final class ConnectionDiagnosticCommandTest extends KernelTestCase +final class DiagnosticCommandTest extends KernelTestCase { public function testExecute(): void { From f13fde686d29f0a07bfb6da824a41a041937678b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Thu, 28 Aug 2025 18:00:47 +0200 Subject: [PATCH 27/27] Fix compatibility with mongodb-odm < 1.12 --- phpstan-baseline.neon | 18 ++++++++ .../DoctrineMongoDBExtension.php | 4 ++ .../DoctrineMongoDBExtensionTest.php | 41 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index cb21d61c..dfc9747d 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -534,6 +534,24 @@ parameters: count: 1 path: tests/DependencyInjection/ConfigurationTest.php + - + message: '#^Call to an undefined method Doctrine\\ODM\\MongoDB\\Configuration\:\:getDefaultKmsProvider\(\)\.$#' + identifier: method.notFound + count: 3 + path: tests/DependencyInjection/DoctrineMongoDBExtensionTest.php + + - + message: '#^Call to an undefined method Doctrine\\ODM\\MongoDB\\Configuration\:\:getDefaultMasterKey\(\)\.$#' + identifier: method.notFound + count: 3 + path: tests/DependencyInjection/DoctrineMongoDBExtensionTest.php + + - + message: '#^Call to an undefined method Doctrine\\ODM\\MongoDB\\Configuration\:\:getDriverOptions\(\)\.$#' + identifier: method.notFound + count: 7 + path: tests/DependencyInjection/DoctrineMongoDBExtensionTest.php + - message: '#^Method Doctrine\\Bundle\\MongoDBBundle\\Tests\\DependencyInjection\\DoctrineMongoDBExtensionTest\:\:assertDICDefinitionMethodCall\(\) has parameter \$params with no value type specified in iterable type array\.$#' identifier: missingType.iterableValue diff --git a/src/DependencyInjection/DoctrineMongoDBExtension.php b/src/DependencyInjection/DoctrineMongoDBExtension.php index 7c881b6c..fc64651a 100644 --- a/src/DependencyInjection/DoctrineMongoDBExtension.php +++ b/src/DependencyInjection/DoctrineMongoDBExtension.php @@ -293,6 +293,10 @@ protected function loadDocumentManager(array $documentManager, string|null $defa ]; if (isset($connections[$connectionName]['autoEncryption'])) { + if (! method_exists(ODMConfiguration::class, 'setAutoEncryption')) { + throw new InvalidArgumentException(sprintf('The "autoEncryption" option requires doctrine/mongodb-odm version 2.12 or higher, "%s" installed.', self::getODMVersion())); + } + $autoEncryption = $connections[$connectionName]['autoEncryption']; $methods['setAutoEncryption'] = array_diff_key( $this->normalizeAutoEncryption($autoEncryption, $defaultDB), diff --git a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php index b6560ee9..fb3841be 100644 --- a/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php +++ b/tests/DependencyInjection/DoctrineMongoDBExtensionTest.php @@ -13,6 +13,7 @@ use Doctrine\Bundle\MongoDBBundle\Tests\DependencyInjection\Fixtures\Bundles\DocumentListenerBundle\EventListener\TestAttributeListener; use Doctrine\ODM\MongoDB\Configuration; use Doctrine\ODM\MongoDB\Mapping\Annotations; +use InvalidArgumentException; use MongoDB\Client; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; @@ -488,6 +489,8 @@ public function testUseTransactionalFlush(): void public function testAutoEncryptionWithKeyVaultClientService(): void { + self::requireAutoEncryptionSupportInODM(); + $container = $this->buildMinimalContainer(); $loader = new DoctrineMongoDBExtension(); @@ -538,6 +541,8 @@ public function testAutoEncryptionWithKeyVaultClientService(): void public function testAutoEncryptionWithComplexKmsAndSchemaMap(): void { + self::requireAutoEncryptionSupportInODM(); + $container = $this->buildMinimalContainer(); $loader = new DoctrineMongoDBExtension(); @@ -590,6 +595,8 @@ public function testAutoEncryptionWithComplexKmsAndSchemaMap(): void public function testAutoEncryptionWithExtraOptions(): void { + self::requireAutoEncryptionSupportInODM(); + $container = $this->buildMinimalContainer(); $loader = new DoctrineMongoDBExtension(); @@ -639,6 +646,8 @@ public function testAutoEncryptionWithExtraOptions(): void public function testAutoEncryptionWithEmptyKmsProvider(): void { + self::requireAutoEncryptionSupportInODM(); + $container = $this->buildMinimalContainer(); $loader = new DoctrineMongoDBExtension(); @@ -663,4 +672,36 @@ public function testAutoEncryptionWithEmptyKmsProvider(): void self::assertArrayHasKey('autoEncryption', $driverOptions); self::assertEquals(['aws' => new Definition(stdClass::class)], $driverOptions['autoEncryption']['kmsProviders']); } + + public function testAutoEncryptionMinimumODMVersion(): void + { + if (InstalledVersions::satisfies(new VersionParser(), 'doctrine/mongodb-odm', '>=2.12@dev')) { + self::markTestSkipped('Installed version of doctrine/mongodb-odm does support auto encryption'); + } + + $container = $this->buildMinimalContainer(); + $loader = new DoctrineMongoDBExtension(); + + $config = [ + 'connections' => [ + 'default' => [ + 'autoEncryption' => [ + 'kmsProvider' => ['type' => 'aws'], + ], + ], + ], + 'document_managers' => ['default' => []], + ]; + + self::expectException(InvalidArgumentException::class); + self::expectExceptionMessage('The "autoEncryption" option requires doctrine/mongodb-odm version 2.12 or higher'); + $loader->load([$config], $container); + } + + private static function requireAutoEncryptionSupportInODM(): void + { + if (! InstalledVersions::satisfies(new VersionParser(), 'doctrine/mongodb-odm', '>=2.12@dev')) { + self::markTestSkipped('Installed version of doctrine/mongodb-odm does not support auto encryption'); + } + } }