Merge branch 'pydantic:main' into issue-536-fix

Author: tselepakis

docs/index.md

@@ -1354,7 +1354,9 @@ print(Settings().model_dump())
 
 #### CLI Kebab Case for Arguments
 
-Change whether CLI arguments should use kebab case by enabling `cli_kebab_case`.
+Change whether CLI arguments should use kebab case by enabling `cli_kebab_case`. By default, `cli_kebab_case=True` will
+ignore enum fields, and is equivalent to `cli_kebab_case='no_enums'`. To apply kebab case to everything, including
+enums, use `cli_kebab_case='all'`.
 
 ```py
 import sys
@@ -1857,6 +1859,248 @@ Last, run your application inside a Docker container and supply your newly creat
 docker service create --name pydantic-with-secrets --secret my_secret_data pydantic-app:latest
 ```
 
+## Nested Secrets
+
+The default secrets implementation, `SecretsSettingsSource`, has behaviour that is not always desired or sufficient.
+For example, the default implementation does not support secret fields in nested submodels.
+
+`NestedSecretsSettingsSource` can be used as a drop-in replacement to `SecretsSettingsSource` to adjust the default behaviour.
+All differences are summarized in the table below.
+
+| `SecretsSettingsSource`                                                                                                                                         | `NestedSecretsSettingsSourcee`                                                                                                    |
+|-----------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------|
+| Secret fields must belong to a top level model.                                                                                                                 | Secrets can be fields of nested models.                                                                                           |
+| Secret files can be placed in `secrets_dir`s only.                                                                                                              | Secret files can be placed in subdirectories for nested models.                                                                   |
+| Secret files discovery is based on the same configuration options that are used by `EnvSettingsSource`: `case_sensitive`, `env_nested_delimiter`, `env_prefix`. | Default options are respected, but can be overridden with `secrets_case_sensitive`, `secrets_nested_delimiter`, `secrets_prefix`. |
+| When `secrets_dir` is missing on the file system, a warning is generated.                                                                                       | Use `secrets_dir_missing` options to choose whether to issue warning, raise error, or silently ignore.                            |
+
+### Use Case: Plain Directory Layout
+
+```text
+📂 secrets
+├── 📄 app_key
+└── 📄 db_passwd
+```
+
+In the example below, secrets nested delimiter `'_'` is different from env nested delimiter `'__'`.
+Value for `Settings.db.user` can be passed in env variable `MY_DB__USER`.
+
+```py
+from pydantic import BaseModel, SecretStr
+
+from pydantic_settings import (
+    BaseSettings,
+    NestedSecretsSettingsSource,
+    SettingsConfigDict,
+)
+
+
+class AppSettings(BaseModel):
+    key: SecretStr
+
+
+class DbSettings(BaseModel):
+    user: str
+    passwd: SecretStr
+
+
+class Settings(BaseSettings):
+    app: AppSettings
+    db: DbSettings
+
+    model_config = SettingsConfigDict(
+        env_prefix='MY_',
+        env_nested_delimiter='__',
+        secrets_dir='secrets',
+        secrets_nested_delimiter='_',
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls,
+        init_settings,
+        env_settings,
+        dotenv_settings,
+        file_secret_settings,
+    ):
+        return (
+            init_settings,
+            env_settings,
+            dotenv_settings,
+            NestedSecretsSettingsSource(file_secret_settings),
+        )
+```
+
+### Use Case: Nested Directory Layout
+
+```text
+📂 secrets
+├── 📂 app
+│   └── 📄 key
+└── 📂 db
+    └── 📄 passwd
+```
+```py
+from pydantic import BaseModel, SecretStr
+
+from pydantic_settings import (
+    BaseSettings,
+    NestedSecretsSettingsSource,
+    SettingsConfigDict,
+)
+
+
+class AppSettings(BaseModel):
+    key: SecretStr
+
+
+class DbSettings(BaseModel):
+    user: str
+    passwd: SecretStr
+
+
+class Settings(BaseSettings):
+    app: AppSettings
+    db: DbSettings
+
+    model_config = SettingsConfigDict(
+        env_prefix='MY_',
+        env_nested_delimiter='__',
+        secrets_dir='secrets',
+        secrets_nested_subdir=True,
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls,
+        init_settings,
+        env_settings,
+        dotenv_settings,
+        file_secret_settings,
+    ):
+        return (
+            init_settings,
+            env_settings,
+            dotenv_settings,
+            NestedSecretsSettingsSource(file_secret_settings),
+        )
+```
+
+### Use Case: Multiple Nested Directories
+
+```text
+📂 secrets
+├── 📂 default
+│   ├── 📂 app
+│   │   └── 📄 key
+│   └── 📂 db
+│       └── 📄 passwd
+└── 📂 override
+    ├── 📂 app
+    │   └── 📄 key
+    └── 📂 db
+        └── 📄 passwd
+```
+```py
+from pydantic import BaseModel, SecretStr
+
+from pydantic_settings import (
+    BaseSettings,
+    NestedSecretsSettingsSource,
+    SettingsConfigDict,
+)
+
+
+class AppSettings(BaseModel):
+    key: SecretStr
+
+
+class DbSettings(BaseModel):
+    user: str
+    passwd: SecretStr
+
+
+class Settings(BaseSettings):
+    app: AppSettings
+    db: DbSettings
+
+    model_config = SettingsConfigDict(
+        env_prefix='MY_',
+        env_nested_delimiter='__',
+        secrets_dir=['secrets/default', 'secrets/override'],
+        secrets_nested_subdir=True,
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls,
+        init_settings,
+        env_settings,
+        dotenv_settings,
+        file_secret_settings,
+    ):
+        return (
+            init_settings,
+            env_settings,
+            dotenv_settings,
+            NestedSecretsSettingsSource(file_secret_settings),
+        )
+```
+
+### Configuration Options
+
+#### secrets_dir
+
+Path to secrets directory, same as `SecretsSettingsSource.secrets_dir`. If `list`, the last match wins.
+If `secrets_dir` is passed in both source constructor and model config, values are not merged (constructor wins).
+
+#### secrets_dir_missing
+
+If `secrets_dir` does not exist, original `SecretsSettingsSource` issues a warning.
+However, this may be undesirable, for example if we don't mount Docker Secrets in e.g. dev environment.
+Use `secrets_dir_missing` to choose:
+
+* `'ok'` — do nothing if `secrets_dir` does not exist
+* `'warn'` (default) — print warning, same as `SecretsSettingsSource`
+* `'error'` — raise `SettingsError`
+
+If multiple `secrets_dir` passed, the same `secrets_dir_missing` action applies to each of them.
+
+#### secrets_dir_max_size
+
+Limit the size of `secrets_dir` for security reasons, defaults to `SECRETS_DIR_MAX_SIZE` equal to 16 MiB.
+
+`NestedSecretsSettingsSource` is a thin wrapper around `EnvSettingsSource`,
+which loads all potential secrets on initialization. This could lead to `MemoryError` if we mount
+a large file under `secrets_dir`.
+
+If multiple `secrets_dir` passed, the limit applies to each directory independently.
+
+#### secrets_case_sensitive
+
+Same as `case_sensitive`, but works for secrets only. If not specified, defaults to `case_sensitive`.
+
+#### secrets_nested_delimiter
+
+Same as `env_nested_delimiter`, but works for secrets only. If not specified, defaults to `env_nested_delimiter`.
+This option is used to implement _nested secrets directory_ layout and allows to do even nasty things
+like `/run/secrets/model/delim/nested1/delim/nested2`.
+
+#### secrets_nested_subdir
+
+Boolean flag to turn on _nested secrets directory_ mode, `False` by default. If `True`, sets `secrets_nested_delimiter`
+to `os.sep`. Raises `SettingsError` if `secrets_nested_delimiter` is already specified.
+
+#### secrets_prefix
+
+Secret path prefix, similar to `env_prefix`, but works for secrets only. Defaults to `env_prefix`
+if not specified. Works in both plain and nested directory modes, like
+`'/run/secrets/prefix_model__nested'` and `'/run/secrets/prefix_model/nested'`.
+
+
 ## AWS Secrets Manager
 
 You must set one parameter:

pydantic_settings/__init__.py

@@ -18,6 +18,7 @@
     GoogleSecretManagerSettingsSource,
     InitSettingsSource,
     JsonConfigSettingsSource,
+    NestedSecretsSettingsSource,
     NoDecode,
     PydanticBaseSettingsSource,
     PyprojectTomlConfigSettingsSource,
@@ -48,6 +49,7 @@
     'GoogleSecretManagerSettingsSource',
     'InitSettingsSource',
     'JsonConfigSettingsSource',
+    'NestedSecretsSettingsSource',
     'NoDecode',
     'PydanticBaseSettingsSource',
     'PyprojectTomlConfigSettingsSource',

pydantic_settings/main.py

@@ -7,7 +7,7 @@
 from argparse import Namespace
 from collections.abc import Mapping
 from types import SimpleNamespace
-from typing import Any, ClassVar, TypeVar
+from typing import Any, ClassVar, Literal, TypeVar
 
 from pydantic import ConfigDict
 from pydantic._internal._config import config_keys
@@ -62,7 +62,7 @@ class SettingsConfigDict(ConfigDict, total=False):
     cli_flag_prefix_char: str
     cli_implicit_flags: bool | None
     cli_ignore_unknown_args: bool | None
-    cli_kebab_case: bool | None
+    cli_kebab_case: bool | Literal['all', 'no_enums'] | None
     cli_shortcuts: Mapping[str, str | list[str]] | None
     secrets_dir: PathType | None
     json_file: PathType | None
@@ -185,7 +185,7 @@ def __init__(
         _cli_flag_prefix_char: str | None = None,
         _cli_implicit_flags: bool | None = None,
         _cli_ignore_unknown_args: bool | None = None,
-        _cli_kebab_case: bool | None = None,
+        _cli_kebab_case: bool | Literal['all', 'no_enums'] | None = None,
         _cli_shortcuts: Mapping[str, str | list[str]] | None = None,
         _secrets_dir: PathType | None = None,
         **values: Any,
@@ -272,7 +272,7 @@ def _settings_build_values(
         _cli_flag_prefix_char: str | None = None,
         _cli_implicit_flags: bool | None = None,
         _cli_ignore_unknown_args: bool | None = None,
-        _cli_kebab_case: bool | None = None,
+        _cli_kebab_case: bool | Literal['all', 'no_enums'] | None = None,
         _cli_shortcuts: Mapping[str, str | list[str]] | None = None,
         _secrets_dir: PathType | None = None,
     ) -> dict[str, Any]:
@@ -426,6 +426,7 @@ def _settings_build_values(
 
         if sources:
             state: dict[str, Any] = {}
+            defaults: dict[str, Any] = {}
             states: dict[str, dict[str, Any]] = {}
             for source in sources:
                 if isinstance(source, PydanticBaseSettingsSource):
@@ -435,8 +436,15 @@ def _settings_build_values(
                 source_name = source.__name__ if hasattr(source, '__name__') else type(source).__name__
                 source_state = source()
 
+                if isinstance(source, DefaultSettingsSource):
+                    defaults = source_state
+
                 states[source_name] = source_state
                 state = deep_update(source_state, state)
+
+            # Strip any default values not explicity set before returning final state
+            state = {key: val for key, val in state.items() if key not in defaults or defaults[key] != val}
+
             return state
         else:
             # no one should mean to do this, but I think returning an empty dict is marginally preferable

pydantic_settings/sources/__init__.py

@@ -25,6 +25,7 @@
 from .providers.env import EnvSettingsSource
 from .providers.gcp import GoogleSecretManagerSettingsSource
 from .providers.json import JsonConfigSettingsSource
+from .providers.nested_secrets import NestedSecretsSettingsSource
 from .providers.pyproject import PyprojectTomlConfigSettingsSource
 from .providers.secrets import SecretsSettingsSource
 from .providers.toml import TomlConfigSettingsSource
@@ -53,6 +54,7 @@
     'GoogleSecretManagerSettingsSource',
     'InitSettingsSource',
     'JsonConfigSettingsSource',
+    'NestedSecretsSettingsSource',
     'NoDecode',
     'PathType',
     'PydanticBaseEnvSettingsSource',

pydantic_settings/sources/base.py

@@ -266,12 +266,27 @@ def __init__(
         init_kwarg_names = set(init_kwargs.keys())
         for field_name, field_info in settings_cls.model_fields.items():
             alias_names, *_ = _get_alias_names(field_name, field_info)
-            init_kwarg_name = init_kwarg_names & set(alias_names)
+            # When populate_by_name is True, allow using the field name as an input key,
+            # but normalize to the preferred alias to keep keys consistent across sources.
+            matchable_names = set(alias_names)
+            include_name = settings_cls.model_config.get('populate_by_name', False)
+            if include_name:
+                matchable_names.add(field_name)
+            init_kwarg_name = init_kwarg_names & matchable_names
             if init_kwarg_name:
-                preferred_alias = alias_names[0]
-                preferred_set_alias = next(alias for alias in alias_names if alias in init_kwarg_name)
+                preferred_alias = alias_names[0] if alias_names else field_name
+                # Choose provided key deterministically: prefer the first alias in alias_names order;
+                # fall back to field_name if allowed and provided.
+                provided_key = next((alias for alias in alias_names if alias in init_kwarg_names), None)
+                if provided_key is None and include_name and field_name in init_kwarg_names:
+                    provided_key = field_name
+                # provided_key should not be None here because init_kwarg_name is non-empty
+                assert provided_key is not None
                 init_kwarg_names -= init_kwarg_name
-                self.init_kwargs[preferred_alias] = init_kwargs[preferred_set_alias]
+                self.init_kwargs[preferred_alias] = init_kwargs[provided_key]
+        # Include any remaining init kwargs (e.g., extras) unchanged
+        # Note: If populate_by_name is True and the provided key is the field name, but
+        # no alias exists, we keep it as-is so it can be processed as extra if allowed.
         self.init_kwargs.update({key: val for key, val in init_kwargs.items() if key in init_kwarg_names})
 
         super().__init__(settings_cls)