diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cc41fd2..0a8b160f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,15 @@ jobs: COVERAGE_FILE: .coverage.${{ runner.os }}-py${{ matrix.python }} CONTEXT: ${{ runner.os }}-py${{ matrix.python }} + - name: uninstall deps + run: pip uninstall -y tomlkit PyYAML + + - name: test without deps + run: make test + env: + COVERAGE_FILE: .coverage.${{ runner.os }}-py${{ matrix.python }}-without-deps + CONTEXT: ${{ runner.os }}-py${{ matrix.python }}-without-deps + - run: coverage combine - run: coverage xml diff --git a/docs/index.md b/docs/index.md index d1627c9d..33acda33 100644 --- a/docs/index.md +++ b/docs/index.md @@ -534,6 +534,65 @@ Last, run your application inside a Docker container and supply your newly creat docker service create --name pydantic-with-secrets --secret my_secret_data pydantic-app:latest ``` +## Other settings source + +Other settings sources are available for common configuration files: + +- `TomlConfigSettingsSource` using `toml_file` and toml_file_encoding arguments +- `YamlConfigSettingsSource` using `yaml_file` and yaml_file_encoding arguments +- `JsonConfigSettingsSource` using `json_file` and `json_file_encoding` arguments + +You can also provide multiple files by providing a list of path: +```py +toml_file = ['config.default.toml', 'config.custom.toml'] +``` +To use them, you can use the same mechanism described [here](#customise-settings-sources) + + +```py +from typing import Tuple, Type + +from pydantic import BaseModel + +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, + TomlConfigSettingsSource, +) + + +class Nested(BaseModel): + nested_field: str + + +class Settings(BaseSettings): + foobar: str + nested: Nested + model_config = SettingsConfigDict( + toml_file='config.toml', toml_file_encoding='utf-8' + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (TomlConfigSettingsSource(settings_cls),) +``` + +This will be able to read the following "config.toml" file, located in your working directory: + +```toml +foobar = "Hello" +[nested] +nested_field = "world!" +``` + ## Field value priority In the case where a value is specified for the same `Settings` field in multiple ways, diff --git a/pydantic_settings/__init__.py b/pydantic_settings/__init__.py index 7b99f885..f3e4184c 100644 --- a/pydantic_settings/__init__.py +++ b/pydantic_settings/__init__.py @@ -3,8 +3,11 @@ DotEnvSettingsSource, EnvSettingsSource, InitSettingsSource, + JsonConfigSettingsSource, PydanticBaseSettingsSource, SecretsSettingsSource, + TomlConfigSettingsSource, + YamlConfigSettingsSource, ) from .version import VERSION @@ -13,9 +16,12 @@ 'DotEnvSettingsSource', 'EnvSettingsSource', 'InitSettingsSource', + 'JsonConfigSettingsSource', 'PydanticBaseSettingsSource', 'SecretsSettingsSource', 'SettingsConfigDict', + 'TomlConfigSettingsSource', + 'YamlConfigSettingsSource', '__version__', ) diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index d3ea63f2..a42954a5 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -14,6 +14,7 @@ DotenvType, EnvSettingsSource, InitSettingsSource, + PathType, PydanticBaseSettingsSource, SecretsSettingsSource, ) @@ -28,6 +29,12 @@ class SettingsConfigDict(ConfigDict, total=False): env_nested_delimiter: str | None env_parse_none_str: str | None secrets_dir: str | Path | None + json_file: PathType | None + json_file_encoding: str | None + yaml_file: PathType | None + yaml_file_encoding: str | None + toml_file: PathType | None + toml_file_encoding: str | None # Extend `config_keys` by pydantic settings config keys to @@ -195,6 +202,12 @@ def _settings_build_values( env_ignore_empty=False, env_nested_delimiter=None, env_parse_none_str=None, + json_file=None, + json_file_encoding=None, + yaml_file=None, + yaml_file_encoding=None, + toml_file=None, + toml_file_encoding=None, secrets_dir=None, protected_namespaces=('model_', 'settings_'), ) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 335ba9dd..ff09bd1e 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -2,6 +2,7 @@ import json import os +import sys import warnings from abc import ABC, abstractmethod from collections import deque @@ -19,10 +20,49 @@ from pydantic_settings.utils import path_type_label if TYPE_CHECKING: + if sys.version_info >= (3, 11): + import tomllib + else: + tomllib = None + import tomlkit + import yaml + from pydantic_settings.main import BaseSettings +else: + yaml = None + tomllib = None + tomlkit = None + + +def import_yaml() -> None: + global yaml + if yaml is not None: + return + try: + import yaml + except ImportError as e: + raise ImportError('PyYAML is not installed, run `pip install pydantic-settings[yaml]`') from e + + +def import_toml() -> None: + global tomlkit + global tomllib + if sys.version_info < (3, 11): + if tomlkit is not None: + return + try: + import tomlkit + except ImportError as e: + raise ImportError('tomlkit is not installed, run `pip install pydantic-settings[toml]`') from e + else: + if tomllib is not None: + return + import tomllib DotenvType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]] +PathType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]] +DEFAULT_PATH: PathType = Path('') # This is used as default value for `_env_file` in the `BaseSettings` class and # `env_file` in `DotEnvSettingsSource` so the default can be distinguished from `None`. @@ -664,6 +704,103 @@ def __repr__(self) -> str: ) +class ConfigFileSourceMixin(ABC): + def _read_files(self, files: PathType | None) -> dict[str, Any]: + if files is None: + return {} + if isinstance(files, (str, os.PathLike)): + files = [files] + vars: dict[str, Any] = {} + for file in files: + file_path = Path(file).expanduser() + if file_path.is_file(): + vars.update(self._read_file(file_path)) + return vars + + @abstractmethod + def _read_file(self, path: Path) -> dict[str, Any]: + pass + + +class JsonConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin): + """ + A source class that loads variables from a JSON file + """ + + def __init__( + self, + settings_cls: type[BaseSettings], + json_file: PathType | None = DEFAULT_PATH, + json_file_encoding: str | None = None, + ): + self.json_file_path = json_file if json_file != DEFAULT_PATH else settings_cls.model_config.get('json_file') + self.json_file_encoding = ( + json_file_encoding + if json_file_encoding is not None + else settings_cls.model_config.get('json_file_encoding') + ) + self.json_data = self._read_files(self.json_file_path) + super().__init__(settings_cls, self.json_data) + + def _read_file(self, file_path: Path) -> dict[str, Any]: + with open(file_path, encoding=self.json_file_encoding) as json_file: + return json.load(json_file) + + +class TomlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin): + """ + A source class that loads variables from a JSON file + """ + + def __init__( + self, + settings_cls: type[BaseSettings], + toml_file: PathType | None = DEFAULT_PATH, + toml_file_encoding: str | None = None, + ): + self.toml_file_path = toml_file if toml_file != DEFAULT_PATH else settings_cls.model_config.get('toml_file') + self.toml_file_encoding = ( + toml_file_encoding + if toml_file_encoding is not None + else settings_cls.model_config.get('toml_file_encoding') + ) + self.toml_data = self._read_files(self.toml_file_path) + super().__init__(settings_cls, self.toml_data) + + def _read_file(self, file_path: Path) -> dict[str, Any]: + import_toml() + with open(file_path, mode='rb', encoding=self.toml_file_encoding) as toml_file: + if sys.version_info < (3, 11): + return tomlkit.load(toml_file) + return tomllib.load(toml_file) + + +class YamlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin): + """ + A source class that loads variables from a yaml file + """ + + def __init__( + self, + settings_cls: type[BaseSettings], + yaml_file: PathType | None = DEFAULT_PATH, + yaml_file_encoding: str | None = None, + ): + self.yaml_file_path = yaml_file if yaml_file != DEFAULT_PATH else settings_cls.model_config.get('yaml_file') + self.yaml_file_encoding = ( + yaml_file_encoding + if yaml_file_encoding is not None + else settings_cls.model_config.get('yaml_file_encoding') + ) + self.yaml_data = self._read_files(self.yaml_file_path) + super().__init__(settings_cls, self.yaml_data) + + def _read_file(self, file_path: Path) -> dict[str, Any]: + import_yaml() + with open(file_path, encoding=self.yaml_file_encoding) as yaml_file: + return yaml.safe_load(yaml_file) + + def _get_env_var_key(key: str, case_sensitive: bool = False) -> str: return key if case_sensitive else key.lower() diff --git a/pyproject.toml b/pyproject.toml index c7a0c5ab..d622ea9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,10 @@ dependencies = [ ] dynamic = ['version'] +[project.optional-dependencies] +yaml = ["pyyaml>=6.0.1"] +toml = ["tomlkit>=0.12"] + [project.urls] Homepage = 'https://github.com/pydantic/pydantic-settings' Funding = 'https://github.com/sponsors/samuelcolvin' diff --git a/requirements/linting.in b/requirements/linting.in index 64c9833f..6d25ae58 100644 --- a/requirements/linting.in +++ b/requirements/linting.in @@ -2,4 +2,6 @@ black ruff pyupgrade mypy +types-PyYAML +pyyaml==6.0.1 pre-commit diff --git a/requirements/linting.txt b/requirements/linting.txt index 152c7006..152402ae 100644 --- a/requirements/linting.txt +++ b/requirements/linting.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --output-file=requirements/linting.txt requirements/linting.in @@ -36,16 +36,16 @@ pre-commit==2.21.0 # via -r requirements/linting.in pyupgrade==3.3.2 # via -r requirements/linting.in -pyyaml==6.0 - # via pre-commit +pyyaml==6.0.1 + # via + # -r requirements/linting.in + # pre-commit ruff==0.0.270 # via -r requirements/linting.in tokenize-rt==5.0.0 # via pyupgrade -tomli==2.0.1 - # via - # black - # mypy +types-pyyaml==6.0.12.12 + # via -r requirements/linting.in typing-extensions==4.6.2 # via mypy virtualenv==20.23.0 diff --git a/requirements/pyproject.txt b/requirements/pyproject.txt index 9b64021c..62bdac6a 100644 --- a/requirements/pyproject.txt +++ b/requirements/pyproject.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --output-file=requirements/pyproject.txt pyproject.toml +# pip-compile --extra=toml --extra=yaml --output-file=requirements/pyproject.txt pyproject.toml # annotated-types==0.4.0 # via pydantic @@ -12,6 +12,10 @@ pydantic-core==2.14.5 # via pydantic python-dotenv==0.21.1 # via pydantic-settings (pyproject.toml) +pyyaml==6.0.1 + # via pydantic-settings (pyproject.toml) +tomlkit==0.12.3 + # via pydantic-settings (pyproject.toml) typing-extensions==4.6.2 # via # pydantic diff --git a/tests/test_settings.py b/tests/test_settings.py index b2371953..f4eec598 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,4 +1,5 @@ import dataclasses +import json import os import sys import uuid @@ -30,9 +31,12 @@ DotEnvSettingsSource, EnvSettingsSource, InitSettingsSource, + JsonConfigSettingsSource, PydanticBaseSettingsSource, SecretsSettingsSource, SettingsConfigDict, + TomlConfigSettingsSource, + YamlConfigSettingsSource, ) from pydantic_settings.sources import SettingsError, read_env_file @@ -40,6 +44,14 @@ import dotenv except ImportError: dotenv = None +try: + import yaml +except ImportError: + yaml = None +try: + import tomlkit +except ImportError: + tomlkit = None class SimpleSettings(BaseSettings): @@ -1085,16 +1097,60 @@ def test_read_dotenv_vars_when_env_file_is_none(): ) -@pytest.mark.skipif(dotenv, reason='python-dotenv is installed') -def test_dotenv_not_installed(tmp_path): +@pytest.mark.skipif(yaml, reason='PyYAML is installed') +def test_yaml_not_installed(tmp_path): p = tmp_path / '.env' - p.write_text('a=b') + p.write_text( + """ + foobar: "Hello" + """ + ) class Settings(BaseSettings): - a: str + foobar: str + model_config = SettingsConfigDict(yaml_file=p) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (YamlConfigSettingsSource(settings_cls),) + + with pytest.raises(ImportError, match=r'^PyYAML is not installed, run `pip install pydantic-settings\[yaml\]`$'): + Settings() + + +@pytest.mark.skipif(sys.version_info >= (3, 11) or tomlkit, reason='tomlkit/tomllib is installed') +def test_toml_not_installed(tmp_path): + p = tmp_path / '.env' + p.write_text( + """ + foobar = "Hello" + """ + ) + + class Settings(BaseSettings): + foobar: str + model_config = SettingsConfigDict(toml_file=p) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (TomlConfigSettingsSource(settings_cls),) - with pytest.raises(ImportError, match=r'^python-dotenv is not installed, run `pip install pydantic\[dotenv\]`$'): - Settings(_env_file=p) + with pytest.raises(ImportError, match=r'^tomlkit is not installed, run `pip install pydantic-settings\[toml\]`$'): + Settings() def test_alias_set(env): @@ -1915,3 +1971,264 @@ class Settings(BaseSettings): s = Settings() assert s.data == {'foo': 'bar'} + + +def test_json_file(tmp_path): + p = tmp_path / '.env' + p.write_text( + """ + {"foobar": "Hello", "nested": {"nested_field": "world!"}, "null_field": null} + """ + ) + + class Nested(BaseModel): + nested_field: str + + class Settings(BaseSettings): + model_config = SettingsConfigDict(json_file=p) + foobar: str + nested: Nested + null_field: Union[str, None] + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (JsonConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.foobar == 'Hello' + assert s.nested.nested_field == 'world!' + + +def test_json_no_file(): + class Settings(BaseSettings): + model_config = SettingsConfigDict(json_file=None) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (JsonConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.model_dump() == {} + + +@pytest.mark.skipif(yaml is None, reason='pyYaml is not installed') +def test_yaml_file(tmp_path): + p = tmp_path / '.env' + p.write_text( + """ + foobar: "Hello" + null_field: + nested: + nested_field: "world!" + """ + ) + + class Nested(BaseModel): + nested_field: str + + class Settings(BaseSettings): + foobar: str + nested: Nested + null_field: Union[str, None] + model_config = SettingsConfigDict(yaml_file=p) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (YamlConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.foobar == 'Hello' + assert s.nested.nested_field == 'world!' + + +@pytest.mark.skipif(yaml is None, reason='pyYaml is not installed') +def test_yaml_no_file(): + class Settings(BaseSettings): + model_config = SettingsConfigDict(yaml_file=None) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (YamlConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.model_dump() == {} + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomlkit is None, reason='tomlkit/tomllib is not installed') +def test_toml_file(tmp_path): + p = tmp_path / '.env' + p.write_text( + """ + foobar = "Hello" + + [nested] + nested_field = "world!" + """ + ) + + class Nested(BaseModel): + nested_field: str + + class Settings(BaseSettings): + foobar: str + nested: Nested + model_config = SettingsConfigDict(toml_file=p) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (TomlConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.foobar == 'Hello' + assert s.nested.nested_field == 'world!' + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomlkit is None, reason='tomlkit/tomllib is not installed') +def test_toml_no_file(): + class Settings(BaseSettings): + model_config = SettingsConfigDict(toml_file=None) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (TomlConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.model_dump() == {} + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomlkit is None, reason='tomlkit/tomllib is not installed') +def test_multiple_file_toml(tmp_path): + p1 = tmp_path / '.env.toml1' + p2 = tmp_path / '.env.toml2' + p1.write_text( + """ + toml1=1 + """ + ) + p2.write_text( + """ + toml2=2 + """ + ) + + class Settings(BaseSettings): + toml1: int + toml2: int + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (TomlConfigSettingsSource(settings_cls, toml_file=[p1, p2]),) + + s = Settings() + assert s.model_dump() == {'toml1': 1, 'toml2': 2} + + +@pytest.mark.skipif(yaml is None, reason='pyYAML is not installed') +def test_multiple_file_yaml(tmp_path): + p3 = tmp_path / '.env.yaml3' + p4 = tmp_path / '.env.yaml4' + p3.write_text( + """ + yaml3: 3 + """ + ) + p4.write_text( + """ + yaml4: 4 + """ + ) + + class Settings(BaseSettings): + yaml3: int + yaml4: int + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (YamlConfigSettingsSource(settings_cls, yaml_file=[p3, p4]),) + + s = Settings() + assert s.model_dump() == {'yaml3': 3, 'yaml4': 4} + + +def test_multiple_file_json(tmp_path): + p5 = tmp_path / '.env.json5' + p6 = tmp_path / '.env.json6' + + with open(p5, 'w') as f5: + json.dump({'json5': 5}, f5) + with open(p6, 'w') as f6: + json.dump({'json6': 6}, f6) + + class Settings(BaseSettings): + json5: int + json6: int + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (JsonConfigSettingsSource(settings_cls, json_file=[p5, p6]),) + + s = Settings() + assert s.model_dump() == {'json5': 5, 'json6': 6}