diff --git a/docs/index.md b/docs/index.md index 222a62d5..3ffa6f52 100644 --- a/docs/index.md +++ b/docs/index.md @@ -539,9 +539,10 @@ docker service create --name pydantic-with-secrets --secret my_secret_data pydan Other settings sources are available for common configuration files: +- `JsonConfigSettingsSource` using `json_file` and `json_file_encoding` arguments +- `PyprojectTomlConfigSettingsSource` using *(optional)* `pyproject_toml_depth` and *(optional)* `pyproject_toml_table_header` arguments - `TomlConfigSettingsSource` using `toml_file` argument - `YamlConfigSettingsSource` using `yaml_file` and yaml_file_encoding arguments -- `JsonConfigSettingsSource` using `json_file` and `json_file_encoding` arguments You can also provide multiple files by providing a list of path: ```py @@ -592,6 +593,127 @@ foobar = "Hello" nested_field = "world!" ``` +### pyproject.toml + +"pyproject.toml" is a standardized file for providing configuration values in Python projects. +[PEP 518](https://peps.python.org/pep-0518/#tool-table) defines a `[tool]` table that can be used to provide arbitrary tool configuration. +While encouraged to use the `[tool]` table, `PyprojectTomlConfigSettingsSource` can be used to load variables from any location with in "pyproject.toml" file. + +This is controlled by providing `SettingsConfigDict(pyproject_toml_table_header=tuple[str, ...])` where the value is a tuple of header parts. +By default, `pyproject_toml_table_header=('tool', 'pydantic-settings')` which will load variables from the `[tool.pydantic-settings]` table. + +```python +from typing import Tuple, Type + +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + PyprojectTomlConfigSettingsSource, + SettingsConfigDict, +) + + +class Settings(BaseSettings): + """Example loading values from the table used by default.""" + + field: str + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls),) + + +class SomeTableSettings(Settings): + """Example loading values from a user defined table.""" + + model_config = SettingsConfigDict( + pyproject_toml_table_header=('tool', 'some-table') + ) + + +class RootSettings(Settings): + """Example loading values from the root of a pyproject.toml file.""" + + model_config = SettingsConfigDict(extra='ignore', pyproject_toml_table_header=()) +``` + +This will be able to read the following "pyproject.toml" file, located in your working directory, resulting in `Settings(field='default-table')`, `SomeTableSettings(field='some-table')`, & `RootSettings(field='root')`: + +```toml +field = "root" + +[tool.pydantic-settings] +field = "default-table" + +[tool.some-table] +field = "some-table" +``` + +By default, `PyprojectTomlConfigSettingsSource` will only look for a "pyproject.toml" in the your current working directory. +However, there are two options to change this behavior. + +* `SettingsConfigDict(pyproject_toml_depth=)` can be provided to check `` number of directories **up** in the directory tree for a "pyproject.toml" if one is not found in the current working directory. + By default, no parent directories are checked. +* An explicit file path can be provided to the source when it is instantiated (e.g. `PyprojectTomlConfigSettingsSource(settings_cls, Path('~/.config').resolve() / 'pyproject.toml')`). + If a file path is provided this way, it will be treated as absolute (no other locations are checked). + +```python +from pathlib import Path +from typing import Tuple, Type + +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + PyprojectTomlConfigSettingsSource, + SettingsConfigDict, +) + + +class DiscoverSettings(BaseSettings): + """Example of discovering a pyproject.toml in parent directories in not in `Path.cwd()`.""" + + model_config = SettingsConfigDict(pyproject_toml_depth=2) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls),) + + +class ExplicitFilePathSettings(BaseSettings): + """Example of explicitly providing the path to the file to load.""" + + field: str + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return ( + PyprojectTomlConfigSettingsSource( + settings_cls, Path('~/.config').resolve() / 'pyproject.toml' + ), + ) +``` + ## Field value priority In the case where a value is specified for the same `Settings` field in multiple ways, diff --git a/pydantic_settings/__init__.py b/pydantic_settings/__init__.py index f3e4184c..5f08cc62 100644 --- a/pydantic_settings/__init__.py +++ b/pydantic_settings/__init__.py @@ -5,6 +5,7 @@ InitSettingsSource, JsonConfigSettingsSource, PydanticBaseSettingsSource, + PyprojectTomlConfigSettingsSource, SecretsSettingsSource, TomlConfigSettingsSource, YamlConfigSettingsSource, @@ -17,6 +18,7 @@ 'EnvSettingsSource', 'InitSettingsSource', 'JsonConfigSettingsSource', + 'PyprojectTomlConfigSettingsSource', 'PydanticBaseSettingsSource', 'SecretsSettingsSource', 'SettingsConfigDict', diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 1744820b..c5764fc2 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -34,6 +34,26 @@ class SettingsConfigDict(ConfigDict, total=False): json_file_encoding: str | None yaml_file: PathType | None yaml_file_encoding: str | None + pyproject_toml_depth: int + """ + Number of levels **up** from the current working directory to attempt to find a pyproject.toml + file. + + This is only used when a pyproject.toml file is not found in the current working directory. + """ + + pyproject_toml_table_header: tuple[str, ...] + """ + Header of the TOML table within a pyproject.toml file to use when filling variables. + This is supplied as a `tuple[str, ...]` instead of a `str` to accommodate for headers + containing a `.`. + + For example, `toml_table_header = ("tool", "my.tool", "foo")` can be used to fill variable + values from a table with header `[tool."my.tool".foo]`. + + To use the root table, exclude this config setting or provide an empty tuple. + """ + toml_file: PathType | None diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 06203773..8bfd4f8b 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -783,7 +783,7 @@ def _read_file(self, file_path: Path) -> dict[str, Any]: class TomlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin): """ - A source class that loads variables from a JSON file + A source class that loads variables from a TOML file """ def __init__( @@ -803,6 +803,52 @@ def _read_file(self, file_path: Path) -> dict[str, Any]: return tomllib.load(toml_file) +class PyprojectTomlConfigSettingsSource(TomlConfigSettingsSource): + """ + A source class that loads variables from a `pyproject.toml` file. + """ + + def __init__( + self, + settings_cls: type[BaseSettings], + toml_file: Path | None = None, + ) -> None: + self.toml_file_path = self._pick_pyproject_toml_file( + toml_file, settings_cls.model_config.get('pyproject_toml_depth', 0) + ) + self.toml_table_header: tuple[str, ...] = settings_cls.model_config.get( + 'pyproject_toml_table_header', ('tool', 'pydantic-settings') + ) + self.toml_data = self._read_files(self.toml_file_path) + for key in self.toml_table_header: + self.toml_data = self.toml_data.get(key, {}) + super(TomlConfigSettingsSource, self).__init__(settings_cls, self.toml_data) + + @staticmethod + def _pick_pyproject_toml_file(provided: Path | None, depth: int) -> Path: + """Pick a `pyproject.toml` file path to use. + + Args: + provided: Explicit path provided when instantiating this class. + depth: Number of directories up the tree to check of a pyproject.toml. + + """ + if provided: + return provided.resolve() + rv = Path.cwd() / 'pyproject.toml' + count = 0 + if not rv.is_file(): + child = rv.parent.parent / 'pyproject.toml' + while count < depth: + if child.is_file(): + return child + if str(child.parent) == rv.root: + break # end discovery after checking system root once + child = child.parent.parent / 'pyproject.toml' + count += 1 + return rv + + class YamlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin): """ A source class that loads variables from a yaml file diff --git a/tests/conftest.py b/tests/conftest.py index aec508f5..2118f9dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,14 @@ +from __future__ import annotations + import os +from pathlib import Path +from typing import TYPE_CHECKING import pytest +if TYPE_CHECKING: + from collections.abc import Iterator + class SetEnv: def __init__(self): @@ -20,6 +27,34 @@ def clear(self): os.environ.pop(n) +@pytest.fixture +def cd_tmp_path(tmp_path: Path) -> Iterator[Path]: + """Change directory into the value of the ``tmp_path`` fixture. + + .. rubric:: Example + .. code-block:: python + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + from pathlib import Path + + + def test_something(cd_tmp_path: Path) -> None: + ... + + Returns: + Value of the :fixture:`tmp_path` fixture (a :class:`~pathlib.Path` object). + + """ + prev_dir = Path.cwd() + os.chdir(tmp_path) + try: + yield tmp_path + finally: + os.chdir(prev_dir) + + @pytest.fixture def env(): setenv = SetEnv() diff --git a/tests/test_settings.py b/tests/test_settings.py index 62648f94..5075824c 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -25,6 +25,7 @@ dataclasses as pydantic_dataclasses, ) from pydantic.fields import FieldInfo +from pytest_mock import MockerFixture from typing_extensions import Annotated from pydantic_settings import ( @@ -34,6 +35,7 @@ InitSettingsSource, JsonConfigSettingsSource, PydanticBaseSettingsSource, + PyprojectTomlConfigSettingsSource, SecretsSettingsSource, SettingsConfigDict, TomlConfigSettingsSource, @@ -2185,6 +2187,225 @@ def settings_customise_sources( assert s.model_dump() == {} +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +def test_pyproject_toml_file(cd_tmp_path: Path): + pyproject = cd_tmp_path / 'pyproject.toml' + pyproject.write_text( + """ + [tool.pydantic-settings] + foobar = "Hello" + + [tool.pydantic-settings.nested] + nested_field = "world!" + """ + ) + + class Nested(BaseModel): + nested_field: str + + class Settings(BaseSettings): + foobar: str + nested: Nested + model_config = SettingsConfigDict() + + @classmethod + def settings_customise_sources( + cls, settings_cls: Type[BaseSettings], **_kwargs: PydanticBaseSettingsSource + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.foobar == 'Hello' + assert s.nested.nested_field == 'world!' + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +def test_pyproject_toml_file_explicit(cd_tmp_path: Path): + pyproject = cd_tmp_path / 'child' / 'grandchild' / 'pyproject.toml' + pyproject.parent.mkdir(parents=True) + pyproject.write_text( + """ + [tool.pydantic-settings] + foobar = "Hello" + + [tool.pydantic-settings.nested] + nested_field = "world!" + """ + ) + (cd_tmp_path / 'pyproject.toml').write_text( + """ + [tool.pydantic-settings] + foobar = "fail" + + [tool.pydantic-settings.nested] + nested_field = "fail" + """ + ) + + class Nested(BaseModel): + nested_field: str + + class Settings(BaseSettings): + foobar: str + nested: Nested + model_config = SettingsConfigDict() + + @classmethod + def settings_customise_sources( + cls, settings_cls: Type[BaseSettings], **_kwargs: PydanticBaseSettingsSource + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls, pyproject),) + + s = Settings() + assert s.foobar == 'Hello' + assert s.nested.nested_field == 'world!' + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +def test_pyproject_toml_file_parent(mocker: MockerFixture, tmp_path: Path): + cwd = tmp_path / 'child' / 'grandchild' / 'cwd' + cwd.mkdir(parents=True) + mocker.patch('pydantic_settings.sources.Path.cwd', return_value=cwd) + (cwd.parent.parent / 'pyproject.toml').write_text( + """ + [tool.pydantic-settings] + foobar = "Hello" + + [tool.pydantic-settings.nested] + nested_field = "world!" + """ + ) + (tmp_path / 'pyproject.toml').write_text( + """ + [tool.pydantic-settings] + foobar = "fail" + + [tool.pydantic-settings.nested] + nested_field = "fail" + """ + ) + + class Nested(BaseModel): + nested_field: str + + class Settings(BaseSettings): + foobar: str + nested: Nested + model_config = SettingsConfigDict(pyproject_toml_depth=2) + + @classmethod + def settings_customise_sources( + cls, settings_cls: Type[BaseSettings], **_kwargs: PydanticBaseSettingsSource + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.foobar == 'Hello' + assert s.nested.nested_field == 'world!' + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +def test_pyproject_toml_file_header(cd_tmp_path: Path): + pyproject = cd_tmp_path / 'subdir' / 'pyproject.toml' + pyproject.parent.mkdir() + pyproject.write_text( + """ + [tool.pydantic-settings] + foobar = "Hello" + + [tool.pydantic-settings.nested] + nested_field = "world!" + + [tool."my.tool".foo] + status = "success" + """ + ) + + class Settings(BaseSettings): + status: str + model_config = SettingsConfigDict(extra='forbid', pyproject_toml_table_header=('tool', 'my.tool', 'foo')) + + @classmethod + def settings_customise_sources( + cls, settings_cls: Type[BaseSettings], **_kwargs: PydanticBaseSettingsSource + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls, pyproject),) + + s = Settings() + assert s.status == 'success' + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +@pytest.mark.parametrize('depth', [0, 99]) +def test_pyproject_toml_no_file(cd_tmp_path: Path, depth: int): + class Settings(BaseSettings): + model_config = SettingsConfigDict(pyproject_toml_depth=depth) + + @classmethod + def settings_customise_sources( + cls, settings_cls: Type[BaseSettings], **_kwargs: PydanticBaseSettingsSource + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls),) + + s = Settings() + assert s.model_dump() == {} + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +def test_pyproject_toml_no_file_explicit(tmp_path: Path): + pyproject = tmp_path / 'child' / 'pyproject.toml' + (tmp_path / 'pyproject.toml').write_text('[tool.pydantic-settings]\nfield = "fail"') + + class Settings(BaseSettings): + model_config = SettingsConfigDict() + + field: Optional[str] = None + + @classmethod + def settings_customise_sources( + cls, settings_cls: Type[BaseSettings], **_kwargs: PydanticBaseSettingsSource + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls, pyproject),) + + s = Settings() + assert s.model_dump() == {'field': None} + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +@pytest.mark.parametrize('depth', [0, 1, 2]) +def test_pyproject_toml_no_file_too_shallow(depth: int, mocker: MockerFixture, tmp_path: Path): + cwd = tmp_path / 'child' / 'grandchild' / 'cwd' + cwd.mkdir(parents=True) + mocker.patch('pydantic_settings.sources.Path.cwd', return_value=cwd) + (tmp_path / 'pyproject.toml').write_text( + """ + [tool.pydantic-settings] + foobar = "fail" + + [tool.pydantic-settings.nested] + nested_field = "fail" + """ + ) + + class Nested(BaseModel): + nested_field: Optional[str] = None + + class Settings(BaseSettings): + foobar: Optional[str] = None + nested: Nested = Nested() + model_config = SettingsConfigDict(pyproject_toml_depth=depth) + + @classmethod + def settings_customise_sources( + cls, settings_cls: Type[BaseSettings], **_kwargs: PydanticBaseSettingsSource + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (PyprojectTomlConfigSettingsSource(settings_cls),) + + s = Settings() + assert not s.foobar + assert not s.nested.nested_field + + @pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') def test_multiple_file_toml(tmp_path): p1 = tmp_path / '.env.toml1' diff --git a/tests/test_sources.py b/tests/test_sources.py new file mode 100644 index 00000000..f4ad2303 --- /dev/null +++ b/tests/test_sources.py @@ -0,0 +1,99 @@ +"""Test pydantic_settings.sources.""" + +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + +import pytest + +from pydantic_settings.main import BaseSettings, SettingsConfigDict +from pydantic_settings.sources import PyprojectTomlConfigSettingsSource + +try: + import tomli +except ImportError: + tomli = None + +if TYPE_CHECKING: + from pathlib import Path + + from pytest_mock import MockerFixture + + +MODULE = 'pydantic_settings.sources' + +SOME_TOML_DATA = """ +field = "top-level" + +[some] +[some.table] +field = "some" + +[other.table] +field = "other" +""" + + +class SimpleSettings(BaseSettings): + """Simple settings.""" + + model_config = SettingsConfigDict(pyproject_toml_depth=1, pyproject_toml_table_header=('some', 'table')) + + +@pytest.mark.skipif(sys.version_info <= (3, 11) and tomli is None, reason='tomli/tomllib is not installed') +class TestPyprojectTomlConfigSettingsSource: + """Test PyprojectTomlConfigSettingsSource.""" + + def test___init__(self, mocker: MockerFixture, tmp_path: Path) -> None: + """Test __init__.""" + mocker.patch(f'{MODULE}.Path.cwd', return_value=tmp_path) + pyproject = tmp_path / 'pyproject.toml' + pyproject.write_text(SOME_TOML_DATA) + obj = PyprojectTomlConfigSettingsSource(SimpleSettings) + assert obj.toml_table_header == ('some', 'table') + assert obj.toml_data == {'field': 'some'} + assert obj.toml_file_path == tmp_path / 'pyproject.toml' + + def test___init___explicit(self, mocker: MockerFixture, tmp_path: Path) -> None: + """Test __init__ explicit file.""" + mocker.patch(f'{MODULE}.Path.cwd', return_value=tmp_path) + pyproject = tmp_path / 'child' / 'pyproject.toml' + pyproject.parent.mkdir() + pyproject.write_text(SOME_TOML_DATA) + obj = PyprojectTomlConfigSettingsSource(SimpleSettings, pyproject) + assert obj.toml_table_header == ('some', 'table') + assert obj.toml_data == {'field': 'some'} + assert obj.toml_file_path == pyproject + + def test___init___explicit_missing(self, mocker: MockerFixture, tmp_path: Path) -> None: + """Test __init__ explicit file missing.""" + mocker.patch(f'{MODULE}.Path.cwd', return_value=tmp_path) + pyproject = tmp_path / 'child' / 'pyproject.toml' + obj = PyprojectTomlConfigSettingsSource(SimpleSettings, pyproject) + assert obj.toml_table_header == ('some', 'table') + assert not obj.toml_data + assert obj.toml_file_path == pyproject + + @pytest.mark.parametrize('depth', [0, 99]) + def test___init___no_file(self, depth: int, mocker: MockerFixture, tmp_path: Path) -> None: + """Test __init__ no file.""" + + class Settings(BaseSettings): + model_config = SettingsConfigDict(pyproject_toml_depth=depth) + + mocker.patch(f'{MODULE}.Path.cwd', return_value=tmp_path / 'foo') + obj = PyprojectTomlConfigSettingsSource(Settings) + assert obj.toml_table_header == ('tool', 'pydantic-settings') + assert not obj.toml_data + assert obj.toml_file_path == tmp_path / 'foo' / 'pyproject.toml' + + def test___init___parent(self, mocker: MockerFixture, tmp_path: Path) -> None: + """Test __init__ parent directory.""" + mocker.patch(f'{MODULE}.Path.cwd', return_value=tmp_path / 'child') + pyproject = tmp_path / 'pyproject.toml' + pyproject.write_text(SOME_TOML_DATA) + obj = PyprojectTomlConfigSettingsSource(SimpleSettings) + assert obj.toml_table_header == ('some', 'table') + assert obj.toml_data == {'field': 'some'} + assert obj.toml_file_path == tmp_path / 'pyproject.toml'