Feat: free-threaded python support (#3526)

Co-authored-by: Bernát Gábor <[email protected]>

Author: robsdedude

docs/changelog/3391.feature.rst

@@ -0,0 +1,3 @@
+Add support for free-threaded python builds.
+Factors like ``py313t`` will only pick builds with the GIL disabled while factors without trailing ``t`` will only pick
+builds without no-GIL support.

docs/config.rst

@@ -542,11 +542,12 @@ Base options
             - ✅
             - ✅
             - ❌
+        *   - PYTHON_GIL
+            - ✅
+            - ✅
+            - ✅
 
-
-
-   More environment variable-related information
-   can be found in :ref:`environment variable substitutions`.
+   More environment variable-related information can be found in :ref:`environment variable substitutions`.
 
 .. conf::
    :keys: set_env, setenv
@@ -834,6 +835,7 @@ Python options
    Python version for a tox environment. If not specified, the virtual environments factors (e.g. name part) will be
    used to automatically set one. For example, ``py310`` means ``python3.10``, ``py3`` means ``python3`` and ``py``
    means ``python``. If the name does not match this pattern the same Python version tox is installed into will be used.
+   A base interpreter ending with ``t`` means that only free threaded Python implementations are accepted.
 
     .. versionchanged:: 3.1
 
@@ -866,6 +868,29 @@ Python options
 
    The Python executable from within the tox environment.
 
+.. conf::
+   :keys: py_dot_ver
+   :constant:
+   :version_added: 4.0.10
+
+   Major.Minor version of the Python interpreter in the tox environment (e.g., ``3.13``).
+
+.. conf::
+   :keys: py_impl
+   :constant:
+   :version_added: 4.0.10
+
+   Name of the Python implementation in the tox environment in lowercase (e.g., ``cpython``, ``pypy``).
+
+.. conf::
+   :keys: py_free_threaded
+   :constant:
+   :version_added: 4.26
+
+   ``True`` if the Python interpreter in the tox environment is an experimental free-threaded CPython build,
+   else ``False``.
+
+
 Python run
 ~~~~~~~~~~
 .. conf::

src/tox/session/env_select.py

@@ -136,7 +136,7 @@ class _ToxEnvInfo:
     package_skip: tuple[str, Skip] | None = None  #: if set the creation of the packaging environment failed
 
 
-_DYNAMIC_ENV_FACTORS = re.compile(r"(pypy|py|cython|)((\d(\.\d+(\.\d+)?)?)|\d+)?")
+_DYNAMIC_ENV_FACTORS = re.compile(r"(pypy|py|cython|)(((\d(\.\d+(\.\d+)?)?)|\d+)t?)?")
 _PY_PRE_RELEASE_FACTOR = re.compile(r"alpha|beta|rc\.\d+")
 
 

src/tox/tox_env/api.py

@@ -222,6 +222,7 @@ def _default_pass_env(self) -> list[str]:  # noqa: PLR6301
             "FORCE_COLOR",  # force color output
             "NO_COLOR",  # disable color output
             "NETRC",  # used by pip and netrc modules
+            "PYTHON_GIL",  # allows controlling python gil
         ]
         if sys.stdout.isatty():  # if we're on a interactive shell pass on the TERM
             env.append("TERM")

src/tox/tox_env/python/api.py

@@ -5,9 +5,11 @@
 import logging
 import re
 import sys
+import sysconfig
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, List, NamedTuple, cast
+from typing import TYPE_CHECKING, Any, List, NamedTuple
 
 from virtualenv.discovery.py_spec import PythonSpec
 
@@ -26,13 +28,15 @@ class VersionInfo(NamedTuple):
     serial: int
 
 
-class PythonInfo(NamedTuple):
+@dataclass(frozen=True)
+class PythonInfo:
     implementation: str
     version_info: VersionInfo
     version: str
     is_64: bool
     platform: str
     extra: dict[str, Any]
+    free_threaded: bool = False
 
     @property
     def version_no_dot(self) -> str:
@@ -51,11 +55,14 @@ def version_dot(self) -> str:
     r"""
     ^(?!py$)                                               # don't match 'py' as it doesn't provide any info
     (?P<impl>py|pypy|cpython|jython|graalpy|rustpython|ironpython) # the interpreter; most users will simply use 'py'
-    (?P<version>[2-9]\.?[0-9]?[0-9]?)?$                    # the version; one of: MAJORMINOR, MAJOR.MINOR
+    (?:
+    (?P<version>[2-9]\.?[0-9]?[0-9]?)                      # the version; one of: MAJORMINOR, MAJOR.MINOR
+    (?P<threaded>t?)                                       # version followed by t for free-threading
+    )?$
     """,
     re.VERBOSE,
 )
-PY_FACTORS_RE_EXPLICIT_VERSION = re.compile(r"^((?P<impl>cpython|pypy)-)?(?P<version>[2-9]\.[0-9]+)$")
+PY_FACTORS_RE_EXPLICIT_VERSION = re.compile(r"^((?P<impl>cpython|pypy)-)?(?P<version>[2-9]\.[0-9]+)(?P<threaded>t?)$")
 
 
 class Python(ToxEnv, ABC):
@@ -100,6 +107,7 @@ def validate_base_python(value: list[str]) -> list[str]:
         )
         self.conf.add_constant("py_dot_ver", "<python major>.<python minor>", value=self.py_dot_ver)
         self.conf.add_constant("py_impl", "python implementation", value=self.py_impl)
+        self.conf.add_constant("py_free_threaded", "is no-gil interpreted", value=self.py_free_threaded)
 
     def _default_set_env(self) -> dict[str, str]:
         env = super()._default_set_env()
@@ -111,6 +119,9 @@ def _default_set_env(self) -> dict[str, str]:
     def py_dot_ver(self) -> str:
         return self.base_python.version_dot
 
+    def py_free_threaded(self) -> bool:
+        return self.base_python.free_threaded
+
     def py_impl(self) -> str:
         return self.base_python.impl_lower
 
@@ -145,7 +156,7 @@ def extract_base_python(cls, env_name: str) -> str | None:
         match = PY_FACTORS_RE_EXPLICIT_VERSION.match(env_name)
         if match:
             found = match.groupdict()
-            candidates.append(f"{'pypy' if found['impl'] == 'pypy' else ''}{found['version']}")
+            candidates.append(f"{'pypy' if found['impl'] == 'pypy' else ''}{found['version']}{found['threaded']}")
         else:
             for factor in env_name.split("-"):
                 match = PY_FACTORS_RE.match(factor)
@@ -163,7 +174,8 @@ def _python_spec_for_sys_executable(cls) -> PythonSpec:
         implementation = sys.implementation.name
         version = sys.version_info
         bits = "64" if sys.maxsize > 2**32 else "32"
-        string_spec = f"{implementation}{version.major}{version.minor}-{bits}"
+        threaded = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") == 1 else ""
+        string_spec = f"{implementation}{version.major}{version.minor}{threaded}-{bits}"
         return PythonSpec.from_string_spec(string_spec)
 
     @classmethod
@@ -186,7 +198,7 @@ def _validate_base_python(
                         spec_base = cls.python_spec_for_path(path)
                 if any(
                     getattr(spec_base, key) != getattr(spec_name, key)
-                    for key in ("implementation", "major", "minor", "micro", "architecture")
+                    for key in ("implementation", "major", "minor", "micro", "architecture", "free_threaded")
                     if getattr(spec_name, key) is not None
                 ):
                     msg = f"env name {env_name} conflicting with base python {base_python}"
@@ -290,7 +302,7 @@ def base_python(self) -> PythonInfo:
                 raise Skip(msg)
             raise NoInterpreter(base_pythons)
 
-        return cast("PythonInfo", self._base_python)
+        return self._base_python
 
     def _get_env_journal_python(self) -> dict[str, Any]:
         return {

src/tox/tox_env/python/virtual_env/api.py

@@ -146,6 +146,7 @@ def _get_python(self, base_python: list[str]) -> PythonInfo | None:  # noqa: ARG
             is_64=(interpreter.architecture == 64),  # noqa: PLR2004
             platform=interpreter.platform,
             extra={"executable": Path(interpreter.system_executable).resolve()},
+            free_threaded=interpreter.free_threaded,
         )
 
     def prepend_env_var_path(self) -> list[Path]:

tests/conftest.py

@@ -2,6 +2,7 @@
 
 import os
 import sys
+import sysconfig
 from pathlib import Path
 from typing import TYPE_CHECKING, Callable, Iterator, Protocol, Sequence
 from unittest.mock import patch
@@ -100,6 +101,7 @@ def get_python(self: VirtualEnv, base_python: list[str]) -> PythonInfo | None:
                 is_64=True,
                 platform=sys.platform,
                 extra={"executable": Path(sys.executable)},
+                free_threaded=sysconfig.get_config_var("Py_GIL_DISABLED") == 1,
             )
 
         mocker.patch.object(VirtualEnv, "_get_python", get_python)

tests/session/cmd/test_show_config.py

@@ -135,7 +135,7 @@ def test_pass_env_config_default(tox_project: ToxProjectCreator, stdout_is_atty:
         + (["PROGRAMDATA"] if is_win else [])
         + (["PROGRAMFILES"] if is_win else [])
         + (["PROGRAMFILES(x86)"] if is_win else [])
-        + ["REQUESTS_CA_BUNDLE", "SSL_CERT_FILE"]
+        + ["PYTHON_GIL", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE"]
         + (["SYSTEMDRIVE", "SYSTEMROOT", "TEMP"] if is_win else [])
         + (["TERM"] if stdout_is_atty else [])
         + (["TMP", "USERPROFILE"] if is_win else ["TMPDIR"])

tests/session/test_env_select.py

@@ -261,10 +261,15 @@ def test_matches_combined_env(env_name: str, tox_project: ToxProjectCreator) ->
         "pypy312",
         "py3",
         "py3.12",
+        "py3.12t",
         "py312",
+        "py312t",
         "3",
+        "3t",
         "3.12",
+        "3.12t",
         "3.12.0",
+        "3.12.0t",
     ],
 )
 def test_dynamic_env_factors_match(env: str) -> None:

tests/tox_env/python/test_python_api.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sys
+import sysconfig
 from types import SimpleNamespace
 from typing import TYPE_CHECKING, Callable
 from unittest.mock import patch
@@ -81,30 +82,56 @@ def test_diff_msg_no_diff() -> None:
     ("env", "base_python"),
     [
         ("py3", "py3"),
+        ("py3t", "py3t"),
         ("py311", "py311"),
+        ("py311t", "py311t"),
         ("py3.12", "py3.12"),
+        ("py3.12t", "py3.12t"),
         ("pypy2", "pypy2"),
+        ("pypy2t", "pypy2t"),
         ("rustpython3", "rustpython3"),
+        ("rustpython3t", "rustpython3t"),
         ("graalpy", "graalpy"),
+        ("graalpyt", None),
         ("jython", "jython"),
+        ("jythont", None),
         ("cpython3.8", "cpython3.8"),
+        ("cpython3.8t", "cpython3.8t"),
         ("ironpython2.7", "ironpython2.7"),
+        ("ironpython2.7t", "ironpython2.7t"),
         ("functional-py310", "py310"),
+        ("functional-py310t", "py310t"),
         ("bar-pypy2-foo", "pypy2"),
+        ("bar-foo2t-py2", "py2"),
+        ("bar-pypy2t-foo", "pypy2t"),
         ("py", None),
+        ("pyt", None),
         ("django-32", None),
+        ("django-32t", None),
         ("eslint-8.3", None),
+        ("eslint-8.3t", None),
         ("py-310", None),
+        ("py-310t", None),
         ("py3000", None),
+        ("py3000t", None),
         ("4.foo", None),
+        ("4.foot", None),
         ("310", None),
+        ("310t", None),
         ("5", None),
+        ("5t", None),
         ("2000", None),
+        ("2000t", None),
         ("4000", None),
+        ("4000t", None),
         ("3.10", "3.10"),
+        ("3.10t", "3.10t"),
         ("3.9", "3.9"),
+        ("3.9t", "3.9t"),
         ("2.7", "2.7"),
+        ("2.7t", "2.7t"),
         ("pypy-3.10", "pypy3.10"),
+        ("pypy-3.10t", "pypy3.10t"),
     ],
     ids=lambda a: "|".join(a) if isinstance(a, list) else str(a),
 )
@@ -294,13 +321,24 @@ def test_usedevelop_with_nonexistent_basepython(tox_project: ToxProjectCreator)
 
 
 @pytest.mark.parametrize(
-    ("impl", "major", "minor", "arch"),
+    ("impl", "major", "minor", "arch", "free_threaded"),
     [
-        ("cpython", 3, 12, 64),
-        ("pypy", 3, 9, 32),
+        ("cpython", 3, 12, 64, None),
+        ("cpython", 3, 13, 64, True),
+        ("cpython", 3, 13, 64, False),
+        ("pypy", 3, 9, 32, None),
     ],
 )
-def test_python_spec_for_sys_executable(impl: str, major: int, minor: int, arch: int, mocker: MockerFixture) -> None:
+def test_python_spec_for_sys_executable(  # noqa: PLR0913
+    impl: str, major: int, minor: int, arch: int, free_threaded: bool | None, mocker: MockerFixture
+) -> None:
+    get_config_var_ = sysconfig.get_config_var
+
+    def get_config_var(name: str) -> object:
+        if name == "Py_GIL_DISABLED":
+            return free_threaded
+        return get_config_var_(name)
+
     version_info = SimpleNamespace(major=major, minor=minor, micro=5, releaselevel="final", serial=0)
     implementation = SimpleNamespace(
         name=impl,
@@ -312,8 +350,10 @@ def test_python_spec_for_sys_executable(impl: str, major: int, minor: int, arch:
     mocker.patch.object(sys, "version_info", version_info)
     mocker.patch.object(sys, "implementation", implementation)
     mocker.patch.object(sys, "maxsize", 2**arch // 2 - 1)
+    mocker.patch.object(sysconfig, "get_config_var", get_config_var)
     spec = Python._python_spec_for_sys_executable()  # noqa: SLF001
     assert spec.implementation == impl
     assert spec.major == major
     assert spec.minor == minor
     assert spec.architecture == arch
+    assert spec.free_threaded == bool(free_threaded)

tests/tox_env/python/test_python_runner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import sys
+import sysconfig
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -152,11 +153,16 @@ def test_config_skip_missing_interpreters(
     assert result.code == (0 if expected else -1)
 
 
+SYS_PY_VER = "".join(str(i) for i in sys.version_info[0:2]) + (
+    "t" if sysconfig.get_config_var("Py_GIL_DISABLED") == 1 else ""
+)
+
+
 @pytest.mark.parametrize(
     ("skip", "env", "retcode"),
     [
-        ("true", f"py{''.join(str(i) for i in sys.version_info[0:2])}", 0),
-        ("false", f"py{''.join(str(i) for i in sys.version_info[0:2])}", 0),
+        ("true", f"py{SYS_PY_VER}", 0),
+        ("false", f"py{SYS_PY_VER}", 0),
         ("true", "py31", -1),
         ("false", "py31", 1),
         ("true", None, 0),
@@ -169,8 +175,7 @@ def test_skip_missing_interpreters_specified_env(
     env: str | None,
     retcode: int,
 ) -> None:
-    py_ver = "".join(str(i) for i in sys.version_info[0:2])
-    project = tox_project({"tox.ini": f"[tox]\nenvlist=py31,py{py_ver}\n[testenv]\nusedevelop=true"})
+    project = tox_project({"tox.ini": f"[tox]\nenvlist=py31,py{SYS_PY_VER}\n[testenv]\nusedevelop=true"})
     args = [f"--skip-missing-interpreters={skip}"]
     if env:
         args += ["-e", env]

tox.toml

@@ -1,5 +1,5 @@
 requires = ["tox>=4.24.1"]
-env_list = ["fix", "3.14", "3.13", "3.12", "3.11", "3.10", "3.9", "cov", "type", "docs", "pkg_meta"]
+env_list = ["fix", "3.14t", "3.14", "3.13", "3.12", "3.11", "3.10", "3.9", "cov", "type", "docs", "pkg_meta"]
 skip_missing_interpreters = true
 
 [env_run_base]