From 05be1972366d3426f8be92bc44f628cbf486035f Mon Sep 17 00:00:00 2001 From: Test Date: Wed, 3 Jun 2026 15:44:51 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(env):=20let=20users=20pin=20th?= =?UTF-8?q?e=20build=20environment=20location?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build always created its isolated environment in a random temporary directory and deleted it on exit. That left three needs unmet, all raised in #655: builds on read-only containers that only expose a small writable path, inspecting the environment of a failed build, and giving compilation caches like ccache and sccache a stable path (a build-environment path that changes every run shows up as a new file and misses the cache). Until now the only workarounds were nudging TMPDIR, which moves the parent directory but keeps the random name and still deletes it, or hand-rolling a venv with --no-isolation. Add an env_dir argument to DefaultIsolatedEnv, surfaced as the --env-dir option and the BUILD_ENV_DIR environment variable. The location must be empty so build never clobbers existing data; build removes it after a successful build and keeps it after a failure for inspection. Removing on success keeps the location reusable across the default sdist-then-wheel run and across repeated builds, which is what the caching use case needs. --- docs/changelog/655.feature.rst | 4 ++ docs/explanation/how-it-works.rst | 6 +++ docs/how-to/troubleshooting.rst | 21 +++----- docs/reference/api.rst | 10 ++++ docs/reference/cli.rst | 5 ++ docs/reference/environment-variables.rst | 7 ++- src/build/__main__.py | 34 +++++++++++- src/build/_exceptions.py | 2 +- src/build/env.py | 21 ++++++-- tests/test_env.py | 66 ++++++++++++++++++++++++ tests/test_main.py | 34 ++++++++++-- 11 files changed, 184 insertions(+), 26 deletions(-) create mode 100644 docs/changelog/655.feature.rst diff --git a/docs/changelog/655.feature.rst b/docs/changelog/655.feature.rst new file mode 100644 index 00000000..748d93f3 --- /dev/null +++ b/docs/changelog/655.feature.rst @@ -0,0 +1,4 @@ +Add ``--env-dir PATH`` to put the isolated build environment at a fixed location instead of a temporary directory. +``DefaultIsolatedEnv`` takes the same ``path`` argument. build creates the location if it is missing, refuses a +non-empty location, removes it after a successful build, and keeps it after a failure so you can inspect it. A fixed +path lets compilation caches such as ccache and sccache reuse results across builds - by :user:`gaborbernat`. diff --git a/docs/explanation/how-it-works.rst b/docs/explanation/how-it-works.rst index 65d9086f..c964bf06 100644 --- a/docs/explanation/how-it-works.rst +++ b/docs/explanation/how-it-works.rst @@ -132,6 +132,12 @@ How isolation works The isolated environment has access to the Python standard library, the build backend and its dependencies, and nothing from your development environment except environment variables. +build picks a temporary directory for this environment by default and deletes it after the build. Pass ``--env-dir`` to +choose the location yourself. You might do this to keep a failed environment for inspection, to place the environment on +a writable path inside a read-only container, or to give a fixed path to a compilation cache such as ccache or sccache. +Those caches treat a changed build-environment path as a new file, so a path that changes every run never hits the +cache. + When to disable isolation ========================= diff --git a/docs/how-to/troubleshooting.rst b/docs/how-to/troubleshooting.rst index fad15ae3..0e4478e0 100644 --- a/docs/how-to/troubleshooting.rst +++ b/docs/how-to/troubleshooting.rst @@ -449,24 +449,19 @@ You can test if your build backend works independently: $ pip install your-build-backend $ python -m your_build_backend -Preserve build logs and temporary directories -============================================= +Inspect the build environment +============================= -By default, build cleans up the temporary build directory after completion. To control where the temporary directory is -created (useful for debugging or placing it on a specific filesystem): +build removes its isolated environment when the build ends. To keep it for inspection, point ``--env-dir`` at an empty +location: .. code-block:: console - $ TMPDIR=/tmp/debug-build python -m build + $ python -m build --env-dir .build-env -The build will create its temporary environment inside ``/tmp/debug-build``. The directory is automatically cleaned up -after a successful build, but persists if the build fails, allowing you to inspect the environment for debugging. On -Windows, use ``TEMP`` instead of ``TMPDIR``: - -.. code-block:: console - - $ set TEMP=C:\debug-build - $ python -m build +build removes that location after a successful build and keeps it after a failure, so a failed build leaves the +environment in ``.build-env`` for you to examine. Setting ``TMPDIR`` (or ``TEMP`` on Windows) only moves the temporary +directory; it does not keep the environment around. To see backend logs in real-time, use verbose output: diff --git a/docs/reference/api.rst b/docs/reference/api.rst index 5d507a92..48e3deb5 100644 --- a/docs/reference/api.rst +++ b/docs/reference/api.rst @@ -52,6 +52,16 @@ build dependencies before invoking the backend: env.install(builder.get_requires_for_build("wheel")) builder.build("wheel", "dist/") +Pass ``path`` to create the environment at a fixed location instead of a temporary directory. The location must be +empty; ``DefaultIsolatedEnv`` removes it on a clean exit and keeps it when the block raises: + +.. code-block:: python + + with DefaultIsolatedEnv(path=".build-env") as env: + builder = ProjectBuilder.from_isolated_env(env, ".") + env.install(builder.build_system_requires) + builder.build("wheel", "dist/") + Disabling isolation: .. code-block:: python diff --git a/docs/reference/cli.rst b/docs/reference/cli.rst index 560e8912..66cde1c3 100644 --- a/docs/reference/cli.rst +++ b/docs/reference/cli.rst @@ -31,6 +31,11 @@ By default build will build the package in an isolated environment, but this beh specified in your ``pyproject.toml``, runs the build, and then cleans up the environment. This ensures reproducible builds regardless of what packages are installed in your development environment. +Pass ``--env-dir PATH`` to put the environment at a fixed location instead of a temporary directory. The location must +be empty. build removes it after a successful build and keeps it after a failure so you can inspect it. A fixed path +helps compilation caches like ccache and sccache, which treat a changed build-environment path as a new file and miss +the cache. You cannot combine ``--env-dir`` with ``--no-isolation``. + ****************** Dependency Check ****************** diff --git a/docs/reference/environment-variables.rst b/docs/reference/environment-variables.rst index 4de9e209..89f8dd74 100644 --- a/docs/reference/environment-variables.rst +++ b/docs/reference/environment-variables.rst @@ -239,15 +239,14 @@ On Windows, use ``TEMP``: $ set TEMP=C:\path\to\custom\tmp $ python -m build -The temporary directory persists after build completes, allowing you to inspect build artifacts, logs, and the isolated -environment for debugging. By default, build uses the system's default temporary directory and cleans it up after the -build. +These variables only change *where* build creates the temporary directory. build still removes its environment when the +build ends. To pin the environment to a stable path, or to keep it for inspection, use the ``--env-dir`` option instead. **Common use cases**: -- **Debugging**: Keep temp directory to inspect build logs - **Disk space**: Use a different filesystem with more available space - **Performance**: Use faster storage (e.g., tmpfs on Linux) +- **Restricted filesystems**: Point at the writable part of a read-only container ``TMPDIR`` only relocates the base directory; build still appends a random suffix, so the extraction path differs on every run. For compiler caches such as ``ccache`` or ``sccache``, which key on the source path, that random suffix is a diff --git a/src/build/__main__.py b/src/build/__main__.py index dc67f9ee..c523e257 100644 --- a/src/build/__main__.py +++ b/src/build/__main__.py @@ -88,6 +88,7 @@ class _Args(argparse.Namespace): dependency_constraints_txt: str | None skip_dependency_check: bool sdist_extract_dir: str | None + env_dir: str | None _COLORS = { @@ -197,10 +198,11 @@ def _bootstrap_build_env( skip_dependency_check: bool, dependency_constraints_txt: os.PathLike[str] | None, installer: _env.Installer, + env_dir: str | None = None, runner: SubprocessRunner | None = None, ) -> Iterator[ProjectBuilder]: if isolation: - with DefaultIsolatedEnv(installer=installer) as env: + with DefaultIsolatedEnv(installer=installer, path=env_dir) as env: make_builder = partial(ProjectBuilder.from_isolated_env, env, srcdir) if runner: make_builder = partial(make_builder, runner=runner) @@ -252,6 +254,7 @@ def _build( skip_dependency_check: bool, dependency_constraints_txt: os.PathLike[str] | None, installer: _env.Installer, + env_dir: str | None = None, ) -> str: with _bootstrap_build_env( isolation, @@ -261,6 +264,7 @@ def _build( skip_dependency_check, dependency_constraints_txt, installer, + env_dir, pyproject_hooks.quiet_subprocess_runner if _ctx.verbosity < 0 else None, ) as builder: return builder.build(distribution, outdir, config_settings) @@ -311,6 +315,7 @@ def build_package( skip_dependency_check: bool = False, dependency_constraints_txt: os.PathLike[str] | None = None, installer: _env.Installer = 'pip', + env_dir: str | None = None, ) -> list[str]: """ Run the build process. @@ -321,6 +326,7 @@ def build_package( :param config_settings: Configuration settings to be passed to the backend :param isolation: Isolate the build in a separate environment :param skip_dependency_check: Do not perform the dependency check + :param env_dir: Location of the isolated build environment (a temporary directory is used when not set) """ built: list[str] = [] for distribution in distributions: @@ -333,6 +339,7 @@ def build_package( skip_dependency_check, dependency_constraints_txt, installer, + env_dir, ) built.append(os.path.basename(out)) return built @@ -348,6 +355,7 @@ def build_package_via_sdist( dependency_constraints_txt: os.PathLike[str] | None = None, installer: _env.Installer = 'pip', sdist_extract_dir: StrPath | None = None, + env_dir: str | None = None, ) -> list[str]: """ Build a sdist and then the specified distributions from it. @@ -360,13 +368,22 @@ def build_package_via_sdist( :param skip_dependency_check: Do not perform the dependency check :param sdist_extract_dir: Directory to extract the intermediate sdist into; a temporary directory is used and removed afterwards when ``None`` + :param env_dir: Location of the isolated build environment (a temporary directory is used when not set) """ if 'sdist' in distributions: msg = 'Only binary distributions are allowed but sdist was specified' raise ValueError(msg) sdist = _build( - isolation, srcdir, outdir, 'sdist', config_settings, skip_dependency_check, dependency_constraints_txt, installer + isolation, + srcdir, + outdir, + 'sdist', + config_settings, + skip_dependency_check, + dependency_constraints_txt, + installer, + env_dir, ) sdist_name = os.path.basename(sdist) @@ -384,6 +401,7 @@ def build_package_via_sdist( skip_dependency_check, dependency_constraints_txt, installer, + env_dir, ) built.append(os.path.basename(out)) return [sdist_name, *built] @@ -398,6 +416,7 @@ def _build_metadata( skip_dependency_check: bool = False, dependency_constraints_txt: os.PathLike[str] | None = None, installer: _env.Installer = 'pip', + env_dir: str | None = None, ) -> list[str]: import packaging.metadata @@ -416,6 +435,7 @@ def run_subprocess(cmd: Sequence[StrPath], cwd: str | None = None, extra_environ skip_dependency_check, dependency_constraints_txt, installer, + env_dir, runner=run_subprocess, ) as builder, tempfile.TemporaryDirectory() as tempdir, @@ -591,6 +611,12 @@ def __call__( help='disable building the project in an isolated virtual environment. ' 'Build dependencies must be installed separately when this option is used', ) + install_group.add_argument( + '--env-dir', + help='create the isolated build environment at this location instead of a temporary directory. The location ' + 'must be empty; it is removed on success and kept on failure so it can be inspected', + metavar='PATH', + ) install_group.add_argument( '--dependency-constraints-txt', help='constrain build dependencies using a constraints.txt when installing dependencies', @@ -640,6 +666,9 @@ def main(cli_args: Sequence[str], prog: str | None = None) -> None: parser.prog = prog args = cast('_Args', parser.parse_args(cli_args)) + if args.env_dir is not None and args.no_isolation: + parser.error('--env-dir: not allowed with --no-isolation') + _setup_cli(verbosity=args.verbosity) sdist_input = os.path.isfile(args.srcdir) and os.fspath(args.srcdir).lower().endswith('.tar.gz') @@ -650,6 +679,7 @@ def main(cli_args: Sequence[str], prog: str | None = None) -> None: skip_dependency_check=args.skip_dependency_check, dependency_constraints_txt=args.dependency_constraints_txt, installer=args.installer, + env_dir=args.env_dir, ) if args.outdir is not None: diff --git a/src/build/_exceptions.py b/src/build/_exceptions.py index 021f33c6..36ec66b9 100644 --- a/src/build/_exceptions.py +++ b/src/build/_exceptions.py @@ -10,7 +10,7 @@ class BuildException(Exception): """ - Exception raised by :class:`build.ProjectBuilder`. + Exception raised by ``build`` when a build cannot proceed. """ diff --git a/src/build/env.py b/src/build/env.py index 80d29dbb..25d08cf1 100644 --- a/src/build/env.py +++ b/src/build/env.py @@ -43,7 +43,7 @@ from . import _ctx from ._compat.importlib import metadata as importlib_metadata from ._ctx import run_subprocess -from ._exceptions import FailedProcessError +from ._exceptions import BuildException, FailedProcessError from ._util import check_dependency @@ -111,12 +111,24 @@ def __init__( self, *, installer: Installer = 'pip', + path: str | None = None, ) -> None: self.installer: Installer = installer + self._requested_path = path def __enter__(self) -> Self: - try: + if self._requested_path is None: path = tempfile.mkdtemp(prefix='build-env-') + else: + path = self._requested_path + if os.path.isdir(path): + with os.scandir(path) as entries: + if next(entries, None) is not None: + msg = f'Build environment location is not empty: {path}' + raise BuildException(msg) + os.makedirs(path, exist_ok=True) + + try: # Call ``realpath`` to prevent spurious warning from being emitted # that the venv location has changed on Windows for the venv impl. # The username is DOS-encoded in the output of tempfile - the location is the same @@ -143,7 +155,10 @@ def __enter__(self) -> Self: return self def __exit__(self, *args: object) -> None: - shutil.rmtree(self._path, ignore_errors=True) + # A temporary location is always cleaned up. A caller-specified location is kept when the build fails so + # its environment can be inspected, but removed on success so the same location can be reused. + if self._requested_path is None or args[0] is None: + shutil.rmtree(self._path, ignore_errors=True) @property def path(self) -> str: diff --git a/tests/test_env.py b/tests/test_env.py index c176ceb7..becd53e0 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MIT from __future__ import annotations +import contextlib import importlib.util import logging import os @@ -10,6 +11,7 @@ import sys import sysconfig import typing +import unittest.mock from pathlib import Path from types import SimpleNamespace @@ -634,3 +636,67 @@ def test_pythonpath_does_not_interfere_with_outer_pip( env.install({'flit_core'}, _fresh=True) assert subprocess.check_call([env.python_executable, '-c', 'import flit_core']) == 0 + + +@pytest.fixture +def mock_env_create(mocker: pytest_mock.MockerFixture) -> unittest.mock.MagicMock: + return mocker.patch('build.env._PipBackend.create', autospec=True) + + +def test_env_dir_created_at_requested_location( + mock_env_create: unittest.mock.MagicMock, + tmp_path: pathlib.Path, +) -> None: + target = tmp_path / 'nested' / 'build-env' + + with build.env.DefaultIsolatedEnv(path=str(target)) as env: + assert env.path == os.path.realpath(target) + assert os.path.isdir(env.path) + + mock_env_create.assert_called_once() + + +@pytest.mark.usefixtures('mock_env_create') +@pytest.mark.parametrize( + ('use_path', 'fail', 'kept_after'), + [ + pytest.param(False, False, False, id='temporary-success-removed'), + pytest.param(False, True, False, id='temporary-failure-removed'), + pytest.param(True, False, False, id='requested-success-removed'), + pytest.param(True, True, True, id='requested-failure-kept'), + ], +) +def test_env_cleanup( + tmp_path: pathlib.Path, + use_path: bool, + fail: bool, + kept_after: bool, +) -> None: + target = str(tmp_path / 'build-env') if use_path else None + + created_path = '' + with contextlib.suppress(RuntimeError), build.env.DefaultIsolatedEnv(path=target) as env: + created_path = env.path + if fail: + msg = 'boom' + raise RuntimeError(msg) + + assert os.path.exists(created_path) is kept_after + + +def test_env_dir_rejects_non_empty_location(tmp_path: pathlib.Path) -> None: + tmp_path.joinpath('sentinel').touch() + + with ( + pytest.raises(build.BuildException, match='Build environment location is not empty'), + build.env.DefaultIsolatedEnv(path=str(tmp_path)), + ): + raise AssertionError + + assert tmp_path.joinpath('sentinel').exists() + + +@pytest.mark.usefixtures('mock_env_create') +def test_env_dir_accepts_existing_empty_location(tmp_path: pathlib.Path) -> None: + with build.env.DefaultIsolatedEnv(path=str(tmp_path)) as env: + assert env.path == os.path.realpath(tmp_path) diff --git a/tests/test_main.py b/tests/test_main.py index 46be3500..ce297d9c 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -56,6 +56,7 @@ class BuildKwargs(TypedDict): skip_dependency_check: bool dependency_constraints_txt: str | None installer: str | None + env_dir: str | None def make_kwargs( @@ -66,6 +67,7 @@ def make_kwargs( skip_dependency_check: bool = False, dependency_constraints_txt: str | None = None, installer: str | None = None, + env_dir: str | None = None, ) -> BuildKwargs: return { 'distributions': distributions if distributions is not None else ['wheel'], @@ -74,6 +76,7 @@ def make_kwargs( 'skip_dependency_check': skip_dependency_check, 'dependency_constraints_txt': dependency_constraints_txt, 'installer': installer, + 'env_dir': env_dir, } @@ -141,6 +144,13 @@ def make_kwargs( 'build_package_via_sdist', id='dependency-constraints-txt', ), + pytest.param( + ['--env-dir', 'build-env'], + (cwd, out), + make_kwargs(env_dir='build-env'), + 'build_package_via_sdist', + id='env-dir', + ), ], ) def test_parse_args( @@ -164,6 +174,21 @@ def test_parse_args( raise ValueError(msg) +def test_env_dir_flag_forwarded(mocker: pytest_mock.MockerFixture) -> None: + build_package_via_sdist = mocker.patch('build.__main__.build_package_via_sdist', return_value=['something']) + + build.__main__.main(['--env-dir', 'build-env']) + + assert build_package_via_sdist.call_args.kwargs['env_dir'] == 'build-env' + + +def test_env_dir_conflicts_with_no_isolation(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit): + build.__main__.main(['--env-dir', 'build-env', '--no-isolation']) + + assert '--env-dir: not allowed with --no-isolation' in capsys.readouterr().err + + def test_prog() -> None: out = io.StringIO() @@ -238,10 +263,10 @@ def test_build_package_passes_config_settings_to_build(mocker: pytest_mock.Mocke build_cmd.assert_has_calls( [ unittest.mock.call( - False, package_test_flit, '.', 'sdist', config_settings, True, pathlib.Path('constraints.txt'), 'uv' + False, package_test_flit, '.', 'sdist', config_settings, True, pathlib.Path('constraints.txt'), 'uv', None ), unittest.mock.call( - False, package_test_flit, '.', 'wheel', config_settings, True, pathlib.Path('constraints.txt'), 'uv' + False, package_test_flit, '.', 'wheel', config_settings, True, pathlib.Path('constraints.txt'), 'uv', None ), ] ) @@ -279,7 +304,9 @@ def test_build_package_via_sdist_passes_config_settings_to_build(mocker: pytest_ assert extractall.call_args.args[0] == 'temp-sdist-dir' build_cmd.assert_has_calls( [ - unittest.mock.call(False, 'src', 'dist', 'sdist', config_settings, True, pathlib.Path('constraints.txt'), 'uv'), + unittest.mock.call( + False, 'src', 'dist', 'sdist', config_settings, True, pathlib.Path('constraints.txt'), 'uv', None + ), unittest.mock.call( False, os.path.join('temp-sdist-dir', 'demo-1.0.0'), @@ -289,6 +316,7 @@ def test_build_package_via_sdist_passes_config_settings_to_build(mocker: pytest_ True, pathlib.Path('constraints.txt'), 'uv', + None, ), ] )