# HG changeset patch # User Filipe Laíns # Date 1733719242 0 # Node ID b0dbc944fb7b5f7df9ae370a247405d5c7c181ee # Parent 550046479ed6e9c01ea14ae7156002f5ec815ae5 Bug 1935621 - Fix virtual environment sysconfig path calculation r=firefox-build-system-reviewers,ahochheiden Signed-off-by: Filipe Laíns Signed-off-by: Filipe Laíns Signed-off-by: Filipe Laíns Differential Revision: https://phabricator.services.mozilla.com/D231480 --- a/python/mach/mach/site.py +++ b/python/mach/mach/site.py @@ -12,16 +12,17 @@ import json import os import platform import shutil import site import subprocess import sys import sysconfig import tempfile +import warnings from contextlib import contextmanager from pathlib import Path from typing import Callable, Optional from mach.requirements import ( MachEnvRequirements, UnexpectedFlexibleRequirementException, ) @@ -814,43 +815,85 @@ class CommandSiteManager: self._metadata, ) class PythonVirtualenv: """Calculates paths of interest for general python virtual environments""" def __init__(self, prefix): - if _is_windows: - self.bin_path = os.path.join(prefix, "Scripts") - self.python_path = os.path.join(self.bin_path, "python.exe") + self.prefix = os.path.realpath(prefix) + self.paths = self._get_sysconfig_paths(self.prefix) + + # Name of the Python executable to use in virtual environments. + # An executable with the same name as sys.executable might not exist in + # virtual environments. An executable with 'python' as the steam — + # without version numbers or ABI flags — will always be present in + # virtual environments, so we use that. + python_exe_name = "python" + sysconfig.get_config_var("EXE") + + self.bin_path = self.paths["scripts"] + self.python_path = os.path.join(self.bin_path, python_exe_name) + + @staticmethod + def _get_sysconfig_paths(prefix): + """Calculate the sysconfig paths of a virtual environment in the given prefix. + + The virtual environment MUST be using the same Python distribution as us. + """ + # Determine the sysconfig scheme used in virtual environments + if "venv" in sysconfig.get_scheme_names(): + # A 'venv' scheme was added in Python 3.11 to allow users to + # calculate the paths for a virtual environment, since the default + # scheme may not always be the same as used on virtual environments. + # Some common examples are the system Python distributed by macOS, + # Debian, and Fedora. + # For more information, see https://github.com/python/cpython/issues/89576 + venv_scheme = "venv" + elif os.name == "nt": + # We know that before the 'venv' scheme was added, on Windows, + # the 'nt' scheme was used in virtual environments. + venv_scheme = "nt" + elif os.name == "posix": + # We know that before the 'venv' scheme was added, on POSIX, + # the 'posix_prefix' scheme was used in virtual environments. + venv_scheme = "posix_prefix" else: - self.bin_path = os.path.join(prefix, "bin") - self.python_path = os.path.join(self.bin_path, "python") - self.prefix = os.path.realpath(prefix) + # This should never happen with upstream Python, as the 'venv' + # scheme should always be available on >=3.11, and no other + # platforms are supported by the upstream on older Python versions. + # + # Since the 'venv' scheme isn't available, and we have no knowledge + # of this platform/distribution, fallback to the default scheme. + # + # Hitting this will likely be the result of running a custom Python + # distribution targetting a platform that is not supported by the + # upstream. + # In this case, unless the Python vendor patched the Python + # distribution in such a way as the default scheme may not always be + # the same scheme, using the default scheme should be correct. + # If the vendor did patch Python as such, to work around this issue, + # I would recommend them to define a 'venv' scheme that matches + # the layout used on virtual environments in their Python distribution. + # (rec. signed Filipe Laíns — upstream sysconfig maintainer) + venv_scheme = sysconfig.get_default_scheme() + warnings.warn( + f"Unknown platform '{os.name}', using the default install scheme '{venv_scheme}'. " + "If this is incorrect, please ask your Python vendor to add a 'venv' sysconfig scheme " + "(see https://github.com/python/cpython/issues/89576, or check the code comment).", + stacklevel=2, + ) + # Build the sysconfig config_vars dictionary for the virtual environment. + venv_vars = sysconfig.get_config_vars().copy() + venv_vars["base"] = venv_vars["platbase"] = prefix + # Get sysconfig paths for the virtual environment. + return sysconfig.get_paths(venv_scheme, vars=venv_vars) - @functools.lru_cache(maxsize=None) def resolve_sysconfig_packages_path(self, sysconfig_path): - # macOS uses a different default sysconfig scheme based on whether it's using the - # system Python or running in a virtualenv. - # Manually define the scheme (following the implementation in - # "sysconfig._get_default_scheme()") so that we're always following the - # code path for a virtualenv directory structure. - if os.name == "posix": - scheme = "posix_prefix" - else: - scheme = os.name - - sysconfig_paths = sysconfig.get_paths(scheme) - data_path = Path(sysconfig_paths["data"]) - path = Path(sysconfig_paths[sysconfig_path]) - relative_path = path.relative_to(data_path) - - # Path to virtualenv's "site-packages" directory for provided sysconfig path - return os.path.normpath(os.path.normcase(Path(self.prefix) / relative_path)) + return self.paths[sysconfig_path] def site_packages_dirs(self): dirs = [] if sys.platform.startswith("win"): dirs.append(os.path.normpath(os.path.normcase(self.prefix))) purelib = self.resolve_sysconfig_packages_path("purelib") platlib = self.resolve_sysconfig_packages_path("platlib")