From 639000e6317fba7d6a9994dfe1cea0011160558c Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 21 Mar 2026 12:34:27 -0700 Subject: [PATCH 01/28] force full venv --- .bazelrc | 1 + python/private/py_executable.bzl | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 49f98ad7a1..9dd13e9b2a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -54,3 +54,4 @@ common --incompatible_no_implicit_file_export build --lockfile_mode=update +try-import user.bazelrc diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 805f95fa4c..a837cb25ea 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -375,6 +375,7 @@ def _create_executable( # NOTE: --build_python_zip defaults to true on Windows build_zip_enabled = read_possibly_native_flag(ctx, "build_python_zip") + build_zip_enabled = False # When --build_python_zip is enabled, then the zip file becomes # one of the default outputs. @@ -540,7 +541,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root # compatible with full venv. # TODO: Use non-build_python_zip codepath for Windows if is_windows: - create_full_venv = False + create_full_venv = True elif not rp_config.bazel_8_or_later and not is_bootstrap_script: # Full venv for Bazel 7 + system_python is disabled because packaging # it using build_python_zip=true or rules_pkg breaks. From 8eaafca12d689baf0a4dd94664820a6e6749a6a0 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 3 Apr 2026 19:46:07 -0700 Subject: [PATCH 02/28] feat: make windows use full venvs --- .../private/hermetic_runtime_repo_setup.bzl | 5 + python/private/py_executable.bzl | 232 ++++++++++++------ python/private/py_runtime_info.bzl | 12 +- python/private/py_runtime_rule.bzl | 2 + python/private/python_bootstrap_template.txt | 143 +++++++---- python/private/zipapp/zip_main_template.py | 17 +- tests/bootstrap_impls/BUILD.bazel | 2 +- .../run_binary_zip_yes_test.sh | 4 +- tests/bootstrap_impls/sys_path_order_test.py | 1 - .../system_python_nodeps_test.py | 2 + 10 files changed, 283 insertions(+), 137 deletions(-) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index d860983e22..e8b52b602c 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -240,6 +240,11 @@ def define_hermetic_runtime_toolchain_impl( _IS_FREETHREADED_YES: "cpython-{major}{minor}t".format(**version_dict), _IS_FREETHREADED_NO: "cpython-{major}{minor}".format(**version_dict), }), + # On Windows, a symlink-style venv requires supporting .dll files. + venv_bin_files = select({ + "@platforms//os:windows": native.glob(include=["*.dll", "*.pdb"]), + "//conditions:default": [], + }) ) py_runtime_pair( diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index a837cb25ea..60abc5d5f3 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -475,7 +475,7 @@ WARNING: Target: {} # The interpreter is added this late in the process so that it isn't # added to the zipped files. if venv and venv.interpreter: - extra_runfiles = extra_runfiles.merge(ctx.runfiles([venv.interpreter])) + extra_runfiles = extra_runfiles.merge(venv.interpreter_runfiles) return struct( # depset[File] of additional files that should be included as default # outputs. @@ -524,25 +524,103 @@ def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): # * https://github.com/python/cpython/blob/main/Modules/getpath.py # * https://github.com/python/cpython/blob/main/Lib/site.py def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root_to_sys_path, extra_deps): - venv = "_{}.venv".format(output_prefix.lstrip("_")) - - # The pyvenv.cfg file must be present to trigger the venv site hooks. - # Because it's paths are expected to be absolute paths, we can't reliably - # put much in it. See https://github.com/python/cpython/issues/83650 - pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv)) - ctx.actions.write(pyvenv_cfg, "") + venv_root = "_{}.venv".format(output_prefix.lstrip("_")) + runtime = runtime_details.effective_runtime + if runtime.interpreter: + interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) + else: + interpreter_actual_path = runtime.interpreter_path - is_bootstrap_script = BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT is_windows = target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints) + if is_windows: + venv_details = _create_venv_windows(ctx, venv_root = venv_root, + interpreter_actual_path = interpreter_actual_path, + runtime = runtime, + ) + else: + venv_details = _create_venv_unixy(ctx, venv_root= venv_root, + interpreter_actual_path = interpreter_actual_path, + runtime = runtime, + ) - create_full_venv = True + site_packages = "{}/{}".format(venv_root, venv_details.venv_site_packages) + + pth = ctx.actions.declare_file("{}/bazel.pth".format(site_packages)) + ctx.actions.write(pth, "import _bazel_site_init\n") + + site_init = ctx.actions.declare_file("{}/_bazel_site_init.py".format(site_packages)) + computed_subs = ctx.actions.template_dict() + computed_subs.add_joined("%imports%", imports, join_with = ":", map_each = _map_each_identity) + ctx.actions.expand_template( + template = runtime.site_init_template, + output = site_init, + substitutions = { + "%add_runfiles_root_to_sys_path%": add_runfiles_root_to_sys_path, + "%coverage_tool%": _get_coverage_tool_runfiles_path(ctx, runtime), + "%import_all%": "True" if read_possibly_native_flag(ctx, "python_import_all_repositories") else "False", + "%site_init_runfiles_path%": runfiles_root_path(ctx, site_init.short_path), + "%workspace_name%": ctx.workspace_name, + }, + computed_substitutions = computed_subs, + ) + + venv_dir_map = { + VenvSymlinkKind.BIN: venv_details.bin_dir, + VenvSymlinkKind.LIB: site_packages, + } + venv_app_files = create_venv_app_files( + ctx, + deps = collect_deps(ctx, extra_deps), + venv_dir_map = venv_dir_map, + ) + + files_without_interpreter = [pth, site_init] + venv_app_files.venv_files + if venv_details.pyvenv_cfg: + files_without_interpreter.append(venv_details.pyvenv_cfg) + + return struct( + # File or None; the `bin/python3` executable in the venv. + # None if a full venv isn't created. + interpreter = venv_details.interpreter, + # Files in the venv that need to be created for the interpreter to work + interpreter_runfiles = venv_details.interpreter_runfiles, + # bool; True if the venv should be recreated at runtime + recreate_venv_at_runtime = venv_details.recreate_venv_at_runtime, + # Runfiles root relative path or absolute path + interpreter_actual_path = interpreter_actual_path, + files_without_interpreter = files_without_interpreter, + # string; venv-relative path to the site-packages directory. + venv_site_packages = venv_details.venv_site_packages, + # string; runfiles-root relative path to venv root. + venv_root = runfiles_root_path( + ctx, + paths.join( + py_internal.get_label_repo_runfiles_path(ctx.label), + venv_root, + ), + ), + # venv files for user library dependencies (files that are specific + # to the executable bootstrap and python runtime aren't here). + # `root_symlinks` should be used, otherwise, with symlinks files always go + # to `_main` prefix, and binaries from non-root module become broken. + lib_runfiles = ctx.runfiles( + root_symlinks = venv_app_files.runfiles_symlinks, + ), + ) +def _create_venv_unixy(ctx, *, venv_root, runtime, interpreter_actual_path): + interpreter_runfiles = builders.RunfilesBuilder() + if runtime.interpreter: + interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) + else: + interpreter_actual_path = runtime.interpreter_path + + is_bootstrap_script = BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT + create_full_venv = True # The legacy build_python_zip codepath (enabled by default on windows) isn't # compatible with full venv. # TODO: Use non-build_python_zip codepath for Windows - if is_windows: - create_full_venv = True - elif not rp_config.bazel_8_or_later and not is_bootstrap_script: + if not rp_config.bazel_8_or_later and not is_bootstrap_script: # Full venv for Bazel 7 + system_python is disabled because packaging # it using build_python_zip=true or rules_pkg breaks. # * Using build_python_zip=true breaks because the legacy zipapp support @@ -558,24 +636,18 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root # The pyvenv.cfg file must be present to trigger the venv site hooks. # Because it's paths are expected to be absolute paths, we can't reliably # put much in it. See https://github.com/python/cpython/issues/83650 - pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv)) + pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv_root)) ctx.actions.write(pyvenv_cfg, "") else: pyvenv_cfg = None - runtime = runtime_details.effective_runtime venvs_use_declare_symlink_enabled = ( VenvsUseDeclareSymlinkFlag.get_value(ctx) == VenvsUseDeclareSymlinkFlag.YES ) - recreate_venv_at_runtime = False - - if runtime.interpreter: - interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) - else: - interpreter_actual_path = runtime.interpreter_path - bin_dir = "{}/bin".format(venv) + recreate_venv_at_runtime = False + bin_dir = "{}/bin".format(venv_root) if create_full_venv: # Some wrappers around the interpreter (e.g. pyenv) use the program # name to decide what to do, so preserve the name. @@ -604,7 +676,6 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root from_ = paths.dirname(runfiles_root_path(ctx, interpreter.short_path)), to = interpreter_actual_path, ) - ctx.actions.symlink(output = interpreter, target_path = rel_path) else: interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) @@ -627,67 +698,75 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root version += "t" venv_site_packages = "lib/python{}/site-packages".format(version) - site_packages = "{}/{}".format(venv, venv_site_packages) - pth = ctx.actions.declare_file("{}/bazel.pth".format(site_packages)) - ctx.actions.write(pth, "import _bazel_site_init\n") - - site_init = ctx.actions.declare_file("{}/_bazel_site_init.py".format(site_packages)) - computed_subs = ctx.actions.template_dict() - computed_subs.add_joined("%imports%", imports, join_with = ":", map_each = _map_each_identity) - ctx.actions.expand_template( - template = runtime.site_init_template, - output = site_init, - substitutions = { - "%add_runfiles_root_to_sys_path%": add_runfiles_root_to_sys_path, - "%coverage_tool%": _get_coverage_tool_runfiles_path(ctx, runtime), - "%import_all%": "True" if read_possibly_native_flag(ctx, "python_import_all_repositories") else "False", - "%site_init_runfiles_path%": runfiles_root_path(ctx, site_init.short_path), - "%workspace_name%": ctx.workspace_name, - }, - computed_substitutions = computed_subs, + return _venv_details( + pyvenv_cfg = pyvenv_cfg, + venv_site_packages = venv_site_packages, + bin_dir = bin_dir, + recreate_venv_at_runtime = recreate_venv_at_runtime, + interpreter_runfiles = interpreter_runfiles.build(ctx), ) - venv_dir_map = { - VenvSymlinkKind.BIN: bin_dir, - VenvSymlinkKind.LIB: site_packages, - } - venv_app_files = create_venv_app_files( - ctx, - deps = collect_deps(ctx, extra_deps), - venv_dir_map = venv_dir_map, - ) +def _create_venv_windows(ctx, *, venv_root, runtime, interpreter_actual_path): + interpreter_runfiles = builders.RunfilesBuilder() - files_without_interpreter = [pth, site_init] + venv_app_files.venv_files - if pyvenv_cfg: - files_without_interpreter.append(pyvenv_cfg) + # Some wrappers around the interpreter (e.g. pyenv) use the program + # name to decide what to do, so preserve the name. + py_exe_basename = paths.basename(interpreter_actual_path) + bin_dir = "{}/Scripts".format(venv_root) + if runtime.interpreter: + interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename)) + interpreter_runfiles.add(interpreter) + ctx.actions.symlink(output = interpreter, target_file = runtime.interpreter) + for f in runtime.venv_bin_files: + venv_path = "{}/{}".format(bin_dir, f.basename) + venv_file = ctx.actions.declare_file(venv_path) + ctx.actions.symlink(output = venv_file, target_file = f) + interpreter_runfiles.add(venv_file) + else: + interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) + interpreter_runfiles.add(interpreter) + ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path) + + # See site.py logic: Windows uses a version/build agnostic site-packages path + venv_site_packages = "Lib/site-packages" + + return _venv_details( + interpreter = interpreter, + pyvenv_cfg = None, + venv_site_packages = venv_site_packages, + bin_dir = bin_dir, + recreate_venv_at_runtime = True, + interpreter_runfiles = interpreter_runfiles.build(ctx), + ) +def _venv_details(*, + interpreter, + pyvenv_cfg, + venv_site_packages, + bin_dir, + recreate_venv_at_runtime, + interpreter_runfiles, + ): + """Helper to create a struct of platform-specific venv details.""" return struct( - # File or None; the `bin/python3` executable in the venv. - # None if a full venv isn't created. + # File; the `bin/python` executable (or equivalent) within the venv. interpreter = interpreter, - # bool; True if the venv should be recreated at runtime - recreate_venv_at_runtime = recreate_venv_at_runtime, - # Runfiles root relative path or absolute path - interpreter_actual_path = interpreter_actual_path, - files_without_interpreter = files_without_interpreter, - # string; venv-relative path to the site-packages directory. + # File|None; the pyvenv.cfg file, if any. May be none, in which case, + # it's expected that one will be created at runtime. + pyvenv_cfg = pyvenv_cfg, + # str; venv-relative path to the site-packages directory venv_site_packages = venv_site_packages, - # string; runfiles-root relative path to venv root. - venv_root = runfiles_root_path( - ctx, - paths.join( - py_internal.get_label_repo_runfiles_path(ctx.label), - venv, - ), - ), - # venv files for user library dependencies (files that are specific - # to the executable bootstrap and python runtime aren't here). - # `root_symlinks` should be used, otherwise, with symlinks files always go - # to `_main` prefix, and binaries from non-root module become broken. - lib_runfiles = ctx.runfiles( - root_symlinks = venv_app_files.runfiles_symlinks, - ), + # str; ctx-relative path to the venv's bin directory. + bin_dir = bin_dir, + # bool; True if the venv needs to be recreated at runtime (because the + # build-time construction isn't sufficient). False if the build-time + # constructed venv is sufficient. + recreate_venv_at_runtime = recreate_venv_at_runtime , + # runfiles; runfiles for interpreter-specific files in the venv. + interpreter_runfiles = interpreter_runfiles, ) +) + def _map_each_identity(v): return v @@ -777,6 +856,7 @@ def _create_stage1_bootstrap( else: resolve_python_binary_at_runtime = "1" + subs = { "%interpreter_args%": "\n".join(ctx.attr.interpreter_args), "%is_zipfile%": "1" if is_for_zip else "0", diff --git a/python/private/py_runtime_info.bzl b/python/private/py_runtime_info.bzl index af4e7f0596..267a6e0e98 100644 --- a/python/private/py_runtime_info.bzl +++ b/python/private/py_runtime_info.bzl @@ -66,7 +66,8 @@ def _PyRuntimeInfo_init( zip_main_template = None, abi_flags = "", site_init_template = None, - supports_build_time_venv = True): + supports_build_time_venv = True, + venv_bin_files = None): if (interpreter_path and interpreter) or (not interpreter_path and not interpreter): fail("exactly one of interpreter or interpreter_path must be specified") @@ -120,6 +121,7 @@ def _PyRuntimeInfo_init( "stub_shebang": stub_shebang, "supports_build_time_venv": supports_build_time_venv, "zip_main_template": zip_main_template, + "venv_bin_files": venv_bin_files, } PyRuntimeInfo, _unused_raw_py_runtime_info_ctor = provider( @@ -355,6 +357,14 @@ The following substitutions are made during template expansion: :::{versionadded} 0.33.0 ::: +""", +"venv_bin_files": """ +:type: list[File] + +Files that should be added to the venv's `bin/` (or platform-specific equivalent) +directory (using the file's basename). + +:::{versionadded} VERSION_NEXT_FEATURE """, }, ) diff --git a/python/private/py_runtime_rule.bzl b/python/private/py_runtime_rule.bzl index 09e245a58e..b7455f5664 100644 --- a/python/private/py_runtime_rule.bzl +++ b/python/private/py_runtime_rule.bzl @@ -130,6 +130,7 @@ def _py_runtime_impl(ctx): abi_flags = abi_flags, site_init_template = ctx.file.site_init_template, supports_build_time_venv = ctx.attr.supports_build_time_venv, + venv_bin_files = ctx.files.venv_bin_files, )) providers = [ @@ -379,6 +380,7 @@ The {obj}`PyRuntimeInfo.zip_main_template` field. "_python_version_flag": attr.label( default = labels.PYTHON_VERSION, ), + "venv_bin_files": attr.label_list(allow_files=True), }, ), ) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 4efd46690a..fd75bff8e1 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -91,6 +91,8 @@ else: def is_windows(): return os.name == 'nt' +BIN_DIR_NAME = "bin" if not is_windows() else "Scripts" + def get_windows_path_with_unc_prefix(path): """Adds UNC prefix after getting a normalized absolute Windows path. @@ -131,12 +133,6 @@ def get_windows_path_with_unc_prefix(path): # os.path.abspath returns a normalized absolute path return unicode_prefix + os.path.abspath(path) -def has_windows_executable_extension(path): - return path.endswith('.exe') or path.endswith('.com') or path.endswith('.bat') - -if PYTHON_BINARY and is_windows() and not has_windows_executable_extension(PYTHON_BINARY): - PYTHON_BINARY = PYTHON_BINARY + '.exe' - def search_path(name): """Finds a file in a given search path.""" search_path = os.getenv('PATH', os.defpath).split(os.pathsep) @@ -150,31 +146,34 @@ def search_path(name): def find_python_binary(runfiles_root): """Finds the real Python binary if it's not a normal absolute path.""" if PYTHON_BINARY: - return find_binary(runfiles_root, PYTHON_BINARY) + bin_path = find_binary(runfiles_root, PYTHON_BINARY) else: - return find_binary(runfiles_root, PYTHON_BINARY_ACTUAL) + bin_path = find_binary(runfiles_root, PYTHON_BINARY_ACTUAL) + return bin_path def print_verbose(*args, mapping=None, values=None): - if os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE"): - if mapping is not None: - for key, value in sorted((mapping or {}).items()): - print( - "bootstrap: stage 1:", - *(list(args) + ["{}={}".format(key, repr(value))]), - file=sys.stderr, - flush=True - ) - elif values is not None: - for i, v in enumerate(values): - print( - "bootstrap: stage 1:", - *(list(args) + ["[{}] {}".format(i, repr(v))]), - file=sys.stderr, - flush=True - ) - else: - print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True) + if not os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE"): + return + + if mapping is not None: + for key, value in sorted((mapping or {}).items()): + print( + "bootstrap: stage 1:", + *(list(args) + ["{}={}".format(key, repr(value))]), + file=sys.stderr, + flush=True + ) + elif values is not None: + for i, v in enumerate(values): + print( + "bootstrap: stage 1:", + *(list(args) + ["[{}] {}".format(i, repr(v))]), + file=sys.stderr, + flush=True + ) + else: + print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True) def find_binary(runfiles_root, bin_name): """Finds the real binary if it's not a normal absolute path.""" @@ -286,56 +285,90 @@ def create_runfiles_root(): # important that deletion code be in sync with this directory structure return os.path.join(temp_dir, 'runfiles') -def _create_venv(runfiles_root): - runfiles_venv = join(runfiles_root, dirname(dirname(PYTHON_BINARY))) +def _create_venv(runfiles_root, delete_dirs): + rel_runfiles_venv = dirname(dirname(PYTHON_BINARY)) + runfiles_venv = join(runfiles_root, rel_runfiles_venv) + print_verbose("create_venv: runfiles venv:", runfiles_venv) if EXTRACT_ROOT: - venv = join(EXTRACT_ROOT, runfiles_venv) + venv = join(EXTRACT_ROOT, rel_runfiles_venv) os.makedirs(venv, exist_ok=True) - cleanup_dir = None else: import tempfile venv = tempfile.mkdtemp("", f"bazel.{basename(runfiles_venv)}.") - cleanup_dir = venv + delete_dirs.append(venv) + + print_verbose("create_venv: created venv:", venv) python_exe_actual = find_binary(runfiles_root, PYTHON_BINARY_ACTUAL) + if python_exe_actual is None: + raise AssertionError('Could not find python binary: ' + repr(PYTHON_BINARY_ACTUAL)) # See stage1_bootstrap_template.sh for details on this code path. In short, # this handles when the build-time python version doesn't match runtime # and if the initially resolved python_exe_actual is a wrapper script. if RESOLVE_PYTHON_BINARY_AT_RUNTIME: + venv_src = venv.replace("\\", "\\\\") # Escape backslashes src = f""" import sys, site print(sys.executable) -print(site.getsitepackages(["{venv}"])[-1]) +print(sys.base_prefix) +print(site.getsitepackages(["{venv_src}"])[-1]) """ + print_verbose("prog:", src) output = subprocess.check_output([python_exe_actual, "-I"], shell=True, encoding = "utf8", input=src) output = output.strip().split("\n") python_exe_actual = output[0] - venv_site_packages = output[1] + python_home = output[1] if is_windows() else None + venv_site_packages = output[2] os.makedirs(dirname(venv_site_packages), exist_ok=True) runfiles_venv_site_packages = join(runfiles_venv, VENV_REL_SITE_PACKAGES) else: - python_exe_actual = find_binary(runfiles_root, PYTHON_BINARY_ACTUAL) + python_home = dirname(python_exe_actual) if is_windows else None venv_site_packages = join(venv, "lib") runfiles_venv_site_packages = join(runfiles_venv, "lib") - if python_exe_actual is None: - raise AssertionError('Could not find python binary: ' + repr(PYTHON_BINARY_ACTUAL)) - - venv_bin = join(venv, "bin") + venv_bin = join(venv, BIN_DIR_NAME) try: os.mkdir(venv_bin) except FileExistsError as e: pass - # Match the basename; some tools, e.g. pyvenv key off the executable name venv_python_exe = join(venv_bin, os.path.basename(python_exe_actual)) _symlink_exist_ok(from_=venv_python_exe, to=python_exe_actual) + + # Windows requires supporting .dll files in the venv bin dir, but + # they aren't known in advance when the interpreter is resolved at runtime. + if RESOLVE_PYTHON_BINARY_AT_RUNTIME and is_windows(): + files = os.listdir(python_home) + for f in files: + if not f.endswith((".dll", ".pdb")): continue + venv_path = join(venv, BIN_DIR_NAME, f) + target = join(python_home, f) + _symlink_exist_ok(from_=venv_path, to=target) + + # Re-add all the entries under bin/, which includes supporting + # .dll files when under windows. + runfiles_venv_bin = join(runfiles_venv, BIN_DIR_NAME) + for f_basename in os.listdir(runfiles_venv_bin): + venv_path = join(venv, BIN_DIR_NAME, f_basename) + target = join(runfiles_venv_bin, f_basename) + _symlink_exist_ok(from_=venv_path, to=target) + _symlink_exist_ok(from_=join(venv, "lib"), to=join(runfiles_venv, "lib")) _symlink_exist_ok(from_=venv_site_packages, to=runfiles_venv_site_packages) - _symlink_exist_ok(from_=join(venv, "pyvenv.cfg"), to=join(runfiles_venv, "pyvenv.cfg")) - return cleanup_dir, venv_python_exe + + if is_windows(): + print_verbose("create_venv: pyvenv.cfg home: ", python_home) + venv_pyvenv_cfg = join(venv, "pyvenv.cfg") + with open(venv_pyvenv_cfg, "w") as fp: + # Until Windows supports a build-time generated venv using symlinks + # to directories, we have to write the full, absolute, path to PYTHONHOME + # so that support directories (e.g. DLLs, libs) can be found. + fp.write("home = {}\n".format(python_home)) + else: + _symlink_exist_ok(from_=join(venv, "pyvenv.cfg"), to=join(runfiles_venv, "pyvenv.cfg")) + return venv_python_exe def runfiles_envvar(runfiles_root): """Finds the runfiles manifest or the runfiles directory. @@ -428,6 +461,7 @@ def execute_file(python_program, main_filename, args, env, runfiles_root, print_verbose("run: subproc: argv:", values=argv) ret_code = subprocess.call( argv, env=env, cwd=workspace) + print_verbose("run: subproc: exit code:", ret_code) if delete_dirs: for delete_dir in delete_dirs: @@ -463,12 +497,14 @@ def main(): print_verbose("initial cwd:", os.getcwd()) print_verbose("initial environ:", mapping=os.environ) print_verbose("initial sys.path:", values=sys.path) - print_verbose("STAGE2_BOOTSTRAP:", STAGE2_BOOTSTRAP) + print_verbose("IS_ZIPFILE:", IS_ZIPFILE) print_verbose("PYTHON_BINARY:", PYTHON_BINARY) print_verbose("PYTHON_BINARY_ACTUAL:", PYTHON_BINARY_ACTUAL) - print_verbose("IS_ZIPFILE:", IS_ZIPFILE) print_verbose("RECREATE_VENV_AT_RUNTIME:", RECREATE_VENV_AT_RUNTIME) - print_verbose("WORKSPACE_NAME :", WORKSPACE_NAME ) + print_verbose("RESOLVE_PYTHON_BINARY_AT_RUNTIME:", RESOLVE_PYTHON_BINARY_AT_RUNTIME) + print_verbose("STAGE2_BOOTSTRAP:", STAGE2_BOOTSTRAP) + print_verbose("VENV_REL_SITE_PACKAGES:", VENV_REL_SITE_PACKAGES) + print_verbose("WORKSPACE_NAME:", WORKSPACE_NAME ) print_verbose("bootstrap sys.executable:", sys.executable) print_verbose("bootstrap sys._base_executable:", sys._base_executable) print_verbose("bootstrap sys.version:", sys.version) @@ -516,20 +552,19 @@ def main(): assert os.access(main_filename, os.R_OK), \ 'Cannot exec() %r: file not readable.' % main_filename - python_program = find_python_binary(runfiles_root) - if python_program is None: - raise AssertionError("Could not find python binary: {} or {}".format( - repr(PYTHON_BINARY), - repr(PYTHON_BINARY_ACTUAL) - )) - if RECREATE_VENV_AT_RUNTIME: # When the venv is created at runtime, python_program is PYTHON_BINARY_ACTUAL # so we have to re-point it to the symlink in the venv - venv, python_program = _create_venv(runfiles_root) - delete_dirs.append(venv) + python_program = _create_venv(runfiles_root, delete_dirs) else: python_program = find_python_binary(runfiles_root) + if python_program is None: + raise AssertionError("Could not find python binary: {} or {}".format( + repr(PYTHON_BINARY), + repr(PYTHON_BINARY_ACTUAL) + )) + + python_program = get_windows_path_with_unc_prefix(python_program) # Some older Python versions on macOS (namely Python 3.7) may unintentionally # leave this environment variable set after starting the interpreter, which diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index e997110a5c..3c30d560af 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -112,6 +112,13 @@ def has_windows_executable_extension(path): ): _PYTHON_BINARY_VENV = _PYTHON_BINARY_VENV + ".exe" +if ( + _PYTHON_BINARY_ACTUAL + and is_windows() + and not has_windows_executable_extension(_PYTHON_BINARY_ACTUAL) +): + _PYTHON_BINARY_ACTUAL = _PYTHON_BINARY_ACTUAL + ".exe" + def search_path(name): """Finds a file in a given search path.""" @@ -223,7 +230,11 @@ def execute_file( # - When running in a zip file, we need to clean up the # workspace after the process finishes so control must return here. try: - subprocess_argv = [python_program, main_filename] + args + subprocess_argv = [ + python_program, + f"-XRULES_PYTHON_ZIP_DIR={os.path.dirname(runfiles_root)}", + main_filename, + ] + args print_verbose("subprocess argv:", values=subprocess_argv) print_verbose("subprocess env:", mapping=env) print_verbose("subprocess cwd:", workspace) @@ -280,7 +291,7 @@ def main(): # When a venv is used, the `bin/python3` symlink may need to be created. # This case occurs when "create venv at runtime" or "resolve python at # runtime" modes are enabled. - if not os.path.lexists(python_program): + if not os.path.exists(python_program): # The venv bin/python3 interpreter should always be under runfiles, but # double check. We don't want to accidentally create symlinks elsewhere if not python_program.startswith(runfiles_root): @@ -289,6 +300,8 @@ def main(): ) symlink_to = find_binary(runfiles_root, _PYTHON_BINARY_ACTUAL) os.makedirs(os.path.dirname(python_program), exist_ok=True) + if os.path.lexists(python_program): + os.remove(python_program) try: os.symlink(symlink_to, python_program) except OSError as e: diff --git a/tests/bootstrap_impls/BUILD.bazel b/tests/bootstrap_impls/BUILD.bazel index ab3148db00..47363d4d93 100644 --- a/tests/bootstrap_impls/BUILD.bazel +++ b/tests/bootstrap_impls/BUILD.bazel @@ -128,7 +128,7 @@ py_reconfig_test( ) py_reconfig_test( - name = "sys_path_order_bootstrap_system_python_test", + name = "sys_path_order_sys_test", srcs = ["sys_path_order_test.py"], bootstrap_impl = "system_python", env = {"BOOTSTRAP": "system_python"}, diff --git a/tests/bootstrap_impls/run_binary_zip_yes_test.sh b/tests/bootstrap_impls/run_binary_zip_yes_test.sh index ca278083dd..77fe4d3609 100755 --- a/tests/bootstrap_impls/run_binary_zip_yes_test.sh +++ b/tests/bootstrap_impls/run_binary_zip_yes_test.sh @@ -34,8 +34,8 @@ actual=$($bin) # How we detect if a zip file was executed from depends on which bootstrap # is used. # bootstrap_impl=script outputs RULES_PYTHON_ZIP_DIR: -# bootstrap_impl=system_python outputs file:.*Bazel.runfiles -expected_pattern="RULES_PYTHON_ZIP_DIR:/\|file:.*Bazel.runfiles" +# bootstrap_impl=system_python outputs file:.*Bazel.runfiles (or .exe.runfiles on Windows) +expected_pattern="RULES_PYTHON_ZIP_DIR:/\|file:.*Bazel.runfiles\|file:.*\.exe\.runfiles" if ! (echo "$actual" | grep "$expected_pattern" ) >/dev/null; then echo "expected output to match: $expected_pattern" echo "but got: $actual" diff --git a/tests/bootstrap_impls/sys_path_order_test.py b/tests/bootstrap_impls/sys_path_order_test.py index 9ae03bb129..657354f977 100644 --- a/tests/bootstrap_impls/sys_path_order_test.py +++ b/tests/bootstrap_impls/sys_path_order_test.py @@ -17,7 +17,6 @@ import sys import unittest - class SysPathOrderTest(unittest.TestCase): def test_sys_path_order(self): last_stdlib = None diff --git a/tests/bootstrap_impls/system_python_nodeps_test.py b/tests/bootstrap_impls/system_python_nodeps_test.py index 7dc46d6e73..03c7a91479 100644 --- a/tests/bootstrap_impls/system_python_nodeps_test.py +++ b/tests/bootstrap_impls/system_python_nodeps_test.py @@ -1 +1,3 @@ print("Hello, world") +import pathlib +print(pathlib) From 27732f69f609030a0544637de2699f844bd6fc8a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 3 Apr 2026 20:01:31 -0700 Subject: [PATCH 03/28] fix syntax error, sys_path order test --- python/private/py_executable.bzl | 2 -- tests/bootstrap_impls/sys_path_order_test.py | 10 +++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 60abc5d5f3..7e6ed897b5 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -765,8 +765,6 @@ def _venv_details(*, # runfiles; runfiles for interpreter-specific files in the venv. interpreter_runfiles = interpreter_runfiles, ) -) - def _map_each_identity(v): return v diff --git a/tests/bootstrap_impls/sys_path_order_test.py b/tests/bootstrap_impls/sys_path_order_test.py index 657354f977..c20503300b 100644 --- a/tests/bootstrap_impls/sys_path_order_test.py +++ b/tests/bootstrap_impls/sys_path_order_test.py @@ -32,9 +32,13 @@ def test_sys_path_order(self): # error messages are more informative. categorized_paths = [] for i, value in enumerate(sys.path): - # The runtime's root repo may be added to sys.path, but it - # counts as a user directory, not stdlib directory. - if value in (sys.prefix, sys.base_prefix): + # On windows, the `pythonXY.zip` entry shows up as `$venv/Scripts/pythonXY.zip` + # While it's technically part of the venv, it's considered the stdlib. + if os.name == 'nt' and re.search("python.*[.]zip$", value): + category = "stdlib" + elif value in (sys.prefix, sys.base_prefix): + # The runtime's root repo may be added to sys.path, but it + # counts as a user directory, not stdlib directory. category = "user" elif value.startswith(sys.base_prefix): # The runtime's site-package directory might be called From e89f1c95d49bce4948235c35a71843b5ef4e5b9f Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 3 Apr 2026 20:05:39 -0700 Subject: [PATCH 04/28] allow empty dll glob --- python/private/hermetic_runtime_repo_setup.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index e8b52b602c..2c286a721a 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -242,7 +242,10 @@ def define_hermetic_runtime_toolchain_impl( }), # On Windows, a symlink-style venv requires supporting .dll files. venv_bin_files = select({ - "@platforms//os:windows": native.glob(include=["*.dll", "*.pdb"]), + "@platforms//os:windows": native.glob( + include=["*.dll", "*.pdb"], + allow_empty=True, + ), "//conditions:default": [], }) ) From 1f47ab5f03f6c1e7bc736b7c5c414c0e982c2307 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 3 Apr 2026 23:37:20 -0700 Subject: [PATCH 05/28] fix is_windows bug, address gemini comments --- python/private/py_executable.bzl | 11 ++++----- python/private/python_bootstrap_template.txt | 25 ++++++++++---------- python/private/stage2_bootstrap_template.py | 15 +++++------- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 7e6ed897b5..59b1fa95e0 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -375,7 +375,9 @@ def _create_executable( # NOTE: --build_python_zip defaults to true on Windows build_zip_enabled = read_possibly_native_flag(ctx, "build_python_zip") - build_zip_enabled = False + if is_windows: + # The legacy build_python_zip codepath isn't compatible with full venvs on Windows. + build_zip_enabled = False # When --build_python_zip is enabled, then the zip file becomes # one of the default outputs. @@ -610,11 +612,6 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root def _create_venv_unixy(ctx, *, venv_root, runtime, interpreter_actual_path): interpreter_runfiles = builders.RunfilesBuilder() - if runtime.interpreter: - interpreter_actual_path = runfiles_root_path(ctx, runtime.interpreter.short_path) - else: - interpreter_actual_path = runtime.interpreter_path - is_bootstrap_script = BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT create_full_venv = True # The legacy build_python_zip codepath (enabled by default on windows) isn't @@ -761,7 +758,7 @@ def _venv_details(*, # bool; True if the venv needs to be recreated at runtime (because the # build-time construction isn't sufficient). False if the build-time # constructed venv is sufficient. - recreate_venv_at_runtime = recreate_venv_at_runtime , + recreate_venv_at_runtime = recreate_venv_at_runtime, # runfiles; runfiles for interpreter-specific files in the venv. interpreter_runfiles = interpreter_runfiles, ) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 2c1ac1c6db..bb6e06f4e0 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -87,11 +87,9 @@ if is_running_from_zip(): else: import re -# Return True if running on Windows -def is_windows(): - return os.name == 'nt' +IS_WINDOWS = os.name == "nt" -BIN_DIR_NAME = "bin" if not is_windows() else "Scripts" +BIN_DIR_NAME = "bin" if not IS_WINDOWS else "Scripts" def get_windows_path_with_unc_prefix(path): """Adds UNC prefix after getting a normalized absolute Windows path. @@ -102,7 +100,7 @@ def get_windows_path_with_unc_prefix(path): # No need to add prefix for non-Windows platforms. # And \\?\ doesn't work in python 2 or on mingw - if not is_windows() or sys.version_info[0] < 3: + if not IS_WINDOWS or sys.version_info[0] < 3: return path # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been @@ -224,18 +222,18 @@ def find_runfiles_root(main_rel_path): # On Windows, the path may contain both forward and backslashes. # Normalize to the OS separator because the regex used later assumes # the OS-specific separator. - if is_windows(): + if IS_WINDOWS: stub_filename = stub_filename.replace("/", os.sep) if not os.path.isabs(stub_filename): stub_filename = os.path.join(os.getcwd(), stub_filename) while True: - runfiles_root = stub_filename + ('.exe' if is_windows() else '') + '.runfiles' + runfiles_root = stub_filename + ('.exe' if IS_WINDOWS else '') + '.runfiles' if os.path.isdir(runfiles_root): return runfiles_root - runfiles_pattern = r'(.*\.runfiles)' + (r'\\' if is_windows() else '/') + '.*' + runfiles_pattern = r'(.*\.runfiles)' + (r'\\' if IS_WINDOWS else '/') + '.*' matchobj = re.match(runfiles_pattern, stub_filename) if matchobj: return matchobj.group(1) @@ -326,12 +324,13 @@ print(site.getsitepackages(["{venv_src}"])[-1]) encoding = "utf8", input=src) output = output.strip().split("\n") python_exe_actual = output[0] - python_home = output[1] if is_windows() else None + python_home = output[1] if IS_WINDOWS else None venv_site_packages = output[2] os.makedirs(dirname(venv_site_packages), exist_ok=True) runfiles_venv_site_packages = join(runfiles_venv, VENV_REL_SITE_PACKAGES) else: - python_home = dirname(python_exe_actual) if is_windows else None + # On unixy, Python can find home based on the symlink. + python_home = dirname(python_exe_actual) if IS_WINDOWS else None venv_site_packages = join(venv, "lib") runfiles_venv_site_packages = join(runfiles_venv, "lib") @@ -346,7 +345,7 @@ print(site.getsitepackages(["{venv_src}"])[-1]) # Windows requires supporting .dll files in the venv bin dir, but # they aren't known in advance when the interpreter is resolved at runtime. - if RESOLVE_PYTHON_BINARY_AT_RUNTIME and is_windows(): + if RESOLVE_PYTHON_BINARY_AT_RUNTIME and IS_WINDOWS: files = os.listdir(python_home) for f in files: if not f.endswith((".dll", ".pdb")): continue @@ -365,7 +364,7 @@ print(site.getsitepackages(["{venv_src}"])[-1]) _symlink_exist_ok(from_=join(venv, "lib"), to=join(runfiles_venv, "lib")) _symlink_exist_ok(from_=venv_site_packages, to=runfiles_venv_site_packages) - if is_windows(): + if IS_WINDOWS: print_verbose("create_venv: pyvenv.cfg home: ", python_home) venv_pyvenv_cfg = join(venv, "pyvenv.cfg") with open(venv_pyvenv_cfg, "w") as fp: @@ -460,7 +459,7 @@ def execute_file(python_program, main_filename, args, env, runfiles_root, # can't execv because we need control to return here. This only # happens for targets built in the host config. # - if not (is_windows() or workspace or delete_dirs): + if not (IS_WINDOWS or workspace or delete_dirs): _run_execv(python_program, argv, env) print_verbose("run: subproc: environ:", mapping=os.environ) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 3c2d6807cc..614e620c0d 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -67,7 +67,7 @@ def get_build_data(self): from python.runfiles import runfiles rlocation_path = self.BUILD_DATA_FILE path = runfiles.Create().Rlocation(rlocation_path) - if is_windows(): + if IS_WINDOWS: path = os.path.normpath(path) try: # Use utf-8-sig to handle Windows BOM @@ -84,17 +84,14 @@ def get_build_data(self): sys.modules["bazel_binary_info"] = BazelBinaryInfoModule("bazel_binary_info") - -# Return True if running on Windows -def is_windows(): - return os.name == "nt" +IS_WINDOWS = os.name == "nt" def get_windows_path_with_unc_prefix(path): path = path.strip() # No need to add prefix for non-Windows platforms. - if not is_windows() or sys.version_info[0] < 3: + if not IS_WINDOWS or sys.version_info[0] < 3: return path # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been @@ -197,11 +194,11 @@ def find_runfiles_root(main_rel_path): stub_filename = os.path.join(os.getcwd(), stub_filename) while True: - module_space = stub_filename + (".exe" if is_windows() else "") + ".runfiles" + module_space = stub_filename + (".exe" if IS_WINDOWS else "") + ".runfiles" if os.path.isdir(module_space): return module_space - runfiles_pattern = r"(.*\.runfiles)" + (r"\\" if is_windows() else "/") + ".*" + runfiles_pattern = r"(.*\.runfiles)" + (r"\\" if IS_WINDOWS else "/") + ".*" matchobj = re.match(runfiles_pattern, stub_filename) if matchobj: return matchobj.group(1) @@ -454,7 +451,7 @@ def main(): # runfiles root if MAIN_PATH: main_rel_path = MAIN_PATH - if is_windows(): + if IS_WINDOWS: main_rel_path = main_rel_path.replace("/", os.sep) runfiles_root = find_runfiles_root(main_rel_path) From 04fd4eea70d353694a6f67b6c41e375161d2daf4 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 3 Apr 2026 23:43:19 -0700 Subject: [PATCH 06/28] format code --- python/private/hermetic_runtime_repo_setup.bzl | 6 +++--- python/private/py_executable.bzl | 16 ++++++++++------ python/private/py_runtime_info.bzl | 18 +++++++++--------- python/private/py_runtime_rule.bzl | 2 +- python/private/stage2_bootstrap_template.py | 4 ++-- tests/bootstrap_impls/sys_path_order_test.py | 3 ++- .../system_python_nodeps_test.py | 1 + 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 2c286a721a..538ac6f37b 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -243,11 +243,11 @@ def define_hermetic_runtime_toolchain_impl( # On Windows, a symlink-style venv requires supporting .dll files. venv_bin_files = select({ "@platforms//os:windows": native.glob( - include=["*.dll", "*.pdb"], - allow_empty=True, + include = ["*.dll", "*.pdb"], + allow_empty = True, ), "//conditions:default": [], - }) + }), ) py_runtime_pair( diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 59b1fa95e0..2db0f5f0eb 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -535,12 +535,16 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root is_windows = target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints) if is_windows: - venv_details = _create_venv_windows(ctx, venv_root = venv_root, + venv_details = _create_venv_windows( + ctx, + venv_root = venv_root, interpreter_actual_path = interpreter_actual_path, runtime = runtime, ) else: - venv_details = _create_venv_unixy(ctx, venv_root= venv_root, + venv_details = _create_venv_unixy( + ctx, + venv_root = venv_root, interpreter_actual_path = interpreter_actual_path, runtime = runtime, ) @@ -614,6 +618,7 @@ def _create_venv_unixy(ctx, *, venv_root, runtime, interpreter_actual_path): interpreter_runfiles = builders.RunfilesBuilder() is_bootstrap_script = BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT create_full_venv = True + # The legacy build_python_zip codepath (enabled by default on windows) isn't # compatible with full venv. # TODO: Use non-build_python_zip codepath for Windows @@ -736,14 +741,14 @@ def _create_venv_windows(ctx, *, venv_root, runtime, interpreter_actual_path): interpreter_runfiles = interpreter_runfiles.build(ctx), ) -def _venv_details(*, +def _venv_details( + *, interpreter, pyvenv_cfg, venv_site_packages, bin_dir, recreate_venv_at_runtime, - interpreter_runfiles, - ): + interpreter_runfiles): """Helper to create a struct of platform-specific venv details.""" return struct( # File; the `bin/python` executable (or equivalent) within the venv. @@ -851,7 +856,6 @@ def _create_stage1_bootstrap( else: resolve_python_binary_at_runtime = "1" - subs = { "%interpreter_args%": "\n".join(ctx.attr.interpreter_args), "%is_zipfile%": "1" if is_for_zip else "0", diff --git a/python/private/py_runtime_info.bzl b/python/private/py_runtime_info.bzl index 267a6e0e98..8fdbd7bfe2 100644 --- a/python/private/py_runtime_info.bzl +++ b/python/private/py_runtime_info.bzl @@ -120,8 +120,8 @@ def _PyRuntimeInfo_init( "stage2_bootstrap_template": stage2_bootstrap_template, "stub_shebang": stub_shebang, "supports_build_time_venv": supports_build_time_venv, - "zip_main_template": zip_main_template, "venv_bin_files": venv_bin_files, + "zip_main_template": zip_main_template, } PyRuntimeInfo, _unused_raw_py_runtime_info_ctor = provider( @@ -336,6 +336,14 @@ to meet two criteria: :::{versionadded} 1.5.0 ::: +""", + "venv_bin_files": """ +:type: list[File] + +Files that should be added to the venv's `bin/` (or platform-specific equivalent) +directory (using the file's basename). + +:::{versionadded} VERSION_NEXT_FEATURE """, "zip_main_template": """ :type: File @@ -357,14 +365,6 @@ The following substitutions are made during template expansion: :::{versionadded} 0.33.0 ::: -""", -"venv_bin_files": """ -:type: list[File] - -Files that should be added to the venv's `bin/` (or platform-specific equivalent) -directory (using the file's basename). - -:::{versionadded} VERSION_NEXT_FEATURE """, }, ) diff --git a/python/private/py_runtime_rule.bzl b/python/private/py_runtime_rule.bzl index b7455f5664..dfc915f463 100644 --- a/python/private/py_runtime_rule.bzl +++ b/python/private/py_runtime_rule.bzl @@ -361,6 +361,7 @@ See {obj}`PyRuntimeInfo.supports_build_time_venv` for docs. """, default = True, ), + "venv_bin_files": attr.label_list(allow_files = True), "zip_main_template": attr.label( default = "//python/private/zipapp:zip_main_template", allow_single_file = True, @@ -380,7 +381,6 @@ The {obj}`PyRuntimeInfo.zip_main_template` field. "_python_version_flag": attr.label( default = labels.PYTHON_VERSION, ), - "venv_bin_files": attr.label_list(allow_files=True), }, ), ) diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 614e620c0d..dec356d3ac 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -186,8 +186,8 @@ def find_runfiles_root(main_rel_path): # not found. These can be correctly set for a parent Python process, but # inherited by the child, and not correct for it. Later bootstrap code # assumes they're are correct if set. - os.environ.pop('RUNFILES_DIR', None) - os.environ.pop('RUNFILES_MANIFEST_FILE', None) + os.environ.pop("RUNFILES_DIR", None) + os.environ.pop("RUNFILES_MANIFEST_FILE", None) stub_filename = sys.argv[0] if not os.path.isabs(stub_filename): diff --git a/tests/bootstrap_impls/sys_path_order_test.py b/tests/bootstrap_impls/sys_path_order_test.py index c20503300b..8c5911e715 100644 --- a/tests/bootstrap_impls/sys_path_order_test.py +++ b/tests/bootstrap_impls/sys_path_order_test.py @@ -17,6 +17,7 @@ import sys import unittest + class SysPathOrderTest(unittest.TestCase): def test_sys_path_order(self): last_stdlib = None @@ -34,7 +35,7 @@ def test_sys_path_order(self): for i, value in enumerate(sys.path): # On windows, the `pythonXY.zip` entry shows up as `$venv/Scripts/pythonXY.zip` # While it's technically part of the venv, it's considered the stdlib. - if os.name == 'nt' and re.search("python.*[.]zip$", value): + if os.name == "nt" and re.search("python.*[.]zip$", value): category = "stdlib" elif value in (sys.prefix, sys.base_prefix): # The runtime's root repo may be added to sys.path, but it diff --git a/tests/bootstrap_impls/system_python_nodeps_test.py b/tests/bootstrap_impls/system_python_nodeps_test.py index 03c7a91479..36474a1be5 100644 --- a/tests/bootstrap_impls/system_python_nodeps_test.py +++ b/tests/bootstrap_impls/system_python_nodeps_test.py @@ -1,3 +1,4 @@ print("Hello, world") import pathlib + print(pathlib) From 34c7b37adf7ac3a9215941c81403b2bc4c53f44f Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 3 Apr 2026 23:52:38 -0700 Subject: [PATCH 07/28] fix missing arg, test_basic_windows analysis test --- python/private/py_executable.bzl | 1 + tests/base_rules/py_executable_base_tests.bzl | 11 ++--------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 2db0f5f0eb..826a843a1b 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -701,6 +701,7 @@ def _create_venv_unixy(ctx, *, venv_root, runtime, interpreter_actual_path): venv_site_packages = "lib/python{}/site-packages".format(version) return _venv_details( + interpreter = interpreter, pyvenv_cfg = pyvenv_cfg, venv_site_packages = venv_site_packages, bin_dir = bin_dir, diff --git a/tests/base_rules/py_executable_base_tests.bzl b/tests/base_rules/py_executable_base_tests.bzl index f0e2ae9faf..67cfabe630 100644 --- a/tests/base_rules/py_executable_base_tests.bzl +++ b/tests/base_rules/py_executable_base_tests.bzl @@ -44,19 +44,12 @@ def _test_basic_windows(name, config): impl = _test_basic_windows_impl, target = name + "_subject", config_settings = { - # NOTE: The default for this flag is based on the Bazel host OS, not - # the target platform. For windows, it defaults to true, so force - # it to that to match behavior when this test runs on other - # platforms. - # Pass value to both native and starlark versions of the flag until - # the native one is removed. - labels.BUILD_PYTHON_ZIP: True, "//command_line_option:cpu": "windows_x86_64", "//command_line_option:crosstool_top": CROSSTOOL_TOP, "//command_line_option:extra_execution_platforms": [platform_targets.WINDOWS_X86_64], "//command_line_option:extra_toolchains": [CC_TOOLCHAIN], "//command_line_option:platforms": [platform_targets.WINDOWS_X86_64], - } | maybe_builtin_build_python_zip("true"), + }, attr_values = {}, ) @@ -64,7 +57,7 @@ def _test_basic_windows_impl(env, target): target = env.expect.that_target(target) target.executable().path().contains(".exe") target.runfiles().contains_predicate(matching.str_endswith( - target.meta.format_str("/{name}.zip"), + target.meta.format_str("/{name}"), )) target.runfiles().contains_predicate(matching.str_endswith( target.meta.format_str("/{name}.exe"), From 5344aa0a3d1bc2fb9fff0b4e2ad4cc785b852068 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 00:05:26 -0700 Subject: [PATCH 08/28] fix test_debugger --- tests/base_rules/py_executable_base_tests.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/base_rules/py_executable_base_tests.bzl b/tests/base_rules/py_executable_base_tests.bzl index 67cfabe630..69c64d64ae 100644 --- a/tests/base_rules/py_executable_base_tests.bzl +++ b/tests/base_rules/py_executable_base_tests.bzl @@ -239,9 +239,9 @@ def _test_debugger_impl(env, targets): # #3481: Ensure that venv site-packages is setup correctly, if the dependency is coming # from pip integration. - env.expect.that_target(targets.target_venv).runfiles().contains_at_least([ - "{workspace}/{package}/_{name}.venv/lib/python3.13/site-packages/{test_name}_debugger_venv.py", - ]) + env.expect.that_target(targets.target_venv).runfiles().contains_predicate( + matching.str_endswith("site-packages/test_debugger_debugger_venv.py") + ) # 3. Subject exec From 6fc07d7fb39b7c99424922324ca231d04e91756d Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 00:28:54 -0700 Subject: [PATCH 09/28] fix/handle missing interpreter in bin --- python/private/py_executable.bzl | 3 ++- python/private/python_bootstrap_template.txt | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 826a843a1b..0c13741b02 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -662,7 +662,6 @@ def _create_venv_unixy(ctx, *, venv_root, runtime, interpreter_actual_path): # needed or used at runtime. However, the zip code uses the interpreter # File object to figure out some paths. interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename)) - ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path)) elif runtime.interpreter: @@ -671,6 +670,7 @@ def _create_venv_unixy(ctx, *, venv_root, runtime, interpreter_actual_path): # in runfiles is always a symlink. An RBE implementation, for example, # may choose to write what symlink() points to instead. interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) + interpreter_runfiles.add(interpreter) rel_path = relative_path( # dirname is necessary because a relative symlink is relative to @@ -681,6 +681,7 @@ def _create_venv_unixy(ctx, *, venv_root, runtime, interpreter_actual_path): ctx.actions.symlink(output = interpreter, target_path = rel_path) else: interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) + interpreter_runfiles.add(interpreter) ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path) else: interpreter = None diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index bb6e06f4e0..3ae9995dc1 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -144,10 +144,9 @@ def search_path(name): def find_python_binary(runfiles_root): """Finds the real Python binary if it's not a normal absolute path.""" if PYTHON_BINARY: - bin_path = find_binary(runfiles_root, PYTHON_BINARY) + return find_binary(runfiles_root, PYTHON_BINARY) else: - bin_path = find_binary(runfiles_root, PYTHON_BINARY_ACTUAL) - return bin_path + return find_binary(runfiles_root, PYTHON_BINARY_ACTUAL) def print_verbose(*args, mapping=None, values=None): @@ -353,13 +352,16 @@ print(site.getsitepackages(["{venv_src}"])[-1]) target = join(python_home, f) _symlink_exist_ok(from_=venv_path, to=target) - # Re-add all the entries under bin/, which includes supporting - # .dll files when under windows. runfiles_venv_bin = join(runfiles_venv, BIN_DIR_NAME) - for f_basename in os.listdir(runfiles_venv_bin): - venv_path = join(venv, BIN_DIR_NAME, f_basename) - target = join(runfiles_venv_bin, f_basename) - _symlink_exist_ok(from_=venv_path, to=target) + # The runfiles bin directory may not exist if + # supports_build_time_venv=False and the interpreter is resolved at runtime. + if os.path.exists(runfiles_venv_bin): + # Re-add all the entries under bin/, which includes supporting + # .dll files when under windows. + for f_basename in os.listdir(runfiles_venv_bin): + venv_path = join(venv, BIN_DIR_NAME, f_basename) + target = join(runfiles_venv_bin, f_basename) + _symlink_exist_ok(from_=venv_path, to=target) _symlink_exist_ok(from_=join(venv, "lib"), to=join(runfiles_venv, "lib")) _symlink_exist_ok(from_=venv_site_packages, to=runfiles_venv_site_packages) From a77e5429b7af36834dcc47a73e72b86273100dd8 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 00:32:10 -0700 Subject: [PATCH 10/28] remove windows build_python_zip_true test --- .../transition/multi_version_tests.bzl | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/tests/config_settings/transition/multi_version_tests.bzl b/tests/config_settings/transition/multi_version_tests.bzl index 3bb69f2f59..2b4f73a225 100644 --- a/tests/config_settings/transition/multi_version_tests.bzl +++ b/tests/config_settings/transition/multi_version_tests.bzl @@ -120,27 +120,6 @@ def _test_py_binary_windows_build_python_zip_false_impl(env, target): _tests.append(_test_py_binary_windows_build_python_zip_false) -def _test_py_binary_windows_build_python_zip_true(name): - _setup_py_binary_windows( - name, - build_python_zip = True, - impl = _test_py_binary_windows_build_python_zip_true_impl, - ) - -def _test_py_binary_windows_build_python_zip_true_impl(env, target): - default_outputs = env.expect.that_target(target).default_outputs() - - # TODO: These outputs aren't correct. The outputs shouldn't - # have the "_" prefix on them (those are coming from the underlying - # wrapped binary). - default_outputs.contains_exactly([ - "{package}/{test_name}_subject.exe", - "{package}/{test_name}_subject.py", - "{package}/{test_name}_subject.zip", - ]) - -_tests.append(_test_py_binary_windows_build_python_zip_true) - def multi_version_test_suite(name): test_suite( name = name, From d08770be204111bb39a488b70e9f71d43f3e1901 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 00:36:56 -0700 Subject: [PATCH 11/28] format --- .bazelrc.deleted_packages | 1 - tests/base_rules/py_executable_base_tests.bzl | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index 2d8a8075fa..fb5d2ef0bb 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -17,7 +17,6 @@ common --deleted_packages=examples/multi_python_versions/requirements common --deleted_packages=examples/multi_python_versions/tests common --deleted_packages=examples/pip_parse common --deleted_packages=examples/pip_parse_vendored -common --deleted_packages=examples/pip_repository_annotations common --deleted_packages=gazelle common --deleted_packages=gazelle/examples/bzlmod_build_file_generation common --deleted_packages=gazelle/examples/bzlmod_build_file_generation/other_module/other_module/pkg diff --git a/tests/base_rules/py_executable_base_tests.bzl b/tests/base_rules/py_executable_base_tests.bzl index 69c64d64ae..d6b5aedf0c 100644 --- a/tests/base_rules/py_executable_base_tests.bzl +++ b/tests/base_rules/py_executable_base_tests.bzl @@ -240,7 +240,7 @@ def _test_debugger_impl(env, targets): # #3481: Ensure that venv site-packages is setup correctly, if the dependency is coming # from pip integration. env.expect.that_target(targets.target_venv).runfiles().contains_predicate( - matching.str_endswith("site-packages/test_debugger_debugger_venv.py") + matching.str_endswith("site-packages/test_debugger_debugger_venv.py"), ) # 3. Subject exec From 815cacb34fe5548610200c5d71ef57d046254dd4 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 00:38:57 -0700 Subject: [PATCH 12/28] udno deleted packages change --- .bazelrc.deleted_packages | 1 + 1 file changed, 1 insertion(+) diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index fb5d2ef0bb..2d8a8075fa 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages @@ -17,6 +17,7 @@ common --deleted_packages=examples/multi_python_versions/requirements common --deleted_packages=examples/multi_python_versions/tests common --deleted_packages=examples/pip_parse common --deleted_packages=examples/pip_parse_vendored +common --deleted_packages=examples/pip_repository_annotations common --deleted_packages=gazelle common --deleted_packages=gazelle/examples/bzlmod_build_file_generation common --deleted_packages=gazelle/examples/bzlmod_build_file_generation/other_module/other_module/pkg From fb51a467dd9ca495ccd583b2b752bb7a0d006483 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 16:32:48 -0700 Subject: [PATCH 13/28] make zippapp + windows + venv work --- python/private/py_executable.bzl | 13 ++- python/private/py_executable_info.bzl | 8 ++ python/private/zipapp/py_zipapp_rule.bzl | 7 ++ python/private/zipapp/zip_main_template.py | 104 +++++++++++------- ...m_python_zipapp_external_bootstrap_test.sh | 10 ++ 5 files changed, 100 insertions(+), 42 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 0c13741b02..090a87faf1 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -494,6 +494,8 @@ WARNING: Target: {} app_runfiles = app_runfiles.build(ctx), # File|None; the venv `bin/python3` file, if any. venv_python_exe = venv.interpreter if venv else None, + # runfiles; runfiles in the venv for the interpreter + venv_interpreter_runfiles = venv.interpreter_runfiles, ) def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): @@ -1193,6 +1195,7 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = stage2_bootstrap = exec_result.stage2_bootstrap, app_runfiles = app_runfiles, venv_python_exe = exec_result.venv_python_exe, + venv_interpreter_runfiles = exec_result.venv_interpreter_runfiles, interpreter_args = ctx.attr.interpreter_args, ) @@ -1739,6 +1742,7 @@ def _create_providers( stage2_bootstrap, app_runfiles, venv_python_exe, + venv_interpreter_runfiles, interpreter_args): """Creates the providers an executable should return. @@ -1792,14 +1796,15 @@ def _create_providers( create_instrumented_files_info(ctx), _create_run_environment_info(ctx, inherited_environment), PyExecutableInfo( - main = main_py, - runfiles_without_exe = runfiles_details.runfiles_without_exe, + app_runfiles = app_runfiles, build_data_file = runfiles_details.build_data_file, + interpreter_args = interpreter_args, interpreter_path = runtime_details.executable_interpreter_path, + main = main_py, + runfiles_without_exe = runfiles_details.runfiles_without_exe, stage2_bootstrap = stage2_bootstrap, - app_runfiles = app_runfiles, + venv_interpreter_runfiles = venv_interpreter_runfiles, venv_python_exe = venv_python_exe, - interpreter_args = interpreter_args, ), ] diff --git a/python/private/py_executable_info.bzl b/python/private/py_executable_info.bzl index defbd3a05b..4ebc2d4884 100644 --- a/python/private/py_executable_info.bzl +++ b/python/private/py_executable_info.bzl @@ -87,5 +87,13 @@ mode is not enabled. :::{versionadded} 1.9.0 ::: """, + "venv_interpreter_runfiles": """ +:type: runfiles | None + +Runfiles that are specific to the interpreter within the venv. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""" }, ) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index a6ab485fc4..67eca12c93 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -39,6 +39,11 @@ def _create_zipapp_main_py(ctx, py_runtime, py_executable, stage2_bootstrap): "%python_binary_actual%": python_binary_actual_path, "%stage2_bootstrap%": runfiles_root_path(ctx, stage2_bootstrap.short_path), "%workspace_name%": ctx.workspace_name, + "%EXTRACT_DIR%": paths.join( + (ctx.label.repo_name or "_main"), + ctx.label.package, + ctx.label.name, + ), }, ) return zip_main_py @@ -96,6 +101,8 @@ def _create_zip(ctx, py_runtime, py_executable, stage2_bootstrap): runfiles.add(py_runtime.files) if py_executable.venv_python_exe: runfiles.add(py_executable.venv_python_exe) + if py_executable.venv_interpreter_runfiles: + runfiles.add(py_executable.venv_interpreter_runfiles) runfiles.add(py_executable.app_runfiles) runfiles.add(stage2_bootstrap) diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index 3c30d560af..d00745206d 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -22,7 +22,9 @@ del sys.path[0] import os +from os.path import dirname, join import shutil +import stat import subprocess import tempfile import zipfile @@ -35,7 +37,10 @@ # executable to use. _PYTHON_BINARY_ACTUAL = "%python_binary_actual%" _WORKSPACE_NAME = "%workspace_name%" +# relative path under EXTRACT_ROOT to extract to. +EXTRACT_DIR = "%EXTRACT_DIR%" +EXTRACT_ROOT = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") def print_verbose(*args, mapping=None, values=None): if bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")): @@ -181,19 +186,25 @@ def extract_zip(zip_path, dest_dir): target = f.read() os.remove(file_path) os.symlink(target, file_path) + ##os.chmod(file_path, stat.S_IWRITE) # Of those, we set the lower 12 bits, which are the # file mode bits (since the file type bits can't be set by chmod anyway). elif attrs != 0: # Rumor has it these can be 0 for zips created on Windows. - os.chmod(file_path, attrs & 0o7777) + # Add the write bit to ensure the files can be deleted during cleanup and + # overwritten by subsequent invocations. + os.chmod(file_path, (attrs & 0o7777) | stat.S_IWRITE) # Create the runfiles tree by extracting the zip file def create_runfiles_root(): - temp_dir = tempfile.mkdtemp("", "Bazel.runfiles_") - extract_zip(os.path.dirname(__file__), temp_dir) + if EXTRACT_ROOT: + extract_root = join(EXTRACT_ROOT, EXTRACT_DIR) + else: + extract_root = tempfile.mkdtemp("", "Bazel.runfiles_") + extract_zip(os.path.dirname(__file__), extract_root) # IMPORTANT: Later code does `rm -fr` on dirname(runfiles_root) -- it's # important that deletion code be in sync with this directory structure - return os.path.join(temp_dir, "runfiles") + return os.path.join(extract_root, "runfiles") def execute_file( @@ -230,23 +241,61 @@ def execute_file( # - When running in a zip file, we need to clean up the # workspace after the process finishes so control must return here. try: - subprocess_argv = [ - python_program, - f"-XRULES_PYTHON_ZIP_DIR={os.path.dirname(runfiles_root)}", - main_filename, - ] + args - print_verbose("subprocess argv:", values=subprocess_argv) + subprocess_argv = [python_program] + if not EXTRACT_ROOT: + subprocess_argv.append(f"-XRULES_PYTHON_ZIP_DIR={os.path.dirname(runfiles_root)}") + subprocess_argv.append(main_filename) + subprocess_argv += args print_verbose("subprocess env:", mapping=env) print_verbose("subprocess cwd:", workspace) + print_verbose("subprocess argv:", values=subprocess_argv) ret_code = subprocess.call(subprocess_argv, env=env, cwd=workspace) sys.exit(ret_code) finally: - # NOTE: dirname() is called because create_runfiles_root() creates a - # sub-directory within a temporary directory, and we want to remove the - # whole temporary directory. - ##shutil.rmtree(os.path.dirname(runfiles_root), True) - pass - + if not EXTRACT_ROOT: + # NOTE: dirname() is called because create_runfiles_root() creates a + # sub-directory within a temporary directory, and we want to remove the + # whole temporary directory. + extract_root = os.path.dirname(runfiles_root) + print_verbose("cleanup: rmtree: ", extract_root) + shutil.rmtree(extract_root, True) + + +def finish_venv_setup(runfiles_root): + python_program = os.path.join(runfiles_root, _PYTHON_BINARY_VENV) + # When a venv is used, the `bin/python3` symlink may need to be created. + # This case occurs when "create venv at runtime" or "resolve python at + # runtime" modes are enabled. + if not os.path.exists(python_program): + # The venv bin/python3 interpreter should always be under runfiles, but + # double check. We don't want to accidentally create symlinks elsewhere + if not python_program.startswith(runfiles_root): + raise AssertionError( + "Program's venv binary not under runfiles: {python_program}" + ) + symlink_to = find_binary(runfiles_root, _PYTHON_BINARY_ACTUAL) + os.makedirs(dirname(python_program), exist_ok=True) + if os.path.lexists(python_program): + os.remove(python_program) + try: + os.symlink(symlink_to, python_program) + except OSError as e: + raise Exception( + f"Unable to create venv python interpreter symlink: {python_program} -> {symlink_to}" + ) from e + venv_root = dirname(dirname(python_program)) + pyvenv_cfg = join(venv_root, "pyvenv.cfg") + if not os.path.exists(pyvenv_cfg): + print_verbose("finish_venv_setup: create pyvenv.cfg:", pyvenv_cfg) + python_home = join(runfiles_root, dirname(_PYTHON_BINARY_ACTUAL)) + print_verbose("finish_venv_setup: pyvenv.cfg home:", python_home) + with open(pyvenv_cfg, "w") as fp: + # Until Windows supports a build-time generated venv using symlinks + # to directories, we have to write the full, absolute, path to PYTHONHOME + # so that support directories (e.g. DLLs, libs) can be found. + fp.write("home = {}\n".format(python_home)) + + return python_program def main(): print_verbose("running zip main bootstrap") @@ -287,28 +336,7 @@ def main(): ) if _PYTHON_BINARY_VENV: - python_program = os.path.join(runfiles_root, _PYTHON_BINARY_VENV) - # When a venv is used, the `bin/python3` symlink may need to be created. - # This case occurs when "create venv at runtime" or "resolve python at - # runtime" modes are enabled. - if not os.path.exists(python_program): - # The venv bin/python3 interpreter should always be under runfiles, but - # double check. We don't want to accidentally create symlinks elsewhere - if not python_program.startswith(runfiles_root): - raise AssertionError( - "Program's venv binary not under runfiles: {python_program}" - ) - symlink_to = find_binary(runfiles_root, _PYTHON_BINARY_ACTUAL) - os.makedirs(os.path.dirname(python_program), exist_ok=True) - if os.path.lexists(python_program): - os.remove(python_program) - try: - os.symlink(symlink_to, python_program) - except OSError as e: - raise Exception( - f"Unable to create venv python interpreter symlink: {python_program} -> {symlink_to}" - ) from e - + python_program = finish_venv_setup(runfiles_root) else: python_program = find_binary(runfiles_root, _PYTHON_BINARY_ACTUAL) if python_program is None: diff --git a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh index 21c6741197..baac98c6b5 100755 --- a/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh +++ b/tests/py_zipapp/system_python_zipapp_external_bootstrap_test.sh @@ -15,4 +15,14 @@ ZIPAPP="${ZIPAPP/.exe/.zip}" export RULES_PYTHON_BOOTSTRAP_VERBOSE=1 # We're testing the invocation of `__main__.py`, so we have to # manually pass the zipapp to python. +set +e "$PYTHON" "$ZIPAPP" +exit_code=$? +set +x + +if [[ "$exit_code" -ne 0 ]]; then + echo "===============" + echo "Invocation failed, exit code: $exit_code" + echo "===============" + exit "$exit_code" +fi From 84951231d92c62be511b58e7bfd19e3641a23ffe Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 4 Apr 2026 17:12:06 -0700 Subject: [PATCH 14/28] remove extraneous zip_main logic --- python/private/zipapp/zip_main_template.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index d00745206d..8333bc566c 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -106,25 +106,6 @@ def get_windows_path_with_unc_prefix(path): return unicode_prefix + os.path.abspath(path) -def has_windows_executable_extension(path): - return path.endswith(".exe") or path.endswith(".com") or path.endswith(".bat") - - -if ( - _PYTHON_BINARY_VENV - and is_windows() - and not has_windows_executable_extension(_PYTHON_BINARY_VENV) -): - _PYTHON_BINARY_VENV = _PYTHON_BINARY_VENV + ".exe" - -if ( - _PYTHON_BINARY_ACTUAL - and is_windows() - and not has_windows_executable_extension(_PYTHON_BINARY_ACTUAL) -): - _PYTHON_BINARY_ACTUAL = _PYTHON_BINARY_ACTUAL + ".exe" - - def search_path(name): """Finds a file in a given search path.""" search_path = os.getenv("PATH", os.defpath).split(os.pathsep) @@ -186,7 +167,6 @@ def extract_zip(zip_path, dest_dir): target = f.read() os.remove(file_path) os.symlink(target, file_path) - ##os.chmod(file_path, stat.S_IWRITE) # Of those, we set the lower 12 bits, which are the # file mode bits (since the file type bits can't be set by chmod anyway). elif attrs != 0: # Rumor has it these can be 0 for zips created on Windows. From 98aa09db5eb30211438c3da4f1174afdf31d3fec Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 16:03:31 -0700 Subject: [PATCH 15/28] remove chmod+w, cleanup print_verbose --- python/private/zipapp/zip_main_template.py | 45 +++++++++++----------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index 18fe6bbfbc..bea422d5d2 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -47,27 +47,28 @@ def print_verbose(*args, mapping=None, values=None): - if bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")): - if mapping is not None: - for key, value in sorted((mapping or {}).items()): - print( - "bootstrap: stage 1:", - *args, - f"{key}={value!r}", - file=sys.stderr, - flush=True, - ) - elif values is not None: - for i, v in enumerate(values): - print( - "bootstrap: stage 1:", - *args, - f"[{i}] {v!r}", - file=sys.stderr, - flush=True, - ) - else: - print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True) + if not bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")): + return + if mapping is not None: + for key, value in sorted((mapping or {}).items()): + print( + "bootstrap: stage 1:", + *args, + f"{key}={value!r}", + file=sys.stderr, + flush=True, + ) + elif values is not None: + for i, v in enumerate(values): + print( + "bootstrap: stage 1:", + *args, + f"[{i}] {v!r}", + file=sys.stderr, + flush=True, + ) + else: + print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True) @@ -180,7 +181,7 @@ def extract_zip(zip_path, dest_dir): elif attrs != 0: # Rumor has it these can be 0 for zips created on Windows. # Add the write bit to ensure the files can be deleted during cleanup and # overwritten by subsequent invocations. - os.chmod(file_path, (attrs & 0o7777) | stat.S_IWRITE) + os.chmod(file_path, attrs & 0o7777) # Create the runfiles tree by extracting the zip file From 27088fe506f29d74fded6d63d8f72526615cad41 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 16:22:48 -0700 Subject: [PATCH 16/28] normpath, fix venv.interpreter_runfiles NPE --- python/private/py_executable.bzl | 6 +++--- python/private/zipapp/zip_main_template.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 090a87faf1..6f8a391bba 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -476,7 +476,7 @@ WARNING: Target: {} # The interpreter is added this late in the process so that it isn't # added to the zipped files. - if venv and venv.interpreter: + if venv and venv.interpreter_runfiles: extra_runfiles = extra_runfiles.merge(venv.interpreter_runfiles) return struct( # depset[File] of additional files that should be included as default @@ -494,8 +494,8 @@ WARNING: Target: {} app_runfiles = app_runfiles.build(ctx), # File|None; the venv `bin/python3` file, if any. venv_python_exe = venv.interpreter if venv else None, - # runfiles; runfiles in the venv for the interpreter - venv_interpreter_runfiles = venv.interpreter_runfiles, + # runfiles|None; runfiles in the venv for the interpreter + venv_interpreter_runfiles = venv.interpreter_runfiles if venv else None, ) def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv): diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index bea422d5d2..684f4f5181 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -46,6 +46,21 @@ IS_WINDOWS = os.name == "nt" +EXTRACT_ROOT = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") + +# Change the paths with Unix-style forward slashes to backslashes for windows. +# Windows usually transparently rewrites them, but e.g. `\\?\` paths require +# backslashes to be properly understood by Windows APIs. +if IS_WINDOWS: + from os.path import normpath + _STAGE2_BOOTSTRAP = normpath(_STAGE2_BOOTSTRAP) + if _PYTHON_BINARY_VENV: + _PYTHON_BINARY_VENV = normpath(_PYTHON_BINARY_VENV) + _PYTHON_BINARY_ACTUAL = normpath(_PYTHON_BINARY_ACTUAL) + EXTRACT_DIR = normpath(EXTRACT_DIR) + if EXTRACT_ROOT: + EXTRACT_ROOT = normpath(EXTRACT_ROOT) + def print_verbose(*args, mapping=None, values=None): if not bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")): return From 1c069b9b556e274a0852a4c0bd3d668bcd992283 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 16:41:28 -0700 Subject: [PATCH 17/28] special test configs --- .bazelrc | 3 +++ .bazelrc.specialized_configs | 12 ++++++++++++ tests/toolchains/defs.bzl | 1 + 3 files changed, 16 insertions(+) create mode 100644 .bazelrc.specialized_configs diff --git a/.bazelrc b/.bazelrc index 9dd13e9b2a..89d499795d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -42,6 +42,7 @@ common --enable_bzlmod # Local disk cache greatly speeds up builds if the regular cache is lost common --disk_cache=~/.cache/bazel/bazel-disk-cache + # Additional config to use for readthedocs builds. # See .readthedocs.yml for additional flags that can only be determined from # the runtime environment. @@ -54,4 +55,6 @@ common --incompatible_no_implicit_file_export build --lockfile_mode=update +import %workspace%/.bazelrc.specialized_configs + try-import user.bazelrc diff --git a/.bazelrc.specialized_configs b/.bazelrc.specialized_configs new file mode 100644 index 0000000000..46334e0a95 --- /dev/null +++ b/.bazelrc.specialized_configs @@ -0,0 +1,12 @@ + +# Helper config to run most tests without waiting an inordinate amount +# of time or freezing the system +common:fast-tests --build_tests_only=true +common:fast-tests --build_tag_filters=-large,-enormous,-integration-test +common:fast-tests --test_tag_filters=-large,-enormous,-integration-test + +# Helper config for running a single test locally and investigating resulting state +common:testone --test_output=streamed +common:testone --test_strategy=standalone +common:testone --spawn_strategy=standalone +common:testone --strategy=standalone diff --git a/tests/toolchains/defs.bzl b/tests/toolchains/defs.bzl index 25863d18c4..fb4b3beb94 100644 --- a/tests/toolchains/defs.bzl +++ b/tests/toolchains/defs.bzl @@ -57,4 +57,5 @@ def define_toolchain_tests(name): deps = ["//python/runfiles"], data = ["//tests/support:current_build_settings"], target_compatible_with = select(target_compatible_with), + size = "large", ) From fc800e80f67936d83e3d7a6e5acd7fc83244ba04 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 17:11:38 -0700 Subject: [PATCH 18/28] fix zipapp bootstrap for windows --- python/private/zipapp/py_zipapp_rule.bzl | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index b5578918d1..fd9c682f67 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl @@ -222,18 +222,23 @@ def _py_zipapp_executable_impl(ctx): if target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints): executable = ctx.actions.declare_file(ctx.label.name + ".exe") - python_exe = py_executable.venv_python_exe - if python_exe: - python_exe_path = runfiles_root_path(ctx, python_exe.short_path) - elif py_runtime.interpreter: - python_exe_path = runfiles_root_path(ctx, py_runtime.interpreter.short_path) + # The zipapp is an opaque zip file, so the Bazel Python launcher doesn't + # know how to look inside it to find the Python interpreter. This means + # we can only use system paths or programs on PATH to bootstrap. + if py_runtime.interpreter_path: + bootstrap_python_path = py_runtime.interpreter_path else: - python_exe_path = py_runtime.interpreter_path + # A special value the Bazel Python launcher recognized to skip + # lookup in the runfiles and uses `python.exe` from PATH. + bootstrap_python_path = "python" create_windows_exe_launcher( ctx, output = executable, - python_binary_path = python_exe_path, + # The path to a python to use to invoke e.g. `python.exe foo.zip` + python_binary_path = bootstrap_python_path, + # Tell the launcher to invoke `python_binary_path` on itself + # after removing its file extension and appending `.zip`. use_zip_file = True, ) default_outputs = [executable, zip_file] From d087713e33c5a5317b26cd67555d9d46d00fef61 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 19:38:30 -0700 Subject: [PATCH 19/28] cleanup python_bootstrap_template.txt a bit --- python/private/python_bootstrap_template.txt | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 3ae9995dc1..41d7054d63 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -5,13 +5,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import sys +# Generated file from @rules_python//python/private:python_bootstrap_template.txt -import os from os.path import dirname, join, basename -import subprocess -import uuid +import os import shutil +import subprocess +import sys # NOTE: The sentinel strings are split (e.g., "%stage2" + "_bootstrap%") so that # the substitution logic won't replace them. This allows runtime detection of @@ -77,10 +77,7 @@ else: ADDITIONAL_INTERPRETER_ARGS = os.environ.get("RULES_PYTHON_ADDITIONAL_INTERPRETER_ARGS", "") EXTRACT_ROOT = os.environ.get("RULES_PYTHON_EXTRACT_ROOT") -def is_running_from_zip(): - return IS_ZIPFILE - -if is_running_from_zip(): +if IS_ZIPFILE: import shutil import tempfile import zipfile @@ -397,7 +394,7 @@ def runfiles_envvar(runfiles_root): return ('RUNFILES_DIR', runfiles) # If running from a zip, there's no manifest file. - if is_running_from_zip(): + if IS_ZIPFILE: return ('RUNFILES_DIR', runfiles_root) # Look for the runfiles "output" manifest, argv[0] + ".runfiles_manifest" @@ -531,7 +528,7 @@ def main(): delete_dirs = [] - if is_running_from_zip(): + if IS_ZIPFILE: runfiles_root = create_runfiles_root() # NOTE: dirname() is called because create_runfiles_root() creates a # sub-directory within a temporary directory, and we want to remove the @@ -584,7 +581,7 @@ def main(): new_env.update((key, val) for key, val in os.environ.items() if key not in new_env) workspace = None - if is_running_from_zip(): + if IS_ZIPFILE: # If RUN_UNDER_RUNFILES equals 1, it means we need to # change directory to the right runfiles directory. # (So that the data files are accessible) From e0722d00fad1dbca8ec31515ba33f68a1c823a2e Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 19:51:45 -0700 Subject: [PATCH 20/28] fix repl test --- tests/repl/repl_test.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/repl/repl_test.py b/tests/repl/repl_test.py index 01d0442922..5b333c0739 100644 --- a/tests/repl/repl_test.py +++ b/tests/repl/repl_test.py @@ -21,24 +21,53 @@ foo = 1234 """ +IS_WINDOWS = os.name == "nt" class ReplTest(unittest.TestCase): def setUp(self): - self.repl = rfiles.Rlocation("rules_python/python/bin/repl") + rpath = "rules_python/python/bin/repl" + if IS_WINDOWS: + rpath += ".exe" + self.repl = rfiles.Rlocation(rpath) assert self.repl + if IS_WINDOWS: + self.repl = os.path.normpath(self.repl) def run_code_in_repl(self, lines: Iterable[str], *, env=None) -> str: """Runs the lines of code in the REPL and returns the text output.""" + input="\n".join(lines) try: return subprocess.check_output( [self.repl], text=True, stderr=subprocess.STDOUT, - input="\n".join(lines), + input=input, env=env, ).strip() except subprocess.CalledProcessError as error: raise RuntimeError(f"Failed to run the REPL:\n{error.stdout}") from error + except Exception as exc: + if env: + env_str = "\n".join(f"{key}={value!r}" for key, value in sorted(env.items())) + else: + env_str = "" + if isinstance(exc, subprocess.CalledProcessError): + stdout = exc.stdout + else: + stdout = "" + exc.add_note(f""" +===== env start ===== +{env_str} +===== env end ===== +===== input start ===== +{input} +===== input end ===== +commmand: {self.repl} +===== stdout start ===== +{stdout} +===== stdout end ===== +""") + raise def test_repl_version(self): """Validates that we can successfully execute arbitrary code on the REPL.""" From 78c62b2a81b052b6610f63c6ff8fed2afcc75d9c Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 19:55:49 -0700 Subject: [PATCH 21/28] doc missing arg --- python/private/py_executable.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 6f8a391bba..89e2604526 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -1775,6 +1775,8 @@ def _create_providers( stage2_bootstrap: File; the stage 2 bootstrap script. app_runfiles: runfiles; the runfiles for the application (deps, etc). venv_python_exe: File; the python executable in the venv. + venv_interpreter_runfiles: runfiles; runfiles specific to the interpreter + for the venv. interpreter_args: list of strings; arguments to pass to the interpreter. Returns: From bc9d1212e3e77a4e40524af4f6e971c4f1b5fd65 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 19:59:21 -0700 Subject: [PATCH 22/28] format code --- python/private/py_executable_info.bzl | 16 ++++++++-------- python/private/zipapp/zip_main_template.py | 7 ++++--- tests/repl/repl_test.py | 13 +++++++++---- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/python/private/py_executable_info.bzl b/python/private/py_executable_info.bzl index 4ebc2d4884..c2ce6c6092 100644 --- a/python/private/py_executable_info.bzl +++ b/python/private/py_executable_info.bzl @@ -77,6 +77,14 @@ implementation isn't being used. :::{versionadded} 1.9.0 ::: +""", + "venv_interpreter_runfiles": """ +:type: runfiles | None + +Runfiles that are specific to the interpreter within the venv. + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, "venv_python_exe": """ :type: File | None @@ -87,13 +95,5 @@ mode is not enabled. :::{versionadded} 1.9.0 ::: """, - "venv_interpreter_runfiles": """ -:type: runfiles | None - -Runfiles that are specific to the interpreter within the venv. - -:::{versionadded} VERSION_NEXT_FEATURE -::: -""" }, ) diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index 684f4f5181..d59c4bb4c8 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -22,13 +22,12 @@ del sys.path[0] import os -from os.path import dirname, join import shutil import stat import subprocess import tempfile import zipfile -from os.path import dirname, join, basename +from os.path import basename, dirname, join # runfiles-root-relative path _STAGE2_BOOTSTRAP = "%stage2_bootstrap%" @@ -53,6 +52,7 @@ # backslashes to be properly understood by Windows APIs. if IS_WINDOWS: from os.path import normpath + _STAGE2_BOOTSTRAP = normpath(_STAGE2_BOOTSTRAP) if _PYTHON_BINARY_VENV: _PYTHON_BINARY_VENV = normpath(_PYTHON_BINARY_VENV) @@ -61,6 +61,7 @@ if EXTRACT_ROOT: EXTRACT_ROOT = normpath(EXTRACT_ROOT) + def print_verbose(*args, mapping=None, values=None): if not bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")): return @@ -86,7 +87,6 @@ def print_verbose(*args, mapping=None, values=None): print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True) - def get_windows_path_with_unc_prefix(path): """Adds UNC prefix after getting a normalized absolute Windows path. @@ -308,6 +308,7 @@ def finish_venv_setup(runfiles_root): return python_program + def main(): print_verbose("running zip main bootstrap") print_verbose("initial argv:", values=sys.argv) diff --git a/tests/repl/repl_test.py b/tests/repl/repl_test.py index 5b333c0739..319dab561a 100644 --- a/tests/repl/repl_test.py +++ b/tests/repl/repl_test.py @@ -23,6 +23,7 @@ IS_WINDOWS = os.name == "nt" + class ReplTest(unittest.TestCase): def setUp(self): rpath = "rules_python/python/bin/repl" @@ -35,7 +36,7 @@ def setUp(self): def run_code_in_repl(self, lines: Iterable[str], *, env=None) -> str: """Runs the lines of code in the REPL and returns the text output.""" - input="\n".join(lines) + input = "\n".join(lines) try: return subprocess.check_output( [self.repl], @@ -48,14 +49,17 @@ def run_code_in_repl(self, lines: Iterable[str], *, env=None) -> str: raise RuntimeError(f"Failed to run the REPL:\n{error.stdout}") from error except Exception as exc: if env: - env_str = "\n".join(f"{key}={value!r}" for key, value in sorted(env.items())) + env_str = "\n".join( + f"{key}={value!r}" for key, value in sorted(env.items()) + ) else: env_str = "" if isinstance(exc, subprocess.CalledProcessError): stdout = exc.stdout else: stdout = "" - exc.add_note(f""" + exc.add_note( + f""" ===== env start ===== {env_str} ===== env end ===== @@ -66,7 +70,8 @@ def run_code_in_repl(self, lines: Iterable[str], *, env=None) -> str: ===== stdout start ===== {stdout} ===== stdout end ===== -""") +""" + ) raise def test_repl_version(self): From 887ebfb61275675c53505d149b471dc18ad668e2 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Apr 2026 20:18:17 -0700 Subject: [PATCH 23/28] fix some windows 10 settings for local_toolchains --- tests/integration/local_toolchains/.bazelrc | 2 ++ tests/integration/local_toolchains/MODULE.bazel | 4 +++- tests/integration/local_toolchains/pbs_archive.bzl | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/local_toolchains/.bazelrc b/tests/integration/local_toolchains/.bazelrc index aed08b0790..0fbb7678d1 100644 --- a/tests/integration/local_toolchains/.bazelrc +++ b/tests/integration/local_toolchains/.bazelrc @@ -1,4 +1,6 @@ +startup --windows_enable_symlinks common --action_env=RULES_PYTHON_BZLMOD_DEBUG=1 +common --repo_env=RULES_PYTHON_BZLMOD_DEBUG=1 common --lockfile_mode=off test --test_output=errors # Windows requires these for multi-python support: diff --git a/tests/integration/local_toolchains/MODULE.bazel b/tests/integration/local_toolchains/MODULE.bazel index c818942748..fe90fa235a 100644 --- a/tests/integration/local_toolchains/MODULE.bazel +++ b/tests/integration/local_toolchains/MODULE.bazel @@ -37,6 +37,8 @@ local_runtime_repo( pbs_archive = use_repo_rule("//:pbs_archive.bzl", "pbs_archive") +# "pbs" means "python-build-standalone" +# This maps the different platform runtimes to URLS and SHAs pbs_archive( name = "pbs_runtime", sha256 = { @@ -47,7 +49,7 @@ pbs_archive( urls = { "linux": "https://github.com/astral-sh/python-build-standalone/releases/download/20250918/cpython-3.13.7+20250918-x86_64-unknown-linux-gnu-install_only.tar.gz", "mac os x": "https://github.com/astral-sh/python-build-standalone/releases/download/20250918/cpython-3.13.7+20250918-aarch64-apple-darwin-install_only.tar.gz", - "windows server 2022": "https://github.com/astral-sh/python-build-standalone/releases/download/20250918/cpython-3.13.7+20250918-x86_64-pc-windows-msvc-install_only.tar.gz", + "windows": "https://github.com/astral-sh/python-build-standalone/releases/download/20250918/cpython-3.13.7+20250918-x86_64-pc-windows-msvc-install_only.tar.gz", }, ) diff --git a/tests/integration/local_toolchains/pbs_archive.bzl b/tests/integration/local_toolchains/pbs_archive.bzl index 8bd0c1eb10..7d817b1b40 100644 --- a/tests/integration/local_toolchains/pbs_archive.bzl +++ b/tests/integration/local_toolchains/pbs_archive.bzl @@ -16,6 +16,10 @@ def _pbs_archive_impl(repository_ctx): urls = repository_ctx.attr.urls sha256s = repository_ctx.attr.sha256 + # os.name for windows contain build and version; simplify it + if "windows" in os_name: + os_name = "windows" + if os_name not in urls: fail("Unsupported OS: '{}'. Available OSs are: {}".format( os_name, From 085bc24c16e9b18d7ecbab04922f7a6344dcaf9f Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 6 Apr 2026 00:30:56 -0700 Subject: [PATCH 24/28] fix local toolchain --- python/private/hermetic_runtime_repo_setup.bzl | 5 ++++- python/private/local_runtime_repo.bzl | 12 ++++++------ python/private/local_runtime_repo_setup.bzl | 10 ++++++++++ python/private/py_executable.bzl | 11 ++++++----- python/private/repo_utils.bzl | 11 +++++++++-- 5 files changed, 35 insertions(+), 14 deletions(-) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 538ac6f37b..724c7aa3e1 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -243,7 +243,10 @@ def define_hermetic_runtime_toolchain_impl( # On Windows, a symlink-style venv requires supporting .dll files. venv_bin_files = select({ "@platforms//os:windows": native.glob( - include = ["*.dll", "*.pdb"], + include = ["*.dll"], + allow_empty = False, + ) + native.glob( + include = ["*.pdb"], allow_empty = True, ), "//conditions:default": [], diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 6e39152c89..37b7d2b130 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -74,6 +74,12 @@ def _norm_path(path): def _symlink_libraries(rctx, logger, libraries, shlib_suffix): """Symlinks the shared libraries into the lib/ directory. + Individual files are symlinked instead of the whole directory because + shared_lib_dirs contains multiple search paths for the shared libraries, + and the python files may be missing from any of those directories, and + any of those directories may include non-python runtime libraries, + as would be the case if LIBDIR were, for example, /usr/lib. + Args: rctx: A repository_ctx object logger: A repo_utils.logger object @@ -81,12 +87,6 @@ def _symlink_libraries(rctx, logger, libraries, shlib_suffix): shlib_suffix: Optional. Ensure that the generated symlinks end with this suffix. Returns: A list of library paths (under lib/) linked by the action. - - Individual files are symlinked instead of the whole directory because - shared_lib_dirs contains multiple search paths for the shared libraries, - and the python files may be missing from any of those directories, and - any of those directories may include non-python runtime libraries, - as would be the case if LIBDIR were, for example, /usr/lib. """ result = [] for source in libraries: diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 0922181ffe..2af9320e88 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -153,6 +153,16 @@ def define_local_runtime_toolchain_impl( implementation_name = implementation_name, abi_flags = abi_flags, pyc_tag = "{}-{}{}{}".format(implementation_name, major, minor, abi_flags), + venv_bin_files = select({ + "@platforms//os:windows": native.glob( + include = ["lib/*.dll"], + allow_empty = False, + ) + native.glob( + include = ["lib/*.pdb"], + allow_empty=True, + ), + "//conditions:default": [], + }), ) py_runtime_pair( diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 89e2604526..3bff0d2224 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -723,16 +723,17 @@ def _create_venv_windows(ctx, *, venv_root, runtime, interpreter_actual_path): interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename)) interpreter_runfiles.add(interpreter) ctx.actions.symlink(output = interpreter, target_file = runtime.interpreter) - for f in runtime.venv_bin_files: - venv_path = "{}/{}".format(bin_dir, f.basename) - venv_file = ctx.actions.declare_file(venv_path) - ctx.actions.symlink(output = venv_file, target_file = f) - interpreter_runfiles.add(venv_file) else: interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) interpreter_runfiles.add(interpreter) ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path) + for f in runtime.venv_bin_files: + venv_path = "{}/{}".format(bin_dir, f.basename) + venv_file = ctx.actions.declare_file(venv_path) + ctx.actions.symlink(output = venv_file, target_file = f) + interpreter_runfiles.add(venv_file) + # See site.py logic: Windows uses a version/build agnostic site-packages path venv_site_packages = "Lib/site-packages" diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl index 00f43f3521..234e3e5ba8 100644 --- a/python/private/repo_utils.bzl +++ b/python/private/repo_utils.bzl @@ -311,12 +311,19 @@ def _which_unchecked(mrctx, binary_name): ) def _which_describe_failure(binary_name, path): + if "\\" in path or ";" in path: + path_parts = path.split(";") + else: + path_parts = path.split(":") + for i, v in enumerate(path_parts): + path_parts[i] = " [{}]: {}".format(i, v) return ( "Unable to find the binary '{binary_name}' on PATH.\n" + - " PATH = {path}" + " PATH entries:\n" + + "{path_str}" ).format( binary_name = binary_name, - path = path, + path_str = "\n".join(path_parts) ) def _mkdir(mrctx, path): From 132174a5c516af0537b79766ed9d21259db2d908 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 6 Apr 2026 00:46:16 -0700 Subject: [PATCH 25/28] fix glob for windows+linux --- python/private/hermetic_runtime_repo_setup.bzl | 11 +++++++---- python/private/local_runtime_repo_setup.bzl | 13 ++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 724c7aa3e1..78ce4fd0a2 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -243,10 +243,13 @@ def define_hermetic_runtime_toolchain_impl( # On Windows, a symlink-style venv requires supporting .dll files. venv_bin_files = select({ "@platforms//os:windows": native.glob( - include = ["*.dll"], - allow_empty = False, - ) + native.glob( - include = ["*.pdb"], + include = [ + "*.dll", + # The pdb files just provide debugging information + "*.pdb", + ], + # This must be true because glob empty-ness is checked + # during loading phase, before select() filters it out. allow_empty = True, ), "//conditions:default": [], diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 2af9320e88..8f4dedc029 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -155,11 +155,14 @@ def define_local_runtime_toolchain_impl( pyc_tag = "{}-{}{}{}".format(implementation_name, major, minor, abi_flags), venv_bin_files = select({ "@platforms//os:windows": native.glob( - include = ["lib/*.dll"], - allow_empty = False, - ) + native.glob( - include = ["lib/*.pdb"], - allow_empty=True, + include = [ + "*.dll", + # The pdb files just provide debugging information + "*.pdb", + ], + # This must be true because glob empty-ness is checked + # during loading phase, before select() filters it out. + allow_empty = True, ), "//conditions:default": [], }), From 5c51f06f330b10cf90d501b31c392e558aaf67c9 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 6 Apr 2026 00:46:57 -0700 Subject: [PATCH 26/28] format --- python/private/repo_utils.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl index 234e3e5ba8..a558fa08e1 100644 --- a/python/private/repo_utils.bzl +++ b/python/private/repo_utils.bzl @@ -323,7 +323,7 @@ def _which_describe_failure(binary_name, path): "{path_str}" ).format( binary_name = binary_name, - path_str = "\n".join(path_parts) + path_str = "\n".join(path_parts), ) def _mkdir(mrctx, path): From 399a462cb42961f72f96236dc9f8dc0c42d28749 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 6 Apr 2026 00:51:07 -0700 Subject: [PATCH 27/28] remove defunct comment --- python/private/zipapp/zip_main_template.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/private/zipapp/zip_main_template.py b/python/private/zipapp/zip_main_template.py index d59c4bb4c8..df2c01cb82 100644 --- a/python/private/zipapp/zip_main_template.py +++ b/python/private/zipapp/zip_main_template.py @@ -194,8 +194,6 @@ def extract_zip(zip_path, dest_dir): # Of those, we set the lower 12 bits, which are the # file mode bits (since the file type bits can't be set by chmod anyway). elif attrs != 0: # Rumor has it these can be 0 for zips created on Windows. - # Add the write bit to ensure the files can be deleted during cleanup and - # overwritten by subsequent invocations. os.chmod(file_path, attrs & 0o7777) From d771b76dfbf43a72c80c6b0e27dd0c37b99ec306 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 6 Apr 2026 01:09:18 -0700 Subject: [PATCH 28/28] fix dll include again --- python/private/local_runtime_repo_setup.bzl | 4 ++-- python/private/py_executable.bzl | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl index 8f4dedc029..5cb7bda200 100644 --- a/python/private/local_runtime_repo_setup.bzl +++ b/python/private/local_runtime_repo_setup.bzl @@ -156,9 +156,9 @@ def define_local_runtime_toolchain_impl( venv_bin_files = select({ "@platforms//os:windows": native.glob( include = [ - "*.dll", + "lib/*.dll", # The pdb files just provide debugging information - "*.pdb", + "lib/*.pdb", ], # This must be true because glob empty-ness is checked # during loading phase, before select() filters it out. diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 3bff0d2224..739990ee7d 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -728,6 +728,8 @@ def _create_venv_windows(ctx, *, venv_root, runtime, interpreter_actual_path): interpreter_runfiles.add(interpreter) ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path) + # NOTE: The .dll files must exist, however, they may not be known at build time + # if the interpreter is resolved at runtime. for f in runtime.venv_bin_files: venv_path = "{}/{}".format(bin_dir, f.basename) venv_file = ctx.actions.declare_file(venv_path)