diff --git a/.bazelrc b/.bazelrc index 24676574e6..2545f30f80 100644 --- a/.bazelrc +++ b/.bazelrc @@ -41,3 +41,7 @@ common --incompatible_python_disallow_native_rules common --incompatible_no_implicit_file_export build --lockfile_mode=update + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false diff --git a/examples/build_file_generation/.bazelrc b/examples/build_file_generation/.bazelrc index f1ae44fac8..e14ed7c9f6 100644 --- a/examples/build_file_generation/.bazelrc +++ b/examples/build_file_generation/.bazelrc @@ -8,3 +8,7 @@ build --enable_runfiles common --noenable_bzlmod common --enable_workspace common --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc index 28a44a7523..a18c7e2551 100644 --- a/examples/bzlmod/.bazelrc +++ b/examples/bzlmod/.bazelrc @@ -24,3 +24,7 @@ test --test_output=errors --enable_runfiles # Windows requires these for multi-python support: build --enable_runfiles common:bazel7.x --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/examples/multi_python_versions/.bazelrc b/examples/multi_python_versions/.bazelrc index 97a973bd85..39ed55aff4 100644 --- a/examples/multi_python_versions/.bazelrc +++ b/examples/multi_python_versions/.bazelrc @@ -5,3 +5,7 @@ build --enable_runfiles coverage --java_runtime_version=remotejdk_11 common:bazel7.x --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/examples/pip_parse/.bazelrc b/examples/pip_parse/.bazelrc index f263a1744d..1bdae98af6 100644 --- a/examples/pip_parse/.bazelrc +++ b/examples/pip_parse/.bazelrc @@ -1,3 +1,7 @@ # https://docs.bazel.build/versions/main/best-practices.html#using-the-bazelrc-file try-import %workspace%/user.bazelrc common --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/examples/pip_parse_vendored/.bazelrc b/examples/pip_parse_vendored/.bazelrc index a6ea2d9138..e846b1ac11 100644 --- a/examples/pip_parse_vendored/.bazelrc +++ b/examples/pip_parse_vendored/.bazelrc @@ -8,3 +8,7 @@ build --enable_runfiles common --noenable_bzlmod common --enable_workspace common --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/examples/pip_repository_annotations/.bazelrc b/examples/pip_repository_annotations/.bazelrc index 9397bd31b8..82c7a1dbd2 100644 --- a/examples/pip_repository_annotations/.bazelrc +++ b/examples/pip_repository_annotations/.bazelrc @@ -7,3 +7,7 @@ common --noenable_bzlmod common --enable_workspace common --legacy_external_runfiles=false common --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/gazelle/.bazelrc b/gazelle/.bazelrc index 9a38133e9d..0f46b5b584 100644 --- a/gazelle/.bazelrc +++ b/gazelle/.bazelrc @@ -16,3 +16,7 @@ build --@rules_python//python/config_settings:incompatible_default_to_explicit_i build --enable_runfiles common:bazel7.x --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/gazelle/examples/bzlmod_build_file_generation/.bazelrc b/gazelle/examples/bzlmod_build_file_generation/.bazelrc index 31097b41de..5ab484ee67 100644 --- a/gazelle/examples/bzlmod_build_file_generation/.bazelrc +++ b/gazelle/examples/bzlmod_build_file_generation/.bazelrc @@ -14,3 +14,7 @@ common:bazel7.x --incompatible_python_disallow_native_rules # rules_python code. In the BCR presubmits, this override is removed # and the bazel_dep version of rules_python is used. common --override_module=rules_python=../../../ + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/python/private/build_data_writer.ps1 b/python/private/build_data_writer.ps1 index db7a48e676..652f00057a 100644 --- a/python/private/build_data_writer.ps1 +++ b/python/private/build_data_writer.ps1 @@ -1,17 +1,24 @@ $OutputPath = $env:OUTPUT - -Add-Content -Path $OutputPath -Value "TARGET $env:TARGET" -Add-Content -Path $OutputPath -Value "CONFIG_MODE $env:CONFIG_MODE" -Add-Content -Path $OutputPath -Value "STAMPED $env:STAMPED" +$Lines = @( + "TARGET $env:TARGET", + "CONFIG_MODE $env:CONFIG_MODE", + "STAMPED $env:STAMPED" +) $VersionFilePath = $env:VERSION_FILE -if (-not [string]::IsNullOrEmpty($VersionFilePath)) { - Get-Content -Path $VersionFilePath | Add-Content -Path $OutputPath +if (-not [string]::IsNullOrEmpty($VersionFilePath) -and (Test-Path $VersionFilePath)) { + $Lines += Get-Content -Path $VersionFilePath } $InfoFilePath = $env:INFO_FILE -if (-not [string]::IsNullOrEmpty($InfoFilePath)) { - Get-Content -Path $InfoFilePath | Add-Content -Path $OutputPath +if (-not [string]::IsNullOrEmpty($InfoFilePath) -and (Test-Path $InfoFilePath)) { + $Lines += Get-Content -Path $InfoFilePath } +# Use .NET to write file to avoid PowerShell encoding/locking quirks +# We use UTF8 without BOM for compatibility with how the bash script writes (and +# what consumers expect). +$Utf8NoBom = New-Object System.Text.UTF8Encoding $False +[System.IO.File]::WriteAllLines($OutputPath, $Lines, $Utf8NoBom) + exit 0 diff --git a/python/private/py_console_script_gen.bzl b/python/private/py_console_script_gen.bzl index de016036b2..528ef8c4fb 100644 --- a/python/private/py_console_script_gen.bzl +++ b/python/private/py_console_script_gen.bzl @@ -55,6 +55,9 @@ def _py_console_script_gen_impl(ctx): mnemonic = "PyConsoleScriptBinaryGen", progress_message = "Generating py_console_script_binary main: %{label}", executable = ctx.executable._tool, + env = { + "RULES_PYTHON_BOOTSTRAP_VERBOSE": "1", + }, ) return [DefaultInfo( diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index f2d5a42fda..3b065c6ea5 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -381,6 +381,13 @@ def Main(): print_verbose("initial cwd:", os.getcwd()) print_verbose("initial environ:", mapping=os.environ) print_verbose("initial sys.path:", values=sys.path) + print_verbose("STAGE2_BOOTSTRAP:", STAGE2_BOOTSTRAP) + print_verbose("PYTHON_BINARY:", PYTHON_BINARY) + print_verbose("PYTHON_BINARY_ACTUAL:", PYTHON_BINARY_ACTUAL) + print_verbose("IS_ZIPFILE:", IS_ZIPFILE) + print_verbose("RECREATE_VENV_AT_RUNTIME:", RECREATE_VENV_AT_RUNTIME) + print_verbose("WORKSPACE_NAME :", WORKSPACE_NAME ) + args = sys.argv[1:] new_env = {} @@ -391,6 +398,7 @@ def Main(): # matters if `_main` doesn't exist (which can occur if a binary # is packaged and needs no artifacts from the main repo) main_rel_path = os.path.normpath(STAGE2_BOOTSTRAP) + print_verbose("main_rel_path:", main_rel_path) if IsRunningFromZip(): module_space = CreateModuleSpace() @@ -399,6 +407,8 @@ def Main(): module_space = FindModuleSpace(main_rel_path) delete_module_space = False + print_verbose("runfiles root:", module_space) + if os.environ.get("RULES_PYTHON_TESTING_TELL_MODULE_SPACE"): new_env["RULES_PYTHON_TESTING_MODULE_SPACE"] = module_space @@ -412,6 +422,20 @@ def Main(): main_filename = os.path.join(module_space, main_rel_path) main_filename = GetWindowsPathWithUNCPrefix(main_filename) + if not os.path.exists(main_filename): + import pathlib + + manifest_path = os.environ["RUNFILES_MANIFEST_FILE"] + manifest = pathlib.Path(manifest_path) + manifest_text = manifest.read_text() + print_verbose("RUNFILES MANIFEST:\n", manifest_text) + + rf = pathlib.Path(module_space) + lines = [] + for x in rf.glob("**"): + lines.append(str(x)) + lines = "\n".join(lines) + raise Exception(f"{main_filename} not found, lines:\n{lines}") assert os.path.exists(main_filename), \ 'Cannot exec() %r: file not found.' % main_filename assert os.access(main_filename, os.R_OK), \ diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py index 959e631ad1..4eecb5b174 100644 --- a/python/private/stage2_bootstrap_template.py +++ b/python/private/stage2_bootstrap_template.py @@ -48,6 +48,7 @@ COVERAGE_INSTRUMENTED = "%coverage_instrumented%" == "1" # runfiles-root-relative path to a file with binary-specific build information +# It uses forward slashes, so must be converted for proper usage on Windows. BUILD_DATA_FILE = "%build_data_file%" # ===== Template substitutions end ===== @@ -64,9 +65,17 @@ def get_build_data(self): import runfiles except ImportError: from python.runfiles import runfiles - path = runfiles.Create().Rlocation(self.BUILD_DATA_FILE) - with open(path) as fp: - return fp.read() + rlocation_path = self.BUILD_DATA_FILE + if is_windows(): + rlocation_path = rlocation_path.replace("/", "\\") + path = runfiles.Create().Rlocation(rlocation_path) + try: + with open(path) as fp: + return fp.read() + except Exception as exc: + if hasattr(exc, "add_note"): + exc.add_note(f"runfiles lookup path: {rlocation_path}") + raise sys.modules["bazel_binary_info"] = BazelBinaryInfoModule("bazel_binary_info") diff --git a/tests/build_data/BUILD.bazel b/tests/build_data/BUILD.bazel index 64db005f51..599e04b875 100644 --- a/tests/build_data/BUILD.bazel +++ b/tests/build_data/BUILD.bazel @@ -21,5 +21,6 @@ genrule( name = "tool_build_data", outs = ["tool_build_data.txt"], cmd = "$(location :print_build_data) > $(OUTS)", + tags = ["manual"], tools = [":print_build_data"], ) diff --git a/tests/integration/compile_pip_requirements/.bazelrc b/tests/integration/compile_pip_requirements/.bazelrc index b85f03bcb6..be2ceedf66 100644 --- a/tests/integration/compile_pip_requirements/.bazelrc +++ b/tests/integration/compile_pip_requirements/.bazelrc @@ -3,3 +3,7 @@ test --test_output=errors # Windows requires these for multi-python support: build --enable_runfiles common:bazel7.x --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/tests/integration/compile_pip_requirements_test_from_external_repo/.bazelrc b/tests/integration/compile_pip_requirements_test_from_external_repo/.bazelrc index ab10c8caf7..80d2b0d97d 100644 --- a/tests/integration/compile_pip_requirements_test_from_external_repo/.bazelrc +++ b/tests/integration/compile_pip_requirements_test_from_external_repo/.bazelrc @@ -1,2 +1,6 @@ test --test_output=errors common:bazel7.x --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/tests/integration/local_toolchains/.bazelrc b/tests/integration/local_toolchains/.bazelrc index aed08b0790..6a920b180c 100644 --- a/tests/integration/local_toolchains/.bazelrc +++ b/tests/integration/local_toolchains/.bazelrc @@ -6,3 +6,7 @@ build --enable_runfiles common:bazel7.x --incompatible_python_disallow_native_rules build --//:py=local common --announce_rc + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/tests/integration/pip_parse/.bazelrc b/tests/integration/pip_parse/.bazelrc index a74909297d..3d3d8bd67f 100644 --- a/tests/integration/pip_parse/.bazelrc +++ b/tests/integration/pip_parse/.bazelrc @@ -6,3 +6,7 @@ build --enable_runfiles try-import %workspace%/user.bazelrc common:bazel7.x --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/tests/integration/py_cc_toolchain_registered/.bazelrc b/tests/integration/py_cc_toolchain_registered/.bazelrc index fb31561892..e5a95490e6 100644 --- a/tests/integration/py_cc_toolchain_registered/.bazelrc +++ b/tests/integration/py_cc_toolchain_registered/.bazelrc @@ -1,3 +1,7 @@ # This aids debugging on failure build --toolchain_resolution_debug=python common:bazel7.x --incompatible_python_disallow_native_rules + +# See issue 3567. Disable implicit python zip creation. +common --build_python_zip=false +common --@rules_python//python/config_settings:build_python_zip=false \ No newline at end of file diff --git a/tests/pypi/whl_installer/wheel_installer_test.py b/tests/pypi/whl_installer/wheel_installer_test.py index 7040b0cfd8..91adddf15a 100644 --- a/tests/pypi/whl_installer/wheel_installer_test.py +++ b/tests/pypi/whl_installer/wheel_installer_test.py @@ -63,7 +63,9 @@ def setUp(self) -> None: shutil.copy(os.path.join("examples", "wheel", self.wheel_name), self.wheel_dir) def tearDown(self): - shutil.rmtree(self.wheel_dir) + # On windows, the wheel file remains open, so gives an error upon + # deletion for some reason. + shutil.rmtree(self.wheel_dir, ignore_errors=True) def test_wheel_exists(self) -> None: wheel_installer._extract_wheel(