From e9388a57a9ebb5e5db5d99c7432654a18b0a3c59 Mon Sep 17 00:00:00 2001 From: Chongkai Zhu Date: Sun, 14 Dec 2025 10:40:21 +0800 Subject: [PATCH 1/2] Make the Unix build truly relocatable as CPython patch --- cpython-unix/build-cpython.sh | 84 +++++++++++-------- ...h-cpython-relocatable-sysconfig-3.10.patch | 55 ++++++++++++ ...h-cpython-relocatable-sysconfig-3.13.patch | 42 ++++++++++ 3 files changed, 145 insertions(+), 36 deletions(-) create mode 100644 cpython-unix/patch-cpython-relocatable-sysconfig-3.10.patch create mode 100644 cpython-unix/patch-cpython-relocatable-sysconfig-3.13.patch diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 2afeb134..d8612c89 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -174,6 +174,12 @@ else patch -p1 -i ${ROOT}/patch-ctypes-callproc-legacy.patch fi +if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then + patch -p1 -i ${ROOT}/patch-cpython-relocatable-sysconfig-3.13.patch +elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_10}" ]; then + patch -p1 -i ${ROOT}/patch-cpython-relocatable-sysconfig-3.10.patch +fi + # On Windows, CPython looks for the Tcl/Tk libraries relative to the base prefix, # which we want. But on Unix, it doesn't. This patch applies similar behavior on Unix, # thereby ensuring that the Tcl/Tk libraries are found in the correct location. @@ -858,6 +864,7 @@ fi # that a) it works on as many machines as possible b) doesn't leak details # about the build environment, which is non-portable. cat > ${ROOT}/hack_sysconfig.py << EOF +import ast import json import os import sys @@ -869,7 +876,7 @@ FREETHREADED = sysconfig.get_config_var("Py_GIL_DISABLED") MAJMIN = ".".join([str(sys.version_info[0]), str(sys.version_info[1])]) LIB_SUFFIX = "t" if FREETHREADED else "" PYTHON_CONFIG = os.path.join(ROOT, "install", "bin", "python%s-config" % MAJMIN) -PLATFORM_CONFIG = os.path.join(ROOT, sysconfig.get_config_var("LIBPL").lstrip("/")) +PLATFORM_CONFIG = sysconfig.get_config_var("LIBPL") MAKEFILE = os.path.join(PLATFORM_CONFIG, "Makefile") SYSCONFIGDATA = os.path.join( ROOT, @@ -900,51 +907,56 @@ def replace_in_all(search, replace): replace_in_file(SYSCONFIGDATA, search, replace) -def replace_in_sysconfigdata(search, replace, keys): - """Replace a string in the sysconfigdata file for select keys.""" - with open(SYSCONFIGDATA, "rb") as fh: - data = fh.read() +def _find_build_time_vars_assign(module): + """Return the Assign node for 'build_time_vars = {...}' or None.""" + for node in module.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "build_time_vars": + return node + return None - globals_dict = {} - locals_dict = {} - exec(data, globals_dict, locals_dict) - build_time_vars = locals_dict['build_time_vars'] - for key in keys: - if key in build_time_vars: - build_time_vars[key] = build_time_vars[key].replace(search, replace) +def _patch_build_time_vars(assign_node, keys, search, replace): + """Patch plain string values for the given keys inside the dict literal.""" + if not isinstance(assign_node.value, ast.Dict): + return - with open(SYSCONFIGDATA, "wb") as fh: - fh.write(b'# system configuration generated and used by the sysconfig module\n') - fh.write(('build_time_vars = %s' % json.dumps(build_time_vars, indent=4, sort_keys=True)).encode("utf-8")) - fh.close() + d: ast.Dict = assign_node.value + for key_node, val_node in zip(d.keys, d.values): + k = key_node.value + if k in keys and isinstance(val_node, ast.Constant) and isinstance(val_node.value, str): + val_node.value = val_node.value.replace(search, replace) -def format_sysconfigdata(): - """Reformat the sysconfigdata file to avoid implicit string concatenations. +def replace_in_sysconfigdata(search, replace, keys): + """Replace a string in the sysconfigdata file for select keys.""" + with open(SYSCONFIGDATA, "r", encoding="utf-8") as fh: + source = fh.read() - In some Python versions, the sysconfigdata file contains implicit string - concatenations that extend over multiple lines, which make string replacement - much harder. This function reformats the file to avoid this issue. + module = ast.parse(source) + assign = _find_build_time_vars_assign(module) + if assign is None: + # Nothing to do if build_time_vars isn't present. + return - See: https://github.com/python/cpython/blob/a03efb533a58fd13fb0cc7f4a5c02c8406a407bd/Mac/BuildScript/build-installer.py#L1360C1-L1385C15. - """ - with open(SYSCONFIGDATA, "rb") as fh: - data = fh.read() + # Patch the dict + _patch_build_time_vars(assign, keys, search, replace) - globals_dict = {} - locals_dict = {} - exec(data, globals_dict, locals_dict) - build_time_vars = locals_dict['build_time_vars'] + # Compute the textual prefix up to (but not including) the start of build_time_vars. + lines = source.splitlines() + start_line = assign.lineno # 1-based + head_text = "\n".join(lines[: start_line - 1]) - with open(SYSCONFIGDATA, "wb") as fh: - fh.write(b'# system configuration generated and used by the sysconfig module\n') - fh.write(('build_time_vars = %s' % json.dumps(build_time_vars, indent=4, sort_keys=True)).encode("utf-8")) - fh.close() + # Unparse the patched assignment + assign_src = ast.unparse(assign) + # Rewrite: preserved head + patched assignment + trailing newline + new_source = head_text + ("\n" if head_text and not head_text.endswith("\n") else "") + new_source += assign_src + "\n" -# Format sysconfig to ensure that string replacements take effect. -format_sysconfigdata() + with open(SYSCONFIGDATA, "w", encoding="utf-8") as fh: + fh.write(new_source) # Remove `-Werror=unguarded-availability-new` from `CFLAGS` and `CPPFLAGS`. # These flags are passed along when building extension modules. In that context, @@ -1038,7 +1050,7 @@ metadata = { "python_paths_abstract": sysconfig.get_paths(expand=False), "python_exe": "install/bin/python%s%s" % (sysconfig.get_python_version(), sys.abiflags), "python_major_minor_version": sysconfig.get_python_version(), - "python_stdlib_platform_config": sysconfig.get_config_var("LIBPL").lstrip("/"), + "python_stdlib_platform_config": sysconfig.get_config_var("LIBPL"), "python_config_vars": {k: str(v) for k, v in sysconfig.get_config_vars().items()}, } diff --git a/cpython-unix/patch-cpython-relocatable-sysconfig-3.10.patch b/cpython-unix/patch-cpython-relocatable-sysconfig-3.10.patch new file mode 100644 index 00000000..d20b01c9 --- /dev/null +++ b/cpython-unix/patch-cpython-relocatable-sysconfig-3.10.patch @@ -0,0 +1,55 @@ +diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py +index daf9f000060..ed70b771e56 100644 +--- a/Lib/sysconfig.py ++++ b/Lib/sysconfig.py +@@ -1,6 +1,7 @@ + """Access to Python's configuration information.""" + + import os ++import re + import sys + from os.path import pardir, realpath + +@@ -407,9 +408,29 @@ def _get_sysconfigdata_name(): + ) + + ++def _print_config_dict(d, stream): ++ # Build a regex that matches the prefix at the start of the ++ # string or following a whitespace character. ++ prefix = d.get("prefix", "/install") ++ install_prefix_pattern = re.compile(r"(^|\s)" + re.escape(prefix)) ++ # The replacement string. Use a backreference \1 to keep the ++ # whitespace if it was present. ++ relocatable_path = r"\1{installed_base}" ++ ++ print ("{", file=stream) ++ for k, v in sorted(d.items()): ++ if isinstance(v, str): ++ replaced, count = install_prefix_pattern.subn(relocatable_path, v) ++ if count: ++ value_literal = "f" + repr(replaced) ++ print(f" {k!r}: {value_literal},", file=stream) ++ continue ++ print(f" {k!r}: {v!r},", file=stream) ++ print ("}", file=stream) ++ ++ + def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" +- import pprint + vars = {} + # load the installed Makefile: + makefile = get_makefile_filename() +@@ -463,8 +484,10 @@ def _generate_posix_vars(): + with open(destfile, 'w', encoding='utf8') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') ++ f.write('from pathlib import Path\n') ++ f.write('installed_base = str(Path(__file__).resolve().parent.parent.parent)\n') + f.write('build_time_vars = ') +- pprint.pprint(vars, stream=f) ++ _print_config_dict(vars, stream=f) + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w', encoding='utf8') as f: diff --git a/cpython-unix/patch-cpython-relocatable-sysconfig-3.13.patch b/cpython-unix/patch-cpython-relocatable-sysconfig-3.13.patch new file mode 100644 index 00000000..06363852 --- /dev/null +++ b/cpython-unix/patch-cpython-relocatable-sysconfig-3.13.patch @@ -0,0 +1,42 @@ +diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py +index d7257b9d2d0..1209757b45b 100644 +--- a/Lib/sysconfig/__main__.py ++++ b/Lib/sysconfig/__main__.py +@@ -1,4 +1,5 @@ + import os ++import re + import sys + from sysconfig import ( + _ALWAYS_STR, +@@ -151,8 +152,22 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): + + + def _print_config_dict(d, stream): ++ # Build a regex that matches the prefix at the start of the ++ # string or following a whitespace character. ++ prefix = d.get("prefix", "/install") ++ install_prefix_pattern = re.compile(r"(^|\s)" + re.escape(prefix)) ++ # The replacement string. Use a backreference \1 to keep the ++ # whitespace if it was present. ++ relocatable_path = r"\1{installed_base}" ++ + print ("{", file=stream) + for k, v in sorted(d.items()): ++ if isinstance(v, str): ++ replaced, count = install_prefix_pattern.subn(relocatable_path, v) ++ if count: ++ value_literal = "f" + repr(replaced) ++ print(f" {k!r}: {value_literal},", file=stream) ++ continue + print(f" {k!r}: {v!r},", file=stream) + print ("}", file=stream) + +@@ -212,6 +227,8 @@ def _generate_posix_vars(): + with open(destfile, 'w', encoding='utf8') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') ++ f.write('from pathlib import Path\n') ++ f.write('installed_base = str(Path(__file__).resolve().parent.parent.parent)\n') + f.write('build_time_vars = ') + _print_config_dict(vars, stream=f) + From 7436194c1178a1c9ee00f3a36d62f8674c155cd9 Mon Sep 17 00:00:00 2001 From: Chongkai Zhu Date: Thu, 18 Dec 2025 03:11:39 +0800 Subject: [PATCH 2/2] use a trick to get_config_var as before --- cpython-unix/build-cpython.sh | 3 ++- cpython-unix/build.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index d8612c89..1baa4d09 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -1031,6 +1031,7 @@ extension_suffixes.append(".abi3.so") extension_suffixes.append(".so") +libpl_path = sysconfig.get_config_var("LIBPL") metadata = { "python_abi_tag": sys.abiflags, "python_implementation_cache_tag": sys.implementation.cache_tag, @@ -1050,7 +1051,7 @@ metadata = { "python_paths_abstract": sysconfig.get_paths(expand=False), "python_exe": "install/bin/python%s%s" % (sysconfig.get_python_version(), sys.abiflags), "python_major_minor_version": sysconfig.get_python_version(), - "python_stdlib_platform_config": sysconfig.get_config_var("LIBPL"), + "python_stdlib_platform_config": libpl_path[libpl_path.find("install"):], "python_config_vars": {k: str(v) for k, v in sysconfig.get_config_vars().items()}, } diff --git a/cpython-unix/build.py b/cpython-unix/build.py index fd9810c6..bcb4174f 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -682,7 +682,8 @@ def python_build_info( } if info.get("build-mode") == "shared": - shared_dir = extra_metadata["python_config_vars"]["DESTSHARED"].strip("/") + shared_dir = extra_metadata["python_config_vars"]["DESTSHARED"] + shared_dir = shared_dir[shared_dir.find("install"):] extension_suffix = extra_metadata["python_config_vars"]["EXT_SUFFIX"] entry["shared_lib"] = "%s/%s%s" % (shared_dir, extension, extension_suffix)