From 6ea40818ba63db706bd758f6b42729998ad237ac Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 14:16:04 -0700 Subject: [PATCH 01/12] android: migrate plugin build.gradle to Kotlin DSL Faithful, behavior-preserving port of the per-ABI task graph (gradle-download-task, tarTree/untar, copyOpt, zipSitePackages, dart-bridge download+rename, packaging, abiFilters) to build.gradle.kts. Verified green: run_example builds and runs on the arm64 emulator with identical jniLibs and Python output (numpy works). Prerequisite for the native-split tasks. --- .../android/build.gradle | 176 ----------------- .../android/build.gradle.kts | 182 ++++++++++++++++++ 2 files changed, 182 insertions(+), 176 deletions(-) delete mode 100644 src/serious_python_android/android/build.gradle create mode 100644 src/serious_python_android/android/build.gradle.kts diff --git a/src/serious_python_android/android/build.gradle b/src/serious_python_android/android/build.gradle deleted file mode 100644 index b6dd6e28..00000000 --- a/src/serious_python_android/android/build.gradle +++ /dev/null @@ -1,176 +0,0 @@ -group 'com.flet.serious_python_android' -version '3.0.0' - -// Python runtime versions come from the generated python_versions.properties -// (a snapshot of python-build's manifest.json — see serious_python's -// `gen_version_tables`). SERIOUS_PYTHON_VERSION selects the version; everything -// else derives from the table. The per-field env vars are escape hatches. -def _pv = new Properties() -file('python_versions.properties').withInputStream { _pv.load(it) } -def python_version = System.getenv('SERIOUS_PYTHON_VERSION') ?: _pv['default_python_version'] -def python_full_version = System.getenv('SERIOUS_PYTHON_FULL_VERSION') ?: _pv["${python_version}.full_version"] -def python_build_date = System.getenv('SERIOUS_PYTHON_BUILD_DATE') ?: _pv['python_build_release_date'] -def dart_bridge_version = System.getenv('DART_BRIDGE_VERSION') ?: _pv['dart_bridge_version'] -if (python_full_version == null) { - def known = _pv.keySet().findAll { it.endsWith('.full_version') }.collect { it - '.full_version' } - throw new GradleException("serious_python: unknown SERIOUS_PYTHON_VERSION '${python_version}'. Supported: ${known.join(', ')}") -} - -buildscript { - repositories { - google() - mavenCentral() - } - - dependencies { - // The Android Gradle Plugin knows how to build native code with the NDK. - classpath 'com.android.tools.build:gradle:8.11.1' - classpath 'de.undercouch:gradle-download-task:5.6.0' - } -} - -rootProject.allprojects { - repositories { - google() - mavenCentral() - } -} - -apply plugin: 'com.android.library' -apply plugin: 'de.undercouch.download' - -android { - namespace "com.flet.serious_python_android" - - // Bumping the plugin compileSdkVersion requires all clients of this plugin - // to bump the version in their app. - compileSdkVersion 36 - - // No native code is compiled here — libdart_bridge.so is downloaded as a - // pre-built artifact from flet-dev/dart-bridge releases (see the - // downloadDartBridge_$abi tasks below) and dropped into jniLibs. - - compileOptions { - sourceCompatibility JavaVersion.VERSION_17 - targetCompatibility JavaVersion.VERSION_17 - } - - defaultConfig { - minSdkVersion 21 - - ndk { - // python-build dropped 32-bit Android in 3.13 (PEP 738), so the - // python-android-dart--armeabi-v7a tarball only exists for 3.12. - if (python_version == '3.12') { - abiFilters 'arm64-v8a', 'armeabi-v7a', 'x86_64' - } else { - abiFilters 'arm64-v8a', 'x86_64' - } - } - } - - packagingOptions { - doNotStrip "*/arm64-v8a/libpython*.so" - doNotStrip "*/armeabi-v7a/libpython*.so" - doNotStrip "*/x86/libpython*.so" - doNotStrip "*/x86_64/libpython*.so" - } -} - -import de.undercouch.gradle.tasks.download.Download - -def fletCacheRoot = System.getenv('FLET_CACHE_DIR') -def pythonCacheDir = new File( - fletCacheRoot ? new File(fletCacheRoot) : new File(System.getProperty('user.home'), '.flet/cache'), - "python-build/v${python_full_version}" -) -def dartBridgeCacheDir = new File( - fletCacheRoot ? new File(fletCacheRoot) : new File(System.getProperty('user.home'), '.flet/cache'), - "dart-bridge/v${dart_bridge_version}" -) - -task copyBuildDist(type: Copy) { - def srcDir = System.getenv('SERIOUS_PYTHON_BUILD_DIST') - if (srcDir != null) { - from srcDir - into 'src/main/jniLibs' - } -} - -// Loop through abiFilters -def packageTasks = [] -android.defaultConfig.ndk.abiFilters.each { abi -> - - def srcDir = System.getenv('SERIOUS_PYTHON_SITE_PACKAGES') - if (srcDir == null || srcDir.allWhitespace) { - throw new InvalidUserDataException("SERIOUS_PYTHON_SITE_PACKAGES environment variable is not set.") - } - - packageTasks.add("zipSitePackages_$abi") - packageTasks.add("copyOpt_$abi") - - tasks.register("jniCleanUp_$abi", Delete) { - delete "src/main/jniLibs/$abi" - } - - tasks.register("downloadDistArchive_$abi", Download) { - src "https://github.com/flet-dev/python-build/releases/download/${python_build_date}/python-android-dart-${python_full_version}-${abi}.tar.gz" - dest new File(pythonCacheDir, "python-android-dart-${python_full_version}-${abi}.tar.gz") - onlyIfModified true - useETag "all" - tempAndMove true - doFirst { dest.parentFile.mkdirs() } - } - tasks.register("untarFile_$abi", Copy) { - from tarTree(tasks.named("downloadDistArchive_$abi").get().dest) - into "src/main/jniLibs/$abi" - dependsOn "jniCleanUp_$abi", "downloadDistArchive_$abi" - } - - tasks.register("copyOpt_$abi", Copy) { - from fileTree(dir: "$srcDir/$abi/opt", include: ["**/*.so"]) - into "src/main/jniLibs/$abi" - eachFile { - path = name - } - includeEmptyDirs = false - dependsOn "jniCleanUp_$abi" - } - - tasks.register("zipSitePackages_$abi", Zip) { - from fileTree(dir: "$srcDir/$abi") - archiveFileName = "libpythonsitepackages.so" - destinationDirectory = file("src/main/jniLibs/$abi") - dependsOn "jniCleanUp_$abi", "untarFile_$abi" - } - - // dart-bridge ships a per-(abi × Python-minor-version) prebuilt .so. The - // binary's DT_NEEDED is libpython3.X.so (version-specific), so the bridge - // .so MUST match the libpython bundled in the same APK. We download from - // the pinned dart_bridge_version release into a cache shared across - // builds, then drop it as `libdart_bridge.so` (no version suffix) so the - // Dart side can DynamicLibrary.open by a stable short name. - tasks.register("downloadDartBridge_$abi", Download) { - src "https://github.com/flet-dev/dart-bridge/releases/download/v${dart_bridge_version}/libdart_bridge-android-${abi}-py${python_version}.so" - dest new File(dartBridgeCacheDir, "libdart_bridge-android-${abi}-py${python_version}.so") - onlyIfModified true - useETag "all" - tempAndMove true - doFirst { dest.parentFile.mkdirs() } - } - tasks.register("copyDartBridge_$abi", Copy) { - from tasks.named("downloadDartBridge_$abi").get().dest - into "src/main/jniLibs/$abi" - rename '.*', 'libdart_bridge.so' - dependsOn "downloadDartBridge_$abi", "jniCleanUp_$abi" - } - packageTasks.add("copyDartBridge_$abi") -} - -if (System.getenv('SERIOUS_PYTHON_BUILD_DIST')) { - task copyOrUntar(dependsOn: 'copyBuildDist') -} else { - task copyOrUntar(dependsOn: packageTasks) -} - -preBuild.dependsOn copyOrUntar diff --git a/src/serious_python_android/android/build.gradle.kts b/src/serious_python_android/android/build.gradle.kts new file mode 100644 index 00000000..3e8c2f89 --- /dev/null +++ b/src/serious_python_android/android/build.gradle.kts @@ -0,0 +1,182 @@ +import com.android.build.gradle.LibraryExtension +import de.undercouch.gradle.tasks.download.Download +import org.gradle.api.InvalidUserDataException +import java.io.File +import java.util.Properties + +buildscript { + repositories { + google() + mavenCentral() + } + dependencies { + // The Android Gradle Plugin knows how to build native code with the NDK. + classpath("com.android.tools.build:gradle:8.11.1") + classpath("de.undercouch:gradle-download-task:5.6.0") + } +} + +group = "com.flet.serious_python_android" +version = "3.0.0" + +rootProject.allprojects { + repositories { + google() + mavenCentral() + } +} + +apply(plugin = "com.android.library") +apply(plugin = "de.undercouch.download") + +// Python runtime versions come from the generated python_versions.properties +// (a snapshot of python-build's manifest.json — see serious_python's +// `gen_version_tables`). SERIOUS_PYTHON_VERSION selects the version; everything +// else derives from the table. The per-field env vars are escape hatches. +val pv = Properties().apply { + file("python_versions.properties").inputStream().use { load(it) } +} +val pythonVersion: String = System.getenv("SERIOUS_PYTHON_VERSION") ?: pv.getProperty("default_python_version") +val pythonFullVersion: String? = System.getenv("SERIOUS_PYTHON_FULL_VERSION") ?: pv.getProperty("$pythonVersion.full_version") +val pythonBuildDate: String = System.getenv("SERIOUS_PYTHON_BUILD_DATE") ?: pv.getProperty("python_build_release_date") +val dartBridgeVersion: String = System.getenv("DART_BRIDGE_VERSION") ?: pv.getProperty("dart_bridge_version") +if (pythonFullVersion == null) { + val known = pv.keys.map { it.toString() }.filter { it.endsWith(".full_version") }.map { it.removeSuffix(".full_version") } + throw GradleException("serious_python: unknown SERIOUS_PYTHON_VERSION '$pythonVersion'. Supported: ${known.joinToString(", ")}") +} + +// python-build dropped 32-bit Android in 3.13 (PEP 738), so the +// python-android-dart--armeabi-v7a tarball only exists for 3.12. +val abis: List = if (pythonVersion == "3.12") + listOf("arm64-v8a", "armeabi-v7a", "x86_64") +else + listOf("arm64-v8a", "x86_64") + +configure { + namespace = "com.flet.serious_python_android" + + // Bumping the plugin compileSdk requires all clients of this plugin to bump too. + compileSdk = 36 + + // No native code is compiled here — libdart_bridge.so is downloaded as a + // pre-built artifact from flet-dev/dart-bridge releases (see the + // downloadDartBridge_$abi tasks below) and dropped into jniLibs. + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + defaultConfig { + minSdk = 21 + ndk { + abiFilters.addAll(abis) + } + } + + packaging { + jniLibs { + keepDebugSymbols += setOf( + "*/arm64-v8a/libpython*.so", + "*/armeabi-v7a/libpython*.so", + "*/x86/libpython*.so", + "*/x86_64/libpython*.so", + ) + } + } +} + +val fletCacheRoot: String? = System.getenv("FLET_CACHE_DIR") +val cacheBase: File = if (fletCacheRoot != null) File(fletCacheRoot) else File(System.getProperty("user.home"), ".flet/cache") +val pythonCacheDir = File(cacheBase, "python-build/v$pythonFullVersion") +val dartBridgeCacheDir = File(cacheBase, "dart-bridge/v$dartBridgeVersion") + +tasks.register("copyBuildDist") { + val srcDir = System.getenv("SERIOUS_PYTHON_BUILD_DIST") + if (srcDir != null) { + from(srcDir) + into("src/main/jniLibs") + } +} + +val siteSrcDir: String? = System.getenv("SERIOUS_PYTHON_SITE_PACKAGES") + +// Loop through abiFilters +val packageTasks = mutableListOf() +for (abi in abis) { + if (siteSrcDir == null || siteSrcDir.isBlank()) { + throw InvalidUserDataException("SERIOUS_PYTHON_SITE_PACKAGES environment variable is not set.") + } + + packageTasks.add("zipSitePackages_$abi") + packageTasks.add("copyOpt_$abi") + + tasks.register("jniCleanUp_$abi") { + delete("src/main/jniLibs/$abi") + } + + val distFile = File(pythonCacheDir, "python-android-dart-$pythonFullVersion-$abi.tar.gz") + tasks.register("downloadDistArchive_$abi") { + src("https://github.com/flet-dev/python-build/releases/download/$pythonBuildDate/python-android-dart-$pythonFullVersion-$abi.tar.gz") + dest(distFile) + onlyIfModified(true) + useETag("all") + tempAndMove(true) + doFirst { distFile.parentFile.mkdirs() } + } + tasks.register("untarFile_$abi") { + from(tarTree(distFile)) + into("src/main/jniLibs/$abi") + dependsOn("jniCleanUp_$abi", "downloadDistArchive_$abi") + } + + tasks.register("copyOpt_$abi") { + from(fileTree("$siteSrcDir/$abi/opt") { include("**/*.so") }) + into("src/main/jniLibs/$abi") + eachFile { path = name } + includeEmptyDirs = false + dependsOn("jniCleanUp_$abi") + } + + tasks.register("zipSitePackages_$abi") { + from(fileTree("$siteSrcDir/$abi")) + archiveFileName.set("libpythonsitepackages.so") + destinationDirectory.set(file("src/main/jniLibs/$abi")) + dependsOn("jniCleanUp_$abi", "untarFile_$abi") + } + + // dart-bridge ships a per-(abi × Python-minor-version) prebuilt .so. The + // binary's DT_NEEDED is libpython3.X.so (version-specific), so the bridge + // .so MUST match the libpython bundled in the same APK. We download from + // the pinned dart_bridge_version release into a cache shared across + // builds, then drop it as `libdart_bridge.so` (no version suffix) so the + // Dart side can DynamicLibrary.open by a stable short name. + val bridgeFile = File(dartBridgeCacheDir, "libdart_bridge-android-$abi-py$pythonVersion.so") + tasks.register("downloadDartBridge_$abi") { + src("https://github.com/flet-dev/dart-bridge/releases/download/v$dartBridgeVersion/libdart_bridge-android-$abi-py$pythonVersion.so") + dest(bridgeFile) + onlyIfModified(true) + useETag("all") + tempAndMove(true) + doFirst { bridgeFile.parentFile.mkdirs() } + } + tasks.register("copyDartBridge_$abi") { + from(bridgeFile) + into("src/main/jniLibs/$abi") + rename(".*", "libdart_bridge.so") + dependsOn("downloadDartBridge_$abi", "jniCleanUp_$abi") + } + packageTasks.add("copyDartBridge_$abi") +} + +val copyOrUntar = tasks.register("copyOrUntar") { + if (System.getenv("SERIOUS_PYTHON_BUILD_DIST") != null) { + dependsOn("copyBuildDist") + } else { + dependsOn(packageTasks) + } +} + +tasks.named("preBuild") { + dependsOn(copyOrUntar) +} From e62be85c94569094adaae44f328f0fbdd4753981 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 14:16:04 -0700 Subject: [PATCH 02/12] android: add _sp_bootstrap.py native-module finder (Phase E) sys.meta_path finder that resolves relocated CPython extension modules from their .soref markers (jniLibs lib loaded by basename). Reads markers via frozen zipimport.get_data (zip-resident) or open() (extracted), imports only builtin/frozen machinery so it runs before any native is resolvable. Host-tested: zip + extracted-dir probes, idempotent install, pure imports fall through. Not yet wired into the build. --- .../python/_sp_bootstrap.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 src/serious_python_android/python/_sp_bootstrap.py diff --git a/src/serious_python_android/python/_sp_bootstrap.py b/src/serious_python_android/python/_sp_bootstrap.py new file mode 100644 index 00000000..27d1ed99 --- /dev/null +++ b/src/serious_python_android/python/_sp_bootstrap.py @@ -0,0 +1,99 @@ +"""serious_python Android import bootstrap. + +Installed by the dart-bridge embedder *before* ``site`` runs, via the fixed call:: + + import _sp_bootstrap; _sp_bootstrap.install() + +It registers a ``sys.meta_path`` finder that resolves native CPython extension +modules which the build relocated into ``jniLibs//`` as real ``lib.so`` +files (loaded by basename through the Android linker namespace, exactly like +``libdart_bridge.so``). Pure ``.py``/``.pyc`` modules are left to ``zipimport`` / +``FileFinder`` — this finder returns ``None`` for them. + +For every relocated extension the build leaves a ``.soref`` marker at the module's +original path; its content is the ``lib.so`` filename. The marker is read +**lazily** in ``find_spec`` via the frozen ``zipimport`` ``get_data`` API (for zip +entries) or a plain ``open`` (for entries extracted to disk, e.g. ``extract.zip``). + +CRITICAL: this module must load and run *before any native module is resolvable*, +so it imports **only builtin/frozen** machinery — ``sys``, ``zipimport``, +``importlib.machinery`` — and never ``zipfile``/``struct``/``zlib`` (which would be +a chicken-and-egg: those are themselves native). +""" + +import sys +import zipimport +from importlib.machinery import ExtensionFileLoader, ModuleSpec + +_MARKER_SUFFIX = ".soref" +_installed = False + + +class _SorefFinder: + """meta_path finder: dotted name -> jniLibs lib via its ``.soref`` marker.""" + + def __init__(self): + # Cache one zipimporter per zip sys.path entry. Value is None for entries + # that are not zips (plain directories) so we don't retry zipimporter(). + self._zi_cache = {} + + def _zipimporter(self, entry): + try: + return self._zi_cache[entry] + except KeyError: + try: + zi = zipimport.zipimporter(entry) + except Exception: + zi = None # not a zip (e.g. a directory) + self._zi_cache[entry] = zi + return zi + + def _read_marker(self, member): + """Return the soname recorded in ``member`` (.soref), or None if absent. + + Probes every current ``sys.path`` entry: zip entries via the frozen + ``zipimport.get_data`` (known member, no native deps), directory entries + via a plain ``open`` (covers packages unpacked from ``extract.zip``). + """ + for entry in sys.path: + if not entry: + continue + zi = self._zipimporter(entry) + if zi is not None: + try: + return zi.get_data(member) # archive-relative member path + except Exception: + continue + else: + # Directory entry: try the marker as a real file on disk. + path = entry + "/" + member + try: + with open(path, "rb") as f: + return f.read() + except OSError: + continue + return None + + def find_spec(self, fullname, path=None, target=None): + member = fullname.replace(".", "/") + _MARKER_SUFFIX + data = self._read_marker(member) + if data is None: + return None # not a relocated native module -> let others handle it + soname = data.decode("utf-8").strip() + # origin = bare soname -> create_dynamic dlopen()s it by basename, which the + # app's linker namespace resolves from the APK (modern packaging). + loader = ExtensionFileLoader(fullname, soname) + return ModuleSpec(fullname, loader, origin=soname) + + +def install(): + """Insert the finder at the front of ``sys.meta_path`` (idempotent).""" + global _installed + if _installed: + return + for f in sys.meta_path: + if isinstance(f, _SorefFinder): + _installed = True + return + sys.meta_path.insert(0, _SorefFinder()) + _installed = True From 411e891821af3ff6cf8871a9be0a1e0321abac72 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 14:26:20 -0700 Subject: [PATCH 03/12] android: split native modules to jniLibs, pure code to stored zips (Phase A/B) The Kotlin split tasks replace zipSitePackages: they relocate every tagged CPython extension .so (stdlib lib-dynload + site-packages) to jniLibs//lib.so (dotted name, '.'->'-', readable & injective), leaving a .soref marker (content = lib name) at the module's path in ABI-common stored zips. stdlib bundle is cracked post-untar; libpythonbundle.so/libpythonsitepackages.so no longer ship. Pure code -> stdlib.zip/sitepackages.zip (assets, noCompress); allowlisted packages (SERIOUS_PYTHON_ANDROID_EXTRACT_PACKAGES) -> extract.zip, excluded from sitepackages.zip. _sp_bootstrap.py embedded at stdlib.zip root. Verified by artifact inspection: 53 stdlib + 19 numpy natives mangled in jniLibs, markers present with matching libs, zips stored, no fake-zip .so. Runtime wiring (D/F) pending, so not yet runnable end-to-end. --- src/serious_python_android/android/.gitignore | 1 + .../android/build.gradle.kts | 129 +++++++++++++++++- 2 files changed, 124 insertions(+), 6 deletions(-) diff --git a/src/serious_python_android/android/.gitignore b/src/serious_python_android/android/.gitignore index a4f3c175..642cbe42 100644 --- a/src/serious_python_android/android/.gitignore +++ b/src/serious_python_android/android/.gitignore @@ -8,3 +8,4 @@ /captures .cxx /src/main/jniLibs +/src/main/assets diff --git a/src/serious_python_android/android/build.gradle.kts b/src/serious_python_android/android/build.gradle.kts index 3e8c2f89..8832e19e 100644 --- a/src/serious_python_android/android/build.gradle.kts +++ b/src/serious_python_android/android/build.gradle.kts @@ -3,6 +3,10 @@ import de.undercouch.gradle.tasks.download.Download import org.gradle.api.InvalidUserDataException import java.io.File import java.util.Properties +import java.util.zip.CRC32 +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipOutputStream buildscript { repositories { @@ -84,6 +88,12 @@ configure { ) } } + + // Keep the stdlib/sitepackages/extract zips stored (uncompressed) in the APK so + // zipimport can read members without zlib. + androidResources { + noCompress.add("zip") + } } val fletCacheRoot: String? = System.getenv("FLET_CACHE_DIR") @@ -101,6 +111,48 @@ tasks.register("copyBuildDist") { val siteSrcDir: String? = System.getenv("SERIOUS_PYTHON_SITE_PACKAGES") +// ---- native split ----------------------------------------------------------- +// Relocate CPython extension modules to jniLibs//lib.so (loaded by +// basename via the linker namespace), leaving a .soref marker (content = the lib +// name) at each module's path in the pure zip. Pure code ships in ABI-common +// stored zips; path-hungry packages from SERIOUS_PYTHON_ANDROID_EXTRACT_PACKAGES +// are moved whole into extract.zip and excluded from sitepackages.zip. +val extractPackages: List = (System.getenv("SERIOUS_PYTHON_ANDROID_EXTRACT_PACKAGES") ?: "") + .split(",").map { it.trim() }.filter { it.isNotEmpty() } +val primaryAbi = abis.first() // pure zips are ABI-common: build once +val assetsDir = file("src/main/assets") +val bootstrapPy = file("../python/_sp_bootstrap.py") + +val extTag = Regex("""\.(cpython-[^/]+|abi3)\.so$""") // tagged extension module +fun isExtModule(name: String) = extTag.containsMatchIn(name) +fun extDottedName(rel: String): String { // slash-rel path -> dotted import name + val dir = rel.substringBeforeLast('/', "") + val mod = rel.substringAfterLast('/').replace(extTag, "") + return (if (dir.isEmpty()) "" else dir.replace('/', '.') + ".") + mod +} +fun mangledLib(dotted: String) = "lib" + dotted.replace('.', '-') + ".so" +fun sorefPath(rel: String) = rel.replace(extTag, ".soref") +fun isAllowlisted(rel: String) = extractPackages.any { rel == it || rel.startsWith("$it/") } + +// Minimal STORED (uncompressed) zip so members stay readable via zipimport.get_data +// with no zlib at runtime. +class StoredZip(val out: ZipOutputStream) { + fun add(name: String, data: ByteArray) { + val e = ZipEntry(name).apply { + method = ZipEntry.STORED + size = data.size.toLong() + compressedSize = data.size.toLong() + crc = CRC32().apply { update(data) }.value + } + out.putNextEntry(e); out.write(data); out.closeEntry() + } + fun close() = out.close() +} +fun storedZip(f: File): StoredZip { + f.parentFile.mkdirs() + return StoredZip(ZipOutputStream(f.outputStream()).apply { setMethod(ZipOutputStream.STORED) }) +} + // Loop through abiFilters val packageTasks = mutableListOf() for (abi in abis) { @@ -108,7 +160,8 @@ for (abi in abis) { throw InvalidUserDataException("SERIOUS_PYTHON_SITE_PACKAGES environment variable is not set.") } - packageTasks.add("zipSitePackages_$abi") + packageTasks.add("splitStdlib_$abi") + packageTasks.add("splitSitePackages_$abi") packageTasks.add("copyOpt_$abi") tasks.register("jniCleanUp_$abi") { @@ -138,11 +191,75 @@ for (abi in abis) { dependsOn("jniCleanUp_$abi") } - tasks.register("zipSitePackages_$abi") { - from(fileTree("$siteSrcDir/$abi")) - archiveFileName.set("libpythonsitepackages.so") - destinationDirectory.set(file("src/main/jniLibs/$abi")) - dependsOn("jniCleanUp_$abi", "untarFile_$abi") + val jniDir = file("src/main/jniLibs/$abi") + val abiSiteDir = file("$siteSrcDir/$abi") + val bundleFile = File(jniDir, "libpythonbundle.so") + val isPrimary = abi == primaryAbi + + // Crack the stdlib bundle (libpythonbundle.so): modules/*.so -> mangled jniLibs + // (+ .soref markers in stdlib.zip), stdlib/* -> stdlib.zip root, then delete it. + tasks.register("splitStdlib_$abi") { + dependsOn("untarFile_$abi") + doLast { + if (!bundleFile.exists()) throw GradleException("libpythonbundle.so missing in jniLibs/$abi") + val zip = if (isPrimary) storedZip(File(assetsDir, "stdlib.zip")) else null + ZipFile(bundleFile).use { zf -> + val en = zf.entries() + while (en.hasMoreElements()) { + val e = en.nextElement() + if (e.isDirectory) continue + val data = zf.getInputStream(e).readBytes() + val name = e.name + when { + name.startsWith("modules/") -> { + val rel = name.removePrefix("modules/") // top-level module file + when { + isExtModule(rel) -> { + val lib = mangledLib(extDottedName(rel)) + File(jniDir, lib).writeBytes(data) + zip?.add(sorefPath(rel), lib.toByteArray()) + } + rel.endsWith(".so") -> File(jniDir, File(rel).name).writeBytes(data) + else -> zip?.add(rel, data) + } + } + name.startsWith("stdlib/") -> zip?.add(name.removePrefix("stdlib/"), data) + else -> zip?.add(name, data) + } + } + } + zip?.add("_sp_bootstrap.py", bootstrapPy.readBytes()) // finder at zip root + zip?.close() + bundleFile.delete() // fake-zip must not ship + } + } + + // Site-packages tree: tagged .so -> mangled jniLibs (+ .soref markers), pure -> + // sitepackages.zip (or extract.zip if allowlisted); opt/ is left to copyOpt. + tasks.register("splitSitePackages_$abi") { + dependsOn("untarFile_$abi") + mustRunAfter("copyOpt_$abi", "splitStdlib_$abi") + doLast { + jniDir.mkdirs() + val siteZip = if (isPrimary) storedZip(File(assetsDir, "sitepackages.zip")) else null + val extractZip = if (isPrimary) storedZip(File(assetsDir, "extract.zip")) else null + abiSiteDir.walkTopDown().filter { it.isFile }.forEach { f -> + val rel = f.relativeTo(abiSiteDir).path.replace(File.separatorChar, '/') + if (rel == "opt" || rel.startsWith("opt/")) return@forEach // dep libs -> copyOpt + val zip = if (isAllowlisted(rel)) extractZip else siteZip + when { + isExtModule(rel) -> { + val lib = mangledLib(extDottedName(rel)) + f.copyTo(File(jniDir, lib), overwrite = true) + zip?.add(sorefPath(rel), lib.toByteArray()) + } + rel.endsWith(".so") -> f.copyTo(File(jniDir, f.name), overwrite = true) // untagged -> dep + else -> zip?.add(rel, f.readBytes()) + } + } + siteZip?.close() + extractZip?.close() + } } // dart-bridge ships a per-(abi × Python-minor-version) prebuilt .so. The From 884514d4368df5d9bed3ba1f1c9ba5d6a1e8d839 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 14:49:47 -0700 Subject: [PATCH 04/12] =?UTF-8?q?android:=20runtime=20for=20native-mmap=20?= =?UTF-8?q?flow=20=E2=80=94=20first=20end-to-end=20green=20(Phase=20D)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AndroidPlugin.java: getFilesDir/extractAsset/unzipAsset (AssetManager + java.util.zip) to copy the stored zips to disk and unpack extract.zip. - serious_python_android.dart run(): copy stdlib.zip/sitepackages.zip once (version-keyed), unpack extract.zip; sys.path = [modulePaths, programDir, extract, sitepackages.zip, stdlib.zip]; PYTHONHOME=base. No fake-zip extraction. - _sp_bootstrap.py: resolve soname to an absolute origin under nativeLibraryDir when present (legacy packaging; CPython prepends ./ to a no-slash origin which breaks bare-soname load). - build.gradle.kts: inject interim sitecustomize.py (installs finder during site); declare split jniLibs/assets outputs + always-run so AGP's native-libs merge re-packages incrementally (no flutter clean needed). Verified on arm64 emulator: run_example runs fully — bz2, sqlite, and numpy all work, with pure code from stored zips (zipimport) and ALL native modules loaded via the finder from jniLibs. Legacy packaging still on (natives extracted); dropping it for mmap is next. --- .../android/build.gradle.kts | 11 ++++ .../serious_python_android/AndroidPlugin.java | 46 +++++++++++++++ .../lib/serious_python_android.dart | 58 +++++++++++-------- .../python/_sp_bootstrap.py | 28 +++++++-- 4 files changed, 115 insertions(+), 28 deletions(-) diff --git a/src/serious_python_android/android/build.gradle.kts b/src/serious_python_android/android/build.gradle.kts index 8832e19e..3123215f 100644 --- a/src/serious_python_android/android/build.gradle.kts +++ b/src/serious_python_android/android/build.gradle.kts @@ -200,6 +200,11 @@ for (abi in abis) { // (+ .soref markers in stdlib.zip), stdlib/* -> stdlib.zip root, then delete it. tasks.register("splitStdlib_$abi") { dependsOn("untarFile_$abi") + // The doLast rewrites jniLibs/ (mangled libs in, bundle out); declare it as a + // tracked output and always re-run so AGP's native-libs merge re-packages. + outputs.dir(jniDir) + outputs.dir(assetsDir) + outputs.upToDateWhen { false } doLast { if (!bundleFile.exists()) throw GradleException("libpythonbundle.so missing in jniLibs/$abi") val zip = if (isPrimary) storedZip(File(assetsDir, "stdlib.zip")) else null @@ -229,6 +234,9 @@ for (abi in abis) { } } zip?.add("_sp_bootstrap.py", bootstrapPy.readBytes()) // finder at zip root + // Interim install hook: site (during Py_Initialize) imports this and + // installs the finder. Superseded by the dart-bridge pre-site shim (F). + zip?.add("sitecustomize.py", "import _sp_bootstrap\n_sp_bootstrap.install()\n".toByteArray()) zip?.close() bundleFile.delete() // fake-zip must not ship } @@ -239,6 +247,9 @@ for (abi in abis) { tasks.register("splitSitePackages_$abi") { dependsOn("untarFile_$abi") mustRunAfter("copyOpt_$abi", "splitStdlib_$abi") + outputs.dir(jniDir) + outputs.dir(assetsDir) + outputs.upToDateWhen { false } doLast { jniDir.mkdirs() val siteZip = if (isPrimary) storedZip(File(assetsDir, "sitepackages.zip")) else null diff --git a/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java b/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java index de434d49..6f903b57 100644 --- a/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java +++ b/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java @@ -73,6 +73,52 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull Result result) { ContextWrapper contextWrapper = new ContextWrapper(context); String nativeLibraryDir = contextWrapper.getApplicationInfo().nativeLibraryDir; result.success(nativeLibraryDir); + } else if (call.method.equals("getFilesDir")) { + result.success(context.getFilesDir().getAbsolutePath()); + } else if (call.method.equals("extractAsset")) { + // Stream an APK asset to disk as one whole file (e.g. stdlib.zip). + try { + String asset = call.argument("asset"); + String dest = call.argument("dest"); + java.io.File destFile = new java.io.File(dest); + if (destFile.getParentFile() != null) destFile.getParentFile().mkdirs(); + byte[] buf = new byte[1 << 16]; + try (java.io.InputStream in = context.getAssets().open(asset); + java.io.OutputStream out = new java.io.FileOutputStream(destFile)) { + int n; + while ((n = in.read(buf)) > 0) out.write(buf, 0, n); + } + result.success(dest); + } catch (Exception e) { + result.error("extractAsset", e.getMessage(), null); + } + } else if (call.method.equals("unzipAsset")) { + // Unpack an APK asset zip (e.g. extract.zip) into a directory tree. + try { + String asset = call.argument("asset"); + String destDir = call.argument("dest"); + java.io.File root = new java.io.File(destDir); + byte[] buf = new byte[1 << 16]; + try (java.io.InputStream in = context.getAssets().open(asset); + java.util.zip.ZipInputStream zis = new java.util.zip.ZipInputStream(in)) { + java.util.zip.ZipEntry e; + while ((e = zis.getNextEntry()) != null) { + java.io.File f = new java.io.File(root, e.getName()); + if (e.isDirectory()) { + f.mkdirs(); + } else { + if (f.getParentFile() != null) f.getParentFile().mkdirs(); + try (java.io.OutputStream out = new java.io.FileOutputStream(f)) { + int n; + while ((n = zis.read(buf)) > 0) out.write(buf, 0, n); + } + } + } + } + result.success(destDir); + } catch (Exception e) { + result.error("unzipAsset", e.getMessage(), null); + } } else { result.notImplemented(); } diff --git a/src/serious_python_android/lib/serious_python_android.dart b/src/serious_python_android/lib/serious_python_android.dart index a7947bc5..7c7d4997 100644 --- a/src/serious_python_android/lib/serious_python_android.dart +++ b/src/serious_python_android/lib/serious_python_android.dart @@ -32,38 +32,48 @@ class SeriousPythonAndroid extends SeriousPythonPlatform { List? modulePaths, Map? environmentVariables, bool? sync}) async { - final nativeLibraryDir = - await methodChannel.invokeMethod('getNativeLibraryDir'); - if (nativeLibraryDir == null) { - throw StateError( - 'serious_python: failed to resolve native library dir'); - } - - final bundlePath = '$nativeLibraryDir/libpythonbundle.so'; - if (!await File(bundlePath).exists()) { - throw Exception('Python bundle not found: $bundlePath'); + // Native extension modules now live in jniLibs (loaded by basename via the + // finder); pure code ships in stored Android-asset zips. Copy the zips to disk + // once (version-keyed) and unpack the allowlist payload; PYTHONPATH points at + // the zips so zipimport serves pure modules in place. + final filesDir = await methodChannel.invokeMethod('getFilesDir'); + if (filesDir == null) { + throw StateError('serious_python: failed to resolve files dir'); } + final base = p.join(filesDir, 'flet', 'py'); + final stdlibZip = p.join(base, 'stdlib.zip'); + final siteZip = p.join(base, 'sitepackages.zip'); + final extractDir = p.join(base, 'extract'); final appVersion = await _appVersion(); - final invalidateKey = appVersion != null ? 'app:$appVersion' : null; - - final pythonLibPath = await extractFileZip(bundlePath, - targetPath: 'python_bundle', invalidateKey: invalidateKey); - - final sitePackagesZip = '$nativeLibraryDir/libpythonsitepackages.so'; - String? sitePackagesPath; - if (await File(sitePackagesZip).exists()) { - sitePackagesPath = await extractFileZip(sitePackagesZip, - targetPath: 'python_site_packages', invalidateKey: invalidateKey); + final key = appVersion != null ? 'app:$appVersion' : 'app:dev'; + final marker = File(p.join(base, '.key')); + final upToDate = + await marker.exists() && (await marker.readAsString()) == key; + if (!upToDate) { + await Directory(base).create(recursive: true); + if (await Directory(extractDir).exists()) { + await Directory(extractDir).delete(recursive: true); + } + await methodChannel.invokeMethod( + 'extractAsset', {'asset': 'stdlib.zip', 'dest': stdlibZip}); + await methodChannel.invokeMethod( + 'extractAsset', {'asset': 'sitepackages.zip', 'dest': siteZip}); + await methodChannel.invokeMethod( + 'unzipAsset', {'asset': 'extract.zip', 'dest': extractDir}); + await marker.writeAsString(key); } final programDir = p.dirname(appPath); + // Highest -> lowest precedence. site-packages before stdlib so pip backports + // can override; extract-dir before sitepackages.zip. Natives resolve via the + // finder, not a sys.path entry. final pythonPaths = [ ...?modulePaths, programDir, - '$pythonLibPath/modules', - '$pythonLibPath/stdlib', - if (sitePackagesPath != null) sitePackagesPath, + extractDir, + siteZip, + stdlibZip, ]; final env = { @@ -72,7 +82,7 @@ class SeriousPythonAndroid extends SeriousPythonPlatform { 'PYTHONNOUSERSITE': '1', 'PYTHONUNBUFFERED': '1', 'LC_CTYPE': 'UTF-8', - 'PYTHONHOME': pythonLibPath, + 'PYTHONHOME': base, 'PYTHONPATH': pythonPaths.join(':'), ...?environmentVariables, }; diff --git a/src/serious_python_android/python/_sp_bootstrap.py b/src/serious_python_android/python/_sp_bootstrap.py index 27d1ed99..3ad9d962 100644 --- a/src/serious_python_android/python/_sp_bootstrap.py +++ b/src/serious_python_android/python/_sp_bootstrap.py @@ -22,6 +22,7 @@ """ import sys +import posix # builtin (native-free): read env without importing os import zipimport from importlib.machinery import ExtensionFileLoader, ModuleSpec @@ -29,6 +30,16 @@ _installed = False +def _native_lib_dir(): + # nativeLibraryDir, exported by AndroidPlugin before Py_Initialize. Under legacy + # packaging the mangled libs are extracted there, so an absolute origin lets the + # interpreter dlopen them (some Android CPython builds prepend "./" to a no-slash + # origin, which breaks a bare-soname namespace lookup). Empty under modern + # packaging -> fall back to bare soname (linker-namespace resolution). + v = posix.environ.get(b"ANDROID_NATIVE_LIBRARY_DIR") + return v.decode("utf-8") if v else None + + class _SorefFinder: """meta_path finder: dotted name -> jniLibs lib via its ``.soref`` marker.""" @@ -36,6 +47,7 @@ def __init__(self): # Cache one zipimporter per zip sys.path entry. Value is None for entries # that are not zips (plain directories) so we don't retry zipimporter(). self._zi_cache = {} + self._native_dir = _native_lib_dir() def _zipimporter(self, entry): try: @@ -80,10 +92,18 @@ def find_spec(self, fullname, path=None, target=None): if data is None: return None # not a relocated native module -> let others handle it soname = data.decode("utf-8").strip() - # origin = bare soname -> create_dynamic dlopen()s it by basename, which the - # app's linker namespace resolves from the APK (modern packaging). - loader = ExtensionFileLoader(fullname, soname) - return ModuleSpec(fullname, loader, origin=soname) + # Prefer an absolute origin under nativeLibraryDir when the lib was extracted + # there (legacy packaging); else bare soname for linker-namespace resolution. + origin = soname + if self._native_dir: + cand = self._native_dir + "/" + soname + try: + open(cand, "rb").close() + origin = cand + except OSError: + pass + loader = ExtensionFileLoader(fullname, origin) + return ModuleSpec(fullname, loader, origin=origin) def install(): From 55015ccacad47f6aad1cb55ded2fc6174196cf48 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 14:55:45 -0700 Subject: [PATCH 05/12] =?UTF-8?q?android:=20drop=20useLegacyPackaging=20?= =?UTF-8?q?=E2=80=94=20mmap=20natives=20from=20the=20APK=20(goal=20achieve?= =?UTF-8?q?d)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The finder now loads extension modules via Bionic's APK zip-path (base.apk!/lib//) when libs aren't extracted (modern packaging). AndroidPlugin exports ANDROID_APK_NATIVE_PREFIX (sourceDir + abi); _sp_bootstrap prefers an extracted nativeLibraryDir copy (legacy) else the APK zip-path. The run_example app sets useLegacyPackaging=false. Proven on arm64 emulator: program runs fully (bz2, sqlite, numpy), ZERO native extraction (no .so in app storage or nativeLibraryDir), and /proc//maps shows 32 r-xp executable mappings backed by base.apk — i.e. native modules are mmap'd directly from the APK. Pure code loads from stored zips via zipimport. --- .../run_example/android/app/build.gradle.kts | 8 ++++---- .../serious_python_android/AndroidPlugin.java | 13 +++++++++++-- .../python/_sp_bootstrap.py | 16 ++++++++++++++-- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/serious_python/example/run_example/android/app/build.gradle.kts b/src/serious_python/example/run_example/android/app/build.gradle.kts index 5478ce2d..d73de46a 100644 --- a/src/serious_python/example/run_example/android/app/build.gradle.kts +++ b/src/serious_python/example/run_example/android/app/build.gradle.kts @@ -16,12 +16,12 @@ android { targetCompatibility = JavaVersion.VERSION_17 } - // serious_python bundles libpython*.so. Use legacy (extracted, uncompressed) - // packaging so the embedded interpreter can dlopen them at runtime, and keep - // their symbols so they are not stripped. + // Modern packaging: native libs stay uncompressed/page-aligned in the APK and load + // directly (mmap). The serious_python finder dlopens extension modules via the Bionic + // APK zip-path, so no legacy extraction is needed. packaging { jniLibs { - useLegacyPackaging = true + useLegacyPackaging = false keepDebugSymbols += setOf( "*/arm64-v8a/libpython*.so", "*/armeabi-v7a/libpython*.so", diff --git a/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java b/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java index 6f903b57..5dde6f75 100644 --- a/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java +++ b/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java @@ -38,8 +38,17 @@ public void onAttachedToEngine(@NonNull FlutterPluginBinding flutterPluginBindin channel.setMethodCallHandler(this); this.context = flutterPluginBinding.getApplicationContext(); try { - Os.setenv(ANDROID_NATIVE_LIBRARY_DIR, - new ContextWrapper(this.context).getApplicationInfo().nativeLibraryDir, true); + android.content.pm.ApplicationInfo ai = + new ContextWrapper(this.context).getApplicationInfo(); + Os.setenv(ANDROID_NATIVE_LIBRARY_DIR, ai.nativeLibraryDir, true); + // Under modern packaging (useLegacyPackaging=false) native libs are NOT extracted + // to nativeLibraryDir; they live uncompressed/page-aligned inside the APK and are + // loadable via Bionic's zip-path (apk!/lib//). Export that prefix so + // the finder can dlopen them directly from the APK (mmap, no extraction). + String abi = (android.os.Build.SUPPORTED_ABIS != null + && android.os.Build.SUPPORTED_ABIS.length > 0) + ? android.os.Build.SUPPORTED_ABIS[0] : ""; + Os.setenv("ANDROID_APK_NATIVE_PREFIX", ai.sourceDir + "!/lib/" + abi + "/", true); } catch (Exception e) { // nothing to do } diff --git a/src/serious_python_android/python/_sp_bootstrap.py b/src/serious_python_android/python/_sp_bootstrap.py index 3ad9d962..a7e3b83f 100644 --- a/src/serious_python_android/python/_sp_bootstrap.py +++ b/src/serious_python_android/python/_sp_bootstrap.py @@ -40,6 +40,13 @@ def _native_lib_dir(): return v.decode("utf-8") if v else None +def _apk_native_prefix(): + # base.apk!/lib// — Bionic zip-path to libs mmap'd from the APK under modern + # packaging (useLegacyPackaging=false), where libs are NOT extracted to disk. + v = posix.environ.get(b"ANDROID_APK_NATIVE_PREFIX") + return v.decode("utf-8") if v else None + + class _SorefFinder: """meta_path finder: dotted name -> jniLibs lib via its ``.soref`` marker.""" @@ -48,6 +55,7 @@ def __init__(self): # that are not zips (plain directories) so we don't retry zipimporter(). self._zi_cache = {} self._native_dir = _native_lib_dir() + self._apk_prefix = _apk_native_prefix() def _zipimporter(self, entry): try: @@ -92,8 +100,10 @@ def find_spec(self, fullname, path=None, target=None): if data is None: return None # not a relocated native module -> let others handle it soname = data.decode("utf-8").strip() - # Prefer an absolute origin under nativeLibraryDir when the lib was extracted - # there (legacy packaging); else bare soname for linker-namespace resolution. + # Resolve the lib to an absolute origin (CPython prepends "./" to a no-slash + # origin, which breaks bare-soname loading). Prefer the extracted copy under + # nativeLibraryDir (legacy packaging); else the Bionic APK zip-path (modern + # packaging, mmap'd from the APK, never extracted). origin = soname if self._native_dir: cand = self._native_dir + "/" + soname @@ -102,6 +112,8 @@ def find_spec(self, fullname, path=None, target=None): origin = cand except OSError: pass + if origin == soname and self._apk_prefix: + origin = self._apk_prefix + soname loader = ExtensionFileLoader(fullname, origin) return ModuleSpec(fullname, loader, origin=origin) From 07ee2ec5eb9a1bf667309e684d569549df0d7df4 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 15:11:40 -0700 Subject: [PATCH 06/12] android: wire SERIOUS_PYTHON_DART_BRIDGE_DIST override; finder install via dart-bridge shim (F) - copyDartBridge_$abi uses a local dir of cross-compiled libdart_bridge-android--py.so when SERIOUS_PYTHON_DART_BRIDGE_DIST is set (mirrors SERIOUS_PYTHON_BUILD_DIST), bypassing the GitHub download for dev iteration on the bridge. - Drop the interim sitecustomize injection: the dart-bridge Android shim now installs the finder before site (proven on emulator: green with finder installed solely by the shim). Re-enable the sitecustomize fallback for bridges without the shim (commented in the task). This branch now requires the dart-bridge android-native-mmap bridge (via the override until released). --- .../android/build.gradle.kts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/serious_python_android/android/build.gradle.kts b/src/serious_python_android/android/build.gradle.kts index 3123215f..8d78c087 100644 --- a/src/serious_python_android/android/build.gradle.kts +++ b/src/serious_python_android/android/build.gradle.kts @@ -234,9 +234,9 @@ for (abi in abis) { } } zip?.add("_sp_bootstrap.py", bootstrapPy.readBytes()) // finder at zip root - // Interim install hook: site (during Py_Initialize) imports this and - // installs the finder. Superseded by the dart-bridge pre-site shim (F). - zip?.add("sitecustomize.py", "import _sp_bootstrap\n_sp_bootstrap.install()\n".toByteArray()) + // The dart-bridge Android shim (F) installs the finder before `site`. A + // sitecustomize fallback can be re-enabled for bridges without that shim: + // zip?.add("sitecustomize.py", "import _sp_bootstrap\n_sp_bootstrap.install()\n".toByteArray()) zip?.close() bundleFile.delete() // fake-zip must not ship } @@ -279,7 +279,14 @@ for (abi in abis) { // the pinned dart_bridge_version release into a cache shared across // builds, then drop it as `libdart_bridge.so` (no version suffix) so the // Dart side can DynamicLibrary.open by a stable short name. - val bridgeFile = File(dartBridgeCacheDir, "libdart_bridge-android-$abi-py$pythonVersion.so") + // SERIOUS_PYTHON_DART_BRIDGE_DIST: local-dev override pointing at a dir of + // freshly cross-compiled libdart_bridge-android--py.so, bypassing the + // GitHub release download (mirrors the SERIOUS_PYTHON_BUILD_DIST escape hatch). + val dartBridgeDist = System.getenv("SERIOUS_PYTHON_DART_BRIDGE_DIST") + val bridgeFile = if (dartBridgeDist != null) + File(dartBridgeDist, "libdart_bridge-android-$abi-py$pythonVersion.so") + else + File(dartBridgeCacheDir, "libdart_bridge-android-$abi-py$pythonVersion.so") tasks.register("downloadDartBridge_$abi") { src("https://github.com/flet-dev/dart-bridge/releases/download/v$dartBridgeVersion/libdart_bridge-android-$abi-py$pythonVersion.so") dest(bridgeFile) @@ -292,7 +299,8 @@ for (abi in abis) { from(bridgeFile) into("src/main/jniLibs/$abi") rename(".*", "libdart_bridge.so") - dependsOn("downloadDartBridge_$abi", "jniCleanUp_$abi") + if (dartBridgeDist == null) dependsOn("downloadDartBridge_$abi") + dependsOn("jniCleanUp_$abi") } packageTasks.add("copyDartBridge_$abi") } From e6b0a42f1b84704e984da334fd01bb9a93ec01d2 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 15:23:42 -0700 Subject: [PATCH 07/12] android: bootstrap audit probe + clean stale ABI jniLibs dirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _sp_bootstrap.install(): report any extension module already imported at install time (loaded during core-init before the finder) — verified empty on emulator, so PyConfig core-init pulls only builtin/frozen modules (F's pre-site install is safe). - cleanStaleAbis: remove jniLibs/ dirs for ABIs not in the current set (e.g. stale armeabi-v7a 3.12 leftovers when building 3.14) so old fake-zip .so aren't packaged. Audits on arm64 emulator + AAB: importtime (no pre-finder natives), ABI-split (per-ABI natives, ~33MB pure payload shared ONCE in assets vs per-ABI duplication before), multi-ABI markers resolve in x86_64, allowlist partition+extract+import, 16KB-aligned. --- src/serious_python_android/android/build.gradle.kts | 13 +++++++++++++ src/serious_python_android/python/_sp_bootstrap.py | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/src/serious_python_android/android/build.gradle.kts b/src/serious_python_android/android/build.gradle.kts index 8d78c087..751be497 100644 --- a/src/serious_python_android/android/build.gradle.kts +++ b/src/serious_python_android/android/build.gradle.kts @@ -155,6 +155,19 @@ fun storedZip(f: File): StoredZip { // Loop through abiFilters val packageTasks = mutableListOf() + +// Remove jniLibs/ dirs for ABIs not in the current set (e.g. stale armeabi-v7a +// 3.12 leftovers when building 3.14) so they aren't packaged into the APK/AAB. +val jniLibsRoot = file("src/main/jniLibs") +tasks.register("cleanStaleAbis") { + doLast { + jniLibsRoot.listFiles()?.forEach { d -> + if (d.isDirectory && d.name !in abis) d.deleteRecursively() + } + } +} +packageTasks.add("cleanStaleAbis") + for (abi in abis) { if (siteSrcDir == null || siteSrcDir.isBlank()) { throw InvalidUserDataException("SERIOUS_PYTHON_SITE_PACKAGES environment variable is not set.") diff --git a/src/serious_python_android/python/_sp_bootstrap.py b/src/serious_python_android/python/_sp_bootstrap.py index a7e3b83f..be29d3c6 100644 --- a/src/serious_python_android/python/_sp_bootstrap.py +++ b/src/serious_python_android/python/_sp_bootstrap.py @@ -123,6 +123,14 @@ def install(): global _installed if _installed: return + # Bootstrap audit: any extension module (.so) already imported at this point was + # loaded during interpreter core-init, BEFORE the finder existed — which only works + # if it was builtin/frozen. A non-empty list means that module must be made static + # (PyImport_AppendInittab) or it will fail under modern packaging. Expected: empty. + pre = [n for n, m in sys.modules.items() + if getattr(m, "__file__", None) and str(m.__file__).endswith(".so")] + if pre: + sys.stderr.write("SP_BOOTSTRAP pre-finder native modules: %r\n" % (pre,)) for f in sys.meta_path: if isinstance(f, _SorefFinder): _installed = True From e439ce450aef24a34ccc251aacdcf4853a2fbec4 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 16:34:38 -0700 Subject: [PATCH 08/12] Regenerate version tables from python-build 20260614 (dart_bridge 1.4.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dart run serious_python:gen_version_tables --release-date 20260614 — the 20260614 manifest now pins dart_bridge_version 1.4.0 (the full-API/PyConfig Android bridge that installs the native-module finder before site). Python versions unchanged. Default build now downloads the released v1.4.0 bridge; verified green on the arm64 emulator (sqlite, numpy) with no env overrides. SERIOUS_PYTHON_DART_BRIDGE_DIST remains as a dev escape hatch. --- src/serious_python/lib/src/python_versions.dart | 2 +- src/serious_python_android/android/python_versions.properties | 2 +- src/serious_python_darwin/darwin/python_versions.properties | 2 +- src/serious_python_linux/linux/python_versions.properties | 2 +- src/serious_python_windows/windows/python_versions.properties | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/serious_python/lib/src/python_versions.dart b/src/serious_python/lib/src/python_versions.dart index dad2aa82..eac66b3b 100644 --- a/src/serious_python/lib/src/python_versions.dart +++ b/src/serious_python/lib/src/python_versions.dart @@ -11,7 +11,7 @@ const dartBridgeVersionEnvironmentVariable = "DART_BRIDGE_VERSION"; /// python-build release the bundled runtimes come from (YYYYMMDD). const pythonReleaseDate = "20260614"; -const dartBridgeVersion = "1.3.2"; +const dartBridgeVersion = "1.4.0"; const defaultPythonVersion = "3.14"; class PythonRelease { diff --git a/src/serious_python_android/android/python_versions.properties b/src/serious_python_android/android/python_versions.properties index c72c7f8e..7cc4bbe2 100644 --- a/src/serious_python_android/android/python_versions.properties +++ b/src/serious_python_android/android/python_versions.properties @@ -1,7 +1,7 @@ # GENERATED by `dart run serious_python:gen_version_tables` from # python-build manifest.json (release 20260614). Do not edit by hand. default_python_version=3.14 -dart_bridge_version=1.3.2 +dart_bridge_version=1.4.0 python_build_release_date=20260614 3.12.full_version=3.12.13 3.13.full_version=3.13.14 diff --git a/src/serious_python_darwin/darwin/python_versions.properties b/src/serious_python_darwin/darwin/python_versions.properties index c72c7f8e..7cc4bbe2 100644 --- a/src/serious_python_darwin/darwin/python_versions.properties +++ b/src/serious_python_darwin/darwin/python_versions.properties @@ -1,7 +1,7 @@ # GENERATED by `dart run serious_python:gen_version_tables` from # python-build manifest.json (release 20260614). Do not edit by hand. default_python_version=3.14 -dart_bridge_version=1.3.2 +dart_bridge_version=1.4.0 python_build_release_date=20260614 3.12.full_version=3.12.13 3.13.full_version=3.13.14 diff --git a/src/serious_python_linux/linux/python_versions.properties b/src/serious_python_linux/linux/python_versions.properties index c72c7f8e..7cc4bbe2 100644 --- a/src/serious_python_linux/linux/python_versions.properties +++ b/src/serious_python_linux/linux/python_versions.properties @@ -1,7 +1,7 @@ # GENERATED by `dart run serious_python:gen_version_tables` from # python-build manifest.json (release 20260614). Do not edit by hand. default_python_version=3.14 -dart_bridge_version=1.3.2 +dart_bridge_version=1.4.0 python_build_release_date=20260614 3.12.full_version=3.12.13 3.13.full_version=3.13.14 diff --git a/src/serious_python_windows/windows/python_versions.properties b/src/serious_python_windows/windows/python_versions.properties index c72c7f8e..7cc4bbe2 100644 --- a/src/serious_python_windows/windows/python_versions.properties +++ b/src/serious_python_windows/windows/python_versions.properties @@ -1,7 +1,7 @@ # GENERATED by `dart run serious_python:gen_version_tables` from # python-build manifest.json (release 20260614). Do not edit by hand. default_python_version=3.14 -dart_bridge_version=1.3.2 +dart_bridge_version=1.4.0 python_build_release_date=20260614 3.12.full_version=3.12.13 3.13.full_version=3.13.14 From 3c0ac1e48c52f7519ab352eda77c7b49607a4dba Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 16:43:24 -0700 Subject: [PATCH 09/12] docs: drop useLegacyPackaging consumer instructions; document native-mmap (CHANGELOGs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_example/flask_example READMEs: remove the useLegacyPackaging/keepDebugSymbols packaging block — consumers need no special native packaging now, just minSdk 23+. Document SERIOUS_PYTHON_ANDROID_EXTRACT_PACKAGES for path-hungry packages. - run_example app build.gradle.kts: drop the packaging block entirely (modern packaging is AGP's default at minSdk 23+). Verified green on emulator with natives still mmap'd from base.apk (32 r-xp maps), zero extraction. - serious_python + serious_python_android CHANGELOGs (3.0.0): replace the useLegacyPackaging language with the native-mmap design; bump bundled dart_bridge to 1.4.0. --- src/serious_python/CHANGELOG.md | 5 ++-- .../example/flask_example/README.md | 21 ++++++-------- .../example/run_example/README.md | 28 ++++++++----------- .../run_example/android/app/build.gradle.kts | 17 ++--------- src/serious_python_android/CHANGELOG.md | 5 ++-- 5 files changed, 28 insertions(+), 48 deletions(-) diff --git a/src/serious_python/CHANGELOG.md b/src/serious_python/CHANGELOG.md index fb7aba82..069c0a61 100644 --- a/src/serious_python/CHANGELOG.md +++ b/src/serious_python/CHANGELOG.md @@ -1,7 +1,8 @@ ## 3.0.0 -* **New in-process transport (dart_bridge FFI).** `SeriousPython.run` can now run the embedded interpreter **in-process** through the `dart_bridge` FFI bridge instead of talking to it over a socket. The Python lifecycle (initialize / run / teardown) is absorbed into the `dart_bridge` native library on every platform — `dart_bridge.xcframework` (iOS/macOS), `libdart_bridge.so` (Android/Linux), and `dart_bridge.dll` / `dart_bridge.pyd` (Windows) — and a new `PythonBridge` API exposes a MsgPack control channel plus dedicated binary data channels between Dart and Python. See the `bridge_example` app. The bundled `dart_bridge` is **1.2.3**. -* **Breaking change:** requires Flutter **3.44.2** / Dart 3.12+. The Android plugin moves to AGP **8.11.1**, `compileSdk` **36**, Java **17**, and the Kotlin-DSL Gradle build; the removed `android.bundle.enableUncompressedNativeLibs` flag is replaced with `useLegacyPackaging` + `keepDebugSymbols` for the bundled `libpython*.so`. +* **New in-process transport (dart_bridge FFI).** `SeriousPython.run` can now run the embedded interpreter **in-process** through the `dart_bridge` FFI bridge instead of talking to it over a socket. The Python lifecycle (initialize / run / teardown) is absorbed into the `dart_bridge` native library on every platform — `dart_bridge.xcframework` (iOS/macOS), `libdart_bridge.so` (Android/Linux), and `dart_bridge.dll` / `dart_bridge.pyd` (Windows) — and a new `PythonBridge` API exposes a MsgPack control channel plus dedicated binary data channels between Dart and Python. See the `bridge_example` app. The bundled `dart_bridge` is **1.4.0**. +* **Android native packaging — memory-mapped from the APK.** Python extension modules are relocated into `jniLibs` and loaded **directly from the APK** (mmap, no extraction) by a custom importer that resolves them from `.soref` markers; pure Python ships in stored, ABI-common asset zips read via `zipimport` (no per-ABI duplication). Apps no longer need `useLegacyPackaging` / `keepDebugSymbols` — the brittle per-app packaging config is gone. Set **`SERIOUS_PYTHON_ANDROID_EXTRACT_PACKAGES`** (comma-separated relative paths) to ship path-hungry packages extracted to disk. The dart-bridge Android binary uses the full CPython API (`PyConfig`) to install the importer before `site` runs. +* **Breaking change:** requires Flutter **3.44.2** / Dart 3.12+. The Android plugin moves to AGP **8.11.1**, `compileSdk` **36**, Java **17**, and the Kotlin-DSL Gradle build (`build.gradle.kts`). * Python runtime versions are now a committed snapshot of `flet-dev/python-build`'s date-keyed `manifest.json`, generated by `dart run serious_python:gen_version_tables`. **`SERIOUS_PYTHON_VERSION`** (short, e.g. `3.14`) is the single input — the full CPython version, python-build release date, Pyodide version + platform tag, and `dart_bridge` version all derive from it. `SERIOUS_PYTHON_FULL_VERSION`, `SERIOUS_PYTHON_BUILD_DATE`, and `DART_BRIDGE_VERSION` remain as rarely-needed escape hatches. The native build configs (Android `build.gradle`, Darwin podspec, Linux/Windows `CMakeLists.txt`) read the generated `python_versions.properties`, and a CI job fails if the snapshots drift from the manifest. This replaces the per-config hardcoded defaults and the `flet build`-exported `SERIOUS_PYTHON_FULL_VERSION` / `SERIOUS_PYTHON_BUILD_DATE` introduced in 2.0.0. * Bundle **3.12.13 / 3.13.14 / 3.14.6** (python-build `20260614`); Pyodide **0.27.7 / 0.29.4 / 314.0.0** (314.0.0 GA, up from the 314.0.0a2 in 2.0.0). * Add **`dart run serious_python:main version [--json]`** — prints the serious_python version, the pinned python-build release, and the supported Python / Pyodide / dart_bridge matrix. diff --git a/src/serious_python/example/flask_example/README.md b/src/serious_python/example/flask_example/README.md index ac573f01..24c72c17 100644 --- a/src/serious_python/example/flask_example/README.md +++ b/src/serious_python/example/flask_example/README.md @@ -37,21 +37,16 @@ export SERIOUS_PYTHON_SITE_PACKAGES=$(pwd)/build/site-packages dart run serious_python:main package app/src -p Linux -r -r -r app/src/requirements.txt ``` -Important: to make `serious_python` work in your own Android app, the bundled -`libpython*.so` must be shipped uncompressed and extracted so the embedded -interpreter can `dlopen` them at runtime. In `android/app/build.gradle.kts`: +For Android, no special native-library packaging config is required. +`serious_python` relocates Python extension modules into `jniLibs` and loads them +directly from the APK (memory-mapped, no extraction), and ships pure Python in +stored asset zips. Just use a `minSdk` of 23+ so native libs stay uncompressed and +page-aligned in the APK: ```kotlin android { - packaging { - jniLibs { - useLegacyPackaging = true - } + defaultConfig { + minSdk = 23 } } -``` - -`useLegacyPackaging = true` is the modern replacement (AGP 8.1+) for both the -old `android.bundle.enableUncompressedNativeLibs=false` gradle property and the -`android:extractNativeLibs="true"` manifest attribute, and it covers both APK and -App Bundle builds. See the [public issue](https://issuetracker.google.com/issues/147096055). \ No newline at end of file +``` \ No newline at end of file diff --git a/src/serious_python/example/run_example/README.md b/src/serious_python/example/run_example/README.md index 1ff3c267..463f3765 100644 --- a/src/serious_python/example/run_example/README.md +++ b/src/serious_python/example/run_example/README.md @@ -43,29 +43,23 @@ For Pyodide: dart run serious_python:main package app/src -p Emscripten -r -r -r app/src/requirements.txt ``` -For Android: +For Android, no special native-library packaging config is required. serious_python +relocates Python extension modules into `jniLibs` and loads them directly from the APK +(memory-mapped, no extraction), and ships pure Python in stored asset zips read via +`zipimport`. Just use a `minSdk` of 23+ so native libs stay uncompressed/page-aligned in +the APK: In `android/app/build.gradle.kts`: ```kotlin android { - // serious_python bundles libpython*.so. Use legacy (extracted, uncompressed) - // packaging so the embedded interpreter can dlopen them at runtime, and keep - // their symbols so they are not stripped. - packaging { - jniLibs { - useLegacyPackaging = true - keepDebugSymbols += setOf( - "*/arm64-v8a/libpython*.so", - "*/armeabi-v7a/libpython*.so", - "*/x86/libpython*.so", - "*/x86_64/libpython*.so", - ) - } - } - defaultConfig { minSdk = 23 } } -``` \ No newline at end of file +``` + +To ship a path-hungry package **extracted to disk** instead of inside the zip — for +packages that read bundled data via `__file__` / `pkg_resources` rather than +`importlib.resources` — set `SERIOUS_PYTHON_ANDROID_EXTRACT_PACKAGES` to a comma-separated +list of relative package paths before building. \ No newline at end of file diff --git a/src/serious_python/example/run_example/android/app/build.gradle.kts b/src/serious_python/example/run_example/android/app/build.gradle.kts index d73de46a..f1d3f07b 100644 --- a/src/serious_python/example/run_example/android/app/build.gradle.kts +++ b/src/serious_python/example/run_example/android/app/build.gradle.kts @@ -16,20 +16,9 @@ android { targetCompatibility = JavaVersion.VERSION_17 } - // Modern packaging: native libs stay uncompressed/page-aligned in the APK and load - // directly (mmap). The serious_python finder dlopens extension modules via the Bionic - // APK zip-path, so no legacy extraction is needed. - packaging { - jniLibs { - useLegacyPackaging = false - keepDebugSymbols += setOf( - "*/arm64-v8a/libpython*.so", - "*/armeabi-v7a/libpython*.so", - "*/x86/libpython*.so", - "*/x86_64/libpython*.so", - ) - } - } + // No special native-library packaging is needed: serious_python loads extension + // modules directly from the APK (mmap) via its finder. Modern packaging + // (useLegacyPackaging=false) is AGP's default at minSdk 23+. defaultConfig { // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). diff --git a/src/serious_python_android/CHANGELOG.md b/src/serious_python_android/CHANGELOG.md index 774ca11e..6a296470 100644 --- a/src/serious_python_android/CHANGELOG.md +++ b/src/serious_python_android/CHANGELOG.md @@ -1,7 +1,8 @@ ## 3.0.0 -* **In-process Python (dart_bridge FFI).** The Python lifecycle now runs through `libdart_bridge.so` (from `flet-dev/dart-bridge` **1.2.3**) instead of a socket transport; `extractNativeLibs=true` keeps `libpython*.so` extractable for `dlopen`. -* **Breaking change:** requires Flutter **3.44.2**. Moves to AGP **8.11.1**, Gradle 8.11.1, `compileSdk` **36** and Java **17**; `useLegacyPackaging` + `keepDebugSymbols` for the bundled `libpython*.so` replace the removed `android.bundle.enableUncompressedNativeLibs`. +* **In-process Python (dart_bridge FFI).** The Python lifecycle now runs through `libdart_bridge.so` (from `flet-dev/dart-bridge` **1.4.0**) instead of a socket transport. +* **Native modules are memory-mapped from the APK — no more `useLegacyPackaging`.** Python extension modules (stdlib `lib-dynload` and site-packages) are relocated into `jniLibs//lib.so` and loaded directly from the APK by a custom `sys.meta_path` finder that resolves them from `.soref` markers — no extraction to disk, still ABI-split by the Play Store. Pure Python ships in **stored, ABI-common asset zips** read via `zipimport`, so the stdlib is no longer duplicated per ABI. This replaces the previous scheme of zipping stdlib/site-packages into fake `lib*.so` files (which required `useLegacyPackaging`). The dart-bridge Android binary now uses the full CPython API (`PyConfig`) to install the finder before `site` runs. Set **`SERIOUS_PYTHON_ANDROID_EXTRACT_PACKAGES`** (comma-separated relative paths) to ship path-hungry packages extracted to disk instead. +* **Breaking change:** requires Flutter **3.44.2**. Moves to AGP **8.11.1**, Gradle 8.11.1, `compileSdk` **36**, Java **17**, and the **Kotlin-DSL** Gradle build (`build.gradle.kts`). * `build.gradle` resolves the Python version from the generated `python_versions.properties` (a snapshot of python-build's `manifest.json`): `SERIOUS_PYTHON_VERSION` selects the version; the full version, build date and `dart_bridge` version derive from the table, with `SERIOUS_PYTHON_FULL_VERSION` / `SERIOUS_PYTHON_BUILD_DATE` / `DART_BRIDGE_VERSION` left as escape hatches. Downloads continue to use python-build's date-keyed release scheme. * Remove the scaffold `getPlatformVersion` method. From 1efd909b2578ff6229953c21cc15ea63a3ef9ecc Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 16:55:37 -0700 Subject: [PATCH 10/12] flask_example: add INTERNET permission to main manifest (fix release builds) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Flask app binds a local socket server, so it needs android.permission.INTERNET in all build types. It was only in the debug/profile manifests, so release builds hit 'PermissionError: [Errno 1] Operation not permitted' on socket() (the app isn't in the inet group). Verified: release APK now grants INTERNET and the Flask server starts (Running on http://127.0.0.1:55001) — flask/werkzeug from the zip, _socket via the finder. --- .../flask_example/android/app/src/main/AndroidManifest.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/serious_python/example/flask_example/android/app/src/main/AndroidManifest.xml b/src/serious_python/example/flask_example/android/app/src/main/AndroidManifest.xml index 9658760c..97e1ff4d 100644 --- a/src/serious_python/example/flask_example/android/app/src/main/AndroidManifest.xml +++ b/src/serious_python/example/flask_example/android/app/src/main/AndroidManifest.xml @@ -1,4 +1,7 @@ + + Date: Tue, 16 Jun 2026 17:31:00 -0700 Subject: [PATCH 11/12] android: split-aware APK native prefix (Play Store AAB) + drop dead getNativeLibraryDir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ANDROID_APK_NATIVE_PREFIX now points at whichever installed APK actually contains lib// — base.apk for single-APK builds, the per-ABI config split for Play Store AAB installs (where base.apk has no native libs). Detected by probing each of sourceDir + splitSourceDirs for the always-present libdart_bridge.so. - Remove the unused getNativeLibraryDir method-channel handler (run() uses getFilesDir). Verified with bundletool: built the AAB, build-apks + install-apks the device splits (base.apk + split_config.arm64_v8a.apk), ran green (numpy). adb-root /proc//maps shows 33 r-xp mappings from split_config.arm64_v8a.apk, 0 from base.apk, 0 extracted — natives mmap'd directly from the config split. --- .../serious_python_android/AndroidPlugin.java | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java b/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java index 5dde6f75..54ba8f80 100644 --- a/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java +++ b/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java @@ -44,16 +44,41 @@ public void onAttachedToEngine(@NonNull FlutterPluginBinding flutterPluginBindin // Under modern packaging (useLegacyPackaging=false) native libs are NOT extracted // to nativeLibraryDir; they live uncompressed/page-aligned inside the APK and are // loadable via Bionic's zip-path (apk!/lib//). Export that prefix so - // the finder can dlopen them directly from the APK (mmap, no extraction). + // the finder can dlopen them directly from the APK (mmap, no extraction). For Play + // Store AAB installs the libs are in a per-ABI config split, not base.apk, so pick + // whichever installed APK actually contains lib//. String abi = (android.os.Build.SUPPORTED_ABIS != null && android.os.Build.SUPPORTED_ABIS.length > 0) ? android.os.Build.SUPPORTED_ABIS[0] : ""; - Os.setenv("ANDROID_APK_NATIVE_PREFIX", ai.sourceDir + "!/lib/" + abi + "/", true); + Os.setenv("ANDROID_APK_NATIVE_PREFIX", apkNativePrefix(ai, abi), true); } catch (Exception e) { // nothing to do } } + // Bionic zip-path prefix (!/lib//) of the installed APK that holds the + // native libs. Single-APK builds -> base.apk; Play Store AAB installs -> the + // per-ABI config split (base.apk has no libs then). Detected by probing for the + // always-present libdart_bridge.so. + private static String apkNativePrefix(android.content.pm.ApplicationInfo ai, String abi) { + java.util.List apks = new java.util.ArrayList<>(); + if (ai.sourceDir != null) apks.add(ai.sourceDir); + if (ai.splitSourceDirs != null) { + java.util.Collections.addAll(apks, ai.splitSourceDirs); + } + String member = "lib/" + abi + "/libdart_bridge.so"; + for (String apk : apks) { + try (java.util.zip.ZipFile zf = new java.util.zip.ZipFile(apk)) { + if (zf.getEntry(member) != null) { + return apk + "!/lib/" + abi + "/"; + } + } catch (Exception e) { + // unreadable apk — skip + } + } + return (ai.sourceDir != null ? ai.sourceDir : "") + "!/lib/" + abi + "/"; + } + @Override public void onAttachedToActivity(@NonNull ActivityPluginBinding activityPluginBinding) { PythonActivity.mActivity = activityPluginBinding.getActivity(); @@ -78,10 +103,6 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull Result result) { } catch (Exception e) { result.error("Error", e.getMessage(), null); } - } else if (call.method.equals("getNativeLibraryDir")) { - ContextWrapper contextWrapper = new ContextWrapper(context); - String nativeLibraryDir = contextWrapper.getApplicationInfo().nativeLibraryDir; - result.success(nativeLibraryDir); } else if (call.method.equals("getFilesDir")) { result.success(context.getFilesDir().getAbsolutePath()); } else if (call.method.equals("extractAsset")) { From 74ecf34ca566e4994c9172396f0bd2be3069fa33 Mon Sep 17 00:00:00 2001 From: Feodor Fitsner Date: Tue, 16 Jun 2026 20:40:32 -0700 Subject: [PATCH 12/12] docs(readme): add 'How packaging works' section (per-platform layout) Document, per platform, where the stdlib and site-packages live, how native extension modules are shipped (Android: jniLibs mmap'd from the APK via a custom importer; iOS: xcframeworks + AppleFrameworkLoader; macOS: universal .so; Linux/Windows: on-disk; Web: Pyodide wheels), architecture handling, and how the user's app.zip is produced and extracted at runtime. Replace the stale Android note (enableUncompressedNativeLibs / extractNativeLibs) with the new no-config guidance (minSdk 23+). --- src/serious_python/README.md | 57 ++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/src/serious_python/README.md b/src/serious_python/README.md index ad965893..c1834aab 100644 --- a/src/serious_python/README.md +++ b/src/serious_python/README.md @@ -207,6 +207,47 @@ Additional Python binary packages for iOS and Android can be built with adding a Request additional packages for iOS and Android on [Flet Discussions - Packages](https://github.com/flet-dev/flet/discussions/categories/packages). +## How packaging works + +`dart run serious_python:main package` assembles two things, which the platform plugin then bundles into your Flutter app: + +1. **The CPython runtime + standard library** — a per-target build downloaded from [flet-dev/python-build](https://github.com/flet-dev/python-build) (and, for native extensions, [mobile-forge](https://github.com/flet-dev/mobile-forge)) and bundled by the plugin at build time. +2. **Your app + its dependencies** — your Python sources are zipped into an **asset** (`app/app.zip` by default), and `pip`-installed packages are placed where each platform expects them. + +At runtime the plugin sets `PYTHONHOME` / `PYTHONPATH` (or, on Android, installs a custom importer) so the interpreter finds the stdlib, your dependencies, and your app. + +The on-disk layout differs per platform, mostly because each OS has different rules for shipping **native (compiled) extension modules** — the `.so`/`.pyd`/`.dylib` files inside packages like `numpy`: + +| Platform | Standard library | Site-packages (deps) | Native extension modules | Architectures | +| --- | --- | --- | --- | --- | +| **Android** | `stdlib.zip` asset, read via `zipimport` | `sitepackages.zip` asset, read via `zipimport` | relocated to `jniLibs//`, **memory-mapped from the APK** (no extraction), resolved by a custom importer | natives per-ABI in `jniLibs`; pure zips are ABI-common (shipped once) | +| **iOS** | dir inside the framework resource bundle | dir inside the framework resource bundle | each `.so` wrapped in a signed `.framework` inside an `.xcframework`, loaded via CPython's `AppleFrameworkLoader` (`.fwork` markers) | device `arm64` + simulator `arm64`/`x86_64` xcframework slices | +| **macOS** | dir inside the framework resource bundle | dir (universal) | universal (`lipo`'d `arm64`+`x86_64`) `.so`, loaded directly | `arm64`+`x86_64` merged into fat binaries | +| **Linux** | `/python/` | `/site-packages/` | on-disk `.so` (in `lib-dynload` / package dirs) | one of `x86_64` / `aarch64` per build | +| **Windows** | `/Lib/` | `/site-packages/` | on-disk `.pyd`/`.dll` in `/DLLs/` | `x86_64` | +| **Web** | bundled inside Pyodide | `__pypackages__/` inside `app.zip` | Pyodide WebAssembly wheels | `wasm32` | + +### Your app program (all platforms) + +`package` copies your Python sources into a temp dir (honoring `--exclude` globs, optionally compiling to `.pyc` with `--compile-app`), zips it to `app/app.zip` (override with `-a`/`--asset`), and writes an `app.zip.hash` next to it. At runtime the asset is extracted once to the app's support directory (`/flet/app`), guarded by a hash + an optional `invalidateKey` (typically your app version) so it only re-extracts when the bundle changes — debug builds always re-extract. Your app dir is placed first on `sys.path`; a sibling `__pypackages__/` is also added (so you can vendor pure-Python deps next to your code). + +`pip install` output goes to `build/site-packages` by default (override with the `SERIOUS_PYTHON_SITE_PACKAGES` env var). For mobile, packages are installed **per architecture** (a `sitecustomize.py` shim spoofs the wheel platform tag so the correct mobile wheels resolve), then merged or split per platform as shown above. + +### Android specifics + +- **Pure Python** (stdlib + dependencies) ships in two **stored** (uncompressed) ABI-common zips — `stdlib.zip` and `sitepackages.zip` — copied once to the app's files dir and imported in place via `zipimport`. Final `sys.path` (highest first): your app dir, the extract dir, `sitepackages.zip`, `stdlib.zip`. +- **Native modules** (stdlib `lib-dynload` and site-package extensions) are relocated to `jniLibs//lib.so` and loaded **directly from the APK** (memory-mapped, never extracted to disk); a `sys.meta_path` finder resolves them from `.soref` markers left in the zips. This is why Android needs **no** `useLegacyPackaging` / `keepDebugSymbols` config and the stdlib is **not** duplicated per ABI. +- **Path-hungry packages** (those that read bundled data via `__file__` / `pkg_resources` rather than `importlib.resources`) can be shipped extracted to disk instead of inside the zip — list them (comma-separated relative paths) in `SERIOUS_PYTHON_ANDROID_EXTRACT_PACKAGES`; they go into `extract.zip` and are unpacked to disk at first launch. +- Works for both **single APK** (`flutter build apk`) and **Play Store App Bundles** (per-ABI config splits); under legacy packaging / `minSdk < 23` the same finder falls back to loading from the extracted `nativeLibraryDir`. + +### iOS / macOS specifics + +The CPython runtime, stdlib, and (on iOS) native extensions are bundled into `serious_python_darwin.framework` as resources. On **iOS**, the App Store forbids loose `.dylib`s, so every native extension `.so` is repackaged into a signed `.framework` inside an `.xcframework`, with a `.fwork` text marker left at the module's import path; CPython's `AppleFrameworkLoader` reads the marker and loads the framework binary. On **macOS**, native extensions stay as plain `.so`, merged into universal (`arm64`+`x86_64`) binaries at package time. `PYTHONHOME` is the framework's resource path; `sys.path` includes `/site-packages`, `/stdlib`, and `/stdlib/lib-dynload`. + +### Linux / Windows specifics + +The CPython runtime (`libpython3.so` + `libpython.so` on Linux; `python3.dll` + `python.dll` on Windows), `libdart_bridge`, the stdlib, and native modules are copied next to your app's executable at build time. `PYTHONHOME` is the executable's directory. On Windows, extension modules (`.pyd`) and their dependent DLLs live in `/DLLs/`, which is added to `sys.path`. + ## Platform notes ### Build matrix @@ -238,18 +279,16 @@ MACOSX_DEPLOYMENT_TARGET = 10.15; ### Android -To make `serious_python` work in your own Android app: +No special native-library packaging config is required (see [How packaging works](#how-packaging-works)). serious_python loads native modules directly from the APK and ships pure Python in stored asset zips, so you don't need `useLegacyPackaging`, `keepDebugSymbols`, `extractNativeLibs`, or `android.bundle.enableUncompressedNativeLibs`. Just use a `minSdk` of 23+ so native libs stay uncompressed/page-aligned in the APK: -If you build an App Bundle Edit `android/gradle.properties` and add the flag: - -``` -android.bundle.enableUncompressedNativeLibs=false +```kotlin +android { + defaultConfig { + minSdk = 23 + } +} ``` -If you build an APK Make sure `android/app/src/AndroidManifest.xml` has `android:extractNativeLibs="true"` in the `` tag. - -For more information, see the [public issue](https://issuetracker.google.com/issues/147096055). - ## Troubleshooting ### Detailed logging