Shim split: dart_bridge.c (Dart-callable) + dart_bridge_shim.c (Python module)

Fixes the Linux/Windows/Android bridge_example handshake-timeout failure
by making Python and Dart share the SAME `global_enqueue_handler_func`
cell on every platform.

Before: the wheel-installed dart_bridge.so (cibuildwheel) and the
Flutter-built libflet_bridge.so each compiled a full copy of
dart_bridge.c — two separate binary instances, two separate copies of
the static `global_enqueue_handler_func`. Python's
set_enqueue_handler_func wrote to the wheel's copy; Dart's
DartBridge_EnqueueMessage read libflet_bridge's copy (NULL); messages
were silently dropped.

After: the core lives in libflet_bridge ONLY. The wheel ships a thin
shim that resolves the core's exports at PyInit time via runtime
symbol lookup (dlsym(RTLD_DEFAULT) → dlopen RTLD_GLOBAL fallback on
Linux/macOS, LoadLibrary+GetProcAddress on Windows). One global cell,
visible to both sides.

Changes:
- native/dart_bridge.c: drop the Python-callable methods. Exports
  three symbols for the shim to resolve:
    * dart_bridge_global_enqueue_handler_func (PyObject*, was static)
    * dart_bridge_post_to_dart (helper that wraps Dart_PostCObject_DL
      so the shim doesn't need its own copy of dart_api_dl.c)
    * existing DartBridge_InitDartApiDL + DartBridge_EnqueueMessage
- native/dart_bridge_shim.c (NEW): PyInit_dart_bridge resolves the
  three exports above via shim_sym_lookup(); set_enqueue_handler_func
  writes through the resolved pointer; send_bytes calls through the
  resolved function pointer. ImportError if libflet_bridge isn't
  loaded into the process (catches misconfiguration loudly).
- python/setup.py: compile only dart_bridge_shim.c; add -ldl on Linux
  for the dlopen/dlsym used by the shim. Dropped dart_api_dl.c from
  the wheel's sources (the shim doesn't call Dart_PostCObject_DL
  directly).
- darwin/Classes/dart_bridge_shim.c (symlink): so the Apple static
  link picks both .c files into the framework. Apple's
  dlsym(RTLD_DEFAULT) finds the symbols in-process (smoke-tested:
  macOS bridge_example_macos still passes).
- ci.yml + test-bridge-build.yml: drop continue-on-error from
  bridge_example_linux and bridge_example_windows — they should pass
  now.

Verified locally on macOS: `flutter test integration_test -d macos`
still green. CI verification pending — pushing to dart-bridge will
exercise all three desktop platforms.

Author: FeodorFitsner

.github/workflows/ci.yml

@@ -315,13 +315,6 @@ jobs:
     # Python (no static-link path like on Apple); pull it from the wheel build.
     needs: release_build
     if: startsWith(github.ref, 'refs/tags/v')
-    # KNOWN DEFERRED: symbol visibility between the wheel's dart_bridge.so and
-    # the Flutter-built libflet_bridge.so means Python's enqueue handler is
-    # never seen by Dart's DartBridge_EnqueueMessage (each has its own copy
-    # of global_enqueue_handler_func). Proper fix is the shim split from the
-    # original plan — tracked in bridge-followups.md. continue-on-error so
-    # the test doesn't block tag releases until the split lands.
-    continue-on-error: true
     strategy:
       fail-fast: false
       matrix:
@@ -406,9 +399,6 @@ jobs:
     runs-on: windows-latest
     needs: release_build
     if: startsWith(github.ref, 'refs/tags/v')
-    # Same KNOWN DEFERRED as bridge_example_linux above — symbol visibility
-    # between wheel and libflet_bridge.dll. Tracked in bridge-followups.md.
-    continue-on-error: true
     strategy:
       fail-fast: false
       matrix:

.github/workflows/test-bridge-build.yml

@@ -189,16 +189,6 @@ jobs:
     name: Bridge example Linux ${{ matrix.title }} round-trip (Python ${{ matrix.python_version }})
     runs-on: ${{ matrix.runner }}
     needs: test_wheel_build
-    # KNOWN DEFERRED: the wheel-installed dart_bridge.so and the Flutter-built
-    # libflet_bridge.so are separate binaries with separate copies of
-    # global_enqueue_handler_func. Python's set_enqueue_handler_func writes
-    # to the wheel's copy; Dart's DartBridge_EnqueueMessage reads
-    # libflet_bridge.so's copy, which is always NULL. Test currently times
-    # out at handshake echo. Proper fix is the shim split from the original
-    # plan (dart_bridge_shim.c that dlopens libflet_bridge.so RTLD_GLOBAL) —
-    # tracked in bridge-followups.md. continue-on-error so this matrix
-    # leaves data without blocking the rest of CI.
-    continue-on-error: true
     strategy:
       fail-fast: false
       matrix:
@@ -279,10 +269,6 @@ jobs:
     name: Bridge example Windows round-trip (Python ${{ matrix.python_version }})
     runs-on: windows-latest
     needs: test_wheel_build
-    # Same KNOWN DEFERRED as test_bridge_example_linux above — symbol
-    # visibility between wheel and libflet_bridge.dll. Tracked in
-    # bridge-followups.md.
-    continue-on-error: true
     strategy:
       fail-fast: false
       matrix:

src/serious_python_bridge/darwin/Classes/dart_bridge_shim.c

@@ -0,0 +1 @@
+../../native/dart_bridge_shim.c

src/serious_python_bridge/native/dart_bridge.c

@@ -15,12 +15,21 @@
 #endif
 
 // ---------------------------------------------------------------------------
-// Core: symbols called from Dart via FFI. On platforms with the shared-lib
-// split these live in libflet_bridge; on iOS they're linked statically into
-// the serious_python framework.
+// Core: symbols called from Dart via FFI plus exported helpers the Python-side
+// shim (dart_bridge_shim.c) resolves at runtime via dlsym. Compiled into
+// libflet_bridge.{so,dll,dylib} by the Flutter plugin build. On Apple
+// platforms also compiled into the static archive linked into the
+// serious_python framework alongside dart_bridge_shim.c.
+//
+// The shim NEVER defines its own copy of these symbols — it always looks them
+// up at runtime. That keeps Dart's view of `global_enqueue_handler_func` and
+// the Python shim's view as a single shared cell on every platform.
 // ---------------------------------------------------------------------------
 
-static PyObject* global_enqueue_handler_func = NULL;
+// Exported (non-static) so dart_bridge_shim.c's set_enqueue_handler_func can
+// write to it via dlsym. Initialised to NULL; the shim swaps in a PyObject*
+// callable when Python registers a handler.
+EXPORT PyObject* dart_bridge_global_enqueue_handler_func = NULL;
 
 EXPORT intptr_t DartBridge_InitDartApiDL(void* data) {
     return Dart_InitializeApiDL(data);
@@ -37,7 +46,7 @@ EXPORT void DartBridge_EnqueueMessage(const char* data, size_t len) {
 
     PyGILState_STATE gstate = PyGILState_Ensure();
 
-    if (!global_enqueue_handler_func) {
+    if (!dart_bridge_global_enqueue_handler_func) {
         fprintf(stderr, "[dart_bridge] enqueue handler is not registered\n");
         PyGILState_Release(gstate);
         return;
@@ -50,7 +59,8 @@ EXPORT void DartBridge_EnqueueMessage(const char* data, size_t len) {
         return;
     }
 
-    PyObject* result = PyObject_CallFunctionObjArgs(global_enqueue_handler_func, arg, NULL);
+    PyObject* result = PyObject_CallFunctionObjArgs(
+        dart_bridge_global_enqueue_handler_func, arg, NULL);
     if (!result) {
         PyErr_Print();
     }
@@ -60,49 +70,22 @@ EXPORT void DartBridge_EnqueueMessage(const char* data, size_t len) {
     PyGILState_Release(gstate);
 }
 
-// ---------------------------------------------------------------------------
-// Shim: Python-callable methods exposed by the `dart_bridge` module.
-// ---------------------------------------------------------------------------
-
-static PyObject* set_enqueue_handler_func(PyObject* self, PyObject* args) {
-    PyObject* func;
-
-    if (!PyArg_ParseTuple(args, "O:set_enqueue_handler_func", &func)) {
-        return NULL;
-    }
-
-    if (!PyCallable_Check(func)) {
-        PyErr_SetString(PyExc_TypeError, "parameter must be callable");
-        return NULL;
-    }
-
-    Py_XINCREF(func);
-    Py_XDECREF(global_enqueue_handler_func);
-    global_enqueue_handler_func = func;
-
-    Py_RETURN_NONE;
-}
-
-static PyObject* send_bytes(PyObject* self, PyObject* args) {
-    int64_t port;
-    const char* buffer;
-    Py_ssize_t length;
-
-    if (!PyArg_ParseTuple(args, "Ly#", &port, &buffer, &length)) {
-        return NULL;
-    }
-
+// Exported helper called by the shim's send_bytes(). Keeps the
+// Dart_PostCObject_DL invocation in this translation unit so the shim doesn't
+// need its own copy of dart_api_dl.c. Returns 0 on success, -1 on failure with
+// a Python exception set.
+EXPORT int dart_bridge_post_to_dart(int64_t port, const char* buffer, size_t length) {
     if (port == 0) {
         PyErr_SetString(PyExc_RuntimeError, "Dart port is 0 (invalid)");
-        return NULL;
+        return -1;
     }
 
     // Dart_PostCObject_DL is a function pointer populated by Dart_InitializeApiDL.
     // Calling it before init segfaults; surface a clean error instead.
     if (Dart_PostCObject_DL == NULL) {
         PyErr_SetString(PyExc_RuntimeError,
                         "Dart API DL not initialized (call DartBridge_InitDartApiDL from Dart first)");
-        return NULL;
+        return -1;
     }
 
     Dart_CObject obj;
@@ -113,24 +96,7 @@ static PyObject* send_bytes(PyObject* self, PyObject* args) {
 
     if (!Dart_PostCObject_DL(port, &obj)) {
         PyErr_SetString(PyExc_RuntimeError, "Dart_PostCObject_DL failed");
-        return NULL;
+        return -1;
     }
-
-    Py_RETURN_TRUE;
-}
-
-static PyMethodDef methods[] = {
-    {"send_bytes", send_bytes, METH_VARARGS, "Post a bytes payload to a Dart ReceivePort."},
-    {"set_enqueue_handler_func", set_enqueue_handler_func, METH_VARARGS,
-     "Register the Python callable that receives bytes posted from Dart."},
-    {NULL, NULL, 0, NULL}
-};
-
-static struct PyModuleDef moduledef = {
-    PyModuleDef_HEAD_INIT,
-    "dart_bridge", NULL, -1, methods
-};
-
-PyMODINIT_FUNC PyInit_dart_bridge(void) {
-    return PyModule_Create(&moduledef);
+    return 0;
 }

src/serious_python_bridge/native/dart_bridge_shim.c

@@ -0,0 +1,131 @@
+// Python-callable shim for the dart_bridge module.
+//
+// This file is the *only* source compiled into the dart_bridge wheel built by
+// cibuildwheel. It contains no Dart-callable symbols and no copy of the
+// shared `dart_bridge_global_enqueue_handler_func` cell — instead it resolves
+// the core's exports (defined in dart_bridge.c, linked into libflet_bridge)
+// at PyInit time via dlsym/GetProcAddress. That keeps Dart's view and
+// Python's view of the global as the SAME cell on Linux/Windows/Android.
+//
+// On Apple platforms this same file is also static-linked into the
+// serious_python framework alongside dart_bridge.c — the runtime lookup
+// then resolves to the symbols statically linked into the host binary
+// (dlopen of libflet_bridge is skipped because there's no such file).
+
+#define PY_SSIZE_T_CLEAN
+#define Py_LIMITED_API 0x030c0000
+#include <Python.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#if defined(_WIN32)
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+// Function-pointer + global types resolved at PyInit time.
+typedef int (*PostToDartFn)(int64_t port, const char* buffer, size_t length);
+
+static PyObject** g_handler_slot = NULL;   // points at dart_bridge_global_enqueue_handler_func in libflet_bridge
+static PostToDartFn g_post_to_dart = NULL; // dart_bridge_post_to_dart in libflet_bridge
+
+#if defined(_WIN32)
+static void* shim_sym_lookup(const char* name) {
+    // LoadLibraryA returns the existing handle if flet_bridge.dll is already
+    // loaded by Flutter (single instance, same memory, just bumps refcount).
+    // The DLL search path includes the executable's directory, where Flutter
+    // places plugin DLLs.
+    HMODULE flet = LoadLibraryA("flet_bridge.dll");
+    if (!flet) return NULL;
+    return (void*)GetProcAddress(flet, name);
+}
+#else
+static void* shim_sym_lookup(const char* name) {
+    // RTLD_DEFAULT searches every library already loaded into the process
+    // including ones loaded with RTLD_LOCAL by Dart's DynamicLibrary.open.
+    void* p = dlsym(RTLD_DEFAULT, name);
+    if (p) return p;
+    // Not visible globally (Dart loaded libflet_bridge with RTLD_LOCAL). Try
+    // an explicit RTLD_GLOBAL dlopen so subsequent lookups see it. dlopen of
+    // an already-loaded library returns the existing handle — single instance,
+    // same memory, just promoted into the global namespace.
+#if defined(__APPLE__)
+    void* h = dlopen("libflet_bridge.dylib", RTLD_NOW | RTLD_GLOBAL);
+#else
+    void* h = dlopen("libflet_bridge.so", RTLD_NOW | RTLD_GLOBAL);
+#endif
+    if (!h) return NULL;
+    return dlsym(h, name);
+}
+#endif
+
+static PyObject* set_enqueue_handler_func(PyObject* self, PyObject* args) {
+    PyObject* func;
+
+    if (!PyArg_ParseTuple(args, "O:set_enqueue_handler_func", &func)) {
+        return NULL;
+    }
+    if (!PyCallable_Check(func)) {
+        PyErr_SetString(PyExc_TypeError, "parameter must be callable");
+        return NULL;
+    }
+    if (!g_handler_slot) {
+        PyErr_SetString(PyExc_RuntimeError,
+                        "dart_bridge: libflet_bridge symbol not resolved (was the bridge plugin loaded?)");
+        return NULL;
+    }
+
+    Py_XINCREF(func);
+    Py_XDECREF(*g_handler_slot);
+    *g_handler_slot = func;
+
+    Py_RETURN_NONE;
+}
+
+static PyObject* send_bytes(PyObject* self, PyObject* args) {
+    int64_t port;
+    const char* buffer;
+    Py_ssize_t length;
+
+    if (!PyArg_ParseTuple(args, "Ly#", &port, &buffer, &length)) {
+        return NULL;
+    }
+    if (!g_post_to_dart) {
+        PyErr_SetString(PyExc_RuntimeError,
+                        "dart_bridge: libflet_bridge symbol not resolved (was the bridge plugin loaded?)");
+        return NULL;
+    }
+    if (g_post_to_dart(port, buffer, (size_t)length) != 0) {
+        // Helper sets the exception.
+        return NULL;
+    }
+    Py_RETURN_TRUE;
+}
+
+static PyMethodDef methods[] = {
+    {"send_bytes", send_bytes, METH_VARARGS, "Post a bytes payload to a Dart ReceivePort."},
+    {"set_enqueue_handler_func", set_enqueue_handler_func, METH_VARARGS,
+     "Register the Python callable that receives bytes posted from Dart."},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT,
+    "dart_bridge", NULL, -1, methods
+};
+
+PyMODINIT_FUNC PyInit_dart_bridge(void) {
+    // Resolve the libflet_bridge exports we depend on. Surface a clean
+    // ImportError if the lookup fails — typically means the bridge plugin's
+    // native library wasn't loaded into the process before Python ran.
+    g_handler_slot = (PyObject**)shim_sym_lookup("dart_bridge_global_enqueue_handler_func");
+    g_post_to_dart = (PostToDartFn)shim_sym_lookup("dart_bridge_post_to_dart");
+    if (!g_handler_slot || !g_post_to_dart) {
+        PyErr_SetString(PyExc_ImportError,
+                        "dart_bridge: failed to resolve libflet_bridge symbols "
+                        "(is serious_python_bridge's native library loaded into the process?)");
+        return NULL;
+    }
+    return PyModule_Create(&moduledef);
+}

src/serious_python_bridge/python/setup.py

@@ -1,20 +1,28 @@
+import sys
 from setuptools import Extension, setup
 
 NATIVE_ROOT = "../native"
 
+extra_link_args = []
+# dlopen/dlsym on Linux live in libdl. macOS rolls them into libSystem.
+# Windows uses LoadLibrary/GetProcAddress from kernel32 (implicit).
+if sys.platform.startswith("linux"):
+    extra_link_args.append("-ldl")
+
 setup(
     ext_modules=[
         Extension(
             "dart_bridge",
-            sources=[
-                f"{NATIVE_ROOT}/dart_bridge.c",
-                f"{NATIVE_ROOT}/dart_api/dart_api_dl.c",
-            ],
-            include_dirs=[NATIVE_ROOT],
+            # Shim only — the Dart-callable core and its dart_api_dl.c live in
+            # libflet_bridge (built by the Flutter plugin) and are resolved at
+            # PyInit time via dlsym / GetProcAddress.
+            sources=[f"{NATIVE_ROOT}/dart_bridge_shim.c"],
             # Limited API / abi3: one .so per platform works for any Python
-            # 3.12+. Keep this in lockstep with Py_LIMITED_API in dart_bridge.c.
+            # 3.12+. Keep this in lockstep with Py_LIMITED_API in
+            # dart_bridge_shim.c.
             define_macros=[("Py_LIMITED_API", "0x030c0000")],
             py_limited_api=True,
+            extra_link_args=extra_link_args,
         )
     ],
 )