[GR-70223] Ensure fast execution of GraalPy C extensions with Native Image

ci.jsonnet

@@ -118,10 +118,6 @@
     // -----------------------------------------------------------------------------------------------------------------
     local gate_task_dict = {
         "python-unittest": gpgate + platform_spec(no_jobs) + platform_spec({
-            "linux:amd64:jdk21"          : daily     + t("01:00:00") + provide(GPY_JVM21_STANDALONE),
-            "linux:aarch64:jdk21"        : daily     + t("02:00:00") + provide(GPY_JVM21_STANDALONE),
-            "darwin:aarch64:jdk21"       : daily     + t("01:30:00") + provide(GPY_JVM21_STANDALONE),
-            "windows:amd64:jdk21"        : daily     + t("01:30:00") + provide(GPY_JVM21_STANDALONE),
             "linux:amd64:jdk-latest"     : tier2                     + require(GPY_JVM_STANDALONE),
             "linux:aarch64:jdk-latest"   : tier3                     + provide(GPY_JVM_STANDALONE),
             "darwin:aarch64:jdk-latest"  : tier3                     + provide(GPY_JVM_STANDALONE),
@@ -136,10 +132,6 @@
             "darwin:aarch64:jdk-latest"  : daily     + t("01:00:00"),
         }),
         "python-unittest-multi-context": gpgate + platform_spec(no_jobs) + platform_spec({
-            "linux:amd64:jdk21"          : daily     + t("01:00:00") + require(GPY_JVM21_STANDALONE),
-            "linux:aarch64:jdk21"        : daily     + t("01:30:00") + require(GPY_JVM21_STANDALONE),
-            "darwin:aarch64:jdk21"       : daily     + t("01:00:00") + require(GPY_JVM21_STANDALONE),
-            "windows:amd64:jdk21"        : daily     + t("02:00:00"),
             "linux:amd64:jdk-latest"     : tier2                     + require(GPY_JVM_STANDALONE),
             "linux:aarch64:jdk-latest"   : daily     + t("01:30:00") + require(GPY_JVM_STANDALONE),
             "darwin:aarch64:jdk-latest"  : daily     + t("01:00:00") + require(GPY_JVM_STANDALONE),
@@ -160,10 +152,6 @@
             "linux:amd64:jdk-latest"     : tier2,
         }),
         "python-unittest-standalone": gpgate_maven + platform_spec(no_jobs) + platform_spec({
-            "linux:amd64:jdk21"          : daily     + t("02:00:00") + require(GPY_JVM21_STANDALONE),
-            "linux:aarch64:jdk21"        : daily     + t("02:00:00") + require(GPY_JVM21_STANDALONE),
-            "darwin:aarch64:jdk21"       : daily     + t("02:00:00") + require(GPY_JVM21_STANDALONE),
-            "windows:amd64:jdk21"        : daily     + t("02:00:00") + require(GPY_JVM21_STANDALONE) + batches(2),
             "linux:amd64:jdk-latest"     : tier3                     + require(GPY_JVM_STANDALONE) + require(GRAAL_JDK_LATEST),
             "linux:aarch64:jdk-latest"   : tier3                     + require(GPY_JVM_STANDALONE) + require(GRAAL_JDK_LATEST),
             "darwin:aarch64:jdk-latest"  : tier3                     + require(GPY_JVM_STANDALONE) + require(GRAAL_JDK_LATEST),
@@ -244,17 +232,17 @@
             "windows:amd64:jdk-latest"   : weekly    + t("20:00:00") + require(GPY_NATIVE_STANDALONE),
         }),
         "python-coverage-jacoco-tagged": cov_jacoco_tagged + batches(COVERAGE_SPLIT) + platform_spec(no_jobs) + platform_spec({
-            "linux:amd64:jdk21"          : weekly    + t("20:00:00"),
-            "darwin:aarch64:jdk21"       : weekly    + t("20:00:00"),
-            "windows:amd64:jdk21"        : weekly    + t("20:00:00"),
+            "linux:amd64:jdk-latest"          : weekly    + t("20:00:00"),
+            "darwin:aarch64:jdk-latest"       : weekly    + t("20:00:00"),
+            "windows:amd64:jdk-latest"        : weekly    + t("20:00:00"),
         }),
         "python-coverage-jacoco-base": cov_jacoco_base + platform_spec(no_jobs) + platform_spec({
-            "linux:amd64:jdk21"          : weekly    + t("20:00:00"),
-            "darwin:aarch64:jdk21"       : weekly    + t("20:00:00"),
-            "windows:amd64:jdk21"        : weekly    + t("20:00:00"),
+            "linux:amd64:jdk-latest"          : weekly    + t("20:00:00"),
+            "darwin:aarch64:jdk-latest"       : weekly    + t("20:00:00"),
+            "windows:amd64:jdk-latest"        : weekly    + t("20:00:00"),
         }),
         "python-coverage-truffle": cov_truffle + platform_spec(no_jobs) + platform_spec({
-            "linux:amd64:jdk21"          : weekly    + t("20:00:00"),
+            "linux:amd64:jdk-latest"          : weekly    + t("20:00:00"),
         }),
         "corp-compliance-watchdog": watchdog + platform_spec(no_jobs) + platform_spec({
             "linux:amd64:jdk-latest"     : tier1,
@@ -274,10 +262,20 @@
         }),
         // tests with sandboxed backends for various modules (posix, sha3, compression, pyexpat, ...)
         "python-unittest-sandboxed": gpgate_ee + platform_spec(no_jobs) + platform_spec({
+            "linux:amd64:jdk21"          : daily     + t("01:00:00") + provide(GPY_JVM21_STANDALONE),
+            "linux:aarch64:jdk21"        : daily     + t("02:00:00") + provide(GPY_JVM21_STANDALONE),
+            "darwin:aarch64:jdk21"       : daily     + t("01:00:00") + provide(GPY_JVM21_STANDALONE),
+            "windows:amd64:jdk21"        : daily     + t("01:30:00"),
             "linux:amd64:jdk-latest"     : tier2 + batches(2),
             "linux:aarch64:jdk-latest"   : tier3,
             "darwin:aarch64:jdk-latest"  : tier3,
         }),
+        "python-unittest-multi-context-sandboxed": gpgate_ee + platform_spec(no_jobs) + platform_spec({
+            "linux:amd64:jdk21"          : daily     + t("01:00:00") + require(GPY_JVM21_STANDALONE),
+            "linux:aarch64:jdk21"        : daily     + t("01:30:00") + require(GPY_JVM21_STANDALONE),
+            "darwin:aarch64:jdk21"       : daily     + t("01:00:00") + require(GPY_JVM21_STANDALONE),
+            "windows:amd64:jdk21"        : daily     + t("02:00:00"),
+        }),
         "python-svm-unittest-sandboxed": gpgate_ee + platform_spec(no_jobs) + platform_spec({
             "linux:amd64:jdk-latest"     : tier3 + provide(GPYEE_NATIVE_STANDALONE),
         }),

docs/contributor/IMPLEMENTATION_DETAILS.md

@@ -199,24 +199,27 @@ Below is a rough draft of the types and memory layouts involved and how they con
 ```
     Managed Heap                                        Native Heap
                                                           Stub allocated to represent managed object
-    +------------------------+                          +-------------------------------+
-    | PInt                   |<---+                     | struct PyGC_Head {            |
-    +------------------------+    |                     |    uintptr_t _gc_next         |
-    | BigInteger value       |    |                     |    uintptr_t _gc_prev         |
-    +------------------------+    |                     | }                             |
-                                  |       +------------>| struct GraalPyObject {        |
-    +------------------------+    |       |             |    Py_ssize_t ob_refcnt       |
-    | PythonNativeWrapper    |<---+       |     +------>|    PyObject *ob_type          |
-    +------------------------+------------+     |       |    int32_t handle_table_index |---+
-    |                        +<---+             |       | }                             |   |
-    +------------------------+    |             |       +-------------------------------+   |
-                                  |             |                                           |
-                                  |             |                                           |
-    +------------------------+    |             |                                           |
-+---| PythonObjectReference  |<---+             |                                           |
-|   +------------------------+------------------+                                           |
-|   | boolean gc             |                                                              |
-|   | boolean freeAtCollect  |                         -                                    |
+    +---------------------------+                       +-------------------------------+
++-->| PInt                      |                       | struct PyGC_Head {            |
+|   +---------------------------+                       |    uintptr_t _gc_next         |
+|   | long nativePointer        |---------------+       |    uintptr_t _gc_prev         |
+|   | PythonObjectReference ref |----+          |       | }                             |
+|   | BigInteger value          |    |          +------>| struct GraalPyObject {        |
+|   +---------------------------+    |          |       |    Py_ssize_t ob_refcnt       |
+|                                    |          |       |    PyObject *ob_type          |
+|                                    |          |       |    int32_t handle_table_index |---+
+|                                    |          |       | }                             |   |
+|                                    |          |       +-------------------------------+   |
+|                                    |          |                                           |
+|                                    |          |                                           |
+|                                    |          |                                           |
+|                                    |          |                                           |
+|   +------------------------+       |          |                                           |
++---| PythonObjectReference  |<------+          |                                           |
+    +------------------------+                  |                                           |
+    | long pointer           |------------------+                                           |
+    | boolean gc             |                                                              |
++---| boolean freeAtCollect  |                                                              |
 |   +------------------------+                                                              |
 |                                                                                           |
 |   +------------------------+                                                              |
@@ -244,7 +247,7 @@ Below is a rough draft of the types and memory layouts involved and how they con
 
 ```
 
-Managed objects are associated with `PythonNativeWrapper` subinstances when
+Managed objects store a pointer to native `GraalPyObject` subinstances when
 they go to native, native objects are represented throughout the interpreters
 as `PythonAbstractNativeObject`. Both have associated weak references,
 `PythonObjectReference` and `NativeObjectReference`, respectively.
@@ -261,14 +264,14 @@ Java code and is, for example, passed as an argument.
 
 When a managed object is passed to a native extension code:
 
-* We create  `PythonNativeWrapper`. We create `PythonNativeWrapper` to
-  provide a different interop protocol, and not expose `toNative` and
-  `asPointer` on Python objects. The wrapper is stored inside the
+* We allocate a `GraalPyObject`, `GraalPyVarObject`, or `GraalPyFloatObject` in
+  off-heap memory. The native pointer is stored inside the
   `PythonAbstractObject`, because pointer identity is relied upon by some
   extensions we saw in the wild. (See PythonToNativeNode)
     * If the object was a "primitive" (TruffleString, int, long, boolean) we must
-      box it first into a `PythonAbstractObject`, so we create a `PString` or
-      `PInt` wrapper (or retrieve one for the singletons).
+      first promote the object to an appropriate instance of
+      `PythonAbstractObject` (e.g. `PString` is the promoted object for
+      `TruffleString` and `PInt` for Java `int`, `long`, `boolean`).
     * For container types (such as tuples, lists), when their elements are
       accessed any primitive elements are (like the previous step) boxed into a
       `PythonAbstractObject`.

graalpython/com.oracle.graal.python.annotations/src/com/oracle/graal/python/annotations/CApiExternalFunctionSignatures.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.SOURCE)
+@Target(ElementType.TYPE)
+public @interface CApiExternalFunctionSignatures {
+}

graalpython/com.oracle.graal.python.benchmarks/python/harness.py

@@ -206,6 +206,22 @@ def warmup(cp_index):
         return -1
 
 
+def ensure_packages(**package_specs):
+    import sys, os
+
+    rootdir = os.path.dirname(__file__)
+    while os.path.basename(rootdir) != 'graalpython':
+        rootdir = os.path.dirname(rootdir)
+
+    sys.path.append(os.path.join(
+        rootdir,
+        "com.oracle.graal.python.test",
+        "src",
+    ))
+    from tests import ensure_packages
+    ensure_packages(**package_specs)
+
+
 def ccompile(name, code):
     import sys, os
 
@@ -287,6 +303,7 @@ def get_bench_module(bench_file):
             with _io.FileIO(bench_file, "r") as f:
                 bench_module.__file__ = bench_file
                 bench_module.ccompile = ccompile
+                bench_module.ensure_packages = ensure_packages
                 exec(compile(f.readall(), bench_file, "exec"), bench_module.__dict__)
                 return bench_module