diff --git a/c_src/py_nif.c b/c_src/py_nif.c index 07e4c95..6740095 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -3147,34 +3147,40 @@ static void owngil_execute_create_local_env(py_context_t *ctx) { res->interp_id = PyInterpreterState_GetID(interp); } - /* Create globals dict with builtins and erlang module */ - res->globals = PyDict_New(); + /* Copy globals from context to inherit preloaded code */ + res->globals = PyDict_Copy(ctx->globals); if (res->globals == NULL) { ctx->response_term = enif_make_tuple2(ctx->shared_env, enif_make_atom(ctx->shared_env, "error"), - enif_make_atom(ctx->shared_env, "globals_failed")); + enif_make_atom(ctx->shared_env, "globals_copy_failed")); ctx->response_ok = false; return; } - /* Add __builtins__ */ - PyObject *builtins = PyEval_GetBuiltins(); - if (builtins != NULL) { - PyDict_SetItemString(res->globals, "__builtins__", builtins); + /* Ensure __builtins__ is present */ + if (PyDict_GetItemString(res->globals, "__builtins__") == NULL) { + PyObject *builtins = PyEval_GetBuiltins(); + if (builtins != NULL) { + PyDict_SetItemString(res->globals, "__builtins__", builtins); + } } - /* Add __name__ = '__main__' */ - PyObject *main_name = PyUnicode_FromString("__main__"); - if (main_name != NULL) { - PyDict_SetItemString(res->globals, "__name__", main_name); - Py_DECREF(main_name); + /* Ensure __name__ = '__main__' is set */ + if (PyDict_GetItemString(res->globals, "__name__") == NULL) { + PyObject *main_name = PyUnicode_FromString("__main__"); + if (main_name != NULL) { + PyDict_SetItemString(res->globals, "__name__", main_name); + Py_DECREF(main_name); + } } - /* Add erlang module */ - PyObject *erlang = PyImport_ImportModule("erlang"); - if (erlang != NULL) { - PyDict_SetItemString(res->globals, "erlang", erlang); - Py_DECREF(erlang); + /* Ensure erlang module is available */ + if (PyDict_GetItemString(res->globals, "erlang") == NULL) { + PyObject *erlang = PyImport_ImportModule("erlang"); + if (erlang != NULL) { + PyDict_SetItemString(res->globals, "erlang", erlang); + Py_DECREF(erlang); + } } /* Use the same dict for locals (module-level execution) */ @@ -4933,32 +4939,38 @@ static ERL_NIF_TERM nif_create_local_env(ErlNifEnv *env, int argc, const ERL_NIF } #endif - /* Create globals dict with builtins and erlang module */ - res->globals = PyDict_New(); + /* Copy globals from context to inherit preloaded code */ + res->globals = PyDict_Copy(ctx->globals); if (res->globals == NULL) { py_context_release(&guard); enif_release_resource(res); - return make_error(env, "globals_failed"); + return make_error(env, "globals_copy_failed"); } - /* Add __builtins__ */ - PyObject *builtins = PyEval_GetBuiltins(); - if (builtins != NULL) { - PyDict_SetItemString(res->globals, "__builtins__", builtins); + /* Ensure __builtins__ is present (may not be in subinterpreter mode) */ + if (PyDict_GetItemString(res->globals, "__builtins__") == NULL) { + PyObject *builtins = PyEval_GetBuiltins(); + if (builtins != NULL) { + PyDict_SetItemString(res->globals, "__builtins__", builtins); + } } - /* Add __name__ = '__main__' so defined functions are accessible via __main__ */ - PyObject *main_name = PyUnicode_FromString("__main__"); - if (main_name != NULL) { - PyDict_SetItemString(res->globals, "__name__", main_name); - Py_DECREF(main_name); + /* Ensure __name__ = '__main__' is set */ + if (PyDict_GetItemString(res->globals, "__name__") == NULL) { + PyObject *main_name = PyUnicode_FromString("__main__"); + if (main_name != NULL) { + PyDict_SetItemString(res->globals, "__name__", main_name); + Py_DECREF(main_name); + } } - /* Add erlang module */ - PyObject *erlang = PyImport_ImportModule("erlang"); - if (erlang != NULL) { - PyDict_SetItemString(res->globals, "erlang", erlang); - Py_DECREF(erlang); + /* Ensure erlang module is available */ + if (PyDict_GetItemString(res->globals, "erlang") == NULL) { + PyObject *erlang = PyImport_ImportModule("erlang"); + if (erlang != NULL) { + PyDict_SetItemString(res->globals, "erlang", erlang); + Py_DECREF(erlang); + } } /* Use the same dict for locals (module-level execution) */ diff --git a/docs/preload.md b/docs/preload.md new file mode 100644 index 0000000..682e2b0 --- /dev/null +++ b/docs/preload.md @@ -0,0 +1,123 @@ +# Preload Code + +This guide covers preloading Python code that executes during interpreter initialization. + +## Overview + +The `py_preload` module allows you to define Python code that runs once per interpreter at creation time. The resulting globals (imports, functions, variables) become available to all process-local environments. + +## Use Cases + +- Share common imports across all contexts +- Define utility functions used throughout your application +- Set up configuration or constants +- Avoid repeated initialization overhead + +## API + +| Function | Description | +|----------|-------------| +| `set_code/1` | Set preload code (binary or iolist) | +| `get_code/0` | Get current preload code or `undefined` | +| `clear_code/0` | Remove preload code | +| `has_preload/0` | Check if preload is configured | + +## Basic Usage + +```erlang +%% Set preload code at application startup +py_preload:set_code(<<" +import json +import os + +def shared_helper(x): + return x * 2 + +CONFIG = {'debug': True, 'version': '1.0'} +">>). + +%% Create a context - preload is automatically applied +{ok, Ctx} = py_context:new(#{mode => worker}). + +%% Use preloaded imports +{ok, <<"{\"a\": 1}">>} = py:eval(Ctx, <<"json.dumps({'a': 1})">>). + +%% Use preloaded functions +{ok, 10} = py:eval(Ctx, <<"shared_helper(5)">>). + +%% Use preloaded variables +{ok, true} = py:eval(Ctx, <<"CONFIG['debug']">>). +``` + +## Execution Flow + +``` +Interpreter/Context Creation + │ + ▼ +apply_registered_paths() ← sys.path updates + │ + ▼ +apply_registered_imports() ← module imports + │ + ▼ +apply_preload() ← preload code execution + │ + ▼ +(interpreter ready) + │ + ▼ +create_local_env() ← copies from interpreter globals +``` + +## Process Isolation + +Each process-local environment gets an isolated copy of preloaded globals: + +```erlang +py_preload:set_code(<<"COUNTER = 0">>). + +{ok, Ctx1} = py_context:new(#{mode => worker}). +{ok, Ctx2} = py_context:new(#{mode => worker}). + +%% Modify in Ctx1 +ok = py:exec(Ctx1, <<"COUNTER = 100">>). +{ok, 100} = py:eval(Ctx1, <<"COUNTER">>). + +%% Ctx2 still has original value +{ok, 0} = py:eval(Ctx2, <<"COUNTER">>). +``` + +## Clearing Preload + +Clearing preload only affects new contexts: + +```erlang +py_preload:set_code(<<"PRELOADED = 42">>). + +{ok, Ctx1} = py_context:new(#{mode => worker}). +{ok, 42} = py:eval(Ctx1, <<"PRELOADED">>). + +%% Clear preload +py_preload:clear_code(). + +%% Existing context still has it +{ok, 42} = py:eval(Ctx1, <<"PRELOADED">>). + +%% New context does not +{ok, Ctx2} = py_context:new(#{mode => worker}). +{error, _} = py:eval(Ctx2, <<"PRELOADED">>). +``` + +## Best Practices + +1. **Set preload early** - Configure before creating any contexts +2. **Keep it focused** - Only include truly shared code +3. **Avoid side effects** - Preload runs once per interpreter +4. **Use for imports** - Common imports benefit most from preloading + +## Limitations + +- Changes to preload code don't affect existing contexts +- Same preload applies to all context modes (worker, subinterp, owngil) +- Preload errors during context creation will fail the context diff --git a/src/py_context.erl b/src/py_context.erl index 1b37e0e..171433e 100644 --- a/src/py_context.erl +++ b/src/py_context.erl @@ -421,6 +421,8 @@ init(Parent, Id, Mode) -> %% Apply all registered imports and paths to this interpreter apply_registered_imports(Ref), apply_registered_paths(Ref), + %% Apply preload code (populates globals for process-local envs) + apply_preload(Ref), %% For subinterpreters, create a dedicated event worker EventState = setup_event_worker(Ref, InterpId), %% For thread-model subinterpreters, spawn a dedicated callback handler @@ -518,6 +520,13 @@ apply_registered_paths(Ref) -> Paths -> py_nif:interp_apply_paths(Ref, Paths) end. +%% @private Apply preload code to the interpreter's globals. +%% +%% Called when a new interpreter is created. The preload code populates +%% the context's globals dict, which process-local environments inherit. +apply_preload(Ref) -> + py_preload:apply_preload(Ref). + %% @private create_context(worker) -> py_nif:context_create(worker); diff --git a/src/py_preload.erl b/src/py_preload.erl new file mode 100644 index 0000000..b8dda91 --- /dev/null +++ b/src/py_preload.erl @@ -0,0 +1,110 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Python preload code registry. +%%% +%%% Allows users to preload Python code that executes during interpreter +%%% initialization. The resulting globals become the base namespace that +%%% process-local environments inherit from. +%%% +%%% == Usage == +%%% +%%% ``` +%%% %% At application startup +%%% py_preload:set_code(<<" +%%% import json +%%% import os +%%% +%%% def shared_helper(x): +%%% return x * 2 +%%% +%%% CONFIG = {'debug': True} +%%% ">>). +%%% +%%% %% Later, any context will have these preloaded +%%% {ok, Ctx} = py_context:new(#{mode => worker}), +%%% {ok, 10} = py:eval(Ctx, <<"shared_helper(5)">>). +%%% ''' +%%% +%%% == Storage == +%%% +%%% Uses `persistent_term' for the preload code. Changes only affect +%%% newly created contexts; existing contexts are not modified. +%%% +%%% @end +-module(py_preload). + +-export([ + set_code/1, + get_code/0, + clear_code/0, + has_preload/0, + apply_preload/1 +]). + +-define(PRELOAD_KEY, {py_preload, code}). + +%% @doc Set preload code to be executed once per interpreter at init. +%% +%% The code is executed in the interpreter's `__main__' namespace. +%% All defined functions, variables, and imports become available +%% in process-local environments. +%% +%% @param Code Python code as binary or iolist +-spec set_code(binary() | iolist()) -> ok. +set_code(Code) when is_binary(Code); is_list(Code) -> + persistent_term:put(?PRELOAD_KEY, iolist_to_binary(Code)). + +%% @doc Get the current preload code. +%% +%% @returns The preload code binary, or `undefined' if not set +-spec get_code() -> binary() | undefined. +get_code() -> + try + persistent_term:get(?PRELOAD_KEY) + catch + error:badarg -> undefined + end. + +%% @doc Clear the preload code. +%% +%% New contexts will start with empty globals. Existing contexts +%% are not affected. +-spec clear_code() -> ok. +clear_code() -> + try + persistent_term:erase(?PRELOAD_KEY) + catch + error:badarg -> ok + end, + ok. + +%% @doc Check if preload code is configured. +-spec has_preload() -> boolean(). +has_preload() -> + get_code() =/= undefined. + +%% @doc Apply preload code to a context reference. +%% +%% Called internally by `py_context' during context initialization. +%% Executes the preload code in the context's interpreter. +%% +%% @param Ref NIF context reference +%% @returns `ok' if successful or no preload configured, `{error, Reason}' on failure +-spec apply_preload(reference()) -> ok | {error, term()}. +apply_preload(Ref) -> + case get_code() of + undefined -> ok; + Code -> py_nif:context_exec(Ref, Code) + end. diff --git a/test/py_preload_SUITE.erl b/test/py_preload_SUITE.erl new file mode 100644 index 0000000..598aba1 --- /dev/null +++ b/test/py_preload_SUITE.erl @@ -0,0 +1,249 @@ +%% Copyright 2026 Benoit Chesneau +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%%% @doc Tests for py_preload module. +-module(py_preload_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2 +]). + +-export([ + set_get_clear_test/1, + has_preload_test/1, + preload_variable_inherited_test/1, + preload_function_inherited_test/1, + preload_import_inherited_test/1, + preload_isolation_test/1, + preload_new_context_inherits_test/1, + clear_preload_new_context_test/1 +]). + +all() -> + [ + {group, api_tests}, + {group, inheritance_tests} + ]. + +groups() -> + [ + {api_tests, [sequence], [ + set_get_clear_test, + has_preload_test + ]}, + {inheritance_tests, [sequence], [ + preload_variable_inherited_test, + preload_function_inherited_test, + preload_import_inherited_test, + preload_isolation_test, + preload_new_context_inherits_test, + clear_preload_new_context_test + ]} + ]. + +init_per_suite(Config) -> + {ok, _} = application:ensure_all_started(erlang_python), + Config. + +end_per_suite(_Config) -> + %% Clear any preload code left over + py_preload:clear_code(), + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + %% Ensure clean state before each test + py_preload:clear_code(), + Config. + +end_per_testcase(_TestCase, _Config) -> + %% Clean up after each test + py_preload:clear_code(), + ok. + +%% ============================================================================= +%% API Tests +%% ============================================================================= + +set_get_clear_test(_Config) -> + %% Initially no code is set + undefined = py_preload:get_code(), + + %% Set code + Code = <<"X = 42">>, + ok = py_preload:set_code(Code), + + %% Get code + Code = py_preload:get_code(), + + %% Set different code (overwrite) + Code2 = <<"Y = 100">>, + ok = py_preload:set_code(Code2), + Code2 = py_preload:get_code(), + + %% Clear code + ok = py_preload:clear_code(), + undefined = py_preload:get_code(), + + %% Clear again should not error + ok = py_preload:clear_code(), + undefined = py_preload:get_code(). + +has_preload_test(_Config) -> + %% Initially no preload + false = py_preload:has_preload(), + + %% Set code + ok = py_preload:set_code(<<"X = 1">>), + true = py_preload:has_preload(), + + %% Clear code + ok = py_preload:clear_code(), + false = py_preload:has_preload(). + +%% ============================================================================= +%% Inheritance Tests +%% ============================================================================= + +preload_variable_inherited_test(_Config) -> + %% Set preload code with a variable + ok = py_preload:set_code(<<"PRELOADED = 42">>), + + %% Create context and check variable is available + {ok, Ctx} = py_context:new(#{mode => worker}), + try + {ok, 42} = py:eval(Ctx, <<"PRELOADED">>) + after + py_context:stop(Ctx) + end. + +preload_function_inherited_test(_Config) -> + %% Set preload code with a function + ok = py_preload:set_code(<<" +def double(x): + return x * 2 + +def add(a, b): + return a + b +">>), + + %% Create context and check functions work + {ok, Ctx} = py_context:new(#{mode => worker}), + try + {ok, 10} = py:eval(Ctx, <<"double(5)">>), + {ok, 7} = py:eval(Ctx, <<"add(3, 4)">>) + after + py_context:stop(Ctx) + end. + +preload_import_inherited_test(_Config) -> + %% Set preload code with imports + ok = py_preload:set_code(<<" +import json +import os +">>), + + %% Create context and check imports are available + {ok, Ctx} = py_context:new(#{mode => worker}), + try + %% json should be available + {ok, <<"{\"a\": 1}">>} = py:eval(Ctx, <<"json.dumps({'a': 1})">>), + %% os should be available + {ok, _} = py:eval(Ctx, <<"os.getcwd()">>) + after + py_context:stop(Ctx) + end. + +preload_isolation_test(_Config) -> + %% Set preload code + ok = py_preload:set_code(<<"PRELOADED = 42">>), + + %% Create first context and modify the variable + {ok, Ctx1} = py_context:new(#{mode => worker}), + try + {ok, 42} = py:eval(Ctx1, <<"PRELOADED">>), + ok = py:exec(Ctx1, <<"PRELOADED = 100">>), + {ok, 100} = py:eval(Ctx1, <<"PRELOADED">>), + + %% Create second context - should have original value + {ok, Ctx2} = py_context:new(#{mode => worker}), + try + {ok, 42} = py:eval(Ctx2, <<"PRELOADED">>) + after + py_context:stop(Ctx2) + end + after + py_context:stop(Ctx1) + end. + +preload_new_context_inherits_test(_Config) -> + %% Set preload code + ok = py_preload:set_code(<<" +SHARED_CONFIG = {'debug': True, 'version': '1.0'} + +def get_config_value(key): + return SHARED_CONFIG.get(key) +">>), + + %% Create multiple contexts - all should have preload + Ctxs = [begin + {ok, C} = py_context:new(#{mode => worker}), + C + end || _ <- lists:seq(1, 3)], + + try + lists:foreach(fun(Ctx) -> + {ok, true} = py:eval(Ctx, <<"get_config_value('debug')">>), + {ok, <<"1.0">>} = py:eval(Ctx, <<"get_config_value('version')">>) + end, Ctxs) + after + lists:foreach(fun(Ctx) -> py_context:stop(Ctx) end, Ctxs) + end. + +clear_preload_new_context_test(_Config) -> + %% Set preload code + ok = py_preload:set_code(<<"PRELOADED = 42">>), + + %% Create context with preload + {ok, Ctx1} = py_context:new(#{mode => worker}), + try + {ok, 42} = py:eval(Ctx1, <<"PRELOADED">>) + after + py_context:stop(Ctx1) + end, + + %% Clear preload + ok = py_preload:clear_code(), + + %% New context should NOT have preloaded variable + {ok, Ctx2} = py_context:new(#{mode => worker}), + try + {error, _} = py:eval(Ctx2, <<"PRELOADED">>) + after + py_context:stop(Ctx2) + end.