feature: now we recycle lua thread GC objects for our "light threads" and added lua_thread_cache_max_entries directive to configure the cache size.

Author: agentzh

.travis.yml

@@ -88,7 +88,7 @@ install:
   - git clone https://github.com/openresty/lua-resty-mysql.git ../lua-resty-mysql
   - git clone https://github.com/openresty/lua-resty-string.git ../lua-resty-string
   - git clone https://github.com/openresty/stream-lua-nginx-module.git ../stream-lua-nginx-module
-  - git clone -b v2.1-agentzh https://github.com/openresty/luajit2.git luajit2
+  - git clone -b reset-thread https://github.com/openresty/luajit2.git luajit2
 
 before_script:
   - mysql -uroot -e 'create database ngx_test; grant all on ngx_test.* to "ngx_test"@"%" identified by "ngx_test"; flush privileges;'

README.markdown

@@ -1063,6 +1063,7 @@ Directives
 * [lua_use_default_type](#lua_use_default_type)
 * [lua_malloc_trim](#lua_malloc_trim)
 * [lua_code_cache](#lua_code_cache)
+* [lua_thread_cache_max_entries](#lua_thread_cache_max_entries)
 * [lua_regex_cache_max_entries](#lua_regex_cache_max_entries)
 * [lua_regex_match_limit](#lua_regex_match_limit)
 * [lua_package_path](#lua_package_path)
@@ -1293,6 +1294,27 @@ development as it has a significant negative impact on overall performance. For
 
 [Back to TOC](#directives)
 
+lua_thread_cache_max_entries
+----------------------------
+
+**syntax:** *lua_thread_cache_max_entries <num>*
+
+**default:** *lua_thread_cache_max_entries 1024*
+
+**context:** *http*
+
+Specifies the maximum number of entries allowed in the worker process level lua thread object cache.
+
+This cache recycles the lua thread GC objects among all our "light threads".
+
+A zero value of `<num>` disables the cache.
+
+Note that this feature requires OpenResty's LuaJIT with the new C API `lua_resetthread`.
+
+This feature was first introduced in verson `v0.10.9`.
+
+[Back to TOC](#directives)
+
 lua_regex_cache_max_entries
 ---------------------------
 

doc/HttpLuaModule.wiki

@@ -1029,6 +1029,24 @@ Disabling the Lua code cache is strongly
 discouraged for production use and should only be used during
 development as it has a significant negative impact on overall performance. For example, the performance of a "hello world" Lua example can drop by an order of magnitude after disabling the Lua code cache.
 
+== lua_thread_cache_max_entries ==
+
+'''syntax:''' ''lua_thread_cache_max_entries <num>''
+
+'''default:''' ''lua_thread_cache_max_entries 1024''
+
+'''context:''' ''http''
+
+Specifies the maximum number of entries allowed in the worker process level lua thread object cache.
+
+This cache recycles the lua thread GC objects among all our "light threads".
+
+A zero value of `<num>` disables the cache.
+
+Note that this feature requires OpenResty's LuaJIT with the new C API `lua_resetthread`.
+
+This feature was first introduced in verson `v0.10.9`.
+
 == lua_regex_cache_max_entries ==
 
 '''syntax:''' ''lua_regex_cache_max_entries <num>''

src/ngx_http_lua_common.h

@@ -185,6 +185,13 @@ typedef struct {
 } ngx_http_lua_preload_hook_t;
 
 
+typedef struct {
+    int             ref;
+    lua_State      *co;
+    ngx_queue_t     queue;
+} ngx_http_lua_thread_ref_t;
+
+
 struct ngx_http_lua_main_conf_s {
     lua_State           *lua;
     ngx_pool_cleanup_t  *vm_cleanup;
@@ -203,6 +210,8 @@ struct ngx_http_lua_main_conf_s {
 
     ngx_connection_t    *watcher;  /* for watching the process exit event */
 
+    ngx_int_t            lua_thread_cache_max_entries;
+
 #if (NGX_PCRE)
     ngx_int_t            regex_cache_entries;
     ngx_int_t            regex_cache_max_entries;
@@ -276,6 +285,9 @@ struct ngx_http_lua_main_conf_s {
 
     ngx_flag_t           set_sa_restart;
 
+    ngx_queue_t          free_lua_threads;  /* of ngx_http_lua_thread_ref_t */
+    ngx_queue_t          cached_lua_threads;  /* of ngx_http_lua_thread_ref_t */
+
     unsigned             requires_header_filter:1;
     unsigned             requires_body_filter:1;
     unsigned             requires_capture_filter:1;

src/ngx_http_lua_control.c

@@ -305,6 +305,7 @@ ngx_http_lua_ngx_redirect(lua_State *L)
 static int
 ngx_http_lua_on_abort(lua_State *L)
 {
+    int                           co_ref;
     ngx_http_request_t           *r;
     ngx_http_lua_ctx_t           *ctx;
     ngx_http_lua_co_ctx_t        *coctx = NULL;
@@ -335,18 +336,9 @@ ngx_http_lua_on_abort(lua_State *L)
         return 2;
     }
 
-    ngx_http_lua_coroutine_create_helper(L, r, ctx, &coctx);
-
-    lua_pushlightuserdata(L, ngx_http_lua_lightudata_mask(
-                          coroutines_key));
-    lua_rawget(L, LUA_REGISTRYINDEX);
-    lua_pushvalue(L, -2);
-
-    dd("on_wait thread 1: %p", lua_tothread(L, -1));
-
-    coctx->co_ref = luaL_ref(L, -2);
-    lua_pop(L, 1);
+    ngx_http_lua_coroutine_create_helper(L, r, ctx, &coctx, &co_ref);
 
+    coctx->co_ref = co_ref;
     coctx->is_uthread = 1;
     ctx->on_abort_co_ctx = coctx;
 

src/ngx_http_lua_coroutine.c

@@ -59,7 +59,7 @@ ngx_http_lua_coroutine_create(lua_State *L)
         return luaL_error(L, "no request ctx found");
     }
 
-    return ngx_http_lua_coroutine_create_helper(L, r, ctx, NULL);
+    return ngx_http_lua_coroutine_create_helper(L, r, ctx, NULL, NULL);
 }
 
 
@@ -92,7 +92,7 @@ ngx_http_lua_coroutine_wrap(lua_State *L)
         return luaL_error(L, "no request ctx found");
     }
 
-    ngx_http_lua_coroutine_create_helper(L, r, ctx, &coctx);
+    ngx_http_lua_coroutine_create_helper(L, r, ctx, &coctx, NULL);
 
     coctx->is_wrap = 1;
 
@@ -104,11 +104,12 @@ ngx_http_lua_coroutine_wrap(lua_State *L)
 
 int
 ngx_http_lua_coroutine_create_helper(lua_State *L, ngx_http_request_t *r,
-    ngx_http_lua_ctx_t *ctx, ngx_http_lua_co_ctx_t **pcoctx)
+    ngx_http_lua_ctx_t *ctx, ngx_http_lua_co_ctx_t **pcoctx, int *co_ref)
 {
     lua_State                     *vm;  /* the Lua VM */
     lua_State                     *co;  /* new coroutine to be created */
     ngx_http_lua_co_ctx_t         *coctx; /* co ctx for the new coroutine */
+    ngx_http_lua_main_conf_t      *lmcf;
 
     luaL_argcheck(L, lua_isfunction(L, 1) && !lua_iscfunction(L, 1), 1,
                   "Lua function expected");
@@ -120,7 +121,13 @@ ngx_http_lua_coroutine_create_helper(lua_State *L, ngx_http_request_t *r,
     /* create new coroutine on root Lua state, so it always yields
      * to main Lua thread
      */
-    co = lua_newthread(vm);
+    if (co_ref == NULL) {
+        co = lua_newthread(vm);
+
+    } else {
+        lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module);
+        *co_ref = ngx_http_lua_new_cached_thread(vm, &co, lmcf, 0);
+    }
 
     ngx_http_lua_probe_user_coroutine_create(r, L, co);
 
@@ -151,6 +158,10 @@ ngx_http_lua_coroutine_create_helper(lua_State *L, ngx_http_request_t *r,
 
     lua_xmove(vm, L, 1);    /* move coroutine from main thread to L */
 
+    if (co_ref) {
+        lua_pop(vm, 1);  /* pop coroutines */
+    }
+
     lua_pushvalue(L, 1);    /* copy entry function to top of L*/
     lua_xmove(L, co, 1);    /* move entry function from L to co */
 

src/ngx_http_lua_coroutine.h

@@ -15,7 +15,7 @@
 void ngx_http_lua_inject_coroutine_api(ngx_log_t *log, lua_State *L);
 
 int ngx_http_lua_coroutine_create_helper(lua_State *L, ngx_http_request_t *r,
-    ngx_http_lua_ctx_t *ctx, ngx_http_lua_co_ctx_t **pcoctx);
+    ngx_http_lua_ctx_t *ctx, ngx_http_lua_co_ctx_t **pcoctx, int *co_ref);
 
 
 #endif /* _NGX_HTTP_LUA_COROUTINE_H_INCLUDED_ */

src/ngx_http_lua_module.c

@@ -85,6 +85,13 @@ static ngx_command_t ngx_http_lua_cmds[] = {
       0,
       NULL },
 
+    { ngx_string("lua_thread_cache_max_entries"),
+      NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_num_slot,
+      NGX_HTTP_MAIN_CONF_OFFSET,
+      offsetof(ngx_http_lua_main_conf_t, lua_thread_cache_max_entries),
+      NULL },
+
     { ngx_string("lua_max_running_timers"),
       NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_num_slot,
@@ -793,6 +800,13 @@ ngx_http_lua_init(ngx_conf_t *cf)
                           "the OpenResty releases from https://openresty.org/"
                           "en/download.html)");
         }
+#elif !defined(HAVE_LUA_RESETTHREAD)
+        ngx_log_error(NGX_LOG_ALERT, cf->log, 0,
+                      "detected an old version of OpenResty's LuaJIT missing "
+                      "the lua_resetthread API and thus the "
+                      "performance will be compromised; please upgrade to the "
+                      "latest version of OpenResty's LuaJIT: "
+                      "https://github.com/openresty/luajit2");
 #endif
 
         ngx_http_lua_content_length_hash =
@@ -915,6 +929,7 @@ ngx_http_lua_create_main_conf(ngx_conf_t *cf)
     lmcf->pool = cf->pool;
     lmcf->max_pending_timers = NGX_CONF_UNSET;
     lmcf->max_running_timers = NGX_CONF_UNSET;
+    lmcf->lua_thread_cache_max_entries = NGX_CONF_UNSET;
 #if (NGX_PCRE)
     lmcf->regex_cache_max_entries = NGX_CONF_UNSET;
     lmcf->regex_match_limit = NGX_CONF_UNSET;
@@ -942,7 +957,27 @@ ngx_http_lua_create_main_conf(ngx_conf_t *cf)
 static char *
 ngx_http_lua_init_main_conf(ngx_conf_t *cf, void *conf)
 {
-    ngx_http_lua_main_conf_t *lmcf = conf;
+#ifdef HAVE_LUA_RESETTHREAD
+    ngx_int_t                    i, n;
+    ngx_http_lua_thread_ref_t   *trefs;
+#endif
+
+    ngx_http_lua_main_conf_t     *lmcf = conf;
+
+    if (lmcf->lua_thread_cache_max_entries < 0) {
+        lmcf->lua_thread_cache_max_entries = 1024;
+
+#ifndef HAVE_LUA_RESETTHREAD
+
+    } else if (lmcf->lua_thread_cache_max_entries > 0) {
+        ngx_log_error(NGX_LOG_EMERG, cf->log, 0,
+                      "lua_thread_cache_max_entries has no effect when "
+                      "LuaJIT has no support for the lua_resetthread API "
+                      "(you forgot to use OpenResty's LuaJIT?)");
+        return NGX_CONF_ERROR;
+
+#endif
+    }
 
 #if (NGX_PCRE)
     if (lmcf->regex_cache_max_entries == NGX_CONF_UNSET) {
@@ -976,6 +1011,26 @@ ngx_http_lua_init_main_conf(ngx_conf_t *cf, void *conf)
 
     lmcf->cycle = cf->cycle;
 
+    ngx_queue_init(&lmcf->free_lua_threads);
+    ngx_queue_init(&lmcf->cached_lua_threads);
+
+#ifdef HAVE_LUA_RESETTHREAD
+    n = lmcf->lua_thread_cache_max_entries;
+
+    if (n > 0) {
+        trefs = ngx_palloc(cf->pool, n * sizeof(ngx_http_lua_thread_ref_t));
+        if (trefs == NULL) {
+            return NGX_CONF_ERROR;
+        }
+
+        for (i = 0; i < n; i++) {
+            trefs[i].ref = LUA_NOREF;
+            trefs[i].co = NULL;
+            ngx_queue_insert_head(&lmcf->free_lua_threads, &trefs[i].queue);
+        }
+    }
+#endif
+
     return NGX_CONF_OK;
 }