Skip to content

feat(discovery): add kubernetes discovery data warm-up in master process #13054

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChuanFF wants to merge 1 commit into
base: master
Choose a base branch
from
+93 −56
Open

feat(discovery): add kubernetes discovery data warm-up in master process #13054

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions apisix/discovery/init.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ function discovery.init_worker()
end
end

function discovery.init()
if discovery_type then
for discovery_name, _ in pairs(discovery_type) do
if discovery[discovery_name].init then
discovery[discovery_name].init()
end
end
end
end

return {
version = 0.1,
discovery = discovery
Expand Down
6 changes: 4 additions & 2 deletions apisix/discovery/kubernetes/informer_factory.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ local function watch(httpc, apiserver, informer)
end


local function list_watch(informer, apiserver)
local function list_watch(informer, apiserver, skip_watch)
local ok
local reason, message
local httpc = http.new()
Expand Down Expand Up @@ -301,7 +301,9 @@ local function list_watch(informer, apiserver)
if informer.post_list then
informer:post_list()
end

if skip_watch then
return true
end
core.log.info("begin to watch ", informer.kind)
informer.fetch_state = "watching"
ok, reason, message = watch(httpc, apiserver, informer)
Expand Down
24 changes: 19 additions & 5 deletions apisix/discovery/kubernetes/init.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,18 @@ local _M = {
}


local function start_fetch(handle)
local function start_fetch(handle, immediate)
if immediate then
-- Execute immediately and synchronously without using timer
core.log.info(process.type(), " process immediately pulls the k8s endpoints list")
local ok, status = pcall(handle.list_watch, handle, handle.apiserver, true)
if not ok then
core.log.error("list_watch failed, kind: ", handle.kind,
", reason: RuntimeException, message: ", status)
Copy link

Copilot AI Mar 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The immediate path only handles pcall errors (Lua runtime exceptions) but doesn't check when list_watch returns false (e.g., connection failure, list failure). Compare with the timer-based path below (line 530-536), which checks both not ok and not status. While list_watch does log errors internally, this function loses track of whether the warm-up actually succeeded. Consider also checking not status and logging appropriately, consistent with the timer path.

Suggested change
", reason: RuntimeException, message: ", status)
", reason: RuntimeException, message: ", status)
elseif not status then
core.log.error("list_watch failed, kind: ", handle.kind,
", reason: ListAndWatchFailure, message: list_watch returned false during immediate fetch")

Copilot uses AI. Check for mistakes.
end
return
end

local timer_runner
timer_runner = function(premature)
if premature then
Expand Down Expand Up @@ -558,7 +569,7 @@ local function single_mode_init(conf)
"please check your APISIX version")
end

if process.type() ~= "privileged agent" then
if process.type() == "worker" then
ctx = endpoint_dict
return
end
Expand Down Expand Up @@ -605,7 +616,7 @@ local function single_mode_init(conf)
default_weight = default_weight
}, { __index = endpoints_informer })

start_fetch(ctx)
start_fetch(ctx, process.type() == "master")
end


Expand Down Expand Up @@ -653,7 +664,7 @@ end
local function multiple_mode_init(confs)
ctx = core.table.new(#confs, 0)

if process.type() ~= "privileged agent" then
if process.type() == "worker" then
multiple_mode_worker_init(confs)
return
end
Expand Down Expand Up @@ -716,7 +727,7 @@ local function multiple_mode_init(confs)
end

for _, item in pairs(ctx) do
start_fetch(item)
start_fetch(item, process.type() == "master")
end
end

Expand Down Expand Up @@ -762,6 +773,9 @@ function _M.init_worker()
end


_M.init = _M.init_worker
Copy link

Copilot AI Mar 12, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_M.init = _M.init_worker means both the master warm-up call and the worker/privileged-agent call execute the same function. If init_worker is ever changed independently (e.g., to add worker-specific logic), it would unintentionally affect the master warm-up path too. Consider extracting the shared initialization logic into a separate helper, or at minimum, adding a comment documenting that init is intentionally aliased to init_worker and any changes to one affect the other.

Copilot uses AI. Check for mistakes.


local function dump_endpoints_from_dict(endpoint_dict)
local keys, err = endpoint_dict:get_keys(0)
if err then
Expand Down
5 changes: 5 additions & 0 deletions apisix/init.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,11 @@ function _M.http_init(args)
end
end

local discovery = require("apisix.discovery.init").discovery
if discovery and discovery.init then
discovery.init()
end

xrpc.init()
end

Expand Down
53 changes: 43 additions & 10 deletions t/kubernetes/discovery/kubernetes.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ _EOC_
env MyPort=6443;
env KUBERNETES_SERVICE_HOST=127.0.0.1;
env KUBERNETES_SERVICE_PORT=6443;
env KUBERNETES_CLIENT_TOKEN=$::token_value;
env KUBERNETES_CLIENT_TOKEN_FILE=$::token_file;
env KUBERNETES_CLIENT_TOKEN=$::token_value;
_EOC_

$block->set_value("main_config", $main_config);
Expand Down Expand Up @@ -200,7 +200,7 @@ deployment:
discovery:
kubernetes:
service:
host: "sample.com"
host: "localhost"
shared_size: "2m"
client:
token: ${KUBERNETES_CLIENT_TOKEN}
Expand All @@ -209,7 +209,7 @@ GET /compare
{
"service": {
"schema": "https",
"host": "sample.com",
"host": "localhost",
"port": "${KUBERNETES_SERVICE_PORT}"
},
"client": {
Expand Down Expand Up @@ -276,14 +276,14 @@ discovery:
kubernetes:
- id: "debug"
service:
host: "1.cluster.com"
schema: "http"
host: "127.0.0.1"
port: "6445"
client:
token: ${KUBERNETES_CLIENT_TOKEN}
- id: "release"
service:
schema: "http"
host: "2.cluster.com"
host: "127.0.0.1"
port: "${MyPort}"
client:
token: ${KUBERNETES_CLIENT_TOKEN}
Expand All @@ -295,8 +295,8 @@ GET /compare
{
"id": "debug",
"service": {
"schema": "https",
"host": "1.cluster.com",
"schema": "http",
"host": "127.0.0.1",
"port": "6445"
},
"client": {
Expand All @@ -309,8 +309,8 @@ GET /compare
{
"id": "release",
"service": {
"schema": "http",
"host": "2.cluster.com",
"schema": "https",
"host": "127.0.0.1",
"port": "${MyPort}"
},
"client": {
Expand Down Expand Up @@ -385,6 +385,8 @@ qr/re-read the token value/
--- grep_error_log_out
re-read the token value
re-read the token value
re-read the token value
re-read the token value



Expand Down Expand Up @@ -421,3 +423,34 @@ GET /compare
Content-type: application/json
--- response_body
true



=== TEST 9: test master process immediately pulls k8s endpoints list (warm-up)
--- yaml_config
apisix:
node_listen: 1984
config_center: yaml
deployment:
role: data_plane
role_data_plane:
config_provider: yaml
discovery:
kubernetes:
client:
token: ${KUBERNETES_CLIENT_TOKEN}
--- config
location /t {
content_by_lua_block {
ngx.say("pass")
}
}
--- log_level: info
--- request
GET /t
--- response_body
pass
--- grep_error_log eval
qr/master process immediately pulls the k8s endpoints list/
--- grep_error_log_out
master process immediately pulls the k8s endpoints list
2 changes: 1 addition & 1 deletion t/kubernetes/discovery/kubernetes2.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ _EOC_
my $main_config = $block->main_config // <<_EOC_;
env KUBERNETES_SERVICE_HOST=127.0.0.1;
env KUBERNETES_SERVICE_PORT=6443;
env KUBERNETES_CLIENT_TOKEN=$::token_value;
env KUBERNETES_CLIENT_TOKEN_FILE=$::token_file;
env KUBERNETES_CLIENT_TOKEN=$::token_value;
_EOC_

$block->set_value("main_config", $main_config);
Expand Down
47 changes: 10 additions & 37 deletions t/kubernetes/discovery/kubernetes3.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ _EOC_
my $main_config = $block->main_config // <<_EOC_;
env KUBERNETES_SERVICE_HOST=127.0.0.1;
env KUBERNETES_SERVICE_PORT=6443;
env KUBERNETES_CLIENT_TOKEN=$::token_value;
env KUBERNETES_CLIENT_TOKEN_FILE=$::token_file;
env KUBERNETES_CLIENT_TOKEN=$::token_value;
_EOC_

$block->set_value("main_config", $main_config);
Expand Down Expand Up @@ -246,15 +246,6 @@ _EOC_
content_by_lua_block {
local http = require("resty.http")
local healthcheck_uri = "http://127.0.0.1:7085" .. "/status/ready"
for i = 1, 4 do
local httpc = http.new()
local res, _ = httpc:request_uri(healthcheck_uri, {method = "GET", keepalive = false})
if res.status == 200 then
ngx.status = res.status
return
end
ngx.sleep(1)
end
local httpc = http.new()
local res, _ = httpc:request_uri(healthcheck_uri, {method = "GET", keepalive = false})
ngx.status = res.status
Expand Down Expand Up @@ -533,15 +524,11 @@ GET /dump
=== TEST 7: test pre_list and post_list work for single-k8s with endpoint_slices
--- log_level: info
--- yaml_config eval: $::single_yaml_config
--- extra_init_by_lua
--- extra_init_by_lua_start
local ngx = ngx
local core = require("apisix.core")

local dict = ngx.shared["kubernetes"]
local ok,err = dict:set("dirty_key", true)
if not ok then
core.log.error("set dirty_key to dict fail, err: ", err)
end
assert(dict:set("dirty_key", true))
--- request
GET /ready_check
--- no_error_log
Expand All @@ -556,15 +543,11 @@ kubernetes discovery module found dirty data in shared dict, key: dirty_key
=== TEST 8: test pre_list and post_list work for multi-k8s with endpoint_slices
--- log_level: info
--- yaml_config eval: $::yaml_config
--- extra_init_by_lua
--- extra_init_by_lua_start
local ngx = ngx
local core = require("apisix.core")

local dict = ngx.shared["kubernetes-first"]
local ok,err = dict:set("dirty_key", true)
if not ok then
core.log.error("set dirty_key to dict fail, err: ", err)
end
assert(dict:set("dirty_key", true))
--- request
GET /ready_check
--- no_error_log
Expand Down Expand Up @@ -593,15 +576,11 @@ discovery:
client:
token_file: "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"
watch_endpoint_slices: false
--- extra_init_by_lua
--- extra_init_by_lua_start
local ngx = ngx
local core = require("apisix.core")

local dict = ngx.shared["kubernetes"]
local ok,err = dict:set("dirty_key", true)
if not ok then
core.log.error("set dirty_key to dict fail, err: ", err)
end
assert(dict:set("dirty_key", true))
--- request
GET /ready_check
--- no_error_log
Expand Down Expand Up @@ -639,15 +618,11 @@ discovery:
client:
token_file: "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"
watch_endpoint_slices: false
--- extra_init_by_lua
--- extra_init_by_lua_start
local ngx = ngx
local core = require("apisix.core")

local dict = ngx.shared["kubernetes-first"]
local ok,err = dict:set("dirty_key", true)
if not ok then
core.log.error("set dirty_key to dict fail, err: ", err)
end
assert(dict:set("dirty_key", true))
--- request
GET /ready_check
--- no_error_log
Expand Down Expand Up @@ -688,10 +663,8 @@ discovery:
--- request
GET /ready_check
--- error_code: 503
--- grep_error_log eval
--- error_log eval
qr/connect apiserver failed/
--- grep_error_log_out
connect apiserver failed



Expand Down
2 changes: 1 addition & 1 deletion t/kubernetes/discovery/kubernetes4.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,8 +243,8 @@ _EOC_
my $main_config = $block->main_config // <<_EOC_;
env KUBERNETES_SERVICE_HOST=127.0.0.1;
env KUBERNETES_SERVICE_PORT=6443;
env KUBERNETES_CLIENT_TOKEN=$::token_value;
env KUBERNETES_CLIENT_TOKEN_FILE=$::token_file;
env KUBERNETES_CLIENT_TOKEN=$::token_value;
_EOC_

$block->set_value("main_config", $main_config);
Expand Down
Loading