From 828c4c1011bbe9794fd03c35a4b2b58d258f0359 Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Tue, 3 Dec 2024 14:57:02 +0800 Subject: [PATCH 1/5] fix(clustering/rpc) support `cluster_use_proxy` option for clustering rpc protocol --- kong/clustering/rpc/manager.lua | 11 +++++++++++ .../09-hybrid_mode/10-forward-proxy_spec.lua | 10 +++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/kong/clustering/rpc/manager.lua b/kong/clustering/rpc/manager.lua index c3925c5073c..f3e3e517572 100644 --- a/kong/clustering/rpc/manager.lua +++ b/kong/clustering/rpc/manager.lua @@ -474,6 +474,17 @@ function _M:connect(premature, node_id, host, path, cert, key) local c = assert(client:new(WS_OPTS)) + if self.conf.cluster_use_proxy then + local proxy_opts = parse_proxy_url(self.conf.proxy_server) + opts.proxy_opts = { + wss_proxy = proxy_opts.proxy_url, + wss_proxy_authorization = proxy_opts.proxy_authorization, + } + + ngx_log(ngx_DEBUG, _log_prefix, + "using proxy ", proxy_opts.proxy_url, " to connect control plane") + end + local ok, err = c:connect(uri, opts) if not ok then ngx_log(ngx_ERR, "[rpc] unable to connect to peer: ", err) diff --git a/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua b/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua index a7f11e41059..1dc6d2a2449 100644 --- a/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua +++ b/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua @@ -71,11 +71,13 @@ local proxy_configs = { -- if existing lmdb data is set, the service/route exists and -- test run too fast before the proxy connection is established --- XXX FIXME: enable inc_sync = on -for _, inc_sync in ipairs { "off" } do +for _, v in ipairs({ {"off", "off"}, {"on", "off"}, {"on", "on"}, }) do + local rpc, inc_sync = v[1], v[2] for _, strategy in helpers.each_strategy() do for proxy_desc, proxy_opts in pairs(proxy_configs) do - describe("CP/DP sync through proxy (" .. proxy_desc .. ") works with #" .. strategy .. " inc_sync=" .. inc_sync .. " backend", function() + describe("CP/DP sync through proxy (" .. proxy_desc .. ") works with #" + .. strategy .. " rpc=" .. rpc .. " inc_sync=" .. inc_sync + .. " backend", function() lazy_setup(function() helpers.get_db_utils(strategy) -- runs migrations @@ -87,6 +89,7 @@ for _, strategy in helpers.each_strategy() do db_update_frequency = 0.1, cluster_listen = "127.0.0.1:9005", nginx_conf = "spec/fixtures/custom_nginx.template", + cluster_rpc = rpc, cluster_incremental_sync = inc_sync, })) @@ -108,6 +111,7 @@ for _, strategy in helpers.each_strategy() do proxy_server_ssl_verify = proxy_opts.proxy_server_ssl_verify, lua_ssl_trusted_certificate = proxy_opts.lua_ssl_trusted_certificate, + cluster_rpc = rpc, cluster_incremental_sync = inc_sync, -- this is unused, but required for the template to include a stream {} block From cb3634dc1bd066984c4b52f587b016f82766bfe7 Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Tue, 3 Dec 2024 15:04:03 +0800 Subject: [PATCH 2/5] fix log prefix to `[rpc]` --- kong/clustering/rpc/manager.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kong/clustering/rpc/manager.lua b/kong/clustering/rpc/manager.lua index f3e3e517572..890e60e5f59 100644 --- a/kong/clustering/rpc/manager.lua +++ b/kong/clustering/rpc/manager.lua @@ -481,8 +481,8 @@ function _M:connect(premature, node_id, host, path, cert, key) wss_proxy_authorization = proxy_opts.proxy_authorization, } - ngx_log(ngx_DEBUG, _log_prefix, - "using proxy ", proxy_opts.proxy_url, " to connect control plane") + ngx_log(ngx_DEBUG, "[rpc] using proxy ", proxy_opts.proxy_url, + " to connect control plane") end local ok, err = c:connect(uri, opts) From 3f5c182d1bd8007b6f761422002256ced35c091d Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Tue, 3 Dec 2024 15:38:57 +0800 Subject: [PATCH 3/5] export parse_proxy_url and add tests for `cluster_use_proxy` option --- kong/clustering/rpc/manager.lua | 1 + kong/clustering/utils.lua | 4 ++-- .../09-hybrid_mode/10-forward-proxy_spec.lua | 9 +++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/kong/clustering/rpc/manager.lua b/kong/clustering/rpc/manager.lua index 890e60e5f59..3d08963b468 100644 --- a/kong/clustering/rpc/manager.lua +++ b/kong/clustering/rpc/manager.lua @@ -30,6 +30,7 @@ local cjson_encode = cjson.encode local cjson_decode = cjson.decode local validate_client_cert = clustering_tls.validate_client_cert local CLUSTERING_PING_INTERVAL = constants.CLUSTERING_PING_INTERVAL +local parse_proxy_url = require("kong.clustering.utils").parse_proxy_url local RPC_MATA_V1 = "kong.meta.v1" diff --git a/kong/clustering/utils.lua b/kong/clustering/utils.lua index 5ee56d30baf..ee34e7dce2e 100644 --- a/kong/clustering/utils.lua +++ b/kong/clustering/utils.lua @@ -33,7 +33,7 @@ local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/%s", local _M = {} -local function parse_proxy_url(proxy_server) +function _M.parse_proxy_url(proxy_server) local ret = {} if proxy_server then @@ -84,7 +84,7 @@ function _M.connect_cp(dp, endpoint, protocols) } if conf.cluster_use_proxy then - local proxy_opts = parse_proxy_url(conf.proxy_server) + local proxy_opts = _M.parse_proxy_url(conf.proxy_server) opts.proxy_opts = { wss_proxy = proxy_opts.proxy_url, wss_proxy_authorization = proxy_opts.proxy_authorization, diff --git a/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua b/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua index 1dc6d2a2449..797cf2364f3 100644 --- a/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua +++ b/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua @@ -170,6 +170,15 @@ for _, strategy in helpers.each_strategy() do if auth_on then assert.matches("accepted basic proxy%-authorization", contents) end + + -- check the debug log of the `cluster_use_proxy` option + path = pl_path.join("servroot2", "logs", "error.log") + contents = pl_file.read(path) + if rpc == "on" and inc_sync == "on" then + assert.matches("%[rpc%] using proxy", contents) + else + assert.matches("%[clustering%] using proxy", contents) + end end) end) end) From c9085a84a73158566770d3d4421b7c09462fd1d3 Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Wed, 4 Dec 2024 15:06:57 +0800 Subject: [PATCH 4/5] 09-hybrid_mode/14-dp_privileged_agent_spec.lua:force enable cluster_rpc when cluster_incremental_sync is on --- .../09-hybrid_mode/14-dp_privileged_agent_spec.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/02-integration/09-hybrid_mode/14-dp_privileged_agent_spec.lua b/spec/02-integration/09-hybrid_mode/14-dp_privileged_agent_spec.lua index b0743edb746..1c5e351bf87 100644 --- a/spec/02-integration/09-hybrid_mode/14-dp_privileged_agent_spec.lua +++ b/spec/02-integration/09-hybrid_mode/14-dp_privileged_agent_spec.lua @@ -20,6 +20,7 @@ describe("DP diabled Incremental Sync RPC #" .. strategy, function() cluster_listen = "127.0.0.1:9005", nginx_conf = "spec/fixtures/custom_nginx.template", + cluster_rpc = "on", cluster_incremental_sync = "on", -- ENABLE incremental sync })) From da25e0811d16e06d1e30d860608402a32fcf06aa Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Wed, 4 Dec 2024 16:03:37 +0800 Subject: [PATCH 5/5] using assert.logfile --- .../09-hybrid_mode/10-forward-proxy_spec.lua | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua b/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua index 797cf2364f3..27856b4554e 100644 --- a/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua +++ b/spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua @@ -172,13 +172,9 @@ for _, strategy in helpers.each_strategy() do end -- check the debug log of the `cluster_use_proxy` option - path = pl_path.join("servroot2", "logs", "error.log") - contents = pl_file.read(path) - if rpc == "on" and inc_sync == "on" then - assert.matches("%[rpc%] using proxy", contents) - else - assert.matches("%[clustering%] using proxy", contents) - end + local line = inc_sync == "on" and "[rpc] using proxy" or + "[clustering] using proxy" + assert.logfile("servroot2/logs/error.log").has.line(line, true) end) end) end)