From 433608e93e93452375537cccedf171ce5e98efc3 Mon Sep 17 00:00:00 2001 From: ruanweihong-aaa Date: Fri, 23 Oct 2026 15:56:01 +0800 Subject: [PATCH 1/4] fix: preserve multiline SSL environment values in standalone YAML --- apisix/cli/file.lua | 19 ++- apisix/cli/ssl_yaml.lua | 200 +++++++++++++++++++++++++++++ apisix/core/config_yaml.lua | 9 +- docs/en/latest/deployment-modes.md | 28 ++++ t/cli/test_env_multiline.lua | 170 ++++++++++++++++++++++++ t/cli/test_env_multiline.sh | 19 +++ 6 files changed, 431 insertions(+), 14 deletions(-) create mode 100644 apisix/cli/ssl_yaml.lua create mode 100644 t/cli/test_env_multiline.lua create mode 100755 t/cli/test_env_multiline.sh diff --git a/apisix/cli/file.lua b/apisix/cli/file.lua index 6e03801cf3fd..55fa8db9cc7c 100644 --- a/apisix/cli/file.lua +++ b/apisix/cli/file.lua @@ -57,7 +57,7 @@ local function tab_is_array(t) end -local function var_sub(val) +local function var_sub(val, on_value) local err local var_used = false -- we use '${{var}}' because '$var' and '${var}' are taken @@ -79,7 +79,7 @@ local function var_sub(val) exported_vars[var] = v var_used = true - return v + return on_value and on_value(v) or v end err = "failed to handle configuration: " .. @@ -160,6 +160,15 @@ _M.resolve_conf_var = resolve_conf_var _M.resolve_conf_var_in_text = resolve_conf_var_in_text +-- CLI validation and runtime reload must use the same SSL substitution rules. +local function load_yaml_with_env(text) + return require("apisix.cli.ssl_yaml").load(text, var_sub) +end + + +_M.load_yaml_with_env = load_yaml_with_env + + local function replace_by_reserved_env_vars(conf) -- TODO: support more reserved environment variables local v = getenv("APISIX_DEPLOYMENT_ETCD_HOST") @@ -322,12 +331,10 @@ function _M.read_yaml_conf(apisix_home) local apisix_conf_path = profile:yaml_path("apisix") local apisix_conf_yaml, _ = util.read_file(apisix_conf_path) if apisix_conf_yaml then - -- Pre-parse substitution: `"${{VAR}}"` stays string, `${{VAR}}` infers type from value. - local resolved, err = resolve_conf_var_in_text(apisix_conf_yaml) - if not resolved then + local apisix_conf, err = load_yaml_with_env(apisix_conf_yaml) + if err then return nil, err end - local apisix_conf = yaml.load(resolved) if not apisix_conf then return nil, "invalid apisix.yaml file" end diff --git a/apisix/cli/ssl_yaml.lua b/apisix/cli/ssl_yaml.lua new file mode 100644 index 000000000000..68b4ef276910 --- /dev/null +++ b/apisix/cli/ssl_yaml.lua @@ -0,0 +1,200 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local yaml = require("lyaml") +local parser = require("yaml").parser +local pairs = pairs +local ipairs = ipairs +local type = type +local pcall = pcall +local _M = {} + + +local function is_ssl_field(path) + if path[1] ~= "ssls" or type(path[2]) ~= "number" then + return false + end + if #path == 3 then + return path[3] == "cert" or path[3] == "key" + end + if #path == 4 then + return ((path[3] == "certs" or path[3] == "keys") and type(path[4]) == "number") + or (path[3] == "client" and path[4] == "ca") + end + return false +end + + +-- Visit aliases once per path, but abort discovery on cycles or excessive depth. +-- The caller then uses the existing YAML load, without introducing a new error. +local function walk(value, path, active, visit, parent, key) + if type(value) ~= "table" then + visit(value, is_ssl_field(path), parent, key) + return true + end + if active[value] or #path > 128 then + return false + end + active[value] = true + for k, v in pairs(value) do + visit(k, false) + path[#path + 1] = k + if not walk(v, path, active, visit, value, k) then + return false + end + path[#path] = nil + end + active[value] = nil + return true +end + + +-- A decoded scalar may spell a marker using YAML escapes. Only accept markers +-- that occur literally in that scalar's source span, not user-authored lookalikes. +local function scalar_origins(text, records) + local offsets, index = {}, 0 + for byte = 1, #text do + local b = text:byte(byte) + if b < 128 or b >= 192 then + offsets[index] = byte + index = index + 1 + end + end + offsets[index] = #text + 1 + local origins, ambiguous = {}, {} + for event in parser(text) do + if event.type == "SCALAR" and records[event.value] then + local first = offsets[event.start_mark.index] + local last = offsets[event.end_mark.index] + local source = first and last and text:sub(first, last - 1) + if source and source:find(event.value, 1, true) + and (event.style == "PLAIN" or event.style == "SINGLE_QUOTED" + or event.style == "DOUBLE_QUOTED") then + origins[event.value] = true + else + ambiguous[event.value] = true + end + end + if event.type == "STREAM_END" then + break + end + end + for marker in pairs(ambiguous) do + origins[marker] = nil + end + return origins +end + + +-- Use the same path and provenance checks for discovery and final restoration. +local function inspect_fields(text, records, prefix) + local conf = yaml.load(text) + local origins = scalar_origins(text, records) + local fields, outside = {}, {} + local ok = walk(conf, {}, {}, function(value, allowed, parent, key) + if type(value) ~= "string" then + return + end + for marker in value:gmatch(prefix .. "%d+__") do + if records[marker] then + if allowed and value == marker and origins[marker] then + fields[marker] = fields[marker] or {} + local targets = fields[marker] + targets[#targets + 1] = {parent, key, records[marker].value} + else + outside[marker] = true + end + end + end + end) + if not ok then + return nil + end + -- A shared anchor must not change a non-SSL consumer's substitution behavior. + for marker in pairs(outside) do + fields[marker] = nil + end + return conf, fields +end + + +function _M.load(text, substitute) + local values, multiline = {}, false + local expanded, _, err = substitute(text, function(value) + values[#values + 1] = {value = value} + multiline = multiline or value:find("[\r\n]") ~= nil + return value + end) + if err then + return nil, err + end + if not multiline then + return yaml.load(expanded) + end + + local serial, prefix = 0 + local collision + repeat + serial = serial + 1 + prefix = "__APISIX_SSL_ENV_" .. serial .. "_" + collision = text:find(prefix, 1, true) + for _, record in ipairs(values) do + collision = collision or record.value:find(prefix, 1, true) + end + until not collision + + local records = {} + for i, record in ipairs(values) do + if record.value:find("[\r\n]") then + record.marker = prefix .. i .. "__" + records[record.marker] = record + end + end + local function render(selected) + local index = 0 + return substitute(text, function() + index = index + 1 + local record = values[index] + return selected[record.marker] and record.marker or record.value + end) + end + local ok, _, selected = pcall(inspect_fields, render(records), records, prefix) + if not ok or not selected or not next(selected) then + return yaml.load(expanded) + end + + local parsed, conf, fields = pcall(inspect_fields, render(selected), records, prefix) + if not parsed or not fields then + return yaml.load(expanded) + end + -- Other substitutions can move or synthesize markers. Check every selected + -- occurrence before restoring any value; never scan the restored PEM again. + for marker in pairs(selected) do + if not fields[marker] then + return yaml.load(expanded) + end + end + for marker in pairs(selected) do + for _, target in ipairs(fields[marker]) do + target[1][target[2]] = target[3] + end + end + return conf +end + + +return _M diff --git a/apisix/core/config_yaml.lua b/apisix/core/config_yaml.lua index f05ca8d0acc0..da337e771f90 100644 --- a/apisix/core/config_yaml.lua +++ b/apisix/core/config_yaml.lua @@ -20,7 +20,6 @@ -- @module core.config_yaml local config_local = require("apisix.core.config_local") -local yaml = require("lyaml") local log = require("apisix.core.log") local json = require("apisix.core.json") local new_tab = require("table.new") @@ -98,13 +97,7 @@ local config_yaml = { local raw_config = f:read("*a") f:close() - -- substitute env vars in the raw YAML text before parsing so that - -- YAML quoting decides types (see apisix/cli/file.lua). - local resolved, err = file.resolve_conf_var_in_text(raw_config) - if not resolved then - return nil, err - end - return yaml.load(resolved), nil + return file.load_yaml_with_env(raw_config) end } diff --git a/docs/en/latest/deployment-modes.md b/docs/en/latest/deployment-modes.md index a9002cc2dfc9..b852e80e58f2 100644 --- a/docs/en/latest/deployment-modes.md +++ b/docs/en/latest/deployment-modes.md @@ -363,6 +363,34 @@ routes: More information about using environment variables can be found [here](./admin-api.md#using-environment-variables). +For SSL resources, a complete `${{VAR}}` reference in `cert`, `key`, a `certs` or +`keys` array item, or `client.ca` can contain a multiline PEM value: + +```yaml +ssls: + - id: example + snis: ["example.com"] + cert: "${{SERVER_CERT}}" + key: "${{SERVER_KEY}}" +#END +``` + +The loader preserves the original multiline value in these fields. Single-line +values, including literal `\n` escapes inside double-quoted YAML, continue to use +the existing substitution rules. Other fields also retain their existing YAML +substitution behavior. + +This protection applies to complete scalar references, including unquoted +references in flow-style YAML. It does not add recursive variable expansion or +protect references embedded in larger strings or block scalars. Configurations +that cannot be identified unambiguously use the existing loading behavior; +for example, a variable shared through an anchor with a non-SSL field is not +protected. A whole configuration supplied through one environment variable is +still expanded as YAML, without special handling of PEM content inside it. + +For those cases, SSL fields also support environment-backed +[Secret references](./terminology/secret.md), such as `$ENV://SERVER_CERT`. + #### To `config_provider: json` All of the rules are stored in one file which named `conf/apisix.json`, diff --git a/t/cli/test_env_multiline.lua b/t/cli/test_env_multiline.lua new file mode 100644 index 000000000000..ea47dca0996d --- /dev/null +++ b/t/cli/test_env_multiline.lua @@ -0,0 +1,170 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +local ffi = require("ffi") +ffi.cdef[[int setenv(const char *, const char *, int); int unsetenv(const char *);]] +local file = require("apisix.cli.file") +local yaml = require("lyaml") + +local function set(name, value) + assert(ffi.C.setenv(name, value, 1) == 0) +end + +local pem = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n" +set("CERT_14038", pem) +set("LIST_14038", "[a,\n b]") +set("TEXT_14038", "hello\nworld") +set("NUM_14038", "0123") +set("ESC_14038", (pem:gsub("\n", "\\n"))) +set("BOOL_14038", "true") +set("EMPTY_14038", "") +set("ROOT_14038", "ssls:\n - cert: abc\n") +set("FIELD_14038", "cert") +ffi.C.unsetenv("MISSING_14038") + +local failures, total = 0, 0 +local function test(name, run) + total = total + 1 + local ok, err = pcall(run) + if ok then + print("PASS " .. name) + else + failures = failures + 1 + print("FAIL " .. name .. ": " .. tostring(err)) + end +end + +local function load(text) + local conf, err = file.load_yaml_with_env(text) + assert(conf, err) + return conf +end + +local function old(text) + local expanded, err = file.resolve_conf_var_in_text(text) + assert(expanded, err) + return yaml.load(expanded) +end + +local function check(name, text, expected) + test(name, function() + local conf = load(text) + for path, value in pairs(expected) do + local actual = conf + for key in path:gmatch("[^.]+") do + actual = actual[tonumber(key) or key] + end + assert(actual == value, path .. ": unexpected value") + end + end) +end +local folded = old('cert: "${{CERT_14038}}"').cert +local ssl_pair = {["ssls.1.cert"] = pem, ["ssls.1.key"] = pem} +for _, q in ipairs({'"', "'", ''}) do + check("certificate and key preserve original bytes quote=" .. q, + 'ssls:\n - cert: ' .. q .. '${{CERT_14038}}' .. q + .. '\n key: ' .. q .. '${{CERT_14038}}' .. q .. '\n', ssl_pair) +end + +-- Each row is a separate regression: name, source YAML, expected field values. +local cases = { + {"non-SSL multiline sequences retain their type", 'items: ${{LIST_14038}}', + {["items.1"] = "a", ["items.2"] = "b"}}, + {"ordinary multiline fields retain folding", + 'other: {cert: "${{TEXT_14038}}", key: "${{TEXT_14038}}"}', + {["other.cert"] = "hello world", ["other.key"] = "hello world"}}, + {"unquoted flow references and Unicode", + 'note: 中文\nssls: [{cert: ${{CERT_14038}}, key: ${{CERT_14038}}}]', + {["note"] = "中文", ["ssls.1.cert"] = pem, ["ssls.1.key"] = pem}}, + {"same variable outside SSL is not protected", + 'ssls: [{cert: "${{CERT_14038}}"}]\nother: "${{CERT_14038}}"', + {["ssls.1.cert"] = pem, other = folded}}, + {"multi-certificate arrays and client CA", [[ssls: + - certs: [${{CERT_14038}}, "${{CERT_14038}}"] + keys: [${{CERT_14038}}] + client: {ca: "${{CERT_14038}}"} +]], {["ssls.1.certs.1"] = pem, ["ssls.1.certs.2"] = pem, + ["ssls.1.keys.1"] = pem, ["ssls.1.client.ca"] = pem}}, + {"literal backslash-n certificate keeps old result", + 'ssls: [{cert: "${{ESC_14038}}"}]', {["ssls.1.cert"] = pem}}, + {"single-line types and defaults unchanged", [[n: ${{NUM_14038}} +s: "${{NUM_14038}}" +b: ${{BOOL_14038}} +e: ${{EMPTY_14038}} +d: ${{MISSING_14038:=fallback}} +]], {n = 83, s = "0123", b = true, e = yaml.null, d = "fallback"}}, + {"multiline default is protected only in SSL", + 'ssls:\n - cert: "${{MISSING_14038:=line1\nline2}}"\n', + {["ssls.1.cert"] = "line1\nline2"}}, + {"whole document variable retains legacy behavior", '${{ROOT_14038}}', + {["ssls.1.cert"] = "abc"}}, + {"single-line variable key selects actual SSL field", + 'ssls:\n - ${{FIELD_14038}}: "${{CERT_14038}}"\n', {["ssls.1.cert"] = pem}}, + {"direct block PEM does not change", 'ssls:\n - cert: |\n BEGIN\n DATA\n END\n', + {["ssls.1.cert"] = "BEGIN\nDATA\nEND\n"}}, + {"anchors confined to SSL preserve bytes", + 'ssls:\n - cert: &pem "${{CERT_14038}}"\n key: *pem\n', ssl_pair}, + {"shared non-SSL anchor keeps legacy semantics", + 'shared: &p "${{CERT_14038}}"\nssls:\n - cert: *p\n', + {shared = folded, ["ssls.1.cert"] = folded}}, + {"merge inside SSL preserves supported fields", + 'ssls:\n - &s {cert: "${{CERT_14038}}"}\n - <<: *s\n', + {["ssls.1.cert"] = pem, ["ssls.2.cert"] = pem}}, + {"embedded reference retains old behavior", + 'ssls: [{cert: "before-${{TEXT_14038}}-after"}]', + {["ssls.1.cert"] = "before-hello world-after"}}, +} +for _, case in ipairs(cases) do + check(unpack(case)) +end + +test("lookup failure still reports missing variable", function() + local c, err = file.load_yaml_with_env('ssls: [{cert: "${{MISSING_14038}}"}]') + assert(c == nil and err:find("MISSING_14038", 1, true)) +end) + +test("raw and escaped marker-like user data remain literal", function() + local c = load('a: __APISIX_SSL_ENV_1_1__\n' + .. 'b: "\\x5f_APISIX_SSL_ENV_2_1__"\nssls: [{cert: "${{CERT_14038}}"}]') + assert(c.a == "__APISIX_SSL_ENV_1_1__" and c.b == "__APISIX_SSL_ENV_2_1__") + -- An ambiguous discovery may fall back, but must never overwrite user text. +end) + +test("marker-looking original content is not interpreted again", function() + set("ODD_14038", "first\n__APISIX_SSL_ENV_1_2__\n${{NUM_14038}}\n%1") + local c = load('ssls: [{cert: "${{ODD_14038}}", key: "${{CERT_14038}}"}]') + assert(c.ssls[1].cert == "first\n__APISIX_SSL_ENV_1_2__\n${{NUM_14038}}\n%1") +end) + +test("cyclic aliases do not hang discovery", function() + local t = 'cycle: &a {self: *a}\nssls: [{cert: "${{CERT_14038}}"}]' + local c = load(t) + assert(c.cycle.self == c.cycle) + assert(c.ssls[1].cert == old(t).ssls[1].cert) +end) + +test("top-level scalar retains old folding", function() + assert(load('"${{TEXT_14038}}"') == "hello world") +end) + +test("structural expansion cannot synthesize a protected marker via escapes", function() + set("INJECT_14038", 'text"\n key: "\\x5f_APISIX_SSL_ENV_1_1__') + local t = 'ssls:\n - cert: "${{CERT_14038}}"\n x: "${{INJECT_14038}}"\n' + local c = load(t) + assert(c.ssls[1].key == "__APISIX_SSL_ENV_1_1__") +end) + +print("RESULT " .. (total - failures) .. "/" .. total .. " passed") +assert(failures == 0, failures .. " regression tests failed") diff --git a/t/cli/test_env_multiline.sh b/t/cli/test_env_multiline.sh new file mode 100755 index 000000000000..af8c1b27587d --- /dev/null +++ b/t/cli/test_env_multiline.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +luajit t/cli/test_env_multiline.lua From 0aa090bac8a6fa3ad368a4494d96836a55bebd48 Mon Sep 17 00:00:00 2001 From: ruanweihong-aaa Date: Sun, 25 Oct 2026 22:42:50 +0800 Subject: [PATCH 2/4] fix: localize globals in standalone YAML loader --- apisix/cli/file.lua | 1 + apisix/cli/ssl_yaml.lua | 1 + 2 files changed, 2 insertions(+) diff --git a/apisix/cli/file.lua b/apisix/cli/file.lua index 55fa8db9cc7c..32122808688d 100644 --- a/apisix/cli/file.lua +++ b/apisix/cli/file.lua @@ -25,6 +25,7 @@ local dkjson = require("dkjson") local pl_path = require("pl.path") local pairs = pairs +local require = require local type = type local tonumber = tonumber local getenv = os.getenv diff --git a/apisix/cli/ssl_yaml.lua b/apisix/cli/ssl_yaml.lua index 68b4ef276910..229ac30121e0 100644 --- a/apisix/cli/ssl_yaml.lua +++ b/apisix/cli/ssl_yaml.lua @@ -18,6 +18,7 @@ local yaml = require("lyaml") local parser = require("yaml").parser local pairs = pairs +local next = next local ipairs = ipairs local type = type local pcall = pcall From 024264dc6c6a59dcd5a04482f554b07ee6bdf05d Mon Sep 17 00:00:00 2001 From: ruanweihong-aaa Date: Tue, 27 Oct 2026 02:27:12 +0800 Subject: [PATCH 3/4] fix: warn on unprotected multiline SSL environment values --- apisix/cli/file.lua | 10 ++++++++-- apisix/cli/ssl_yaml.lua | 29 +++++++++++++++++++++++++---- apisix/core/config_yaml.lua | 2 +- docs/en/latest/deployment-modes.md | 2 ++ t/cli/test_env_multiline.lua | 28 ++++++++++++++++++++++++++-- 5 files changed, 62 insertions(+), 9 deletions(-) diff --git a/apisix/cli/file.lua b/apisix/cli/file.lua index 32122808688d..986461637338 100644 --- a/apisix/cli/file.lua +++ b/apisix/cli/file.lua @@ -33,6 +33,7 @@ local str_gmatch = string.gmatch local str_find = string.find local str_sub = string.sub local print = print +local stderr = io.stderr local _M = {} local exported_vars @@ -162,8 +163,13 @@ _M.resolve_conf_var_in_text = resolve_conf_var_in_text -- CLI validation and runtime reload must use the same SSL substitution rules. -local function load_yaml_with_env(text) - return require("apisix.cli.ssl_yaml").load(text, var_sub) +local function warn_yaml_fallback(message) + stderr:write("WARNING: ", message, "\n") +end + + +local function load_yaml_with_env(text, warn) + return require("apisix.cli.ssl_yaml").load(text, var_sub, warn or warn_yaml_fallback) end diff --git a/apisix/cli/ssl_yaml.lua b/apisix/cli/ssl_yaml.lua index 229ac30121e0..4fb313b7a26a 100644 --- a/apisix/cli/ssl_yaml.lua +++ b/apisix/cli/ssl_yaml.lua @@ -105,13 +105,16 @@ end local function inspect_fields(text, records, prefix) local conf = yaml.load(text) local origins = scalar_origins(text, records) - local fields, outside = {}, {} + local fields, outside, ssl = {}, {}, {} local ok = walk(conf, {}, {}, function(value, allowed, parent, key) if type(value) ~= "string" then return end for marker in value:gmatch(prefix .. "%d+__") do if records[marker] then + if allowed then + ssl[marker] = true + end if allowed and value == marker and origins[marker] then fields[marker] = fields[marker] or {} local targets = fields[marker] @@ -123,17 +126,22 @@ local function inspect_fields(text, records, prefix) end end) if not ok then - return nil + return nil, nil, next(ssl) ~= nil end -- A shared anchor must not change a non-SSL consumer's substitution behavior. for marker in pairs(outside) do fields[marker] = nil end + for marker in pairs(ssl) do + if not fields[marker] then + return conf, fields, true + end + end return conf, fields end -function _M.load(text, substitute) +function _M.load(text, substitute, warn) local values, multiline = {}, false local expanded, _, err = substitute(text, function(value) values[#values + 1] = {value = value} @@ -165,6 +173,7 @@ function _M.load(text, substitute) records[record.marker] = record end end + -- Replaying substitution over the same source visits occurrences in the same order. local function render(selected) local index = 0 return substitute(text, function() @@ -173,19 +182,31 @@ function _M.load(text, substitute) return selected[record.marker] and record.marker or record.value end) end - local ok, _, selected = pcall(inspect_fields, render(records), records, prefix) + local function warn_fallback() + if warn then + warn("cannot preserve some multiline SSL environment values; " + .. "using legacy YAML substitution for them (consider $ENV:// references)") + warn = nil + end + end + local ok, _, selected, unprotected = pcall(inspect_fields, render(records), records, prefix) + if unprotected then + warn_fallback() + end if not ok or not selected or not next(selected) then return yaml.load(expanded) end local parsed, conf, fields = pcall(inspect_fields, render(selected), records, prefix) if not parsed or not fields then + warn_fallback() return yaml.load(expanded) end -- Other substitutions can move or synthesize markers. Check every selected -- occurrence before restoring any value; never scan the restored PEM again. for marker in pairs(selected) do if not fields[marker] then + warn_fallback() return yaml.load(expanded) end end diff --git a/apisix/core/config_yaml.lua b/apisix/core/config_yaml.lua index da337e771f90..9a5a42e53ea9 100644 --- a/apisix/core/config_yaml.lua +++ b/apisix/core/config_yaml.lua @@ -97,7 +97,7 @@ local config_yaml = { local raw_config = f:read("*a") f:close() - return file.load_yaml_with_env(raw_config) + return file.load_yaml_with_env(raw_config, log.warn) end } diff --git a/docs/en/latest/deployment-modes.md b/docs/en/latest/deployment-modes.md index b852e80e58f2..d336740273c3 100644 --- a/docs/en/latest/deployment-modes.md +++ b/docs/en/latest/deployment-modes.md @@ -387,6 +387,8 @@ that cannot be identified unambiguously use the existing loading behavior; for example, a variable shared through an anchor with a non-SSL field is not protected. A whole configuration supplied through one environment variable is still expanded as YAML, without special handling of PEM content inside it. +When the loader identifies a multiline SSL reference but cannot safely protect +it, it logs a warning without including the value and retains the existing behavior. For those cases, SSL fields also support environment-backed [Secret references](./terminology/secret.md), such as `$ENV://SERVER_CERT`. diff --git a/t/cli/test_env_multiline.lua b/t/cli/test_env_multiline.lua index ea47dca0996d..1369af8d69f3 100644 --- a/t/cli/test_env_multiline.lua +++ b/t/cli/test_env_multiline.lua @@ -24,6 +24,8 @@ end local pem = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n" set("CERT_14038", pem) +local crlf = pem:gsub("\n", "\r\n") +set("CRLF_14038", crlf) set("LIST_14038", "[a,\n b]") set("TEXT_14038", "hello\nworld") set("NUM_14038", "0123") @@ -76,8 +78,29 @@ for _, q in ipairs({'"', "'", ''}) do check("certificate and key preserve original bytes quote=" .. q, 'ssls:\n - cert: ' .. q .. '${{CERT_14038}}' .. q .. '\n key: ' .. q .. '${{CERT_14038}}' .. q .. '\n', ssl_pair) + check("CRLF certificate quote=" .. q, + 'ssls: [{cert: ' .. q .. '${{CRLF_14038}}' .. q .. '}]', + {["ssls.1.cert"] = crlf}) end +test("warn only for unprotected SSL references, without exposing values", function() + local warnings = {} + local function warn(message) + warnings[#warnings + 1] = message + assert(not message:find(pem, 1, true)) + end + file.load_yaml_with_env('items: ${{LIST_14038}}', warn) + file.load_yaml_with_env('ssls: [{cert: "${{CERT_14038}}"}]', warn) + assert(#warnings == 0) + local text = 'shared: &p "${{CERT_14038}}"\nssls: [{cert: *p}]' + local conf = file.load_yaml_with_env(text, warn) + assert(conf.ssls[1].cert == old(text).ssls[1].cert) + assert(#warnings == 1 and warnings[1]:find("legacy YAML", 1, true)) + conf = file.load_yaml_with_env('shared: &p "${{CERT_14038}}"\n' + .. 'ssls: [{cert: *p}, {key: "${{CERT_14038}}"}]', warn) + assert(conf.ssls[2].key == pem and #warnings == 2) +end) + -- Each row is a separate regression: name, source YAML, expected field values. local cases = { {"non-SSL multiline sequences retain their type", 'items: ${{LIST_14038}}', @@ -162,8 +185,9 @@ end) test("structural expansion cannot synthesize a protected marker via escapes", function() set("INJECT_14038", 'text"\n key: "\\x5f_APISIX_SSL_ENV_1_1__') local t = 'ssls:\n - cert: "${{CERT_14038}}"\n x: "${{INJECT_14038}}"\n' - local c = load(t) - assert(c.ssls[1].key == "__APISIX_SSL_ENV_1_1__") + local warnings = 0 + local c = file.load_yaml_with_env(t, function() warnings = warnings + 1 end) + assert(c.ssls[1].key == "__APISIX_SSL_ENV_1_1__" and warnings == 1) end) print("RESULT " .. (total - failures) .. "/" .. total .. " passed") From 91eefe827a9181fc03e683c7a1e708bf4dea6b97 Mon Sep 17 00:00:00 2001 From: ruanweihong-aaa Date: Tue, 27 Oct 2026 09:49:17 +0800 Subject: [PATCH 4/4] refactor: clarify standalone YAML SSL substitution stages --- apisix/cli/ssl_yaml.lua | 102 +++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 44 deletions(-) diff --git a/apisix/cli/ssl_yaml.lua b/apisix/cli/ssl_yaml.lua index 4fb313b7a26a..5d45100d0672 100644 --- a/apisix/cli/ssl_yaml.lua +++ b/apisix/cli/ssl_yaml.lua @@ -105,43 +105,40 @@ end local function inspect_fields(text, records, prefix) local conf = yaml.load(text) local origins = scalar_origins(text, records) - local fields, outside, ssl = {}, {}, {} + local uses = {} local ok = walk(conf, {}, {}, function(value, allowed, parent, key) if type(value) ~= "string" then return end for marker in value:gmatch(prefix .. "%d+__") do if records[marker] then - if allowed then - ssl[marker] = true - end + local use = uses[marker] or {targets = {}} + uses[marker] = use + use.ssl = use.ssl or allowed if allowed and value == marker and origins[marker] then - fields[marker] = fields[marker] or {} - local targets = fields[marker] - targets[#targets + 1] = {parent, key, records[marker].value} + local targets = use.targets + targets[#targets + 1] = {parent = parent, key = key} else - outside[marker] = true + use.unsafe = true end end end end) - if not ok then - return nil, nil, next(ssl) ~= nil - end -- A shared anchor must not change a non-SSL consumer's substitution behavior. - for marker in pairs(outside) do - fields[marker] = nil - end - for marker in pairs(ssl) do - if not fields[marker] then - return conf, fields, true + local fields, unprotected = {}, false + for marker, use in pairs(uses) do + if ok and not use.unsafe then + fields[marker] = use.targets + elseif use.ssl then + unprotected = true end end - return conf, fields + return {config = conf, fields = fields, unprotected = unprotected} end -function _M.load(text, substitute, warn) +-- Save original values and prepare a repeatable substitution with optional markers. +local function prepare_substitution(text, substitute) local values, multiline = {}, false local expanded, _, err = substitute(text, function(value) values[#values + 1] = {value = value} @@ -151,8 +148,9 @@ function _M.load(text, substitute, warn) if err then return nil, err end + local env = {expanded = expanded} if not multiline then - return yaml.load(expanded) + return env end local serial, prefix = 0 @@ -174,7 +172,7 @@ function _M.load(text, substitute, warn) end end -- Replaying substitution over the same source visits occurrences in the same order. - local function render(selected) + env.render = function(selected) local index = 0 return substitute(text, function() index = index + 1 @@ -182,40 +180,56 @@ function _M.load(text, substitute, warn) return selected[record.marker] and record.marker or record.value end) end - local function warn_fallback() - if warn then - warn("cannot preserve some multiline SSL environment values; " - .. "using legacy YAML substitution for them (consider $ENV:// references)") - warn = nil - end - end - local ok, _, selected, unprotected = pcall(inspect_fields, render(records), records, prefix) - if unprotected then - warn_fallback() + env.records, env.prefix = records, prefix + return env +end + + +-- Discover supported fields, then recheck after expanding all other variables. +-- Return nil to use legacy YAML; the second result requests a warning. +local function restore_ssl(env) + local ok, discovered = pcall(inspect_fields, env.render(env.records), env.records, env.prefix) + if not ok then + return nil end - if not ok or not selected or not next(selected) then - return yaml.load(expanded) + local selected = discovered.fields + if not next(selected) then + return nil, discovered.unprotected end - - local parsed, conf, fields = pcall(inspect_fields, render(selected), records, prefix) - if not parsed or not fields then - warn_fallback() - return yaml.load(expanded) + local parsed, final = pcall(inspect_fields, env.render(selected), env.records, env.prefix) + if not parsed then + return nil, true end -- Other substitutions can move or synthesize markers. Check every selected -- occurrence before restoring any value; never scan the restored PEM again. for marker in pairs(selected) do - if not fields[marker] then - warn_fallback() - return yaml.load(expanded) + if not final.fields[marker] then + return nil, true end end for marker in pairs(selected) do - for _, target in ipairs(fields[marker]) do - target[1][target[2]] = target[3] + for _, target in ipairs(final.fields[marker]) do + target.parent[target.key] = env.records[marker].value end end - return conf + return final.config, discovered.unprotected +end + + +function _M.load(text, substitute, warn) + local env, err = prepare_substitution(text, substitute) + if not env then + return nil, err + end + local conf, unprotected + if env.records then + conf, unprotected = restore_ssl(env) + end + if unprotected and warn then + warn("cannot preserve some multiline SSL environment values; " + .. "using legacy YAML substitution for them (consider $ENV:// references)") + end + return conf or yaml.load(env.expanded) end