fix: preserve multiline SSL environment values in standalone YAML

apisix/cli/file.lua

@@ -25,13 +25,15 @@ local dkjson = require("dkjson")
 local pl_path = require("pl.path")
 
 local pairs = pairs
+local require = require
 local type = type
 local tonumber = tonumber
 local getenv = os.getenv
 local str_gmatch = string.gmatch
 local str_find = string.find
 local str_sub = string.sub
 local print = print
+local stderr = io.stderr
 
 local _M = {}
 local exported_vars
@@ -57,7 +59,7 @@ local function tab_is_array(t)
 end
 
 
-local function var_sub(val)
+local function var_sub(val, on_value)
     local err
     local var_used = false
     -- we use '${{var}}' because '$var' and '${var}' are taken
@@ -79,7 +81,7 @@ local function var_sub(val)
 
             exported_vars[var] = v
             var_used = true
-            return v
+            return on_value and on_value(v) or v
         end
 
         err = "failed to handle configuration: " ..
@@ -160,6 +162,20 @@ _M.resolve_conf_var = resolve_conf_var
 _M.resolve_conf_var_in_text = resolve_conf_var_in_text
 
 
+-- CLI validation and runtime reload must use the same SSL substitution rules.
+local function warn_yaml_fallback(message)
+    stderr:write("WARNING: ", message, "\n")
+end
+
+
+local function load_yaml_with_env(text, warn)
+    return require("apisix.cli.ssl_yaml").load(text, var_sub, warn or warn_yaml_fallback)
+end
+
+
+_M.load_yaml_with_env = load_yaml_with_env
+
+
 local function replace_by_reserved_env_vars(conf)
     -- TODO: support more reserved environment variables
     local v = getenv("APISIX_DEPLOYMENT_ETCD_HOST")
@@ -322,12 +338,10 @@ function _M.read_yaml_conf(apisix_home)
         local apisix_conf_path = profile:yaml_path("apisix")
         local apisix_conf_yaml, _ = util.read_file(apisix_conf_path)
         if apisix_conf_yaml then
-            -- Pre-parse substitution: `"${{VAR}}"` stays string, `${{VAR}}` infers type from value.
-            local resolved, err = resolve_conf_var_in_text(apisix_conf_yaml)
-            if not resolved then
+            local apisix_conf, err = load_yaml_with_env(apisix_conf_yaml)
+            if err then
                 return nil, err
             end
-            local apisix_conf = yaml.load(resolved)
             if not apisix_conf then
                 return nil, "invalid apisix.yaml file"
             end

apisix/cli/ssl_yaml.lua

@@ -0,0 +1,236 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local yaml = require("lyaml")
+local parser = require("yaml").parser
+local pairs = pairs
+local next = next
+local ipairs = ipairs
+local type = type
+local pcall = pcall
+local _M = {}
+
+
+local function is_ssl_field(path)
+    if path[1] ~= "ssls" or type(path[2]) ~= "number" then
+        return false
+    end
+    if #path == 3 then
+        return path[3] == "cert" or path[3] == "key"
+    end
+    if #path == 4 then
+        return ((path[3] == "certs" or path[3] == "keys") and type(path[4]) == "number")
+               or (path[3] == "client" and path[4] == "ca")
+    end
+    return false
+end
+
+
+-- Visit aliases once per path, but abort discovery on cycles or excessive depth.
+-- The caller then uses the existing YAML load, without introducing a new error.
+local function walk(value, path, active, visit, parent, key)
+    if type(value) ~= "table" then
+        visit(value, is_ssl_field(path), parent, key)
+        return true
+    end
+    if active[value] or #path > 128 then
+        return false
+    end
+    active[value] = true
+    for k, v in pairs(value) do
+        visit(k, false)
+        path[#path + 1] = k
+        if not walk(v, path, active, visit, value, k) then
+            return false
+        end
+        path[#path] = nil
+    end
+    active[value] = nil
+    return true
+end
+
+
+-- A decoded scalar may spell a marker using YAML escapes. Only accept markers
+-- that occur literally in that scalar's source span, not user-authored lookalikes.
+local function scalar_origins(text, records)
+    local offsets, index = {}, 0
+    for byte = 1, #text do
+        local b = text:byte(byte)
+        if b < 128 or b >= 192 then
+            offsets[index] = byte
+            index = index + 1
+        end
+    end
+    offsets[index] = #text + 1
+    local origins, ambiguous = {}, {}
+    for event in parser(text) do
+        if event.type == "SCALAR" and records[event.value] then
+            local first = offsets[event.start_mark.index]
+            local last = offsets[event.end_mark.index]
+            local source = first and last and text:sub(first, last - 1)
+            if source and source:find(event.value, 1, true)
+               and (event.style == "PLAIN" or event.style == "SINGLE_QUOTED"
+                    or event.style == "DOUBLE_QUOTED") then
+                origins[event.value] = true
+            else
+                ambiguous[event.value] = true
+            end
+        end
+        if event.type == "STREAM_END" then
+            break
+        end
+    end
+    for marker in pairs(ambiguous) do
+        origins[marker] = nil
+    end
+    return origins
+end
+
+
+-- Use the same path and provenance checks for discovery and final restoration.
+local function inspect_fields(text, records, prefix)
+    local conf = yaml.load(text)
+    local origins = scalar_origins(text, records)
+    local uses = {}
+    local ok = walk(conf, {}, {}, function(value, allowed, parent, key)
+        if type(value) ~= "string" then
+            return
+        end
+        for marker in value:gmatch(prefix .. "%d+__") do
+            if records[marker] then
+                local use = uses[marker] or {targets = {}}
+                uses[marker] = use
+                use.ssl = use.ssl or allowed
+                if allowed and value == marker and origins[marker] then
+                    local targets = use.targets
+                    targets[#targets + 1] = {parent = parent, key = key}
+                else
+                    use.unsafe = true
+                end
+            end
+        end
+    end)
+    -- A shared anchor must not change a non-SSL consumer's substitution behavior.
+    local fields, unprotected = {}, false
+    for marker, use in pairs(uses) do
+        if ok and not use.unsafe then
+            fields[marker] = use.targets
+        elseif use.ssl then
+            unprotected = true
+        end
+    end
+    return {config = conf, fields = fields, unprotected = unprotected}
+end
+
+
+-- Save original values and prepare a repeatable substitution with optional markers.
+local function prepare_substitution(text, substitute)
+    local values, multiline = {}, false
+    local expanded, _, err = substitute(text, function(value)
+        values[#values + 1] = {value = value}
+        multiline = multiline or value:find("[\r\n]") ~= nil
+        return value
+    end)
+    if err then
+        return nil, err
+    end
+    local env = {expanded = expanded}
+    if not multiline then
+        return env
+    end
+
+    local serial, prefix = 0
+    local collision
+    repeat
+        serial = serial + 1
+        prefix = "__APISIX_SSL_ENV_" .. serial .. "_"
+        collision = text:find(prefix, 1, true)
+        for _, record in ipairs(values) do
+            collision = collision or record.value:find(prefix, 1, true)
+        end
+    until not collision
+
+    local records = {}
+    for i, record in ipairs(values) do
+        if record.value:find("[\r\n]") then
+            record.marker = prefix .. i .. "__"
+            records[record.marker] = record
+        end
+    end
+    -- Replaying substitution over the same source visits occurrences in the same order.
+    env.render = function(selected)
+        local index = 0
+        return substitute(text, function()
+            index = index + 1
+            local record = values[index]
+            return selected[record.marker] and record.marker or record.value
+        end)
+    end
+    env.records, env.prefix = records, prefix
+    return env
+end
+
+
+-- Discover supported fields, then recheck after expanding all other variables.
+-- Return nil to use legacy YAML; the second result requests a warning.
+local function restore_ssl(env)
+    local ok, discovered = pcall(inspect_fields, env.render(env.records), env.records, env.prefix)
+    if not ok then
+        return nil
+    end
+    local selected = discovered.fields
+    if not next(selected) then
+        return nil, discovered.unprotected
+    end
+    local parsed, final = pcall(inspect_fields, env.render(selected), env.records, env.prefix)
+    if not parsed then
+        return nil, true
+    end
+    -- Other substitutions can move or synthesize markers. Check every selected
+    -- occurrence before restoring any value; never scan the restored PEM again.
+    for marker in pairs(selected) do
+        if not final.fields[marker] then
+            return nil, true
+        end
+    end
+    for marker in pairs(selected) do
+        for _, target in ipairs(final.fields[marker]) do
+            target.parent[target.key] = env.records[marker].value
+        end
+    end
+    return final.config, discovered.unprotected
+end
+
+
+function _M.load(text, substitute, warn)
+    local env, err = prepare_substitution(text, substitute)
+    if not env then
+        return nil, err
+    end
+    local conf, unprotected
+    if env.records then
+        conf, unprotected = restore_ssl(env)
+    end
+    if unprotected and warn then
+        warn("cannot preserve some multiline SSL environment values; "
+             .. "using legacy YAML substitution for them (consider $ENV:// references)")
+    end
+    return conf or yaml.load(env.expanded)
+end
+
+
+return _M

apisix/core/config_yaml.lua

@@ -20,7 +20,6 @@
 -- @module core.config_yaml
 
 local config_local = require("apisix.core.config_local")
-local yaml         = require("lyaml")
 local log          = require("apisix.core.log")
 local json         = require("apisix.core.json")
 local new_tab      = require("table.new")
@@ -98,13 +97,7 @@ local config_yaml = {
         local raw_config = f:read("*a")
         f:close()
 
-        -- substitute env vars in the raw YAML text before parsing so that
-        -- YAML quoting decides types (see apisix/cli/file.lua).
-        local resolved, err = file.resolve_conf_var_in_text(raw_config)
-        if not resolved then
-            return nil, err
-        end
-        return yaml.load(resolved), nil
+        return file.load_yaml_with_env(raw_config, log.warn)
     end
 }
 

docs/en/latest/deployment-modes.md

@@ -363,6 +363,36 @@ routes:
 
 More information about using environment variables can be found [here](./admin-api.md#using-environment-variables).
 
+For SSL resources, a complete `${{VAR}}` reference in `cert`, `key`, a `certs` or
+`keys` array item, or `client.ca` can contain a multiline PEM value:
+
+```yaml
+ssls:
+  - id: example
+    snis: ["example.com"]
+    cert: "${{SERVER_CERT}}"
+    key: "${{SERVER_KEY}}"
+#END
+```
+
+The loader preserves the original multiline value in these fields. Single-line
+values, including literal `\n` escapes inside double-quoted YAML, continue to use
+the existing substitution rules. Other fields also retain their existing YAML
+substitution behavior.
+
+This protection applies to complete scalar references, including unquoted
+references in flow-style YAML. It does not add recursive variable expansion or
+protect references embedded in larger strings or block scalars. Configurations
+that cannot be identified unambiguously use the existing loading behavior;
+for example, a variable shared through an anchor with a non-SSL field is not
+protected. A whole configuration supplied through one environment variable is
+still expanded as YAML, without special handling of PEM content inside it.
+When the loader identifies a multiline SSL reference but cannot safely protect
+it, it logs a warning without including the value and retains the existing behavior.
+
+For those cases, SSL fields also support environment-backed
+[Secret references](./terminology/secret.md), such as `$ENV://SERVER_CERT`.
+
 #### To `config_provider: json`
 
 All of the rules are stored in one file which named `conf/apisix.json`,