test(cz-cli): replay real CLI history against the parser

packages/cz-cli/package.json

@@ -24,7 +24,8 @@
     "test:mcp": "bun test/mcp-serve-smoke.ts",
     "preview:gateway-prompt": "bun script/preview-gateway-prompt.ts",
     "test:unit": "bun test test/classify-error.test.ts",
-    "test:all": "bun test test/classify-error.test.ts && bun test/e2e-routing.ts && bun test/e2e-help.ts && bun test/e2e.ts",
+    "test:history": "bun test test/history-argv.test.ts test/history-effect.test.ts",
+    "test:all": "bun test test/classify-error.test.ts && bun run test:history && bun test/e2e-routing.ts && bun test/e2e-help.ts && bun test/e2e.ts",
     "build:local": "bun run build --single && mkdir -p dist && cp ../opencode/dist/cz-cli-darwin-arm64/bin/cz-cli dist/cz-cli && cp ../opencode/dist/cz-cli-darwin-arm64/bin/clickzetta-* ../opencode/dist/cz-cli-darwin-arm64/bin/tui-title-brand.ts ../opencode/dist/cz-cli-darwin-arm64/bin/tui-quota-runtime.js ../opencode/dist/cz-cli-darwin-arm64/bin/tui-quota.tsx ../opencode/dist/cz-cli-darwin-arm64/bin/gateway-prompt-view.tsx dist/ && codesign --sign - dist/cz-cli",
     "test:ci": "bun run build:local && CZ_CLI_BIN=./dist/cz-cli bun run test:all"
   },

packages/cz-cli/test/history-argv.test.ts

@@ -0,0 +1,194 @@
+/**
+ * Tier A of the history regression suite: replay every (command, flag-set)
+ * combination real users have run on this branch's lineage and assert the current
+ * parser still accepts it.
+ *
+ * The fixture is generated from `czcli.public.otel_logs` by
+ * script/export-history-matrix.ts (run manually — it needs the profile and
+ * network). Everything here is in-process argv parsing: no handler runs, no
+ * network, no filesystem writes. The historical invocations point at production
+ * lakehouses and include `task delete`, `sql --write` and `profile remove`, so
+ * "parse only" is a safety requirement, not an optimization.
+ *
+ * A failure names the usage count and user count of what broke. When the failure
+ * is a deliberate change, record it in history-regression/known-changes.ts with
+ * the reason rather than deleting the case.
+ */
+import { describe, expect, test } from "bun:test"
+import matrix from "./history-regression/matrix.json" with { type: "json" }
+import { findKnownChange, isExcludedPath } from "./history-regression/known-changes.js"
+import {
+  isDeclaredOption,
+  replayHistorical,
+  type HistoricalFlag,
+  type ReplayResult,
+} from "./support/history-replay.js"
+
+interface MatrixEntry {
+  cmd: string
+  sub?: string
+  sub2?: string
+  flags: HistoricalFlag[]
+  usageCount: number
+  users: number
+  firstSeen: string
+  lastSeen: string
+  positionalCountMin: number
+  versions: string[]
+}
+
+interface ValueCase {
+  cmd: string
+  sub?: string
+  sub2?: string
+  key: string
+  value: string
+  usageCount: number
+}
+
+const entries = matrix.entries as MatrixEntry[]
+const valueCases = matrix.valueCases as ValueCase[]
+
+/** Verdicts that mean "the parser accepted this invocation". */
+const ACCEPTED = new Set<ReplayResult["verdict"]>(["PASS", "HELP_OR_VERSION", "SUBCOMMAND_HELP"])
+
+const pathOf = (entry: { cmd: string; sub?: string; sub2?: string }) =>
+  [entry.cmd, entry.sub, entry.sub2].filter(Boolean).join(" ")
+
+const tokensOf = (entry: { cmd: string; sub?: string; sub2?: string }) =>
+  // `unknown` is not a command: telemetry writes it when the invocation had no
+  // positional token at all (src/run-cli.ts:687), i.e. `cz-cli --version`.
+  [entry.cmd === "unknown" ? undefined : entry.cmd, entry.sub, entry.sub2].filter(Boolean) as string[]
+
+function describeEntry(entry: MatrixEntry, result: ReplayResult): string {
+  return [
+    `${pathOf(entry)} [${entry.flags.map((f) => f.key).join(" ") || "no flags"}]`,
+    `  ${entry.usageCount} invocations by ${entry.users} user(s), ${entry.firstSeen} → ${entry.lastSeen}`,
+    `  versions: ${entry.versions.join(", ") || "unknown"}`,
+    `  replayed: cz-cli ${result.argv.join(" ")}`,
+    `  verdict:  ${result.verdict}${result.message ? ` — ${result.message}` : ""}`,
+    result.missing?.length ? `  missing:  ${result.missing.join(", ")}` : "",
+    `  If this change is deliberate, add it to test/history-regression/known-changes.ts.`,
+  ]
+    .filter(Boolean)
+    .join("\n")
+}
+
+/**
+ * Assert one replayed combination. Excused failures are checked positively — the
+ * verdict must be the one the triage entry claims — so a known change cannot
+ * quietly start failing in a new way.
+ */
+function assertAccepted(entry: MatrixEntry, result: ReplayResult): void {
+  if (ACCEPTED.has(result.verdict)) return
+  const known = findKnownChange(pathOf(entry), result.verdict, result)
+  if (known) return
+  throw new Error(describeEntry(entry, result))
+}
+
+describe("history regression: argv layer", () => {
+  test("fixture covers this branch's lineage", () => {
+    expect(matrix.scope.versionPatterns).toEqual(["1.17.%", "dev-v1.17%", "dev-v2.%"])
+    expect(entries.length).toBeGreaterThan(2000)
+    expect(matrix.scope.invocations).toBeGreaterThan(100_000)
+  })
+
+  // Highest-usage combinations first, so a regression in something 20k users hit
+  // shows up at the top of the failure list rather than after 2,000 rare ones.
+  const ordered = [...entries].sort((a, b) => b.usageCount - a.usageCount)
+
+  for (const [index, entry] of ordered.entries()) {
+    const path = pathOf(entry)
+    const label = `${String(index).padStart(4, "0")} ${path} [${entry.flags.map((f) => f.key).join(",") || "-"}] ×${entry.usageCount}`
+    const excluded = isExcludedPath(path)
+    if (excluded) {
+      test.skip(`${label} — excluded: ${excluded.reason.slice(0, 80)}…`, () => {})
+      continue
+    }
+    test(label, async () => {
+      assertAccepted(entry, await replayHistorical(tokensOf(entry), entry.flags))
+    })
+  }
+})
+
+describe("history regression: option values", () => {
+  // Every value ever passed to an option that declares `choices`. This is the
+  // layer that catches a narrowed enum: the flag still exists and still parses,
+  // but a value users relied on is no longer admitted.
+  for (const [index, item] of valueCases.entries()) {
+    const path = pathOf(item)
+    const label = `${String(index).padStart(4, "0")} ${path} --${item.key}=${item.value} ×${item.usageCount}`
+    const excluded = isExcludedPath(path)
+    if (excluded) {
+      test.skip(`${label} — excluded`, () => {})
+      continue
+    }
+    if (!isDeclaredOption(tokensOf(item), item.key)) {
+      // The option is not declared on this path at all, so the triple is a
+      // normalization artifact rather than a value the command ever accepted.
+      // Whether the option still exists somewhere is Tier A's assertion.
+      test.skip(`${label} — --${item.key} not declared on '${path}' (normalization artifact)`, () => {})
+      continue
+    }
+    test(label, async () => {
+      const flags: HistoricalFlag[] = [{ key: item.key, hadValue: true, value: item.value }]
+      // fillRequired: this probe isolates one option's accepted values, so the
+      // command's other mandatory options are supplied rather than asserted.
+      const result = await replayHistorical(tokensOf(item), flags, { fillRequired: true })
+      if (ACCEPTED.has(result.verdict)) return
+      if (findKnownChange(path, result.verdict, result)) return
+      throw new Error(
+        [
+          `${path} --${item.key}=${item.value} (${item.usageCount} invocations) is no longer accepted`,
+          `  replayed: cz-cli ${result.argv.join(" ")}`,
+          `  verdict:  ${result.verdict}${result.message ? ` — ${result.message}` : ""}`,
+          `  A narrowed choices list is a breaking change; if deliberate, record it in known-changes.ts.`,
+        ].join("\n"),
+      )
+    })
+  }
+})
+
+/**
+ * The triage matcher decides which failures are allowed to stay green, so its
+ * scope has to be exact: an entry excusing one flag must not excuse the next
+ * deletion on the same path. yargs makes that easy to get wrong by reporting
+ * every unknown argument of an invocation in one message.
+ */
+describe("history regression: triage matcher scope", () => {
+  test("excuses the flag it names", () => {
+    expect(findKnownChange("setup", "FLAG_REMOVED", { message: "Unknown argument: partition" })).toBeDefined()
+  })
+
+  test("both spellings of one flag are one name", () => {
+    // yargs lists the camelCase expansion alongside the flag as given.
+    expect(
+      findKnownChange("analytics-agent domain create", "FLAG_REMOVED", {
+        message: "Unknown arguments: sample-question, sampleQuestion",
+      }),
+    ).toBeDefined()
+  })
+
+  test("does not excuse a deletion that failed alongside the known flag", () => {
+    expect(
+      findKnownChange("setup", "FLAG_REMOVED", {
+        message: "Unknown arguments: partition, another-gone-flag, anotherGoneFlag",
+      }),
+    ).toBeUndefined()
+  })
+
+  test("names are matched whole, not as substrings", () => {
+    // The `task list-folders` entry excuses a flag literally named `1` (the value
+    // of `--parent -1`); it must not cover every message that contains a "1".
+    expect(findKnownChange("task list-folders", "FLAG_REMOVED", { message: "Unknown argument: folder1" })).toBeUndefined()
+  })
+
+  test("every missing flag must be named", () => {
+    expect(findKnownChange("sql", "FLAG_NOT_IN_ARGV", { missing: ["no-limit"] })).toBeDefined()
+    expect(findKnownChange("sql", "FLAG_NOT_IN_ARGV", { missing: ["no-limit", "vcluster"] })).toBeUndefined()
+  })
+
+  test("an entry excuses only the verdict it claims", () => {
+    expect(findKnownChange("sql", "FLAG_REMOVED", { message: "Unknown argument: no-limit" })).toBeUndefined()
+  })
+})

packages/cz-cli/test/history-effect.test.ts

@@ -0,0 +1,169 @@
+/**
+ * Tier B of the history regression suite: for the highest-usage commands, assert
+ * the parameter value actually reaches the outbound request.
+ *
+ * Tier A proves a flag still parses. That is not the same as the flag still doing
+ * something: an option can survive strict validation, land in argv, and be ignored
+ * by a handler that no longer reads it. Only the wire shows the difference, so
+ * these cases run the real handler against the fetch boundary and assert on the
+ * request the command actually produced (query string or body).
+ *
+ * Command selection follows usage in matrix.json — the pairs at the top of the
+ * distribution (sql ≈ 255k invocations, job result ≈ 29k, runs logs ≈ 6k,
+ * task/table/analytics-agent families) rather than an even sample.
+ *
+ * Expectations are transcribed from observed payloads, not guessed, which is why
+ * several assert a TRANSLATED value: `--status SUCCESS` becomes
+ * `instanceStatusList:[1]`, `--limit N` becomes `pageSize:N`, `--type lakehouse`
+ * becomes `dsType:1`. Those mappings are the parameter's real effect, and a
+ * translation table that silently loses an entry is exactly the failure this tier
+ * exists to catch.
+ */
+import { beforeEach, describe, expect, setDefaultTimeout, test } from "bun:test"
+import { writeFileSync } from "node:fs"
+import { join } from "node:path"
+import { onFetch, sqlSuccess, stubStudioContext } from "./support/cz-fixtures.js"
+
+const { execute } = await import("../src/execute.ts")
+
+setDefaultTimeout(20_000)
+
+interface Recorded {
+  url: string
+  method: string
+  body: unknown
+}
+
+/** Auth/context plumbing every studio command drives first; not part of any assertion. */
+const CONTEXT_PATHS = /loginSingle|getCurrentUser|serviceInstanceList|listUserWorkspaces/
+
+let requests: Recorded[] = []
+
+beforeEach(() => {
+  writeFileSync(
+    join(process.env.CLICKZETTA_TEST_HOME!, ".clickzetta", "profiles.toml"),
+    [
+      "[profiles.test]",
+      "pat = 'pat'",
+      "workspace = 'ws'",
+      "instance = 'inst'",
+      "service = 'uat-api.clickzetta.com'",
+      "vcluster = 'default'",
+      "schema = 'public'",
+      "analysis_agent_endpoint = 'https://example.clickzetta.com'",
+      "",
+    ].join("\n"),
+  )
+  requests = []
+  stubStudioContext({ workspaceName: "ws" })
+  // One catch-all responder: these assertions are about the REQUEST, so the reply
+  // only has to be well-formed enough to keep the handler moving. The top-level job
+  // shape terminates lakehouse polling immediately instead of sleeping.
+  onFetch({
+    match: () => true,
+    respond: (url, method, body) => {
+      requests.push({ url, method, body })
+      return {
+        code: 0,
+        ...(sqlSuccess(["c"], [[1]]) as Record<string, unknown>),
+        data: {
+          id: 1,
+          fileId: 1,
+          jobId: "j1",
+          total: 1,
+          list: [{ id: 1, fileId: 1, dataFileName: "x", name: "x" }],
+          items: [{ id: 1 }],
+          records: [{ id: 1 }],
+          status: { state: "SUCCEED" },
+        },
+      }
+    },
+  })
+})
+
+async function run(command: string): Promise<{ exitCode: number; wire: string[] }> {
+  const result = await execute(`${command} --profile test --format json`)
+  const wire = requests
+    .filter((request) => !CONTEXT_PATHS.test(request.url))
+    .map((request) => `${request.url} ${request.body === undefined ? "" : JSON.stringify(request.body)}`)
+  return { exitCode: result.exitCode, wire }
+}
+
+/** [command, what the outbound request must contain, note] */
+type Case = [command: string, expected: string, note?: string]
+
+const CASES: Case[] = [
+  // sql — 255k recorded invocations, the single busiest command
+  ['sql "select 1" --limit 7', "LIMIT 8", "row cap is pushed into the SQL as limit+1 (the extra row detects truncation)"],
+  ['sql "select 1" --set cz.sql.timezone=UTC', '"cz.sql.timezone":"UTC"', "query hint"],
+  ['sql "select 1" --vcluster czprobe1', '"virtualCluster":"czprobe1"'],
+  ['sql "select 1" --schema czprobe1', '"defaultNamespace":["ws","czprobe1"]'],
+  ['sql "select 1" --async', '"hybridPollingTimeout":0', "async drops the sync polling window"],
+  ['sql "select ${t}" --variable t=czprobe1', "select czprobe1", "substitution happens before submit"],
+  ['sql "insert into t values(1)" --write', "insert into t values(1)", "--write releases the write guard"],
+  // job — `job result` alone is ~29k invocations
+  ["job result 20260101abc --timeout 2", '"id":"20260101abc"'],
+  ["job status 20260101abc", '"id":"20260101abc"'],
+  ["job profile 20260101abc", "jobId=20260101abc", "id goes in the query string here, not a body"],
+  // runs / attempts
+  ["runs list --page-size 7", '"pageSize":7'],
+  ["runs list --limit 7", '"pageSize":7', "--limit is an alias of --page-size on the wire"],
+  ["runs list --status SUCCESS", '"instanceStatusList":[1]', "enum mapped to the backend status code"],
+  ["runs list --run-type SCHEDULE", '"instanceType":1', "enum mapped to the backend type code"],
+  ["runs detail 4242", '"taskInstanceId":4242'],
+  ["runs logs 4242", '"taskInstanceId":4242'],
+  ["attempts list 4242", '"taskInstanceId":4242'],
+  // task
+  ["task list --like czprobe1", '"fileName":"czprobe1"'],
+  ["task list --page-size 7", '"pageSize":7'],
+  ["task list --page 3", '"page":3'],
+  ["task list --limit 7", '"pageSize":7'],
+  ["task search --name czprobe1", '"fileName":"czprobe1"'],
+  ["task status czprobe1", '"fileName":"czprobe1"', "name resolves to an id through listFiles"],
+  ["task content czprobe1", '"fileName":"czprobe1"'],
+  ["task schedule-info czprobe1", '"fileName":"czprobe1"'],
+  // table / schema / workspace — these compile to SQL
+  ["table describe czprobe1", "DESC EXTENDED czprobe1"],
+  ["table list --like czprobe1", "SHOW TABLES LIKE 'czprobe1'"],
+  ["schema list", "SHOW SCHEMAS"],
+  ["workspace list", "SHOW WORKSPACES"],
+  // analytics-agent
+  ["analytics-agent domain list", "/analytics-agent/domains"],
+  ["analytics-agent metric list --domain-id 9", '"domainIds":[9]'],
+  ["analytics-agent session list --domain-id 3", '"domainId":3'],
+  // datasource / dqc
+  ["datasource list --type lakehouse", '"dsType":1', "enum mapped to the backend datasource type"],
+  ["dqc list", "/clickzetta-dqc/api/v1/rule/list"],
+]
+
+describe("history regression: parameters reach the wire", () => {
+  for (const [command, expected, note] of CASES) {
+    test(`${command} → ${expected}${note ? ` (${note})` : ""}`, async () => {
+      const { exitCode, wire } = await run(command)
+      const joined = wire.join("\n")
+      if (!joined.includes(expected)) {
+        throw new Error(
+          [
+            `cz-cli ${command}`,
+            `  expected the outbound request to contain: ${expected}`,
+            note ? `  (${note})` : "",
+            `  exit code: ${exitCode}`,
+            wire.length === 0
+              ? "  no request was sent at all"
+              : `  requests sent:\n${wire.map((line) => `    ${line.slice(0, 400)}`).join("\n")}`,
+          ]
+            .filter(Boolean)
+            .join("\n"),
+        )
+      }
+    })
+  }
+
+  test("sql refuses a write without --write, before reaching the network", async () => {
+    // The write guard is client-side, so its regression signature is a request that
+    // SHOULD NOT exist. Tier A cannot see this: `--write` parses either way.
+    const { exitCode, wire } = await run('sql "insert into t values(1)"')
+    expect(exitCode).not.toBe(0)
+    expect(wire.filter((line) => line.includes("insert into t"))).toEqual([])
+  })
+})