Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion evals/runner/agent.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ test("buildPrompt pre1 branch names the pre1Path, both skills, and the output co
basicAuth: {username: "connector@example.com", password: "fixture-token"},
bearerToken: "fixture-token",
},
skillBundle: {mode: "full", version: "0.8.0"},
skillBundle: {mode: "full", version: "0.11.0"},
model: "together/deepseek-ai/DeepSeek-V4-Flash-0731",
reasoningEffort: "high",
kind: "pre1",
Expand Down
2 changes: 1 addition & 1 deletion evals/runner/scenario.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ test("loadScenario loads the pre1-directory-proceed scenario (kind pre1, proceed
assert.equal(s.seed, undefined)
assert.equal(s.requiredSourceFiles, undefined)
assert.equal(s.skillBundle.mode, "full")
assert.equal(s.skillBundle.version, "0.8.0")
assert.equal(s.skillBundle.version, "0.11.0")
})

test("loadScenario loads the pre1-noiam-park scenario (kind pre1, park)", () => {
Expand Down
17 changes: 8 additions & 9 deletions evals/runner/skills_bundle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const execFileAsync = promisify(execFile)
const RUN = "evals/runner/run.ts"
const BUNDLE = "evals/skills-bundle/bundle.json"
const SKILLS = ["author-in-app-connector", "read-authoring-contract", "write-connector-source", "build-and-test", "deploy-and-activate", "design-access-model", "source-openapi-spec", "verify-connector-output", "update-and-rollback", "diagnose-authoring-failure"]
const VERSION = "0.8.0"
const VERSION = "0.11.0"

function readBundle(): {version: string; skills: {name: string; version: string; path: string}[]} {
return JSON.parse(readFileSync(BUNDLE, "utf8")) as {version: string; skills: {name: string; version: string; path: string}[]}
Expand Down Expand Up @@ -74,10 +74,10 @@ test("(b) every bundle.json path resolves to an existing file", () => {
// instructions while keeping the marker literals must fail the gate.
const SKILL_LITERALS: Record<string, string[]> = {
"author-in-app-connector": [
"skipped_human_boundary",
"catalog_id", "draft_id", "upload_id", "run_id", "revision_id",
"app_id", "connector_id", "test_run_id", "deployment_instance_id", "activation_url",
"Do not redeem the approval token",
"Do not call `force_sync` in the authoring session",
"write-connector-source",
],
"read-authoring-contract": [
Expand All @@ -89,10 +89,10 @@ const SKILL_LITERALS: Record<string, string[]> = {
],
"deploy-and-activate": [
"deployment_instance_id", "activation_url", "REVISION_STATUS_ACTIVE", "activation_epoch",
"SYNC_STATUS_DONE", "skipped_human_boundary",
"SYNC_STATUS_DONE",
"Do not redeem the approval token",
"Do not call `c1_connector_service_force_sync` during the funnel run",
"Do not call `c1_connector_authoring_list_revision_summaries` during the funnel run",
"Do not call `c1_connector_service_force_sync` in the authoring session",
"Do not call `c1_connector_authoring_list_revision_summaries` before the OWNER activates",
],
"write-connector-source": [
"Do not call fetch",
Expand Down Expand Up @@ -174,8 +174,7 @@ const SKILL_LITERALS: Record<string, string[]> = {
"Invalid token provided",
"ConnectionOK",
"HostCallOK",
"Poll with backoff",
"row after ~10 polls, stop and report",
"wait for the draft test as build-and-test",
"c1_connector_service_get",
"status.lastError",
],
Expand All @@ -198,7 +197,7 @@ test("(c) each SKILL.md carries the locked section markers, content literals, AS
test("(d) the full-mode scenario parses with mode full and the two pinned scenarios keep their locked modes", () => {
const full = loadScenario("evals/scenarios/tier1-directory-full.json")
assert.equal(full.skillBundle.mode, "full")
assert.equal(full.skillBundle.version, "0.8.0")
assert.equal(full.skillBundle.version, "0.11.0")
assert.equal(full.id, "tier1-directory-full")
const none = loadScenario("evals/scenarios/tier1-directory.json")
assert.equal(none.skillBundle.mode, "none")
Expand Down Expand Up @@ -247,7 +246,7 @@ test("(e) CLI end-to-end: full-mode Tier-0 run exits 0 and the record meta carri
const lines = readFileSync(join(dir, records[0]), "utf8").trim().split("\n")
const meta = JSON.parse(lines[0]) as Record<string, unknown>
assert.equal(meta.skill_bundle_mode, "full")
assert.equal(meta.skill_bundle_version, "0.8.0")
assert.equal(meta.skill_bundle_version, "0.11.0")
} finally {
rmSync(dir, {recursive: true, force: true})
}
Expand Down
2 changes: 1 addition & 1 deletion evals/scenarios/pre1-directory-proceed.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
},
"skillBundle": {
"mode": "full",
"version": "0.8.0"
"version": "0.11.0"
},
"model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
"reasoningEffort": "high"
Expand Down
2 changes: 1 addition & 1 deletion evals/scenarios/pre1-noiam-park.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
},
"skillBundle": {
"mode": "full",
"version": "0.8.0"
"version": "0.11.0"
},
"model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
"reasoningEffort": "high"
Expand Down
2 changes: 1 addition & 1 deletion evals/scenarios/tier1-directory-full.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
},
"skillBundle": {
"mode": "full",
"version": "0.8.0"
"version": "0.11.0"
},
"model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
"reasoningEffort": "high",
Expand Down
20 changes: 10 additions & 10 deletions evals/skills-bundle/bundle.json
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
{
"version": "0.8.0",
"version": "0.11.0",
"skills": [
{"name": "author-in-app-connector", "version": "0.3.2", "path": "../../skills/author-in-app-connector/SKILL.md"},
{"name": "read-authoring-contract", "version": "0.2.0", "path": "../../skills/read-authoring-contract/SKILL.md"},
{"name": "write-connector-source", "version": "0.2.0", "path": "../../skills/write-connector-source/SKILL.md"},
{"name": "build-and-test", "version": "0.3.0", "path": "../../skills/build-and-test/SKILL.md"},
{"name": "deploy-and-activate", "version": "0.3.1", "path": "../../skills/deploy-and-activate/SKILL.md"},
{"name": "design-access-model", "version": "0.1.0", "path": "../../skills/design-access-model/SKILL.md"},
{"name": "author-in-app-connector", "version": "0.3.5", "path": "../../skills/author-in-app-connector/SKILL.md"},
{"name": "read-authoring-contract", "version": "0.2.1", "path": "../../skills/read-authoring-contract/SKILL.md"},
{"name": "write-connector-source", "version": "0.2.1", "path": "../../skills/write-connector-source/SKILL.md"},
{"name": "build-and-test", "version": "0.3.1", "path": "../../skills/build-and-test/SKILL.md"},
{"name": "deploy-and-activate", "version": "0.3.4", "path": "../../skills/deploy-and-activate/SKILL.md"},
{"name": "design-access-model", "version": "0.1.1", "path": "../../skills/design-access-model/SKILL.md"},
{"name": "source-openapi-spec", "version": "0.1.0", "path": "../../skills/source-openapi-spec/SKILL.md"},
{"name": "verify-connector-output", "version": "0.1.0", "path": "../../skills/verify-connector-output/SKILL.md"},
{"name": "update-and-rollback", "version": "0.2.1", "path": "../../skills/update-and-rollback/SKILL.md"},
{"name": "diagnose-authoring-failure", "version": "0.1.0", "path": "../../skills/diagnose-authoring-failure/SKILL.md"}
{"name": "verify-connector-output", "version": "0.1.1", "path": "../../skills/verify-connector-output/SKILL.md"},
{"name": "update-and-rollback", "version": "0.2.2", "path": "../../skills/update-and-rollback/SKILL.md"},
{"name": "diagnose-authoring-failure", "version": "0.1.1", "path": "../../skills/diagnose-authoring-failure/SKILL.md"}
]
}
36 changes: 14 additions & 22 deletions skills/author-in-app-connector/SKILL.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
name: author-in-app-connector
description: Use when driving the full in-app connector authoring funnel end to end, from guide read through the human-activation handoff. Do not use when you are already mid-funnel and only need one stage's procedure - invoke the stage skill directly.
version: 0.3.2
version: 0.3.5
---

# author-in-app-connector
Expand Down Expand Up @@ -31,18 +31,12 @@ names below are the exact tenant MCP titles; the served guide abbreviates them
four `required_source_files` (connector.ts, config-schema.json,
runtime-schema.json, capabilities.json) must be true. STOP if any is
false.
Step-boundary note: the served 12-step contract (agent prompt) splits
this differently - its step 2 is upload+PUTs and its step 3 is
finalize+get_draft. The call sets are identical; only the boundary
differs. In this skill S2 is the full upload dance
(create_draft_source_upload + PUTs + finalize) and S3 is the get_draft
gate; "re-run from S2" always means the full upload dance.
4. S4 build: call `c1_connector_authoring_build_bundle`; extract `run_id`.
STOP if empty.
5. S5 build result: wait for the completion notification if your host provides
one; do not poll while waiting. If your session resumes with an outstanding
dispatch and no completion notification, check `get_run` once before
re-dispatching. Direct API callers without host wakeup poll
re-dispatching. Otherwise (direct API callers): poll
`c1_connector_authoring_get_run` with `run_id` every 5-10s, up to ~10 polls.
GATE: `RUN_STATE_SUCCEEDED`; extract the immutable
`revision_id`. STOP on failure - fix source; if the source changed,
Expand All @@ -58,11 +52,10 @@ names below are the exact tenant MCP titles; the served guide abbreviates them
10. S10 evidence: wait for the completion notification if your host provides
one; do not poll while waiting. If your session resumes with an outstanding
dispatch and no completion notification, check `get_test_run_evidence` once
before re-dispatching. Direct API callers without host wakeup poll
before re-dispatching. Otherwise (direct API callers): poll
`c1_connector_authoring_get_test_run_evidence` with
`(catalog_id, revision_id, test_run_id)` every 5-10s, up to ~10 polls.
GATE: `result == CONNECTOR_TEST_RUN_RESULT_PASS` (the PASS enum value;
the eval fixture records the string `"PASS"`). STOP if FAIL - fix
GATE: `result == CONNECTOR_TEST_RUN_RESULT_PASS`. STOP if FAIL - fix
source; if the source changed, re-run from S2 (re-upload) with a FRESH
`test_run_id`, otherwise re-run from the failing step per build-and-test.
11. S11 handoff discipline: call
Expand Down Expand Up @@ -113,27 +106,26 @@ Net-new provider: before S1, run both pre-1 skills - `source-openapi-spec`
## Human boundary

After S11's deploy + mint, present `activation_url` to a human tenant OWNER,
record the handoff table, and STOP. Never redeem the approval token, never
call `force_sync` - S11b/S11c are `skipped_human_boundary`. Never poll
`REVISION_STATUS_ACTIVE` in the funnel run: if a notification later resumes
you, the revision is ACTIVE - record the `activation_epoch` it carries.
Direct API callers verify per deploy-and-activate once the OWNER reports
activation.
record the handoff table, and STOP. Never redeem the approval token.
Do not call `force_sync` in the authoring session; the human/operator runs
it after the OWNER activates. Never poll `REVISION_STATUS_ACTIVE` before
the OWNER activates. If a notification later resumes you, the revision is
already ACTIVE: record its `activation_epoch`, but still do not force-sync
in this authoring session. Direct API callers verify per deploy-and-activate
once the OWNER reports activation.

## Exit criteria

- S0-S11 all pass per the stage table above.
- S11's handoff-discipline gate passes: all 10 handoff fields, no calls after
mint except the handoff write, no redemption.
- The body contains the literal `skipped_human_boundary` and all 10 handoff
field names.
- S11 handoff: all 10 handoff fields, no calls after mint except the
handoff write, no redemption.

## Anti-patterns

- Do not skip a stage.
- Do not fabricate handoff values.
- Do not redeem the approval token.
- Do not call `force_sync` at any point in the funnel run.
- Do not call `force_sync` in the authoring session; the human/operator runs it after the OWNER activates.
- Do not write the handoff before deploy + mint.

## Blocker protocol
Expand Down
8 changes: 8 additions & 0 deletions skills/author-in-app-connector/SOURCES.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,11 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b)
| In-repo SDK declarations | connector-authoring `b7e8a616cbbb1e336b788f807a3810b08ae00bc7` | The `.d.ts` module surface the funnel's source files are authored against. |
| SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). |
| baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. |

## Eval wiring

- Step boundaries: the served 12-step contract (agent prompt) puts upload+PUTs in its step 2 and finalize+get_draft in its step 3. This skill's S2 is the full upload dance (create_draft_source_upload + PUTs + finalize) and S3 is the get_draft gate; the call sets are identical, and "re-run from S2" always means the full upload dance.
- The eval scorer's S11 gate fails a run with any call after mint other than the handoff write; the skill states the rule (no call after mint before the OWNER activates).
- `evals/runner/stages.ts` rejects any `force_sync` in the authoring transcript, including after an activation notification resumes the session. The skill delegates production sync to the human/operator outside that session.
- The scorer's SKIPPED_STAGES disposition records S11b/S11c as `skipped_human_boundary`; the skill states only the behavior (hand off and stop).
- The SKILL.md content literals (the 10 handoff field names, the never-redeem instruction, and the authoring-session `force_sync` prohibition) are enforced by `evals/runner/skills_bundle.test.ts`.
15 changes: 5 additions & 10 deletions skills/build-and-test/SKILL.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
name: build-and-test
description: Use when uploading draft source, building a bundle, or running and awaiting a draft test sync. Do not use when the task is app/connector provisioning or activation - use deploy-and-activate.
version: 0.3.0
version: 0.3.1
---

# build-and-test
Expand Down Expand Up @@ -30,7 +30,7 @@ the exact tenant MCP titles; the served guide abbreviates them.
wait for the completion notification if your host provides one; do not poll
while waiting. If your session resumes with an outstanding dispatch and no
completion notification, check `c1_connector_authoring_get_run` once before
re-dispatching. Direct API callers without host wakeup poll
re-dispatching. Otherwise (direct API callers): poll
`c1_connector_authoring_get_run` with `run_id` every 5-10s, up to ~10 polls;
if no terminal state, STOP and report the run state rather than redispatching
the build. GATE:
Expand All @@ -42,14 +42,12 @@ the exact tenant MCP titles; the served guide abbreviates them.
one; do not poll while waiting. If your session resumes with an outstanding
dispatch and no completion notification, check
`c1_connector_authoring_get_test_run_evidence` once before re-dispatching.
Direct API callers without host wakeup poll
Otherwise (direct API callers): poll
`c1_connector_authoring_get_test_run_evidence` with the full key
`(catalog_id, revision_id, test_run_id)` every 5-10s, up to ~10 polls
(`NotFound` while pending); if no row, STOP and report pending rather than
redispatching the test. GATE:
`result == CONNECTOR_TEST_RUN_RESULT_PASS` (the PASS enum value on the
real surface; the eval fixture records the string `"PASS"`). STOP if the
row reports FAIL -
`result == CONNECTOR_TEST_RUN_RESULT_PASS`. STOP if the row reports FAIL -
read the error field, fix, and re-run from the correct step with a FRESH
`test_run_id`; never re-mint on a failed run.
5. Re-run-from-correct-step rules: a failed build re-runs from the build
Expand All @@ -66,10 +64,7 @@ the exact tenant MCP titles; the served guide abbreviates them.
- S5 passes: `RUN_STATE_SUCCEEDED` and `revision_id` non-empty.
- S9 passes: `test_run_id` non-empty, at least one successful
`c1_connector_authoring_run_draft_test_sync`.
- S10 passes: durable evidence `result == CONNECTOR_TEST_RUN_RESULT_PASS`
(the scorer's S10 gate checks the fixture's string `"PASS"`).
- The body contains the literal `required_source_files`, `upload_targets`,
`required_headers`, `RUN_STATE_SUCCEEDED`, and `test_run_id`.
- S10 passes: durable evidence `result == CONNECTOR_TEST_RUN_RESULT_PASS`.

## Anti-patterns

Expand Down
5 changes: 5 additions & 0 deletions skills/build-and-test/SOURCES.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,8 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b)
| In-repo SDK declarations | connector-authoring `b7e8a616cbbb1e336b788f807a3810b08ae00bc7` | The `.d.ts` module surface the uploaded source is authored against. |
| SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). |
| baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. |

## Eval wiring

- The eval fixture records the draft-test PASS result as the string `"PASS"`; the real surface returns the enum `CONNECTOR_TEST_RUN_RESULT_PASS`, and the scorer's S10 gate checks the fixture string.
- The SKILL.md content literals (`required_source_files`, `upload_targets`, `required_headers`, `RUN_STATE_SUCCEEDED`, `test_run_id`) are enforced by `evals/runner/skills_bundle.test.ts`.
Loading
Loading