diff --git a/evals/runner/agent.test.ts b/evals/runner/agent.test.ts index 25d953d..28000b9 100644 --- a/evals/runner/agent.test.ts +++ b/evals/runner/agent.test.ts @@ -78,7 +78,7 @@ test("buildPrompt pre1 branch names the pre1Path, both skills, and the output co basicAuth: {username: "connector@example.com", password: "fixture-token"}, bearerToken: "fixture-token", }, - skillBundle: {mode: "full", version: "0.8.0"}, + skillBundle: {mode: "full", version: "0.11.0"}, model: "together/deepseek-ai/DeepSeek-V4-Flash-0731", reasoningEffort: "high", kind: "pre1", diff --git a/evals/runner/scenario.test.ts b/evals/runner/scenario.test.ts index c0358c3..5f2af19 100644 --- a/evals/runner/scenario.test.ts +++ b/evals/runner/scenario.test.ts @@ -100,7 +100,7 @@ test("loadScenario loads the pre1-directory-proceed scenario (kind pre1, proceed assert.equal(s.seed, undefined) assert.equal(s.requiredSourceFiles, undefined) assert.equal(s.skillBundle.mode, "full") - assert.equal(s.skillBundle.version, "0.8.0") + assert.equal(s.skillBundle.version, "0.11.0") }) test("loadScenario loads the pre1-noiam-park scenario (kind pre1, park)", () => { diff --git a/evals/runner/skills_bundle.test.ts b/evals/runner/skills_bundle.test.ts index 5df84e5..331e394 100644 --- a/evals/runner/skills_bundle.test.ts +++ b/evals/runner/skills_bundle.test.ts @@ -15,7 +15,7 @@ const execFileAsync = promisify(execFile) const RUN = "evals/runner/run.ts" const BUNDLE = "evals/skills-bundle/bundle.json" const SKILLS = ["author-in-app-connector", "read-authoring-contract", "write-connector-source", "build-and-test", "deploy-and-activate", "design-access-model", "source-openapi-spec", "verify-connector-output", "update-and-rollback", "diagnose-authoring-failure"] -const VERSION = "0.8.0" +const VERSION = "0.11.0" function readBundle(): {version: string; skills: {name: string; version: string; path: string}[]} { return JSON.parse(readFileSync(BUNDLE, "utf8")) as {version: string; skills: {name: string; version: string; path: string}[]} @@ -74,10 +74,10 @@ test("(b) every bundle.json path resolves to an existing file", () => { // instructions while keeping the marker literals must fail the gate. const SKILL_LITERALS: Record = { "author-in-app-connector": [ - "skipped_human_boundary", "catalog_id", "draft_id", "upload_id", "run_id", "revision_id", "app_id", "connector_id", "test_run_id", "deployment_instance_id", "activation_url", "Do not redeem the approval token", + "Do not call `force_sync` in the authoring session", "write-connector-source", ], "read-authoring-contract": [ @@ -89,10 +89,10 @@ const SKILL_LITERALS: Record = { ], "deploy-and-activate": [ "deployment_instance_id", "activation_url", "REVISION_STATUS_ACTIVE", "activation_epoch", - "SYNC_STATUS_DONE", "skipped_human_boundary", + "SYNC_STATUS_DONE", "Do not redeem the approval token", - "Do not call `c1_connector_service_force_sync` during the funnel run", - "Do not call `c1_connector_authoring_list_revision_summaries` during the funnel run", + "Do not call `c1_connector_service_force_sync` in the authoring session", + "Do not call `c1_connector_authoring_list_revision_summaries` before the OWNER activates", ], "write-connector-source": [ "Do not call fetch", @@ -174,8 +174,7 @@ const SKILL_LITERALS: Record = { "Invalid token provided", "ConnectionOK", "HostCallOK", - "Poll with backoff", - "row after ~10 polls, stop and report", + "wait for the draft test as build-and-test", "c1_connector_service_get", "status.lastError", ], @@ -198,7 +197,7 @@ test("(c) each SKILL.md carries the locked section markers, content literals, AS test("(d) the full-mode scenario parses with mode full and the two pinned scenarios keep their locked modes", () => { const full = loadScenario("evals/scenarios/tier1-directory-full.json") assert.equal(full.skillBundle.mode, "full") - assert.equal(full.skillBundle.version, "0.8.0") + assert.equal(full.skillBundle.version, "0.11.0") assert.equal(full.id, "tier1-directory-full") const none = loadScenario("evals/scenarios/tier1-directory.json") assert.equal(none.skillBundle.mode, "none") @@ -247,7 +246,7 @@ test("(e) CLI end-to-end: full-mode Tier-0 run exits 0 and the record meta carri const lines = readFileSync(join(dir, records[0]), "utf8").trim().split("\n") const meta = JSON.parse(lines[0]) as Record assert.equal(meta.skill_bundle_mode, "full") - assert.equal(meta.skill_bundle_version, "0.8.0") + assert.equal(meta.skill_bundle_version, "0.11.0") } finally { rmSync(dir, {recursive: true, force: true}) } diff --git a/evals/scenarios/pre1-directory-proceed.json b/evals/scenarios/pre1-directory-proceed.json index db39ceb..e174f30 100644 --- a/evals/scenarios/pre1-directory-proceed.json +++ b/evals/scenarios/pre1-directory-proceed.json @@ -29,7 +29,7 @@ }, "skillBundle": { "mode": "full", - "version": "0.8.0" + "version": "0.11.0" }, "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731", "reasoningEffort": "high" diff --git a/evals/scenarios/pre1-noiam-park.json b/evals/scenarios/pre1-noiam-park.json index f02c97a..319a367 100644 --- a/evals/scenarios/pre1-noiam-park.json +++ b/evals/scenarios/pre1-noiam-park.json @@ -27,7 +27,7 @@ }, "skillBundle": { "mode": "full", - "version": "0.8.0" + "version": "0.11.0" }, "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731", "reasoningEffort": "high" diff --git a/evals/scenarios/tier1-directory-full.json b/evals/scenarios/tier1-directory-full.json index 353deb5..d591b50 100644 --- a/evals/scenarios/tier1-directory-full.json +++ b/evals/scenarios/tier1-directory-full.json @@ -27,7 +27,7 @@ }, "skillBundle": { "mode": "full", - "version": "0.8.0" + "version": "0.11.0" }, "model": "together/deepseek-ai/DeepSeek-V4-Flash-0731", "reasoningEffort": "high", diff --git a/evals/skills-bundle/bundle.json b/evals/skills-bundle/bundle.json index c3bbe2e..5870990 100644 --- a/evals/skills-bundle/bundle.json +++ b/evals/skills-bundle/bundle.json @@ -1,15 +1,15 @@ { - "version": "0.8.0", + "version": "0.11.0", "skills": [ - {"name": "author-in-app-connector", "version": "0.3.2", "path": "../../skills/author-in-app-connector/SKILL.md"}, - {"name": "read-authoring-contract", "version": "0.2.0", "path": "../../skills/read-authoring-contract/SKILL.md"}, - {"name": "write-connector-source", "version": "0.2.0", "path": "../../skills/write-connector-source/SKILL.md"}, - {"name": "build-and-test", "version": "0.3.0", "path": "../../skills/build-and-test/SKILL.md"}, - {"name": "deploy-and-activate", "version": "0.3.1", "path": "../../skills/deploy-and-activate/SKILL.md"}, - {"name": "design-access-model", "version": "0.1.0", "path": "../../skills/design-access-model/SKILL.md"}, + {"name": "author-in-app-connector", "version": "0.3.5", "path": "../../skills/author-in-app-connector/SKILL.md"}, + {"name": "read-authoring-contract", "version": "0.2.1", "path": "../../skills/read-authoring-contract/SKILL.md"}, + {"name": "write-connector-source", "version": "0.2.1", "path": "../../skills/write-connector-source/SKILL.md"}, + {"name": "build-and-test", "version": "0.3.1", "path": "../../skills/build-and-test/SKILL.md"}, + {"name": "deploy-and-activate", "version": "0.3.4", "path": "../../skills/deploy-and-activate/SKILL.md"}, + {"name": "design-access-model", "version": "0.1.1", "path": "../../skills/design-access-model/SKILL.md"}, {"name": "source-openapi-spec", "version": "0.1.0", "path": "../../skills/source-openapi-spec/SKILL.md"}, - {"name": "verify-connector-output", "version": "0.1.0", "path": "../../skills/verify-connector-output/SKILL.md"}, - {"name": "update-and-rollback", "version": "0.2.1", "path": "../../skills/update-and-rollback/SKILL.md"}, - {"name": "diagnose-authoring-failure", "version": "0.1.0", "path": "../../skills/diagnose-authoring-failure/SKILL.md"} + {"name": "verify-connector-output", "version": "0.1.1", "path": "../../skills/verify-connector-output/SKILL.md"}, + {"name": "update-and-rollback", "version": "0.2.2", "path": "../../skills/update-and-rollback/SKILL.md"}, + {"name": "diagnose-authoring-failure", "version": "0.1.1", "path": "../../skills/diagnose-authoring-failure/SKILL.md"} ] } diff --git a/skills/author-in-app-connector/SKILL.md b/skills/author-in-app-connector/SKILL.md index ebe2f09..8fd6191 100644 --- a/skills/author-in-app-connector/SKILL.md +++ b/skills/author-in-app-connector/SKILL.md @@ -1,7 +1,7 @@ --- name: author-in-app-connector description: Use when driving the full in-app connector authoring funnel end to end, from guide read through the human-activation handoff. Do not use when you are already mid-funnel and only need one stage's procedure - invoke the stage skill directly. -version: 0.3.2 +version: 0.3.5 --- # author-in-app-connector @@ -31,18 +31,12 @@ names below are the exact tenant MCP titles; the served guide abbreviates them four `required_source_files` (connector.ts, config-schema.json, runtime-schema.json, capabilities.json) must be true. STOP if any is false. - Step-boundary note: the served 12-step contract (agent prompt) splits - this differently - its step 2 is upload+PUTs and its step 3 is - finalize+get_draft. The call sets are identical; only the boundary - differs. In this skill S2 is the full upload dance - (create_draft_source_upload + PUTs + finalize) and S3 is the get_draft - gate; "re-run from S2" always means the full upload dance. 4. S4 build: call `c1_connector_authoring_build_bundle`; extract `run_id`. STOP if empty. 5. S5 build result: wait for the completion notification if your host provides one; do not poll while waiting. If your session resumes with an outstanding dispatch and no completion notification, check `get_run` once before - re-dispatching. Direct API callers without host wakeup poll + re-dispatching. Otherwise (direct API callers): poll `c1_connector_authoring_get_run` with `run_id` every 5-10s, up to ~10 polls. GATE: `RUN_STATE_SUCCEEDED`; extract the immutable `revision_id`. STOP on failure - fix source; if the source changed, @@ -58,11 +52,10 @@ names below are the exact tenant MCP titles; the served guide abbreviates them 10. S10 evidence: wait for the completion notification if your host provides one; do not poll while waiting. If your session resumes with an outstanding dispatch and no completion notification, check `get_test_run_evidence` once - before re-dispatching. Direct API callers without host wakeup poll + before re-dispatching. Otherwise (direct API callers): poll `c1_connector_authoring_get_test_run_evidence` with `(catalog_id, revision_id, test_run_id)` every 5-10s, up to ~10 polls. - GATE: `result == CONNECTOR_TEST_RUN_RESULT_PASS` (the PASS enum value; - the eval fixture records the string `"PASS"`). STOP if FAIL - fix + GATE: `result == CONNECTOR_TEST_RUN_RESULT_PASS`. STOP if FAIL - fix source; if the source changed, re-run from S2 (re-upload) with a FRESH `test_run_id`, otherwise re-run from the failing step per build-and-test. 11. S11 handoff discipline: call @@ -113,27 +106,26 @@ Net-new provider: before S1, run both pre-1 skills - `source-openapi-spec` ## Human boundary After S11's deploy + mint, present `activation_url` to a human tenant OWNER, -record the handoff table, and STOP. Never redeem the approval token, never -call `force_sync` - S11b/S11c are `skipped_human_boundary`. Never poll -`REVISION_STATUS_ACTIVE` in the funnel run: if a notification later resumes -you, the revision is ACTIVE - record the `activation_epoch` it carries. -Direct API callers verify per deploy-and-activate once the OWNER reports -activation. +record the handoff table, and STOP. Never redeem the approval token. +Do not call `force_sync` in the authoring session; the human/operator runs +it after the OWNER activates. Never poll `REVISION_STATUS_ACTIVE` before +the OWNER activates. If a notification later resumes you, the revision is +already ACTIVE: record its `activation_epoch`, but still do not force-sync +in this authoring session. Direct API callers verify per deploy-and-activate +once the OWNER reports activation. ## Exit criteria - S0-S11 all pass per the stage table above. -- S11's handoff-discipline gate passes: all 10 handoff fields, no calls after - mint except the handoff write, no redemption. -- The body contains the literal `skipped_human_boundary` and all 10 handoff - field names. +- S11 handoff: all 10 handoff fields, no calls after mint except the + handoff write, no redemption. ## Anti-patterns - Do not skip a stage. - Do not fabricate handoff values. - Do not redeem the approval token. -- Do not call `force_sync` at any point in the funnel run. +- Do not call `force_sync` in the authoring session; the human/operator runs it after the OWNER activates. - Do not write the handoff before deploy + mint. ## Blocker protocol diff --git a/skills/author-in-app-connector/SOURCES.md b/skills/author-in-app-connector/SOURCES.md index 94abb54..b92f603 100644 --- a/skills/author-in-app-connector/SOURCES.md +++ b/skills/author-in-app-connector/SOURCES.md @@ -13,3 +13,11 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b) | In-repo SDK declarations | connector-authoring `b7e8a616cbbb1e336b788f807a3810b08ae00bc7` | The `.d.ts` module surface the funnel's source files are authored against. | | SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). | | baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. | + +## Eval wiring + +- Step boundaries: the served 12-step contract (agent prompt) puts upload+PUTs in its step 2 and finalize+get_draft in its step 3. This skill's S2 is the full upload dance (create_draft_source_upload + PUTs + finalize) and S3 is the get_draft gate; the call sets are identical, and "re-run from S2" always means the full upload dance. +- The eval scorer's S11 gate fails a run with any call after mint other than the handoff write; the skill states the rule (no call after mint before the OWNER activates). +- `evals/runner/stages.ts` rejects any `force_sync` in the authoring transcript, including after an activation notification resumes the session. The skill delegates production sync to the human/operator outside that session. +- The scorer's SKIPPED_STAGES disposition records S11b/S11c as `skipped_human_boundary`; the skill states only the behavior (hand off and stop). +- The SKILL.md content literals (the 10 handoff field names, the never-redeem instruction, and the authoring-session `force_sync` prohibition) are enforced by `evals/runner/skills_bundle.test.ts`. diff --git a/skills/build-and-test/SKILL.md b/skills/build-and-test/SKILL.md index 622cf64..5521757 100644 --- a/skills/build-and-test/SKILL.md +++ b/skills/build-and-test/SKILL.md @@ -1,7 +1,7 @@ --- name: build-and-test description: Use when uploading draft source, building a bundle, or running and awaiting a draft test sync. Do not use when the task is app/connector provisioning or activation - use deploy-and-activate. -version: 0.3.0 +version: 0.3.1 --- # build-and-test @@ -30,7 +30,7 @@ the exact tenant MCP titles; the served guide abbreviates them. wait for the completion notification if your host provides one; do not poll while waiting. If your session resumes with an outstanding dispatch and no completion notification, check `c1_connector_authoring_get_run` once before - re-dispatching. Direct API callers without host wakeup poll + re-dispatching. Otherwise (direct API callers): poll `c1_connector_authoring_get_run` with `run_id` every 5-10s, up to ~10 polls; if no terminal state, STOP and report the run state rather than redispatching the build. GATE: @@ -42,14 +42,12 @@ the exact tenant MCP titles; the served guide abbreviates them. one; do not poll while waiting. If your session resumes with an outstanding dispatch and no completion notification, check `c1_connector_authoring_get_test_run_evidence` once before re-dispatching. - Direct API callers without host wakeup poll + Otherwise (direct API callers): poll `c1_connector_authoring_get_test_run_evidence` with the full key `(catalog_id, revision_id, test_run_id)` every 5-10s, up to ~10 polls (`NotFound` while pending); if no row, STOP and report pending rather than redispatching the test. GATE: - `result == CONNECTOR_TEST_RUN_RESULT_PASS` (the PASS enum value on the - real surface; the eval fixture records the string `"PASS"`). STOP if the - row reports FAIL - + `result == CONNECTOR_TEST_RUN_RESULT_PASS`. STOP if the row reports FAIL - read the error field, fix, and re-run from the correct step with a FRESH `test_run_id`; never re-mint on a failed run. 5. Re-run-from-correct-step rules: a failed build re-runs from the build @@ -66,10 +64,7 @@ the exact tenant MCP titles; the served guide abbreviates them. - S5 passes: `RUN_STATE_SUCCEEDED` and `revision_id` non-empty. - S9 passes: `test_run_id` non-empty, at least one successful `c1_connector_authoring_run_draft_test_sync`. -- S10 passes: durable evidence `result == CONNECTOR_TEST_RUN_RESULT_PASS` - (the scorer's S10 gate checks the fixture's string `"PASS"`). -- The body contains the literal `required_source_files`, `upload_targets`, - `required_headers`, `RUN_STATE_SUCCEEDED`, and `test_run_id`. +- S10 passes: durable evidence `result == CONNECTOR_TEST_RUN_RESULT_PASS`. ## Anti-patterns diff --git a/skills/build-and-test/SOURCES.md b/skills/build-and-test/SOURCES.md index 4d19d80..abdc9c9 100644 --- a/skills/build-and-test/SOURCES.md +++ b/skills/build-and-test/SOURCES.md @@ -13,3 +13,8 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b) | In-repo SDK declarations | connector-authoring `b7e8a616cbbb1e336b788f807a3810b08ae00bc7` | The `.d.ts` module surface the uploaded source is authored against. | | SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). | | baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. | + +## Eval wiring + +- The eval fixture records the draft-test PASS result as the string `"PASS"`; the real surface returns the enum `CONNECTOR_TEST_RUN_RESULT_PASS`, and the scorer's S10 gate checks the fixture string. +- The SKILL.md content literals (`required_source_files`, `upload_targets`, `required_headers`, `RUN_STATE_SUCCEEDED`, `test_run_id`) are enforced by `evals/runner/skills_bundle.test.ts`. diff --git a/skills/deploy-and-activate/SKILL.md b/skills/deploy-and-activate/SKILL.md index 18f3d04..7099310 100644 --- a/skills/deploy-and-activate/SKILL.md +++ b/skills/deploy-and-activate/SKILL.md @@ -1,7 +1,7 @@ --- name: deploy-and-activate description: Use when creating the app, provisioning the connector, configuring credentials, deploying the instance, minting the approval, or verifying activation. Do not use when the task is source upload, building, or draft testing - use build-and-test. -version: 0.3.1 +version: 0.3.4 --- # deploy-and-activate @@ -30,10 +30,9 @@ below are the exact tenant MCP titles; the served guide abbreviates them. Rule stated verbatim: served guide wins on conflict. GATE: base-url, account-email, and api-token all set. STOP if the credentials are missing - an empty `stringValue` deletes the secret field and the draft test - fails. In agent-driven runs the S8 gate reads the config set via the API - (Path A); the human-in-UI path (Path B) applies to real-tenant sessions - where a human configures in the Admin UI and the agent does not set - config itself. + fails. Use Path A when you configure through the API; Path B applies + when a human configures in the Admin UI and you do not set config + yourself. 4. Deploy: call `c1_connector_authoring_deploy_connector_instance`; capture `deployment_instance_id`. GATE: non-empty. STOP if empty. 5. Mint approval: call `c1_connector_authoring_mint_approval_token` with @@ -41,19 +40,18 @@ below are the exact tenant MCP titles; the served guide abbreviates them. GATE: non-empty `activation_url`. STOP if empty. 6. Present `activation_url` to a human tenant OWNER, record the handoff table, and stop - do not wait for activation first. Never redeem the - approval token or attempt activation yourself. S11b/S11c are - `skipped_human_boundary`. + approval token or attempt activation yourself. -## Post-activation reference (NEVER performed in the funnel run) +## After the OWNER activates -The two legs below are reference material for a later post-activation session -or the human/operator. Performing either in the funnel run fails the S11 gate -(any non-handoff call after mint, including `list_revision_summaries` or -`force_sync`). +After the OWNER activates, report the verification result below. Production +sync belongs to the human/operator, not the authoring session. Before +activation, make no call after mint other than the handoff write. - Verification: if your host provides activation notifications, a - notification resuming you means the revision is ACTIVE - record the - `activation_epoch` it carries; do not poll. If you resume with an + notification resuming you means the revision is already ACTIVE - record + the `activation_epoch` it carries; do not poll or force-sync in this + authoring session. If you resume with an outstanding activation and no notification, check `c1_connector_authoring_list_revision_summaries` once; if `REVISION_STATUS_ACTIVE` record `activation_epoch`, otherwise STOP and @@ -64,12 +62,11 @@ or the human/operator. Performing either in the funnel run fails the S11 gate STOP if not ACTIVE - if approval reports `evidence is unsatisfied`, return to the test step and confirm a PASS row binds this revision before minting a fresh approval URL. If no ACTIVE row after ~10 polls, STOP and report. -- Sync leg: the full lifecycle continues with - `c1_connector_service_force_sync` and verification via - `c1_connector_service_get` that `status.status` is `SYNC_STATUS_DONE` (a - subsequent `sync_disabled` is normal). Performed by the human/operator (or - a later post-activation session), NEVER by the agent in the funnel run: the - S11 gate fails any run whose transcript contains a `force_sync` call. +- Sync leg: do not call `c1_connector_service_force_sync` in the authoring + session, even if a notification resumed it. The human/operator runs it + after the OWNER activates, then verifies with `c1_connector_service_get` + that `status.status` is `SYNC_STATUS_DONE` (a subsequent `sync_disabled` + is normal). ## Exit criteria @@ -79,18 +76,12 @@ or the human/operator. Performing either in the funnel run fails the S11 gate - S8 passes: base-url, account-email, and api-token all configured. - S11 passes: deploy + mint succeeded, all 10 handoff fields, no calls after mint except the handoff write, no redemption. -- The body contains the literal `deployment_instance_id`, `activation_url`, - `REVISION_STATUS_ACTIVE`, `activation_epoch`, `SYNC_STATUS_DONE`, and - `skipped_human_boundary`. ## Anti-patterns - Do not redeem the approval token. -- Do not call `c1_connector_service_force_sync` during the funnel run - the - S11 gate fails any run containing it; the sync leg is post-activation - reference only. -- Do not call `c1_connector_authoring_list_revision_summaries` during the funnel run - it is a non-handoff call after mint and fails the S11 gate. -- Do not force-sync before activation. +- Do not call `c1_connector_service_force_sync` in the authoring session; the human/operator runs it after the OWNER activates. +- Do not call `c1_connector_authoring_list_revision_summaries` before the OWNER activates - after mint, the only call is the handoff write. - Do not configure with an empty `stringValue` - it deletes the secret. - Do not ask a human to paste secrets into agent chat. - Do not skip the idempotent provision reuse. diff --git a/skills/deploy-and-activate/SOURCES.md b/skills/deploy-and-activate/SOURCES.md index 1d02e66..15edc3f 100644 --- a/skills/deploy-and-activate/SOURCES.md +++ b/skills/deploy-and-activate/SOURCES.md @@ -13,3 +13,11 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b) | In-repo SDK declarations | connector-authoring `b7e8a616cbbb1e336b788f807a3810b08ae00bc7` | The `config("field-name")` names the `configuration` keys must match (see `examples/http/connector.ts`: `base-url`, `account-email`, `api-token`). | | SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). | | baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. | + +## Eval wiring + +- In agent-driven eval runs the scorer's S8 gate reads the config set via the API (Path A). +- The scorer's S11 gate fails a run with any call after mint other than the handoff write, including `list_revision_summaries` and `force_sync`. +- `evals/runner/stages.ts` rejects any `force_sync` in the authoring transcript, including a notification-resumed session. The locked sync prohibition now covers the entire authoring session; the human/operator runs production sync after the OWNER activates. +- The scorer's SKIPPED_STAGES disposition records S11b/S11c as `skipped_human_boundary`; the skill states only the behavior (hand off and stop). +- The SKILL.md content literals (`deployment_instance_id`, `activation_url`, `REVISION_STATUS_ACTIVE`, `activation_epoch`, `SYNC_STATUS_DONE`, and the never-redeem / never-force_sync / never-list_revision_summaries instructions) are enforced by `evals/runner/skills_bundle.test.ts`. diff --git a/skills/design-access-model/SKILL.md b/skills/design-access-model/SKILL.md index 11ea4ca..1692ba5 100644 --- a/skills/design-access-model/SKILL.md +++ b/skills/design-access-model/SKILL.md @@ -1,7 +1,7 @@ --- name: design-access-model description: Use when designing the access model (resource types, traits, entitlements, grants, provisioning scope) for a net-new provider before authoring connector source. Do not use when you are already mid-funnel or authoring source - invoke write-connector-source instead. -version: 0.1.0 +version: 0.1.1 --- # design-access-model @@ -63,10 +63,8 @@ Emit the `access_model` half of the pre1.json artifact: - `id_compatibility`: table of `{resource_type, id_shape, stable}` rows. - `provisioning`: list of `{resource_type, provisionable, justification}` rows. -## Eval-alignment contract - -- `id_compatibility` must be non-empty. -- Every `provisioning` entry must carry a non-empty `justification`. +`id_compatibility` must be non-empty, and every `provisioning` entry must +carry a non-empty `justification`. ## Exit criteria diff --git a/skills/design-access-model/SOURCES.md b/skills/design-access-model/SOURCES.md index e5ac149..29c64dc 100644 --- a/skills/design-access-model/SOURCES.md +++ b/skills/design-access-model/SOURCES.md @@ -16,3 +16,7 @@ decisions 2-3); the port source below is what this skill quotes. | Worked build — access-model shape | baton-axiomatic-openrouter#1 | The access-model shape (resource types, traits, entitlements, grants) this worked build demonstrates. | | Worked build — access-model shape | baton-axiomatic-litellm#1 | The access-model shape (resource types, traits, entitlements, grants) this worked build demonstrates. | | Worked build — access-model shape | baton-axiomatic-shopify | The access-model shape (resource types, traits, entitlements, grants) this worked build demonstrates. | + +## Eval wiring + +- The pre1 eval checks that `id_compatibility` is non-empty and that every `provisioning` entry carries a non-empty `justification`. diff --git a/skills/diagnose-authoring-failure/SKILL.md b/skills/diagnose-authoring-failure/SKILL.md index 5f44bee..570b5b6 100644 --- a/skills/diagnose-authoring-failure/SKILL.md +++ b/skills/diagnose-authoring-failure/SKILL.md @@ -1,7 +1,7 @@ --- name: diagnose-authoring-failure description: Use when a build, draft test, activation, or production sync fails and you need the symptom-to-cause-to-fix route. Do not use when the connector is healthy and you are verifying sync output - use verify-connector-output; do not use when updating or rolling back a healthy live connector - use update-and-rollback. -version: 0.1.0 +version: 0.1.1 --- # diagnose-authoring-failure @@ -39,10 +39,9 @@ tooling is replaced by the tenant MCP tools named below. ## Draft-test FAIL reading -The evidence row is authoritative: poll -`c1_connector_authoring_get_test_run_evidence` and read the `result` (PASS -or FAIL) and the `error` field. Poll with backoff (e.g. every 5-10s); if no -row after ~10 polls, stop and report `NotFound`/pending. The FAIL reason +The evidence row is authoritative: wait for the draft test as build-and-test +describes, then read `c1_connector_authoring_get_test_run_evidence` - the +`result` (PASS or FAIL) and the `error` field. The FAIL reason lives on the evidence row; it is not always logged when the read activity succeeds but the outcome evaluation returns FAIL. PASS requires all of: @@ -68,13 +67,6 @@ authoritative outcome for a sync. - Every one of the nine common-failures rows routes to its documented fix. - A draft-test FAIL is read from the evidence row, not from completion. - The logs location and the status row are named. -- The body contains the literals `262144 byte compile limit`, - `1048576 byte limit`, `is_secret`, `credential re-entry required`, - `missing type`, `unregistered transport`, - `ticketing.enabled must be true when ticketing is configured`, - `activation evidence is unsatisfied`, `Invalid token provided`, - `ConnectionOK`, `HostCallOK`, `c1_connector_service_get`, and - `status.lastError`. ## Anti-patterns diff --git a/skills/diagnose-authoring-failure/SOURCES.md b/skills/diagnose-authoring-failure/SOURCES.md index 2d6085c..e3219e1 100644 --- a/skills/diagnose-authoring-failure/SOURCES.md +++ b/skills/diagnose-authoring-failure/SOURCES.md @@ -13,3 +13,7 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b) | baton-admin `diagnose-connector-failure` | `6fe6886f607ed0d2e48a616c30e7ce4bffc32489` | The taxonomy approach: start from the symptom, classify the failure surface, collect the smallest evidence, keep the diagnosis focused on symptom/evidence/owner/next-fix/rerun-target, and the output contract (failure summary, classified surface, evidence used, likely owner, one concrete next fix, smallest verification command). | | SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). | | baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. | + +## Eval wiring + +- The SKILL.md content literals (the common-failures error strings, `ConnectionOK`, `HostCallOK`, `c1_connector_service_get`, `status.lastError`) are enforced by `evals/runner/skills_bundle.test.ts`. diff --git a/skills/read-authoring-contract/SKILL.md b/skills/read-authoring-contract/SKILL.md index 765605d..b8c9f82 100644 --- a/skills/read-authoring-contract/SKILL.md +++ b/skills/read-authoring-contract/SKILL.md @@ -1,7 +1,7 @@ --- name: read-authoring-contract description: Use when starting a connector authoring session and you need the authoritative contract before writing any source. Do not use when you already hold a fresh get_authoring_guide response for this session and are mid-funnel. -version: 0.2.0 +version: 0.2.1 --- # read-authoring-contract @@ -49,15 +49,10 @@ abbreviates them (e.g. `get_authoring_guide` for ## Exit criteria - S0 passes: at least one successful - `c1_connector_authoring_get_authoring_guide` call in the transcript. -- The skill body instructs the resume check - (`c1_connector_authoring_list_authored_catalog_entries` + - `c1_connector_authoring_list_drafts`) before any - `c1_connector_authoring_create_draft` call (string check; the scorer's S1 - gate does not enforce it). -- The body contains the literal tool names `list_sdk_types_versions`, - `get_sdk_types`, `list_authored_catalog_entries`, `list_drafts`, and the - literal strings `runtime_pin_matched` and `default_tag`. + `c1_connector_authoring_get_authoring_guide` call. +- The resume check (`c1_connector_authoring_list_authored_catalog_entries` + + `c1_connector_authoring_list_drafts`) runs before any + `c1_connector_authoring_create_draft` call. ## Anti-patterns diff --git a/skills/read-authoring-contract/SOURCES.md b/skills/read-authoring-contract/SOURCES.md index 9a19191..6eb9321 100644 --- a/skills/read-authoring-contract/SOURCES.md +++ b/skills/read-authoring-contract/SOURCES.md @@ -13,3 +13,8 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b) | In-repo SDK declarations | connector-authoring `b7e8a616cbbb1e336b788f807a3810b08ae00bc7` | The `.d.ts` module surface (`@baton/runtime`, `@baton/helpers`, `@baton/types`) the tagged declarations cover. | | SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). | | baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. | + +## Eval wiring + +- The resume check before `create_draft` is a SKILL.md string check; the scorer's S1 gate does not enforce it. +- The SKILL.md content literals (`list_sdk_types_versions`, `get_sdk_types`, `list_authored_catalog_entries`, `list_drafts`, `runtime_pin_matched`, `default_tag`) are enforced by `evals/runner/skills_bundle.test.ts`. diff --git a/skills/update-and-rollback/SKILL.md b/skills/update-and-rollback/SKILL.md index 1a114e4..b53613c 100644 --- a/skills/update-and-rollback/SKILL.md +++ b/skills/update-and-rollback/SKILL.md @@ -1,14 +1,14 @@ --- name: update-and-rollback description: Use when shipping a change to an activated connector (same-catalog rerun) or rolling back to a previously activated revision. Do not use when verifying a healthy connector's sync output - use verify-connector-output; do not use when diagnosing a failed build, draft test, or sync - use diagnose-authoring-failure. -version: 0.2.1 +version: 0.2.2 --- # update-and-rollback -Update and rollback for an activated authored connector. Runs in a -post-activation session - never during the funnel run. Tool names below are -the exact tenant MCP titles. +Update and rollback for an activated authored connector. Runs only after the +OWNER has activated the connector. Tool names below are the exact tenant MCP +titles. ## Update flow (same-catalog rerun) @@ -100,9 +100,6 @@ fix cycles, stop and report the exact error text (see Blocker protocol). `target_revision_id`, `instance_app_id`, `instance_connector_id`, and `approval_token_id`; the target revision is ACTIVE under a strictly greater activation epoch. -- The body contains the literals `serve image does not match the revision-pinned runtime image`, `/api/v1/connector-authoring/rollbacks`, - `target_revision_id`, `approval_token_id`, `activation_epoch`, and - `image digest`. ## Anti-patterns diff --git a/skills/update-and-rollback/SOURCES.md b/skills/update-and-rollback/SOURCES.md index 619fdba..267c1e7 100644 --- a/skills/update-and-rollback/SOURCES.md +++ b/skills/update-and-rollback/SOURCES.md @@ -12,3 +12,7 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b) | c1 Go source (same pin) | c1 `2e5f53eb441a93087d9754085ca17a5061e125ea` | The `SYNC_STATUS_ERROR` / `SYNC_STATUS_DISABLED` terminal-state semantics: `ConnectorStatusToAPI` derives DISABLED only from an ERROR-classified sync; `sync_disabled_reason` distinguishes the data-anomaly auto-pause from deliberate pauses. | | SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). | | baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. | + +## Eval wiring + +- The SKILL.md content literals (the rotation error string, `/api/v1/connector-authoring/rollbacks`, `target_revision_id`, `approval_token_id`, `activation_epoch`, `image digest`) are enforced by `evals/runner/skills_bundle.test.ts`. diff --git a/skills/verify-connector-output/SKILL.md b/skills/verify-connector-output/SKILL.md index 2770d52..5da0eae 100644 --- a/skills/verify-connector-output/SKILL.md +++ b/skills/verify-connector-output/SKILL.md @@ -1,14 +1,14 @@ --- name: verify-connector-output description: Use when verifying a post-activation connector's sync output: resource/entitlement/grant counts, grant wiring, ID stability across re-sync, and the UI spot-check. Do not use when updating or rolling back a live connector - use update-and-rollback; do not use when diagnosing a failed build, draft test, or sync - use diagnose-authoring-failure. -version: 0.1.0 +version: 0.1.1 --- # verify-connector-output -Post-sync verification for an activated authored connector. Runs in a -post-activation session - never during the funnel run. Tool names below are -the exact tenant MCP titles. +Post-sync verification for an activated authored connector. Runs only after +the OWNER has activated the connector. Tool names below are the exact tenant +MCP titles. ## Checklist @@ -57,17 +57,14 @@ Investigate empty or unexpected results; never invent data to make a demo appear - IDs are stable across a re-sync. - The UI spot-check at `/admin/connector///` shows the connector's resources and grants. -- The body contains the literals `SYNC_STATUS_DONE`, `sync_disabled`, - `ID stability`, and `/admin/connector/`. ## Anti-patterns - Never invent data to make a demo appear complete - investigate empty or unexpected results instead. -- Do not assert fixture counts as the expected counts; query the live API +- Do not assert seed-list counts as the expected counts; query the live API for count parity. -- Do not run this during the funnel run - it is a post-activation session - skill. +- Do not run this before the OWNER activates the connector. - Do not skip the grant-wiring check because counts look right. ## Blocker protocol diff --git a/skills/verify-connector-output/SOURCES.md b/skills/verify-connector-output/SOURCES.md index 97acf11..1416dcd 100644 --- a/skills/verify-connector-output/SOURCES.md +++ b/skills/verify-connector-output/SOURCES.md @@ -13,3 +13,7 @@ model memory). Source-of-truth precedence: (a) MCP-served guide, (b) | c1 Go source (same pin) | c1 `2e5f53eb441a93087d9754085ca17a5061e125ea` | The `SYNC_STATUS_DISABLED` semantics: `ConnectorStatusToAPI` derives DISABLED only from an ERROR-classified sync; `sync_disabled_reason` distinguishes the data-anomaly auto-pause (`Sync paused due to significant drop in sync data` prefix) from deliberate pauses (`system`, `system-customer-opt-out`). | | SDK contract (`baton/*.d.ts`) | connector-authoring git tag `v0.0.26` (sync commit `01a69d8d` "Sync baton runtime types for v0.0.26") | The `.d.ts` module surface this skill is authored against. `runtime_pin_matched`: not verifiable offline — no tenant MCP surface is reachable from the authoring env; repo tag v0.0.26 is the served default_tag when the tenant runtime pin matches (see `read-authoring-contract` for the runtime check). | | baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. | + +## Eval wiring + +- The SKILL.md content literals (`SYNC_STATUS_DONE`, `sync_disabled`, `ID stability`, `/admin/connector/`) are enforced by `evals/runner/skills_bundle.test.ts`. diff --git a/skills/write-connector-source/SKILL.md b/skills/write-connector-source/SKILL.md index c91acc9..5c73174 100644 --- a/skills/write-connector-source/SKILL.md +++ b/skills/write-connector-source/SKILL.md @@ -1,7 +1,7 @@ --- name: write-connector-source description: Use when authoring the four-file source contract (connector.ts, config-schema.json, runtime-schema.json, capabilities.json) for an in-app connector draft, before the S2 upload. Do not use when the source files are already uploaded and you are mid-funnel - invoke build-and-test instead. -version: 0.2.0 +version: 0.2.1 --- # write-connector-source @@ -156,9 +156,7 @@ Field-name parity between `config-schema.json` and `runtime-schema.json`; - [ ] Every `config("...")` literal in `connector.ts` declared in BOTH schemas (parity covers config refs, not just the two schemas). - [ ] `ticketing.enabled` - no `ticketing` block the connector code does not back (`ticketing.enabled must be true when ticketing is configured`). -## Eval-alignment contract - -Taught as contract rules, not eval-gaming literals: +## Contract rules - Scope list calls - pass the tenant-scoping query param the API requires structurally in the query object (e.g. `account_id`); a comment mention does not count. - Handle nullable fields - e.g. `user.title` may be null; project defensively. diff --git a/skills/write-connector-source/SOURCES.md b/skills/write-connector-source/SOURCES.md index d904218..9a63f75 100644 --- a/skills/write-connector-source/SOURCES.md +++ b/skills/write-connector-source/SOURCES.md @@ -22,3 +22,7 @@ at the pinned SHA still carry the presence-only capabilities wording. | baton-axiomatic DSL contract | baton-axiomatic `docs/DSL.md` + `runtime/baton/*.d.ts` @ v0.0.26 (`825e5516…`) | The DSL semantics ground truth the in-repo `.d.ts` are synced from. | | baton-admin DSL skill | baton-admin `author-js-dsl-connector` @ `6fe6886f607ed0d2e48a616c30e7ce4bffc32489` | The JS DSL authoring rules this skill's source-file contract aligns with. | | baton-admin DSL skill | baton-admin `author-auth-config-surface` @ `6fe6886f607ed0d2e48a616c30e7ce4bffc32489` | The auth-config surface rules this skill's config-schema contract aligns with. | + +## Eval wiring + +- The Contract rules section mirrors the eval scorer's source checks (tenant-scoped list calls, nullable fields, total-count pagination, dual-schema config declarations, user helper construction, hygiene); they are taught as contract rules, not eval literals.