Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion evals/runner/agent.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ test("buildPrompt pre1 branch names the pre1Path, both skills, and the output co
basicAuth: {username: "connector@example.com", password: "fixture-token"},
bearerToken: "fixture-token",
},
skillBundle: {mode: "full", version: "0.5.0"},
skillBundle: {mode: "full", version: "0.8.0"},
model: "together/deepseek-ai/DeepSeek-V4-Flash-0731",
reasoningEffort: "high",
kind: "pre1",
Expand Down
2 changes: 1 addition & 1 deletion evals/runner/scenario.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ test("loadScenario loads the pre1-directory-proceed scenario (kind pre1, proceed
assert.equal(s.seed, undefined)
assert.equal(s.requiredSourceFiles, undefined)
assert.equal(s.skillBundle.mode, "full")
assert.equal(s.skillBundle.version, "0.5.0")
assert.equal(s.skillBundle.version, "0.8.0")
})

test("loadScenario loads the pre1-noiam-park scenario (kind pre1, park)", () => {
Expand Down
8 changes: 4 additions & 4 deletions evals/runner/skills_bundle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const execFileAsync = promisify(execFile)
const RUN = "evals/runner/run.ts"
const BUNDLE = "evals/skills-bundle/bundle.json"
const SKILLS = ["author-in-app-connector", "read-authoring-contract", "write-connector-source", "build-and-test", "deploy-and-activate", "design-access-model", "source-openapi-spec", "verify-connector-output", "update-and-rollback", "diagnose-authoring-failure"]
const VERSION = "0.5.0"
const VERSION = "0.8.0"

function readBundle(): {version: string; skills: {name: string; version: string; path: string}[]} {
return JSON.parse(readFileSync(BUNDLE, "utf8")) as {version: string; skills: {name: string; version: string; path: string}[]}
Expand Down Expand Up @@ -156,7 +156,7 @@ const SKILL_LITERALS: Record<string, string[]> = {
"Do not redeem the approval token",
"Do not clear runtime fields, call the provisioner directly, or mutate the",
"evidence is unsatisfied",
"if no ACTIVE row after ~10 polls, STOP and report",
"If no ACTIVE row after ~10 polls, STOP and report",
"no terminal status after ~10 polls, STOP and report",
"SYNC_STATUS_ERROR",
"SYNC_STATUS_DISABLED",
Expand Down Expand Up @@ -198,7 +198,7 @@ test("(c) each SKILL.md carries the locked section markers, content literals, AS
test("(d) the full-mode scenario parses with mode full and the two pinned scenarios keep their locked modes", () => {
const full = loadScenario("evals/scenarios/tier1-directory-full.json")
assert.equal(full.skillBundle.mode, "full")
assert.equal(full.skillBundle.version, "0.5.0")
assert.equal(full.skillBundle.version, "0.8.0")
assert.equal(full.id, "tier1-directory-full")
const none = loadScenario("evals/scenarios/tier1-directory.json")
assert.equal(none.skillBundle.mode, "none")
Expand Down Expand Up @@ -247,7 +247,7 @@ test("(e) CLI end-to-end: full-mode Tier-0 run exits 0 and the record meta carri
const lines = readFileSync(join(dir, records[0]), "utf8").trim().split("\n")
const meta = JSON.parse(lines[0]) as Record<string, unknown>
assert.equal(meta.skill_bundle_mode, "full")
assert.equal(meta.skill_bundle_version, "0.5.0")
assert.equal(meta.skill_bundle_version, "0.8.0")
} finally {
rmSync(dir, {recursive: true, force: true})
}
Expand Down
2 changes: 1 addition & 1 deletion evals/scenarios/pre1-directory-proceed.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
},
"skillBundle": {
"mode": "full",
"version": "0.5.0"
"version": "0.8.0"
},
"model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
"reasoningEffort": "high"
Expand Down
2 changes: 1 addition & 1 deletion evals/scenarios/pre1-noiam-park.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
},
"skillBundle": {
"mode": "full",
"version": "0.5.0"
"version": "0.8.0"
},
"model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
"reasoningEffort": "high"
Expand Down
2 changes: 1 addition & 1 deletion evals/scenarios/tier1-directory-full.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
},
"skillBundle": {
"mode": "full",
"version": "0.5.0"
"version": "0.8.0"
},
"model": "together/deepseek-ai/DeepSeek-V4-Flash-0731",
"reasoningEffort": "high",
Expand Down
8 changes: 4 additions & 4 deletions evals/skills-bundle/bundle.json
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
{
"version": "0.5.0",
"version": "0.8.0",
"skills": [
{"name": "author-in-app-connector", "version": "0.3.0", "path": "../../skills/author-in-app-connector/SKILL.md"},
{"name": "author-in-app-connector", "version": "0.3.2", "path": "../../skills/author-in-app-connector/SKILL.md"},
{"name": "read-authoring-contract", "version": "0.2.0", "path": "../../skills/read-authoring-contract/SKILL.md"},
{"name": "write-connector-source", "version": "0.2.0", "path": "../../skills/write-connector-source/SKILL.md"},
{"name": "build-and-test", "version": "0.3.0", "path": "../../skills/build-and-test/SKILL.md"},
{"name": "deploy-and-activate", "version": "0.2.0", "path": "../../skills/deploy-and-activate/SKILL.md"},
{"name": "deploy-and-activate", "version": "0.3.1", "path": "../../skills/deploy-and-activate/SKILL.md"},
{"name": "design-access-model", "version": "0.1.0", "path": "../../skills/design-access-model/SKILL.md"},
{"name": "source-openapi-spec", "version": "0.1.0", "path": "../../skills/source-openapi-spec/SKILL.md"},
{"name": "verify-connector-output", "version": "0.1.0", "path": "../../skills/verify-connector-output/SKILL.md"},
{"name": "update-and-rollback", "version": "0.1.0", "path": "../../skills/update-and-rollback/SKILL.md"},
{"name": "update-and-rollback", "version": "0.2.1", "path": "../../skills/update-and-rollback/SKILL.md"},
{"name": "diagnose-authoring-failure", "version": "0.1.0", "path": "../../skills/diagnose-authoring-failure/SKILL.md"}
]
}
12 changes: 8 additions & 4 deletions skills/author-in-app-connector/SKILL.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
name: author-in-app-connector
description: Use when driving the full in-app connector authoring funnel end to end, from guide read through the human-activation handoff. Do not use when you are already mid-funnel and only need one stage's procedure - invoke the stage skill directly.
version: 0.3.0
version: 0.3.2
---

# author-in-app-connector
Expand Down Expand Up @@ -112,9 +112,13 @@ Net-new provider: before S1, run both pre-1 skills - `source-openapi-spec`

## Human boundary

After S11's deploy + mint, present `activation_url` to a human tenant OWNER
and STOP. Never redeem the approval token, never poll `REVISION_STATUS_ACTIVE`,
never call `force_sync` - S11b/S11c are `skipped_human_boundary`.
After S11's deploy + mint, present `activation_url` to a human tenant OWNER,
record the handoff table, and STOP. Never redeem the approval token, never
call `force_sync` - S11b/S11c are `skipped_human_boundary`. Never poll
`REVISION_STATUS_ACTIVE` in the funnel run: if a notification later resumes
you, the revision is ACTIVE - record the `activation_epoch` it carries.
Direct API callers verify per deploy-and-activate once the OWNER reports
activation.

## Exit criteria

Expand Down
29 changes: 18 additions & 11 deletions skills/deploy-and-activate/SKILL.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
name: deploy-and-activate
description: Use when creating the app, provisioning the connector, configuring credentials, deploying the instance, minting the approval, or verifying activation. Do not use when the task is source upload, building, or draft testing - use build-and-test.
version: 0.2.0
version: 0.3.1
---

# deploy-and-activate
Expand Down Expand Up @@ -39,9 +39,10 @@ below are the exact tenant MCP titles; the served guide abbreviates them.
5. Mint approval: call `c1_connector_authoring_mint_approval_token` with
`expires_in_seconds` in 1-14400 (max 4 hours); capture `activation_url`.
GATE: non-empty `activation_url`. STOP if empty.
6. HARD STOP at the human boundary: present `activation_url` to a human
tenant OWNER and stop. Never redeem the approval token, never attempt
activation yourself. S11b/S11c are `skipped_human_boundary`.
6. Present `activation_url` to a human tenant OWNER, record the handoff
table, and stop - do not wait for activation first. Never redeem the
approval token or attempt activation yourself. S11b/S11c are
`skipped_human_boundary`.

## Post-activation reference (NEVER performed in the funnel run)

Expand All @@ -50,13 +51,19 @@ or the human/operator. Performing either in the funnel run fails the S11 gate
(any non-handoff call after mint, including `list_revision_summaries` or
`force_sync`).

- Verification: after the OWNER confirms approval, poll
`c1_connector_authoring_list_revision_summaries` until the target
revision's status is `REVISION_STATUS_ACTIVE`; record its
`activation_epoch`. GATE: ACTIVE. STOP if not ACTIVE - if approval reports
`evidence is unsatisfied`, return to the test step and confirm a PASS row
binds this revision before minting a fresh approval URL. Poll with
backoff; if no ACTIVE row after ~10 polls, STOP and report.
- Verification: if your host provides activation notifications, a
notification resuming you means the revision is ACTIVE - record the
`activation_epoch` it carries; do not poll. If you resume with an
Comment on lines +54 to +56

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Suggestion (medium confidence): The old text said the notification confirms the target revision's ACTIVE status. The new text says any notification that resumes the agent "means the revision is ACTIVE", and it never asks the agent to check that the notification's revision matches the minted revision_id. That gap matters in update-and-rollback, where several revisions exist. Consider adding: "confirm the notification's revision matches the target revision_id before recording activation_epoch." The same wording appears in update-and-rollback/SKILL.md step 6 and author-in-app-connector/SKILL.md (Human boundary section).

outstanding activation and no notification, check
`c1_connector_authoring_list_revision_summaries` once; if
`REVISION_STATUS_ACTIVE` record `activation_epoch`, otherwise STOP and
report. Otherwise (direct API callers): once the OWNER reports
activation, poll `c1_connector_authoring_list_revision_summaries` every
5-10s, up to ~10 polls, until the target revision's status is
`REVISION_STATUS_ACTIVE`; record its `activation_epoch`. GATE: ACTIVE.
STOP if not ACTIVE - if approval reports `evidence is unsatisfied`, return
to the test step and confirm a PASS row binds this revision before minting
a fresh approval URL. If no ACTIVE row after ~10 polls, STOP and report.
- Sync leg: the full lifecycle continues with
`c1_connector_service_force_sync` and verification via
`c1_connector_service_get` that `status.status` is `SYNC_STATUS_DONE` (a
Expand Down
21 changes: 14 additions & 7 deletions skills/update-and-rollback/SKILL.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
name: update-and-rollback
description: Use when shipping a change to an activated connector (same-catalog rerun) or rolling back to a previously activated revision. Do not use when verifying a healthy connector's sync output - use verify-connector-output; do not use when diagnosing a failed build, draft test, or sync - use diagnose-authoring-failure.
version: 0.1.0
version: 0.2.1
---

# update-and-rollback
Expand All @@ -24,13 +24,20 @@ the exact tenant MCP titles.
(`expires_in_seconds` 1-14400). GATE: non-empty `activation_url`.
5. HARD STOP at the human boundary: present the URL to a human tenant OWNER
and stop. Do not redeem the approval token for activation.
6. After the OWNER activates, poll
`c1_connector_authoring_list_revision_summaries` until the target
revision is `REVISION_STATUS_ACTIVE`; record its `activation_epoch`.
GATE: ACTIVE. STOP if not ACTIVE - if approval reports `evidence is unsatisfied`,
6. After presenting the URL: if your host provides activation
notifications, a notification resuming you means the revision is
ACTIVE: record the `activation_epoch` it carries; do not poll. If you
resume with an outstanding activation and no notification, check
`c1_connector_authoring_list_revision_summaries` once; if
`REVISION_STATUS_ACTIVE` record `activation_epoch`, otherwise STOP and
report. Otherwise (direct API callers): once the OWNER reports
activation, poll `c1_connector_authoring_list_revision_summaries` every
5-10s, up to ~10 polls, until the target revision is
`REVISION_STATUS_ACTIVE`; record its `activation_epoch`. GATE: ACTIVE.
STOP if not ACTIVE - if approval reports `evidence is unsatisfied`,
return to the draft-test step and confirm a fresh PASS row binds this
revision before minting a new approval URL. Poll with backoff (e.g. every
5-10s); if no ACTIVE row after ~10 polls, STOP and report.
revision before minting a new approval URL.
If no ACTIVE row after ~10 polls, STOP and report.
7. Call `c1_connector_service_force_sync`; poll `c1_connector_service_get`
with backoff (e.g. every 5-10s) until `status.status` is
`SYNC_STATUS_DONE`, `SYNC_STATUS_ERROR`, or `SYNC_STATUS_DISABLED`; if
Expand Down
Loading