Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions runner/evaluators/llm/judge-client.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
import { Output, generateText } from 'ai'
import {
extractJsonMiddleware,
generateText,
NoObjectGeneratedError,
Output,
wrapLanguageModel,
} from 'ai'
import { z } from 'zod'
import {
collectOpencodeSessionSnapshot,
Expand Down Expand Up @@ -129,14 +135,27 @@ export async function runJudgeCall(
inputSummary: `promptBytes=${promptBytes}`,
})

const { model: judgeModel, dispose } = createIsolatedOpencodeModel(
const { model: opencodeModel, dispose } = createIsolatedOpencodeModel(
options.model,
{
port: options.port,
cwd: options.cwd,
}
)

// OpenCode's JSON mode is prompt-only, so models often wrap the JSON in
// markdown fences, sometimes after prose or quoted code fences; extract the
// last fenced JSON object so structured output parses on the first call.
const judgeModel = wrapLanguageModel({
model: opencodeModel,
middleware: extractJsonMiddleware({
transform: (text) =>
[...text.matchAll(/```(?:json)?\s*([\s\S]*?)\s*```/gi)]
.map((match) => match[1]?.trim() ?? '')
.findLast((block) => block.startsWith('{')) ?? text.trim(),
}),
})

try {
try {
const response = await runTracedOpencodeCall(
Expand Down Expand Up @@ -167,6 +186,27 @@ export async function runJudgeCall(
}),
}
} catch (structuredOutputError) {
// Recover the verdict from the first call's text before paying for a
// second full judge call with the same prompt.
if (
NoObjectGeneratedError.isInstance(structuredOutputError) &&
structuredOutputError.text
) {
const recoveredOutput = parseJudgeOutputFromText(
structuredOutputError.text
)
if (recoveredOutput.success) {
logOpencodeTrace(
'structured output failed; recovered verdict from response text'
)

return {
summary: recoveredOutput.data.summary,
requirements: recoveredOutput.data.requirements,
}
}
}

logOpencodeTrace(
`structured output failed; trying JSON fallback: ${structuredOutputError instanceof Error ? structuredOutputError.message : String(structuredOutputError)}`
)
Expand Down
2 changes: 1 addition & 1 deletion runner/evaluators/llm/prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ export function buildJudgePrompt(
Decide pass/fail for each criterion based only on the submitted files.

Rules:
- Use only the submitted files as evidence.
- Use only the submitted files as evidence (does not include reference files).
- File paths are part of the submission and may be used as evidence for placement and naming requirements.
- Return exactly one result per declared requirement id.
- Mark passed=false if evidence is missing or contradictory.
Expand Down
119 changes: 119 additions & 0 deletions runner/evaluators/llm/tests/judge-client.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
import { describe, expect, mock, test } from 'bun:test'
import { MockLanguageModelV3 } from 'ai/test'

let responses: string[] = []
let calls = 0

mock.module('runner/utils/opencode-model', () => ({
createIsolatedOpencodeModel: () => ({
model: new MockLanguageModelV3({
doGenerate: async () => {
const text = responses[Math.min(calls, responses.length - 1)] ?? ''
calls += 1
return {
content: [{ type: 'text', text }],
finishReason: { unified: 'stop', raw: 'stop' },
usage: {
inputTokens: { total: 1, noCache: 1, cacheRead: 0, cacheWrite: 0 },
outputTokens: { total: 1, text: 1, reasoning: 0 },
},
warnings: [],
}
},
}),
dispose: async () => {},
}),
}))

mock.module('runner/utils/opencode-trace', () => ({
logOpencodeTrace: () => {},
runTracedOpencodeCall: (
_tracer: unknown,
_label: string,
fn: () => unknown
) => fn(),
startOpencodeCallTracer: () => ({ noteSessionId: () => {}, stop: () => {} }),
}))

mock.module('runner/utils/opencode-session', () => ({
collectOpencodeSessionSnapshot: async () => undefined,
}))

const { runJudgeCall } = await import('../judge-client')

const verdict = JSON.stringify(
{
summary: 'ok',
requirements: [
{ id: 'req-a', passed: true, reason: 'uses withTiming', evidence: [] },
],
},
null,
2
)

async function judge(...answers: string[]) {
responses = answers
calls = 0
const result = await runJudgeCall({
prompt: 'judge this',
model: 'mock/model',
timeout: 5_000,
port: 1,
})
return { result, calls }
}

describe('runJudgeCall fenced output handling', () => {
test('fenced json parses on the first call', async () => {
const { result, calls } = await judge('```json\n' + verdict + '\n```')
expect(calls).toBe(1)
expect(result.requirements[0]?.id).toBe('req-a')
})

test('fence without language tag parses on the first call', async () => {
const { calls } = await judge('```\n' + verdict + '\n```')
expect(calls).toBe(1)
})

test('prose with braces before fenced json parses on the first call', async () => {
const { calls } = await judge(
'Analysis: screens: {...} looks fine.\n\n```json\n' + verdict + '\n```'
)
expect(calls).toBe(1)
})

test('raw json parses on the first call', async () => {
const { calls } = await judge(verdict)
expect(calls).toBe(1)
})

test('prose before raw json is recovered without a second call', async () => {
const { calls } = await judge('Output JSON:\n' + verdict)
expect(calls).toBe(1)
})

test('code snippet fence before the json fence', async () => {
const { calls } = await judge(
'The submission has:\n```tsx\nscale.value = withTiming(1)\n```\n\n```json\n' +
verdict +
'\n```'
)
expect(calls).toBe(1)
})

test('schema-invalid fenced json falls back to a second call', async () => {
const { calls, result } = await judge(
'```json\n{"requirements":[{"id":"req-a"}]}\n```',
'```json\n' + verdict + '\n```'
)
expect(calls).toBe(2)
expect(result.requirements[0]?.passed).toBe(true)
})

test('unparseable output on both calls throws', async () => {
await expect(judge('no json here', 'still none')).rejects.toThrow(
'judge did not return valid requirement output'
)
})
})
Loading