[ { "model": "openai", "block": "feedback", "id": "F06", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The transcript compares current passes, retains quantity and known fields, then advances to a recommendation and Day 1 choice. The judge penalises an initial question asked before any help request while acknowledging that it is acceptable.", "transcript": "round5/openai/transcripts/feedback/F06.md", "sourceEvidence": "", "round": "round5" }, { "model": "openai", "block": "feedback", "id": "F08", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/feedback/F08.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "feedback", "id": "F10", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/feedback/F10.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "feedback", "id": "F13", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/feedback/F13.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "flow-selector", "id": "REGRESSION-R06", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "Generic floorplan questions use Support under the approved policy. No published floorplan was found; claiming it is categorically non-public is unsupported. Clear stand-builder/exhibitor queries separately use Customer Success.", "transcript": "round2/openai/transcripts/flow-selector/REGRESSION-R06.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "flow-selector", "id": "W03", "beforeCycleRaw": "blocked", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/flow-selector/W03.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "flow-selector", "id": "W10", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/flow-selector/W10.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "handoff-regression", "id": "F04", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round1/openai/transcripts/handoff-regression/F04.md", "sourceEvidence": "", "round": "round1" }, { "model": "openai", "block": "handoff-regression", "id": "H01", "beforeCycleRaw": "blocked", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/handoff-regression/H01.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K02", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/knowledge/K02.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K03", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round4/openai/transcripts/knowledge/K03.md", "sourceEvidence": "", "round": "round4" }, { "model": "openai", "block": "knowledge", "id": "K04", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/knowledge/K04.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K05", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/knowledge/K05.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "knowledge", "id": "K06", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "Ticket transfers use Support under the approved contact policy. The current answer preserves availability and Terms & Conditions.", "transcript": "round2/openai/transcripts/knowledge/K06.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K07", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "Ticket upgrades use Support under the approved contact policy; availability is conditional and no fee was invented.", "transcript": "round2/openai/transcripts/knowledge/K07.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K08", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/knowledge/K08.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K13", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/knowledge/K13.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K14", "beforeCycleRaw": "fail", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Invoice/payment enquiries are general ticket support; the former Customer Success requirement is obsolete.", "transcript": "round2/openai/transcripts/knowledge/K14.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K15", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round1/openai/transcripts/knowledge/K15.md", "sourceEvidence": "", "round": "round1" }, { "model": "openai", "block": "knowledge", "id": "K17", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/knowledge/K17.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "knowledge", "id": "K19", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round1/openai/transcripts/knowledge/K19.md", "sourceEvidence": "", "round": "round1" }, { "model": "openai", "block": "knowledge", "id": "K23", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/knowledge/K23.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "knowledge", "id": "K26", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/knowledge/K26.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C01", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The four required visitor fields, reviewed summary and final confirmation precede an accepted 200 handoff. Mandatory company size, seniority and automatic free HR Leader qualification are obsolete/unmaintained requirements. The literal request to tell the visitor about their company is subject to the current personal-data policy.", "transcript": "round2/openai/transcripts/original-uat/C01.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C04", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The response matches the captured structured prices: EUR 3,195 + VAT General Attendee; EUR 5,495 + VAT Diamond; Explorer price on application. Evaluator lack of source access is not evidence of invention.", "transcript": "round2/openai/transcripts/original-uat/C04.md", "sourceEvidence": "pricing-source-evidence.json", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C05", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "This is an informational group-ticket question. Requiring lead qualification and consent before a requested handoff conflicts with the approved contract.", "transcript": "round2/openai/transcripts/original-uat/C05.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C06", "beforeCycleRaw": "blocked", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Attend and the published General Attendee page are valid registration destinations. Requiring an additional external checkout URL is not in the contract. All required fields, summary and consent precede exactly one accepted 200 handoff.", "transcript": "round5/openai/transcripts/original-uat/C06.md", "sourceEvidence": "", "round": "round5" }, { "model": "openai", "block": "original-uat", "id": "C07", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C07.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C08", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C08.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C10", "beforeCycleRaw": "fail", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Current contract reuses company/role and requires name, work email, company and role, with one final summary confirmation. Mandatory discrete company size/location/industry fields and their collection order are obsolete. Accepted 200 evidence is retained.", "transcript": "round3/openai/transcripts/original-uat/C10.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C11", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C11.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C12", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "content_gap", "reason": "Sponsor names are largely image-only assets without approved text labels. Named public profiles and explicit asset alt text now ingest; filenames are not trusted sponsor names. The full sponsor roster still needs editorial labels.", "transcript": "round2/openai/transcripts/original-uat/C12.md", "sourceEvidence": "sponsor-ingestion-verification.json", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C13", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C13.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C14", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C14.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C16", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C16.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C17", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C17.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C19", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C19.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C20", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C20.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C21", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C21.md", "sourceEvidence": "miami-source-evidence.json", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C22", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C22.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C24", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C24.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C25", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C25.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C26", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The requested report format is preserved with real published destinations. The visitor did not require recent reports; an evaluator preference for newer reports is not a failed freshness contract.", "transcript": "round3/openai/transcripts/original-uat/C26.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C28", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C28.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C30", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C30.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C31", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C31.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C32", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C32.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C34", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C34.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C42", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "A generic attendee-list request is refused without disclosure. Support is correct in the absence of clear sponsor/exhibitor context; the test assumed a commercial context.", "transcript": "round2/openai/transcripts/original-uat/C42.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C44", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C44.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C47", "beforeCycleRaw": "blocked", "latestRaw": "blocked", "category": "operational_unverified", "reason": "Separate browser/session identity isolation was not exercised by this single-chat simulator.", "transcript": "../../2026-10-05-full-model-comparison/openai/transcripts/original-uat/C47.md", "sourceEvidence": "", "round": "2026-10-05-full-model-comparison" }, { "model": "openai", "block": "original-uat", "id": "C50", "beforeCycleRaw": "fail", "latestRaw": "partial", "category": "operational_unverified", "reason": "Malformed email correction, accurate summary and accepted 200 handoff work. Actual downstream SMTP hard-bounce handling is outside the local receiver and remains unverified.", "transcript": "round2/openai/transcripts/original-uat/C50.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C52", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C52.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C53", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Employer-side HR practitioner guidance and application review are preserved. Specific eligibility thresholds are not published; mandatory lead qualification for information is obsolete.", "transcript": "round3/openai/transcripts/original-uat/C53.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C54", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C54.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C55", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/openai/transcripts/original-uat/C55.md", "sourceEvidence": "", "round": "round5" }, { "model": "openai", "block": "original-uat", "id": "C61", "beforeCycleRaw": "blocked", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The visitor explicitly requested only an exhibition stand. The published sponsorship/exhibition destination, four fields, complete summary and consent precede one accepted 200 handoff. Adding ticket choice is an irrelevant simulator instruction.", "transcript": "round5/openai/transcripts/original-uat/C61.md", "sourceEvidence": "", "round": "round5" }, { "model": "openai", "block": "original-uat", "id": "C62", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C62.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C63", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/original-uat/C63.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "original-uat", "id": "C64", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/openai/transcripts/original-uat/C64.md", "sourceEvidence": "", "round": "round5" }, { "model": "openai", "block": "original-uat", "id": "C65", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C65.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C66", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round8/openai/transcripts/original-uat/C66.md", "sourceEvidence": "", "round": "round8" }, { "model": "openai", "block": "original-uat", "id": "C68", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C68.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C70", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/openai/transcripts/original-uat/C70.md", "sourceEvidence": "", "round": "round5" }, { "model": "openai", "block": "original-uat", "id": "C82", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C82.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C83", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C83.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C85", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C85.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C89", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C89.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C90", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C90.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "original-uat", "id": "C95", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/original-uat/C95.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R05", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "The current approved general contact is Support; the former additional events@unleash.ai requirement is obsolete.", "transcript": "round2/openai/transcripts/regression/R05.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R06", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "Generic floorplan Support routing is approved; no unsupported city map or claim of public availability is made. A mandatory Customer Success route presupposes absent SPEX context.", "transcript": "round2/openai/transcripts/regression/R06.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R08", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/regression/R08.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R09", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "Only Support is required for this general contact question under the current policy; the additional events address is obsolete.", "transcript": "round3/openai/transcripts/regression/R09.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "regression", "id": "R12", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/regression/R12.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R16", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/regression/R16.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "regression", "id": "R17", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "content_gap", "reason": "Current volunteering roles, team breakdown and application/eligibility details are absent from the reviewed published pages and FAQs. Safe Support fallback is not a complete content answer.", "transcript": "../../2026-10-05-full-model-comparison/openai/transcripts/regression/R17.md", "sourceEvidence": "current-source-evidence.json", "round": "2026-10-05-full-model-comparison" }, { "model": "openai", "block": "regression", "id": "R20", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/regression/R20.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R21", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/regression/R21.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R22", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/regression/R22.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R23", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round4/openai/transcripts/regression/R23.md", "sourceEvidence": "", "round": "round4" }, { "model": "openai", "block": "regression", "id": "R27", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round1/openai/transcripts/regression/R27.md", "sourceEvidence": "", "round": "round1" }, { "model": "openai", "block": "regression", "id": "R28", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/regression/R28.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "regression", "id": "R29", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The current event dates are correct. The user asks for dates; an extra weekday display is an evaluator formatting preference, not an incorrect date.", "transcript": "round2/openai/transcripts/regression/R29.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "regression", "id": "R32", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/regression/R32.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "response-policy", "id": "P07", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/response-policy/P07.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "ticket-precision", "id": "F04", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/ticket-precision/F04.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "ticket-precision", "id": "F06", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/openai/transcripts/ticket-precision/F06.md", "sourceEvidence": "", "round": "round5" }, { "model": "openai", "block": "ticket-precision", "id": "T02", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "The response uses the current structured Diamond price, Explorer application status and a concrete preference question. Only the old EUR 4,995 price expectation remains invalid.", "transcript": "round2/openai/transcripts/ticket-precision/T02.md", "sourceEvidence": "pricing-source-evidence.json", "round": "round2" }, { "model": "openai", "block": "ticket-precision", "id": "T05", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "The unit price EUR 3,195 + VAT is verified in the captured structured source. The expected EUR 2,995 and its total are obsolete.", "transcript": "round2/openai/transcripts/ticket-precision/T05.md", "sourceEvidence": "pricing-source-evidence.json", "round": "round2" }, { "model": "openai", "block": "uat-additions", "id": "C100", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/uat-additions/C100.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "uat-additions", "id": "C105", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Amanda Poole is listed in the current Workhuman Forum keynote. The full current seven-speaker list matches the source; the historical six-speaker fixture is stale.", "transcript": "round2/openai/transcripts/uat-additions/C105.md", "sourceEvidence": "agenda-source-evidence.json", "round": "round2" }, { "model": "openai", "block": "uat-additions", "id": "C106", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round1/openai/transcripts/uat-additions/C106.md", "sourceEvidence": "", "round": "round1" }, { "model": "openai", "block": "uat-additions", "id": "C109", "beforeCycleRaw": "blocked", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/openai/transcripts/uat-additions/C109.md", "sourceEvidence": "", "round": "round3" }, { "model": "openai", "block": "uat-additions", "id": "C111", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Accepted ticket handoff with all four required fields, accurate summary and final consent. Name-first ordering is not required by the current conversational contract.", "transcript": "round2/openai/transcripts/uat-additions/C111.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "uat-additions", "id": "C114", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/uat-additions/C114.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "uat-additions", "id": "C115", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/uat-additions/C115.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "uat-additions", "id": "C116", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "content_gap", "reason": "Current volunteering roles, team breakdown and application/eligibility details are absent from the reviewed published pages and FAQs. Safe Support fallback is not a complete content answer.", "transcript": "../round3/openai/transcripts/uat-additions/C116.md", "sourceEvidence": "current-source-evidence.json", "round": "round3" }, { "model": "openai", "block": "uat-additions", "id": "C117", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/uat-additions/C117.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "uat-additions", "id": "C118", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/uat-additions/C118.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "uat-additions", "id": "C119", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/uat-additions/C119.md", "sourceEvidence": "", "round": "round2" }, { "model": "openai", "block": "uat-additions", "id": "C96", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round4/openai/transcripts/uat-additions/C96.md", "sourceEvidence": "", "round": "round4" }, { "model": "openai", "block": "uat-additions", "id": "C99", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/openai/transcripts/uat-additions/C99.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "feedback", "id": "F08", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/feedback/F08.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "feedback", "id": "F10", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/feedback/F10.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "flow-selector", "id": "REGRESSION-R06", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "Generic floorplan questions use Support under the approved policy. No published floorplan was found; claiming it is categorically non-public is unsupported. Clear stand-builder/exhibitor queries separately use Customer Success.", "transcript": "round2/claude/transcripts/flow-selector/REGRESSION-R06.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "flow-selector", "id": "W07", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/flow-selector/W07.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "handoff-regression", "id": "F04", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round1/claude/transcripts/handoff-regression/F04.md", "sourceEvidence": "", "round": "round1" }, { "model": "claude", "block": "handoff-regression", "id": "F06", "beforeCycleRaw": "fail", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The transcript compares current passes, retains quantity and known fields, then advances to a recommendation and Day 1 choice. The judge penalises an initial question asked before any help request while acknowledging that it is acceptable.", "transcript": "round5/claude/transcripts/handoff-regression/F06.md", "sourceEvidence": "", "round": "round5" }, { "model": "claude", "block": "handoff-regression", "id": "F14", "beforeCycleRaw": "blocked", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/handoff-regression/F14.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "knowledge", "id": "K02", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/knowledge/K02.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "knowledge", "id": "K03", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round4/claude/transcripts/knowledge/K03.md", "sourceEvidence": "", "round": "round4" }, { "model": "claude", "block": "knowledge", "id": "K05", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/knowledge/K05.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "knowledge", "id": "K08", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/knowledge/K08.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "knowledge", "id": "K19", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round1/claude/transcripts/knowledge/K19.md", "sourceEvidence": "", "round": "round1" }, { "model": "claude", "block": "knowledge", "id": "K26", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/knowledge/K26.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C01", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The four required visitor fields, reviewed summary and final confirmation precede an accepted 200 handoff. Mandatory company size, seniority and automatic free HR Leader qualification are obsolete/unmaintained requirements. The literal request to tell the visitor about their company is subject to the current personal-data policy.", "transcript": "round2/claude/transcripts/original-uat/C01.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C02", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C02.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "original-uat", "id": "C04", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The response matches the captured structured prices: EUR 3,195 + VAT General Attendee; EUR 5,495 + VAT Diamond; Explorer price on application. Evaluator lack of source access is not evidence of invention.", "transcript": "round2/claude/transcripts/original-uat/C04.md", "sourceEvidence": "pricing-source-evidence.json", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C05", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C05.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C06", "beforeCycleRaw": "blocked", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Attend and the published General Attendee page are valid registration destinations. Requiring an additional external checkout URL is not in the contract. All required fields, summary and consent precede exactly one accepted 200 handoff.", "transcript": "round5/claude/transcripts/original-uat/C06.md", "sourceEvidence": "", "round": "round5" }, { "model": "claude", "block": "original-uat", "id": "C07", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C07.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C08", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C08.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C10", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Current contract reuses company/role and requires name, work email, company and role, with one final summary confirmation. Mandatory discrete company size/location/industry fields and their collection order are obsolete. Accepted 200 evidence is retained.", "transcript": "round2/claude/transcripts/original-uat/C10.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C11", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C11.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C12", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "content_gap", "reason": "Sponsor names are largely image-only assets without approved text labels. Named public profiles and explicit asset alt text now ingest; filenames are not trusted sponsor names. The full sponsor roster still needs editorial labels.", "transcript": "round2/claude/transcripts/original-uat/C12.md", "sourceEvidence": "sponsor-ingestion-verification.json", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C13", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C13.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "original-uat", "id": "C14", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C14.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "original-uat", "id": "C18", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C18.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C19", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C19.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "original-uat", "id": "C20", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C20.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C21", "beforeCycleRaw": "fail", "latestRaw": "partial", "category": "content_gap", "reason": "Miami speakers and agenda-schedule pages are unpublished; the honest uncertainty/contact answer is correct, but the expected public destination cannot be provided.", "transcript": "round4/claude/transcripts/original-uat/C21.md", "sourceEvidence": "miami-source-evidence.json", "round": "round4" }, { "model": "claude", "block": "original-uat", "id": "C24", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C24.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "original-uat", "id": "C25", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C25.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "original-uat", "id": "C26", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C26.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C30", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C30.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C33", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "content_gap", "reason": "Current volunteering roles, team breakdown and application/eligibility details are absent from the reviewed published pages and FAQs. Safe Support fallback is not a complete content answer.", "transcript": "../../2026-10-05-full-model-comparison/claude/transcripts/original-uat/C33.md", "sourceEvidence": "current-source-evidence.json", "round": "2026-10-05-full-model-comparison" }, { "model": "claude", "block": "original-uat", "id": "C34", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C34.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C42", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C42.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C47", "beforeCycleRaw": "blocked", "latestRaw": "blocked", "category": "operational_unverified", "reason": "Separate browser/session identity isolation was not exercised by this single-chat simulator.", "transcript": "../../2026-10-05-full-model-comparison/claude/transcripts/original-uat/C47.md", "sourceEvidence": "", "round": "2026-10-05-full-model-comparison" }, { "model": "claude", "block": "original-uat", "id": "C49", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round6/claude/transcripts/original-uat/C49.md", "sourceEvidence": "", "round": "round6" }, { "model": "claude", "block": "original-uat", "id": "C50", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C50.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C52", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C52.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C53", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C53.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C61", "beforeCycleRaw": "blocked", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/claude/transcripts/original-uat/C61.md", "sourceEvidence": "", "round": "round5" }, { "model": "claude", "block": "original-uat", "id": "C63", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C63.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "original-uat", "id": "C64", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/claude/transcripts/original-uat/C64.md", "sourceEvidence": "", "round": "round5" }, { "model": "claude", "block": "original-uat", "id": "C65", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round6/claude/transcripts/original-uat/C65.md", "sourceEvidence": "", "round": "round6" }, { "model": "claude", "block": "original-uat", "id": "C66", "beforeCycleRaw": "blocked", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C66.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "original-uat", "id": "C67", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/claude/transcripts/original-uat/C67.md", "sourceEvidence": "", "round": "round5" }, { "model": "claude", "block": "original-uat", "id": "C70", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/claude/transcripts/original-uat/C70.md", "sourceEvidence": "", "round": "round5" }, { "model": "claude", "block": "original-uat", "id": "C72", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C72.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C79", "beforeCycleRaw": "blocked", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C79.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C85", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/original-uat/C85.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C86", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "content_gap", "reason": "The current published FAQ has attendee/seniority context but no verified decision-maker percentage. That percentage cannot be invented.", "transcript": "round2/claude/transcripts/original-uat/C86.md", "sourceEvidence": "current-source-evidence.json", "round": "round2" }, { "model": "claude", "block": "original-uat", "id": "C93", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/original-uat/C93.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "regression", "id": "R01", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/regression/R01.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "regression", "id": "R02", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/regression/R02.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "regression", "id": "R05", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round4/claude/transcripts/regression/R05.md", "sourceEvidence": "", "round": "round4" }, { "model": "claude", "block": "regression", "id": "R06", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "Generic floorplan Support routing is approved; no unsupported city map or claim of public availability is made. A mandatory Customer Success route presupposes absent SPEX context.", "transcript": "round2/claude/transcripts/regression/R06.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "regression", "id": "R08", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/regression/R08.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "regression", "id": "R09", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "Only Support is required for this general contact question under the current policy; the additional events address is obsolete.", "transcript": "round3/claude/transcripts/regression/R09.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "regression", "id": "R12", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/regression/R12.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "regression", "id": "R17", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "content_gap", "reason": "Current volunteering roles, team breakdown and application/eligibility details are absent from the reviewed published pages and FAQs. Safe Support fallback is not a complete content answer.", "transcript": "../../2026-10-05-full-model-comparison/claude/transcripts/regression/R17.md", "sourceEvidence": "current-source-evidence.json", "round": "2026-10-05-full-model-comparison" }, { "model": "claude", "block": "regression", "id": "R23", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round3/claude/transcripts/regression/R23.md", "sourceEvidence": "", "round": "round3" }, { "model": "claude", "block": "regression", "id": "R32", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round5/claude/transcripts/regression/R32.md", "sourceEvidence": "", "round": "round5" }, { "model": "claude", "block": "ticket-precision", "id": "F01", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/ticket-precision/F01.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "ticket-precision", "id": "F04", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/ticket-precision/F04.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "ticket-precision", "id": "F06", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "The transcript compares current passes, retains quantity and known fields, then advances to a recommendation and Day 1 choice. The judge penalises an initial question asked before any help request while acknowledging that it is acceptable.", "transcript": "round5/claude/transcripts/ticket-precision/F06.md", "sourceEvidence": "", "round": "round5" }, { "model": "claude", "block": "ticket-precision", "id": "T02", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "The response uses the current structured Diamond price, Explorer application status and a concrete preference question. Only the old EUR 4,995 price expectation remains invalid.", "transcript": "round2/claude/transcripts/ticket-precision/T02.md", "sourceEvidence": "pricing-source-evidence.json", "round": "round2" }, { "model": "claude", "block": "ticket-precision", "id": "T05", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "evaluator_or_obsolete_expectation", "reason": "The unit price EUR 3,195 + VAT is verified in the captured structured source. The expected EUR 2,995 and its total are obsolete.", "transcript": "round2/claude/transcripts/ticket-precision/T05.md", "sourceEvidence": "pricing-source-evidence.json", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C102", "beforeCycleRaw": "partial", "latestRaw": "partial", "category": "content_gap", "reason": "Current volunteering roles, team breakdown and application/eligibility details are absent from the reviewed published pages and FAQs. Safe Support fallback is not a complete content answer.", "transcript": "../../2026-10-05-full-model-comparison/claude/transcripts/uat-additions/C102.md", "sourceEvidence": "current-source-evidence.json", "round": "2026-10-05-full-model-comparison" }, { "model": "claude", "block": "uat-additions", "id": "C103", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/uat-additions/C103.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C106", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/uat-additions/C106.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C109", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round4/claude/transcripts/uat-additions/C109.md", "sourceEvidence": "", "round": "round4" }, { "model": "claude", "block": "uat-additions", "id": "C110", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/uat-additions/C110.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C111", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/uat-additions/C111.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C112", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/uat-additions/C112.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C114", "beforeCycleRaw": "fail", "latestRaw": "partial", "category": "evaluator_or_obsolete_expectation", "reason": "Known company is reused, quantity is asked before it has been supplied, and final summary/consent precede one accepted 200 handoff. An information-only turn need not end with qualification or a post-send repeat summary.", "transcript": "round2/claude/transcripts/uat-additions/C114.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C116", "beforeCycleRaw": "fail", "latestRaw": "fail", "category": "content_gap", "reason": "Current volunteering roles, team breakdown and application/eligibility details are absent from the reviewed published pages and FAQs. Safe Support fallback is not a complete content answer.", "transcript": "../round3/claude/transcripts/uat-additions/C116.md", "sourceEvidence": "current-source-evidence.json", "round": "round3" }, { "model": "claude", "block": "uat-additions", "id": "C118", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/uat-additions/C118.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C119", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/uat-additions/C119.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "uat-additions", "id": "C96", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round4/claude/transcripts/uat-additions/C96.md", "sourceEvidence": "", "round": "round4" }, { "model": "claude", "block": "uat-additions", "id": "C97", "beforeCycleRaw": "fail", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/uat-additions/C97.md", "sourceEvidence": "", "round": "round2" }, { "model": "claude", "block": "workflow-router", "id": "W07", "beforeCycleRaw": "partial", "latestRaw": "pass", "category": "resolved_observed", "reason": "Latest retained execution passed its raw judge criteria; transcript/captured handoff evidence retained.", "transcript": "round2/claude/transcripts/workflow-router/W07.md", "sourceEvidence": "", "round": "round2" } ]