{
  "method": "Latest retained results from the original benchmark, earlier failure-only fixes and this adaptive cycle; not a fresh full-suite execution. Same itemised exclusions before/after. Partials/content gaps/operational checks are not passes.",
  "models": {
    "openai": {
      "original-uat": {
        "rawBefore": {
          "cases": 95,
          "verdicts": {
            "fail": 19,
            "pass": 44,
            "partial": 29,
            "blocked": 3
          },
          "strictPassPercent": 46.3
        },
        "rawLatest": {
          "cases": 95,
          "verdicts": {
            "partial": 11,
            "pass": 82,
            "blocked": 1,
            "fail": 1
          },
          "strictPassPercent": 86.3
        },
        "adjustedBefore": {
          "cases": 85,
          "verdicts": {
            "pass": 44,
            "fail": 15,
            "partial": 25,
            "blocked": 1
          },
          "strictPassPercent": 51.8
        },
        "adjustedLatest": {
          "cases": 85,
          "verdicts": {
            "pass": 82,
            "partial": 2,
            "blocked": 1
          },
          "strictPassPercent": 96.5
        }
      },
      "all": {
        "rawBefore": {
          "cases": 279,
          "verdicts": {
            "pass": 151,
            "fail": 68,
            "blocked": 6,
            "partial": 54
          },
          "strictPassPercent": 54.1
        },
        "rawLatest": {
          "cases": 279,
          "verdicts": {
            "pass": 239,
            "fail": 21,
            "partial": 18,
            "blocked": 1
          },
          "strictPassPercent": 85.7
        },
        "adjustedBefore": {
          "cases": 244,
          "verdicts": {
            "pass": 151,
            "blocked": 4,
            "fail": 45,
            "partial": 44
          },
          "strictPassPercent": 61.9
        },
        "adjustedLatest": {
          "cases": 244,
          "verdicts": {
            "pass": 239,
            "fail": 2,
            "partial": 2,
            "blocked": 1
          },
          "strictPassPercent": 98.0
        }
      }
    },
    "claude": {
      "original-uat": {
        "rawBefore": {
          "cases": 95,
          "verdicts": {
            "fail": 17,
            "partial": 25,
            "blocked": 5,
            "pass": 48
          },
          "strictPassPercent": 50.5
        },
        "rawLatest": {
          "cases": 95,
          "verdicts": {
            "partial": 8,
            "pass": 81,
            "fail": 5,
            "blocked": 1
          },
          "strictPassPercent": 85.3
        },
        "adjustedBefore": {
          "cases": 86,
          "verdicts": {
            "fail": 12,
            "partial": 22,
            "pass": 48,
            "blocked": 4
          },
          "strictPassPercent": 55.8
        },
        "adjustedLatest": {
          "cases": 86,
          "verdicts": {
            "pass": 81,
            "partial": 3,
            "fail": 1,
            "blocked": 1
          },
          "strictPassPercent": 94.2
        }
      },
      "all": {
        "rawBefore": {
          "cases": 279,
          "verdicts": {
            "pass": 168,
            "fail": 65,
            "partial": 40,
            "blocked": 6
          },
          "strictPassPercent": 60.2
        },
        "rawLatest": {
          "cases": 279,
          "verdicts": {
            "pass": 239,
            "fail": 23,
            "partial": 16,
            "blocked": 1
          },
          "strictPassPercent": 85.7
        },
        "adjustedBefore": {
          "cases": 247,
          "verdicts": {
            "pass": 168,
            "partial": 34,
            "fail": 40,
            "blocked": 5
          },
          "strictPassPercent": 68.0
        },
        "adjustedLatest": {
          "cases": 247,
          "verdicts": {
            "pass": 239,
            "partial": 4,
            "fail": 3,
            "blocked": 1
          },
          "strictPassPercent": 96.8
        }
      }
    }
  },
  "exclusions": [
    {
      "key": [
        "openai",
        "feedback",
        "F07"
      ],
      "review": {
        "model": "openai",
        "block": "feedback",
        "case": "F07",
        "category": "outdated prices",
        "reason": "Only the old fixed 2,995/4,995 prices cause failure; the recorded structured badge source returns 3,195/5,495. Other evaluated flow and cancellation criteria were satisfied.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/feedback/F07.md"
      }
    },
    {
      "key": [
        "claude",
        "feedback",
        "F07"
      ],
      "review": {
        "model": "claude",
        "block": "feedback",
        "case": "F07",
        "category": "outdated prices",
        "reason": "Only the old fixed 2,995/4,995 prices cause failure; the recorded structured badge source returns 3,195/5,495. Other evaluated flow and cancellation criteria were satisfied.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/feedback/F07.md"
      }
    },
    {
      "key": [
        "openai",
        "feedback",
        "F09"
      ],
      "review": {
        "model": "openai",
        "block": "feedback",
        "case": "F09",
        "category": "outdated prices",
        "reason": "Only the old fixed 2,995/4,995 prices cause failure; the recorded structured badge source returns 3,195/5,495. Other evaluated flow and cancellation criteria were satisfied.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/feedback/F09.md"
      }
    },
    {
      "key": [
        "claude",
        "feedback",
        "F09"
      ],
      "review": {
        "model": "claude",
        "block": "feedback",
        "case": "F09",
        "category": "outdated prices",
        "reason": "Only the old fixed 2,995/4,995 prices cause failure; the recorded structured badge source returns 3,195/5,495. Other evaluated flow and cancellation criteria were satisfied.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/feedback/F09.md"
      }
    },
    {
      "key": [
        "openai",
        "ticket-precision",
        "F07"
      ],
      "review": {
        "model": "openai",
        "block": "ticket-precision",
        "case": "F07",
        "category": "outdated prices",
        "reason": "Only the old fixed 2,995/4,995 prices cause failure; the recorded structured badge source returns 3,195/5,495. Other evaluated flow and cancellation criteria were satisfied.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/ticket-precision/F07.md"
      }
    },
    {
      "key": [
        "claude",
        "ticket-precision",
        "F07"
      ],
      "review": {
        "model": "claude",
        "block": "ticket-precision",
        "case": "F07",
        "category": "outdated prices",
        "reason": "Only the old fixed 2,995/4,995 prices cause failure; the recorded structured badge source returns 3,195/5,495. Other evaluated flow and cancellation criteria were satisfied.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/ticket-precision/F07.md"
      }
    },
    {
      "key": [
        "openai",
        "ticket-precision",
        "F09"
      ],
      "review": {
        "model": "openai",
        "block": "ticket-precision",
        "case": "F09",
        "category": "outdated prices",
        "reason": "Only the old fixed 2,995/4,995 prices cause failure; the recorded structured badge source returns 3,195/5,495. Other evaluated flow and cancellation criteria were satisfied.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/ticket-precision/F09.md"
      }
    },
    {
      "key": [
        "claude",
        "ticket-precision",
        "F09"
      ],
      "review": {
        "model": "claude",
        "block": "ticket-precision",
        "case": "F09",
        "category": "outdated prices",
        "reason": "Only the old fixed 2,995/4,995 prices cause failure; the recorded structured badge source returns 3,195/5,495. Other evaluated flow and cancellation criteria were satisfied.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/ticket-precision/F09.md"
      }
    },
    {
      "key": [
        "openai",
        "ticket-precision",
        "T01"
      ],
      "review": {
        "model": "openai",
        "block": "ticket-precision",
        "case": "T01",
        "category": "outdated prices",
        "reason": "Diamond answer matches the recorded structured price, rather than the fixed historical 4,995 expected by this test.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/ticket-precision/T01.md"
      }
    },
    {
      "key": [
        "claude",
        "ticket-precision",
        "T01"
      ],
      "review": {
        "model": "claude",
        "block": "ticket-precision",
        "case": "T01",
        "category": "outdated prices",
        "reason": "Diamond answer matches the recorded structured price, rather than the fixed historical 4,995 expected by this test.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/ticket-precision/T01.md"
      }
    },
    {
      "key": [
        "openai",
        "uat-additions",
        "C98"
      ],
      "review": {
        "model": "openai",
        "block": "uat-additions",
        "case": "C98",
        "category": "conflicting wording",
        "reason": "The blanket no-British-spelling criterion forbids the current approved app-owned personalised CTA. Rewrite the style expectation before scoring this case.",
        "sourceEvidence": "../../../apps/web/src/lib/agendaPlan.ts",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-ai-defect-retest/openai/transcripts/uat-additions/C98.md"
      }
    },
    {
      "key": [
        "claude",
        "uat-additions",
        "C98"
      ],
      "review": {
        "model": "claude",
        "block": "uat-additions",
        "case": "C98",
        "category": "conflicting wording",
        "reason": "The blanket no-British-spelling criterion forbids the current approved app-owned personalised CTA. Rewrite the style expectation before scoring this case.",
        "sourceEvidence": "../../../apps/web/src/lib/agendaPlan.ts",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/uat-additions/C98.md"
      }
    },
    {
      "key": [
        "openai",
        "uat-additions",
        "C104"
      ],
      "review": {
        "model": "openai",
        "block": "uat-additions",
        "case": "C104",
        "category": "outdated roster",
        "reason": "The frozen August six-speaker note contradicts the seven keynote-format sessions in the current live source. Latest answers match that source.",
        "sourceEvidence": "round3/source-refresh-current.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-ai-defect-retest/round3/openai/transcripts/uat-additions/C104.md"
      }
    },
    {
      "key": [
        "claude",
        "uat-additions",
        "C104"
      ],
      "review": {
        "model": "claude",
        "block": "uat-additions",
        "case": "C104",
        "category": "outdated roster",
        "reason": "The frozen August six-speaker note contradicts the seven keynote-format sessions in the current live source. Latest answers match that source.",
        "sourceEvidence": "round3/source-refresh-current.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-ai-defect-retest/round3/claude/transcripts/uat-additions/C104.md"
      }
    },
    {
      "key": [
        "openai",
        "knowledge",
        "K09"
      ],
      "review": {
        "model": "openai",
        "block": "knowledge",
        "case": "K09",
        "category": "outdated contact policy",
        "reason": "Grading penalises support@unleash.ai instead of the former Customer Success/events contact route. Current policy uses Support for these non-SPEX requests.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "partial",
        "rawLatest": "partial",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/knowledge/K09.md"
      }
    },
    {
      "key": [
        "claude",
        "knowledge",
        "K09"
      ],
      "review": {
        "model": "claude",
        "block": "knowledge",
        "case": "K09",
        "category": "outdated contact policy",
        "reason": "Grading penalises support@unleash.ai instead of the former Customer Success/events contact route. Current policy uses Support for these non-SPEX requests.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "partial",
        "rawLatest": "partial",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/knowledge/K09.md"
      }
    },
    {
      "key": [
        "openai",
        "knowledge",
        "K18"
      ],
      "review": {
        "model": "openai",
        "block": "knowledge",
        "case": "K18",
        "category": "outdated contact policy",
        "reason": "Grading penalises support@unleash.ai instead of the former Customer Success/events contact route. Current policy uses Support for these non-SPEX requests.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/knowledge/K18.md"
      }
    },
    {
      "key": [
        "claude",
        "knowledge",
        "K18"
      ],
      "review": {
        "model": "claude",
        "block": "knowledge",
        "case": "K18",
        "category": "outdated contact policy",
        "reason": "Grading penalises support@unleash.ai instead of the former Customer Success/events contact route. Current policy uses Support for these non-SPEX requests.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/knowledge/K18.md"
      }
    },
    {
      "key": [
        "openai",
        "knowledge",
        "K21"
      ],
      "review": {
        "model": "openai",
        "block": "knowledge",
        "case": "K21",
        "category": "outdated contact policy",
        "reason": "Grading penalises support@unleash.ai instead of the former Customer Success/events contact route. Current policy uses Support for these non-SPEX requests.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "partial",
        "rawLatest": "partial",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/knowledge/K21.md"
      }
    },
    {
      "key": [
        "claude",
        "knowledge",
        "K21"
      ],
      "review": {
        "model": "claude",
        "block": "knowledge",
        "case": "K21",
        "category": "outdated contact policy",
        "reason": "Grading penalises support@unleash.ai instead of the former Customer Success/events contact route. Current policy uses Support for these non-SPEX requests.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "partial",
        "rawLatest": "partial",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/knowledge/K21.md"
      }
    },
    {
      "key": [
        "openai",
        "knowledge",
        "K20"
      ],
      "review": {
        "model": "openai",
        "block": "knowledge",
        "case": "K20",
        "category": "outdated privacy policy",
        "reason": "Old criteria require a privacy-page redirect and deny access; the current explicit policy says the bot cannot share stored information and directs the visitor to support@unleash.ai.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/knowledge/K20.md"
      }
    },
    {
      "key": [
        "claude",
        "knowledge",
        "K20"
      ],
      "review": {
        "model": "claude",
        "block": "knowledge",
        "case": "K20",
        "category": "outdated privacy policy",
        "reason": "Old criteria require a privacy-page redirect and deny access; the current explicit policy says the bot cannot share stored information and directs the visitor to support@unleash.ai.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/knowledge/K20.md"
      }
    },
    {
      "key": [
        "claude",
        "knowledge",
        "K17"
      ],
      "review": {
        "model": "claude",
        "block": "knowledge",
        "case": "K17",
        "category": "outdated contact policy",
        "reason": "The failure is solely the superseded complaints email expectation. OpenAI is retained because its sign-off also hits the duplicate-message warning.",
        "sourceEvidence": "../../../apps/web/src/lib/aiResponsePolicy.ts",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/knowledge/K17.md"
      }
    },
    {
      "key": [
        "openai",
        "uat-additions",
        "C97"
      ],
      "review": {
        "model": "openai",
        "block": "uat-additions",
        "case": "C97",
        "category": "outdated consent policy",
        "reason": "The raw failure penalises obtaining explicit final consent after the name arrives; that confirmation is required. Claude is retained for its separate missing agenda/dispatch problems.",
        "sourceEvidence": "review-notes.md",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/uat-additions/C97.md"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C51"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "case": "C51",
        "category": "judge source-evidence gap",
        "reason": "The judge cannot verify prices/benefits from its context; the recorded structured badge source contains the quoted prices and tier summaries. Missing judge evidence is not demonstrated invention.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/openai/transcripts/original-uat/C51.md"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C51"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "case": "C51",
        "category": "judge source-evidence gap",
        "reason": "The judge cannot verify prices/benefits from its context; the recorded structured badge source contains the quoted prices and tier summaries. Missing judge evidence is not demonstrated invention.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/badge-source-evidence.json",
        "rawBefore": "partial",
        "rawLatest": "partial",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/original-uat/C51.md"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C78"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "case": "C78",
        "category": "judge source-evidence gap",
        "reason": "Recorded badgesLookup and FAQ observations substantiate the prices and conditional onsite policy penalised as invented.",
        "sourceEvidence": "registration-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-ai-defect-retest/round2/claude/transcripts/original-uat/C78.md"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C54"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "case": "C54",
        "category": "judge source-evidence gap",
        "reason": "The fabrication allegation is contradicted by the case-specific retrieved package, startup story or transfer-policy evidence. This does not certify ongoing source freshness.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/additional-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/original-uat/C54.md"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C77"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "case": "C77",
        "category": "judge source-evidence gap",
        "reason": "The fabrication allegation is contradicted by the case-specific retrieved package, startup story or transfer-policy evidence. This does not certify ongoing source freshness.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/additional-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/original-uat/C77.md"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C80"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "case": "C80",
        "category": "judge source-evidence gap",
        "reason": "The fabrication allegation is contradicted by the case-specific retrieved package, startup story or transfer-policy evidence. This does not certify ongoing source freshness.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/additional-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/original-uat/C80.md"
      }
    },
    {
      "key": [
        "claude",
        "knowledge",
        "K04"
      ],
      "review": {
        "model": "claude",
        "block": "knowledge",
        "case": "K04",
        "category": "judge source-evidence gap",
        "reason": "The retrieved Terms & Conditions contain the refund tiers and service charge; current Support routing supersedes the old contact criterion. OpenAI remains for its independent incomplete terms explanation.",
        "sourceEvidence": "../2026-10-05-full-model-comparison/refund-volunteer-source-evidence.json",
        "rawBefore": "fail",
        "rawLatest": "fail",
        "transcript": "../2026-10-05-full-model-comparison/claude/transcripts/knowledge/K04.md"
      }
    },
    {
      "key": [
        "claude",
        "workflow-router",
        "W06"
      ],
      "review": {
        "model": "claude",
        "block": "workflow-router",
        "case": "W06",
        "category": "judge summary-evidence gap",
        "reason": "The visible seeded agenda, current recipient summary, explicit confirmation and single accepted agenda-email capture satisfy the current summary requirement; reprinting the entire agenda was not required.",
        "sourceEvidence": "review-notes.md",
        "rawBefore": "fail",
        "rawLatest": "partial",
        "transcript": "../2026-10-05-ai-defect-retest/round2/claude/transcripts/workflow-router/W06.md"
      }
    },
    {
      "key": [
        "claude",
        "flow-selector",
        "W06"
      ],
      "review": {
        "model": "claude",
        "block": "flow-selector",
        "case": "W06",
        "category": "judge summary-evidence gap",
        "reason": "The visible seeded agenda, current recipient summary, explicit confirmation and single accepted agenda-email capture satisfy the current summary requirement; reprinting the entire agenda was not required.",
        "sourceEvidence": "review-notes.md",
        "rawBefore": "fail",
        "rawLatest": "partial",
        "transcript": "../2026-10-05-ai-defect-retest/round2/claude/transcripts/flow-selector/W06.md"
      }
    },
    {
      "key": [
        "openai",
        "feedback",
        "F06"
      ],
      "review": {
        "model": "openai",
        "block": "feedback",
        "id": "F06",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The transcript compares current passes, retains quantity and known fields, then advances to a recommendation and Day 1 choice. The judge penalises an initial question asked before any help request while acknowledging that it is acceptable.",
        "transcript": "round5/openai/transcripts/feedback/F06.md",
        "sourceEvidence": "",
        "round": "round5"
      }
    },
    {
      "key": [
        "openai",
        "flow-selector",
        "REGRESSION-R06"
      ],
      "review": {
        "model": "openai",
        "block": "flow-selector",
        "id": "REGRESSION-R06",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Generic floorplan questions use Support under the approved policy. No published floorplan was found; claiming it is categorically non-public is unsupported. Clear stand-builder/exhibitor queries separately use Customer Success.",
        "transcript": "round2/openai/transcripts/flow-selector/REGRESSION-R06.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "knowledge",
        "K06"
      ],
      "review": {
        "model": "openai",
        "block": "knowledge",
        "id": "K06",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Ticket transfers use Support under the approved contact policy. The current answer preserves availability and Terms & Conditions.",
        "transcript": "round2/openai/transcripts/knowledge/K06.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "knowledge",
        "K07"
      ],
      "review": {
        "model": "openai",
        "block": "knowledge",
        "id": "K07",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Ticket upgrades use Support under the approved contact policy; availability is conditional and no fee was invented.",
        "transcript": "round2/openai/transcripts/knowledge/K07.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "knowledge",
        "K14"
      ],
      "review": {
        "model": "openai",
        "block": "knowledge",
        "id": "K14",
        "beforeCycleRaw": "fail",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Invoice/payment enquiries are general ticket support; the former Customer Success requirement is obsolete.",
        "transcript": "round2/openai/transcripts/knowledge/K14.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C01"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C01",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The four required visitor fields, reviewed summary and final confirmation precede an accepted 200 handoff. Mandatory company size, seniority and automatic free HR Leader qualification are obsolete/unmaintained requirements. The literal request to tell the visitor about their company is subject to the current personal-data policy.",
        "transcript": "round2/openai/transcripts/original-uat/C01.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C04"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C04",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The response matches the captured structured prices: EUR 3,195 + VAT General Attendee; EUR 5,495 + VAT Diamond; Explorer price on application. Evaluator lack of source access is not evidence of invention.",
        "transcript": "round2/openai/transcripts/original-uat/C04.md",
        "sourceEvidence": "pricing-source-evidence.json",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C05"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C05",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "This is an informational group-ticket question. Requiring lead qualification and consent before a requested handoff conflicts with the approved contract.",
        "transcript": "round2/openai/transcripts/original-uat/C05.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C06"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C06",
        "beforeCycleRaw": "blocked",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Attend and the published General Attendee page are valid registration destinations. Requiring an additional external checkout URL is not in the contract. All required fields, summary and consent precede exactly one accepted 200 handoff.",
        "transcript": "round5/openai/transcripts/original-uat/C06.md",
        "sourceEvidence": "",
        "round": "round5"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C10"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C10",
        "beforeCycleRaw": "fail",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Current contract reuses company/role and requires name, work email, company and role, with one final summary confirmation. Mandatory discrete company size/location/industry fields and their collection order are obsolete. Accepted 200 evidence is retained.",
        "transcript": "round3/openai/transcripts/original-uat/C10.md",
        "sourceEvidence": "",
        "round": "round3"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C26"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C26",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The requested report format is preserved with real published destinations. The visitor did not require recent reports; an evaluator preference for newer reports is not a failed freshness contract.",
        "transcript": "round3/openai/transcripts/original-uat/C26.md",
        "sourceEvidence": "",
        "round": "round3"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C42"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C42",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "A generic attendee-list request is refused without disclosure. Support is correct in the absence of clear sponsor/exhibitor context; the test assumed a commercial context.",
        "transcript": "round2/openai/transcripts/original-uat/C42.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C53"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C53",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Employer-side HR practitioner guidance and application review are preserved. Specific eligibility thresholds are not published; mandatory lead qualification for information is obsolete.",
        "transcript": "round3/openai/transcripts/original-uat/C53.md",
        "sourceEvidence": "",
        "round": "round3"
      }
    },
    {
      "key": [
        "openai",
        "original-uat",
        "C61"
      ],
      "review": {
        "model": "openai",
        "block": "original-uat",
        "id": "C61",
        "beforeCycleRaw": "blocked",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The visitor explicitly requested only an exhibition stand. The published sponsorship/exhibition destination, four fields, complete summary and consent precede one accepted 200 handoff. Adding ticket choice is an irrelevant simulator instruction.",
        "transcript": "round5/openai/transcripts/original-uat/C61.md",
        "sourceEvidence": "",
        "round": "round5"
      }
    },
    {
      "key": [
        "openai",
        "regression",
        "R05"
      ],
      "review": {
        "model": "openai",
        "block": "regression",
        "id": "R05",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The current approved general contact is Support; the former additional events@unleash.ai requirement is obsolete.",
        "transcript": "round2/openai/transcripts/regression/R05.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "regression",
        "R06"
      ],
      "review": {
        "model": "openai",
        "block": "regression",
        "id": "R06",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Generic floorplan Support routing is approved; no unsupported city map or claim of public availability is made. A mandatory Customer Success route presupposes absent SPEX context.",
        "transcript": "round2/openai/transcripts/regression/R06.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "regression",
        "R09"
      ],
      "review": {
        "model": "openai",
        "block": "regression",
        "id": "R09",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Only Support is required for this general contact question under the current policy; the additional events address is obsolete.",
        "transcript": "round3/openai/transcripts/regression/R09.md",
        "sourceEvidence": "",
        "round": "round3"
      }
    },
    {
      "key": [
        "openai",
        "regression",
        "R29"
      ],
      "review": {
        "model": "openai",
        "block": "regression",
        "id": "R29",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The current event dates are correct. The user asks for dates; an extra weekday display is an evaluator formatting preference, not an incorrect date.",
        "transcript": "round2/openai/transcripts/regression/R29.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "ticket-precision",
        "T02"
      ],
      "review": {
        "model": "openai",
        "block": "ticket-precision",
        "id": "T02",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The response uses the current structured Diamond price, Explorer application status and a concrete preference question. Only the old EUR 4,995 price expectation remains invalid.",
        "transcript": "round2/openai/transcripts/ticket-precision/T02.md",
        "sourceEvidence": "pricing-source-evidence.json",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "ticket-precision",
        "T05"
      ],
      "review": {
        "model": "openai",
        "block": "ticket-precision",
        "id": "T05",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The unit price EUR 3,195 + VAT is verified in the captured structured source. The expected EUR 2,995 and its total are obsolete.",
        "transcript": "round2/openai/transcripts/ticket-precision/T05.md",
        "sourceEvidence": "pricing-source-evidence.json",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "uat-additions",
        "C105"
      ],
      "review": {
        "model": "openai",
        "block": "uat-additions",
        "id": "C105",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Amanda Poole is listed in the current Workhuman Forum keynote. The full current seven-speaker list matches the source; the historical six-speaker fixture is stale.",
        "transcript": "round2/openai/transcripts/uat-additions/C105.md",
        "sourceEvidence": "agenda-source-evidence.json",
        "round": "round2"
      }
    },
    {
      "key": [
        "openai",
        "uat-additions",
        "C111"
      ],
      "review": {
        "model": "openai",
        "block": "uat-additions",
        "id": "C111",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Accepted ticket handoff with all four required fields, accurate summary and final consent. Name-first ordering is not required by the current conversational contract.",
        "transcript": "round2/openai/transcripts/uat-additions/C111.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "claude",
        "flow-selector",
        "REGRESSION-R06"
      ],
      "review": {
        "model": "claude",
        "block": "flow-selector",
        "id": "REGRESSION-R06",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Generic floorplan questions use Support under the approved policy. No published floorplan was found; claiming it is categorically non-public is unsupported. Clear stand-builder/exhibitor queries separately use Customer Success.",
        "transcript": "round2/claude/transcripts/flow-selector/REGRESSION-R06.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "claude",
        "handoff-regression",
        "F06"
      ],
      "review": {
        "model": "claude",
        "block": "handoff-regression",
        "id": "F06",
        "beforeCycleRaw": "fail",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The transcript compares current passes, retains quantity and known fields, then advances to a recommendation and Day 1 choice. The judge penalises an initial question asked before any help request while acknowledging that it is acceptable.",
        "transcript": "round5/claude/transcripts/handoff-regression/F06.md",
        "sourceEvidence": "",
        "round": "round5"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C01"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "id": "C01",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The four required visitor fields, reviewed summary and final confirmation precede an accepted 200 handoff. Mandatory company size, seniority and automatic free HR Leader qualification are obsolete/unmaintained requirements. The literal request to tell the visitor about their company is subject to the current personal-data policy.",
        "transcript": "round2/claude/transcripts/original-uat/C01.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C04"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "id": "C04",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The response matches the captured structured prices: EUR 3,195 + VAT General Attendee; EUR 5,495 + VAT Diamond; Explorer price on application. Evaluator lack of source access is not evidence of invention.",
        "transcript": "round2/claude/transcripts/original-uat/C04.md",
        "sourceEvidence": "pricing-source-evidence.json",
        "round": "round2"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C06"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "id": "C06",
        "beforeCycleRaw": "blocked",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Attend and the published General Attendee page are valid registration destinations. Requiring an additional external checkout URL is not in the contract. All required fields, summary and consent precede exactly one accepted 200 handoff.",
        "transcript": "round5/claude/transcripts/original-uat/C06.md",
        "sourceEvidence": "",
        "round": "round5"
      }
    },
    {
      "key": [
        "claude",
        "original-uat",
        "C10"
      ],
      "review": {
        "model": "claude",
        "block": "original-uat",
        "id": "C10",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Current contract reuses company/role and requires name, work email, company and role, with one final summary confirmation. Mandatory discrete company size/location/industry fields and their collection order are obsolete. Accepted 200 evidence is retained.",
        "transcript": "round2/claude/transcripts/original-uat/C10.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "claude",
        "regression",
        "R06"
      ],
      "review": {
        "model": "claude",
        "block": "regression",
        "id": "R06",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Generic floorplan Support routing is approved; no unsupported city map or claim of public availability is made. A mandatory Customer Success route presupposes absent SPEX context.",
        "transcript": "round2/claude/transcripts/regression/R06.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    },
    {
      "key": [
        "claude",
        "regression",
        "R09"
      ],
      "review": {
        "model": "claude",
        "block": "regression",
        "id": "R09",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Only Support is required for this general contact question under the current policy; the additional events address is obsolete.",
        "transcript": "round3/claude/transcripts/regression/R09.md",
        "sourceEvidence": "",
        "round": "round3"
      }
    },
    {
      "key": [
        "claude",
        "ticket-precision",
        "F06"
      ],
      "review": {
        "model": "claude",
        "block": "ticket-precision",
        "id": "F06",
        "beforeCycleRaw": "partial",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The transcript compares current passes, retains quantity and known fields, then advances to a recommendation and Day 1 choice. The judge penalises an initial question asked before any help request while acknowledging that it is acceptable.",
        "transcript": "round5/claude/transcripts/ticket-precision/F06.md",
        "sourceEvidence": "",
        "round": "round5"
      }
    },
    {
      "key": [
        "claude",
        "ticket-precision",
        "T02"
      ],
      "review": {
        "model": "claude",
        "block": "ticket-precision",
        "id": "T02",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The response uses the current structured Diamond price, Explorer application status and a concrete preference question. Only the old EUR 4,995 price expectation remains invalid.",
        "transcript": "round2/claude/transcripts/ticket-precision/T02.md",
        "sourceEvidence": "pricing-source-evidence.json",
        "round": "round2"
      }
    },
    {
      "key": [
        "claude",
        "ticket-precision",
        "T05"
      ],
      "review": {
        "model": "claude",
        "block": "ticket-precision",
        "id": "T05",
        "beforeCycleRaw": "fail",
        "latestRaw": "fail",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "The unit price EUR 3,195 + VAT is verified in the captured structured source. The expected EUR 2,995 and its total are obsolete.",
        "transcript": "round2/claude/transcripts/ticket-precision/T05.md",
        "sourceEvidence": "pricing-source-evidence.json",
        "round": "round2"
      }
    },
    {
      "key": [
        "claude",
        "uat-additions",
        "C114"
      ],
      "review": {
        "model": "claude",
        "block": "uat-additions",
        "id": "C114",
        "beforeCycleRaw": "fail",
        "latestRaw": "partial",
        "category": "evaluator_or_obsolete_expectation",
        "reason": "Known company is reused, quantity is asked before it has been supplied, and final summary/consent precede one accepted 200 handoff. An information-only turn need not end with qualification or a post-send repeat summary.",
        "transcript": "round2/claude/transcripts/uat-additions/C114.md",
        "sourceEvidence": "",
        "round": "round2"
      }
    }
  ]
}