Evaluation status register

export type EvaluationConfidence = "live_verified" | "live_guarded" | "needs_live_retest";
export type CoreControl = "implemented" | "guarded" | "needs_validation";

export const EVALUATION_AS_OF = "2026-08-24";

export const TOOL_EVALUATION: Record<string, {
  confidence: EvaluationConfidence;
  rationale: string;
  proof: string;
}> = {
  get_catalog_status: {
    confidence: "needs_live_retest",
    rationale: "The MCP implementation is fixture-tested; the new authenticated operator action still needs preview validation.",
    proof: "Live health confirms the MCP is online and credentialed, but health is not a substitute for a current get_catalog_status result.",
  },
  search_parts_for_vehicle: {
    confidence: "live_verified",
    rationale: "The live vehicle and qualifier path returns catalog rows with visible completeness.",
    proof: "1997 Ford Mustang / Disc Brake Pad Set returned 208 matches, 50 rows, and a material Sub Model clarification.",
  },
  get_part_fitments: {
    confidence: "live_guarded",
    rationale: "Applications are real and qualified, but the current account returned only 50 of 93 rows without a continuation cursor.",
    proof: "Bosch 9619 returned 50 application rows and total 93; the response correctly remained partial.",
  },
  verify_part_fitment: {
    confidence: "needs_live_retest",
    rationale: "Whole-token product-line resolution and details enrichment were repaired after the live GMB failure.",
    proof: "The previous live GMB 730-0003 run returned 12 upstream rows and then discarded them; preview retest is required.",
  },
  lookup_part_interchanges: {
    confidence: "needs_live_retest",
    rationale: "Reverse interchange recovery and target enrichment were added after the source-identity failure.",
    proof: "The previous Kia run returned nine upstream rows and then zero results; Akebono returned ten direct relationships.",
  },
  lookup_part_supersession: {
    confidence: "needs_live_retest",
    rationale: "Planner routing and separately labeled interchange fallback were repaired; a domain-approved explicit chain is still required.",
    proof: "The previous GM run returned 29 upstream rows but no source identity, and noun-form “supersession” queries could route to details.",
  },
  decode_vin_and_search: {
    confidence: "needs_live_retest",
    rationale: "VIN decoding was live-verified, but the HVAC-to-cabin-filter continuation was repaired afterward.",
    proof: "VIN 4T1BK1EB5GU235558 decoded to a 2016 Toyota Avalon but previously skipped the requested part search.",
  },
  get_part_details: {
    confidence: "live_guarded",
    rationale: "The live tool returns details; the repaired UI now needs confirmation for attribute values, units, and renderable asset classification.",
    proof: "Bosch 9619 returned 31 attributes and catalog resources, but the previous UI exposed only counts and a generic URL as an image.",
  },
};

export function toolEvaluationCounts() {
  const values = Object.values(TOOL_EVALUATION);
  return {
    total: values.length,
    verified: values.filter((item) => item.confidence === "live_verified").length,
    guarded: values.filter((item) => item.confidence === "live_guarded").length,
    retest: values.filter((item) => item.confidence === "needs_live_retest").length,
  };
}

const CORPUS_RATIONALE = "codified from the eval corpus; awaiting fixture/live replay proof";

export const CORE_EVALUATION: Record<string, { control: CoreControl; rationale: string }> = {
  "CORE-VIN-01": { control: "needs_validation", rationale: "VIN decode repaired after the live HVAC-continuation failure; preview retest pending." },
  "CORE-VIN-02": { control: "needs_validation", rationale: "Shares the repaired VIN path with CORE-VIN-01; preview retest pending." },
  "CORE-XR-01": { control: "needs_validation", rationale: "Reverse interchange recovery was added after the live Kia zero-result run." },
  "CORE-XR-02": { control: "needs_validation", rationale: "Positive baseline retained; needs a rerun on the repaired interchange path." },
  "CORE-FV-01": { control: "needs_validation", rationale: "Whole-token product-line resolution was repaired after the live GMB failure." },
  "CORE-XR-03": { control: "needs_validation", rationale: "Typed interchange output needs a live completeness check against the Akebono set." },
  "CORE-SS-01": { control: "needs_validation", rationale: "Lifecycle supersession routing was repaired; a domain-approved chain is still required." },
  "CORE-SS-02": { control: "needs_validation", rationale: "Explicit replaces/replaced-by labeling needs a live VW-chain confirmation." },
  "CORE-SS-03": { control: "needs_validation", rationale: "Interchange-vs-supersession labeling needs a live GM rerun." },
  "CORE-PART-01": { control: "needs_validation", rationale: "Bounded fitment and budget behavior need a live multi-turn replay." },
  "CORE-VEH-01": { control: "needs_validation", rationale: "Job-completion coverage beyond pumps and gasket is not yet demonstrated." },
  "CORE-VEH-02": { control: "needs_validation", rationale: "Brand-completeness disclosure needs a live rerun of the novice flow." },
  "CORE-PD-01": { control: "needs_validation", rationale: "Attribute unit fidelity needs confirmation in the repaired details UI." },
  "CORE-PD-02": { control: "needs_validation", rationale: "Direct asset delivery from part details needs a live image render check." },
  "CORE-REL-01": { control: "guarded", rationale: "Closed-world completeness states are enforced; guarded pending broader replay." },
  "CORE-SYM-01": { control: "guarded", rationale: "Symptom-side recommendations stay behind the qualifier guardrail." },
  "CORE-FD-09": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-FD-11": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-FD-12": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-FD-13": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-FD-14": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-FD-15": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-FD-16": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-CC-02": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-CC-03": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-01": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-02": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-03": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-04": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-05": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-06": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-07": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-08": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-WE-10": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-KJ-01": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-01": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-02": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-03": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-04": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-05": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-07": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-08": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-10": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-11": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-13": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-14": { control: "needs_validation", rationale: CORPUS_RATIONALE },
  "CORE-BT-15": { control: "needs_validation", rationale: CORPUS_RATIONALE },
};

export function coreEvaluationCases() {
  // Same cohort boundary as coreEvaluationCounts: the adjudicated 16-case
  // comparison set, excluding corpus-derived workbench rows.
  return Object.entries(CORE_EVALUATION)
    .filter(([, entry]) => entry.rationale !== CORPUS_RATIONALE)
    .map(([id, entry]) => ({ id, control: entry.control, rationale: entry.rationale }));
}

export function coreEvaluationCounts() {
  // The executive scorecard is the original 16-case comparison cohort. The
  // corpus-derived rows support the broader workbench, but are not part of
  // that adjudicated comparison count.
  const values = Object.values(CORE_EVALUATION).filter((entry) => entry.rationale !== CORPUS_RATIONALE);
  return {
    total: values.length,
    implemented: values.filter((entry) => entry.control === "implemented").length,
    guarded: values.filter((entry) => entry.control === "guarded").length,
    needsValidation: values.filter((entry) => entry.control === "needs_validation").length,
  };
}