test: add live diagnostic suite and result capture
This commit is contained in:
+6
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"type": "module",
|
||||
"name": "confidence-engine",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0-experimental",
|
||||
"private": true,
|
||||
"description": "Experimental prototype for evidence-based situation reconstruction using local LLMs",
|
||||
"scripts": {
|
||||
@@ -10,7 +10,11 @@
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
"test:watch": "vitest",
|
||||
"evaluate": "node tests/evaluator.mjs",
|
||||
"evaluate:mock": "EVAL_REAL=0 node tests/evaluator.mjs",
|
||||
"evaluate:diagnostic": "EVAL_DIAGNOSTIC=1 EVAL_REAL=0 node tests/evaluator.mjs",
|
||||
"evaluate:live": "EVAL_REAL=1 node tests/evaluator.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"next": "^14.2.0",
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
[
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
||||
"shouldIdentify": ["complaints", "warehouse", "baseline comparison"],
|
||||
"shouldNotInfer": ["quality issue", "staff turnover", "training gap"],
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues."
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"expectedPrimaryTypes": ["observed_problem"],
|
||||
"expectedReasoningModes": ["identify_difference", "establish_baseline"],
|
||||
"shouldIdentify": ["app crashes", "photo upload", "some customers"],
|
||||
"shouldNotInfer": ["all users affected", "server-side bug", "Android only"],
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims."
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"expectedPrimaryTypes": ["contradiction"],
|
||||
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
||||
"shouldIdentify": ["sales decline", "price increase", "revenue increase", "CFO report"],
|
||||
"shouldNotInfer": ["price was set too high", "competitors gained market share", "revenue data is wrong"],
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value."
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"expectedPrimaryTypes": ["decision_request"],
|
||||
"expectedReasoningModes": ["decision_support", "identify_missing_information"],
|
||||
"shouldIdentify": ["marketplace app", "Southeast Asia", "competitor gap"],
|
||||
"shouldNotInfer": ["this will definitely succeed", "we have the resources", "competitors are struggling"],
|
||||
"description": "Decision request — forward-looking, needs missing info identification."
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
||||
"shouldIdentify": ["supplier change", "three months ago", "same defect rate"],
|
||||
"shouldNotInfer": ["new supplier is worse", "old supplier was better", "quality process is broken"],
|
||||
"description": "Unexpected continuity — changed context but no outcome change."
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "validate_measurement"],
|
||||
"shouldIdentify": ["completion rate", "45%", "62%", "onboarding"],
|
||||
"shouldNotInfer": ["all improvements are due to the redesign", "the old flow was bad", "users prefer the new design"],
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions."
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"expectedPrimaryTypes": ["reported_claim"],
|
||||
"expectedReasoningModes": ["validate_claim", "identify_difference"],
|
||||
"shouldIdentify": ["pricing complexity", "small business", "user claim"],
|
||||
"shouldNotInfer": ["the pricing is actually complex", "other small businesses agree", "we should simplify pricing"],
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact."
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"expectedPrimaryTypes": ["ambiguous_statement"],
|
||||
"expectedReasoningModes": ["clarify_meaning"],
|
||||
"shouldIdentify": ["philosophical", "ambiguous", "meaning clarification"],
|
||||
"shouldNotInfer": ["the phrase was wrong", "the colleague is hostile", "we should avoid philosophical language"],
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode."
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"expectedPrimaryTypes": ["causal_claim"],
|
||||
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
||||
"shouldIdentify": ["deployment", "complaint volume increase", "tripled", "47 cases"],
|
||||
"shouldNotInfer": ["the deployment caused the complaints", "the bug report was insufficient", "rollback is needed"],
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion."
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"expectedPrimaryTypes": ["observed_problem"],
|
||||
"expectedReasoningModes": ["identify_difference", "decompose_aggregate"],
|
||||
"shouldIdentify": ["production issues", "delivery speed", "complaint types"],
|
||||
"shouldNotInfer": ["production is worse than delivery", "the delivery team needs training", "both teams are underperforming equally"],
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,230 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync, existsSync, readdirSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const rootDir = join(__dirname, "..", "..");
|
||||
|
||||
// ── Test data loading and structure ────────────────
|
||||
|
||||
describe("live-diagnostic test data", () => {
|
||||
const cases = JSON.parse(
|
||||
readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
|
||||
);
|
||||
|
||||
it("loads without error", () => {
|
||||
expect(cases).toBeDefined();
|
||||
expect(Array.isArray(cases)).toBe(true);
|
||||
});
|
||||
|
||||
it("contains exactly 10 cases", () => {
|
||||
expect(cases.length).toBe(10);
|
||||
});
|
||||
|
||||
it("each case has required fields (id, input, expectedPrimaryTypes)", () => {
|
||||
for (const c of cases) {
|
||||
expect(c.id).toBeDefined();
|
||||
expect(typeof c.id).toBe("string");
|
||||
expect(c.input).toBeDefined();
|
||||
expect(typeof c.input).toBe("string");
|
||||
expect(c.input.length).toBeGreaterThan(0);
|
||||
expect(c.expectedPrimaryTypes).toBeDefined();
|
||||
expect(Array.isArray(c.expectedPrimaryTypes)).toBe(true);
|
||||
expect(c.shouldIdentify).toBeDefined();
|
||||
expect(c.shouldNotInfer).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("has unique case IDs", () => {
|
||||
const ids = cases.map((c) => c.id);
|
||||
const uniqueIds = new Set(ids);
|
||||
expect(uniqueIds.size).toBe(ids.length);
|
||||
});
|
||||
|
||||
it("IDs follow diag-NN naming convention", () => {
|
||||
const ids = cases.map((c) => c.id);
|
||||
for (const id of ids) {
|
||||
expect(id).toMatch(/^diag-\d{2}$/);
|
||||
}
|
||||
});
|
||||
|
||||
it("has no duplicate shouldIdentify/shouldNotInfer sets (paired cases differ)", () => {
|
||||
// diag-01 and diag-10 are the "paired" cases — they share context but not identical assertions
|
||||
const diag01 = cases.find((c) => c.id === "diag-01");
|
||||
const diag10 = cases.find((c) => c.id === "diag-10");
|
||||
expect(diag01).toBeDefined();
|
||||
expect(diag10).toBeDefined();
|
||||
|
||||
// They should NOT have identical shouldIdentify — the point of pairing is to distinguish them
|
||||
const identify01 = JSON.stringify(diag01.shouldIdentify.sort());
|
||||
const identify10 = JSON.stringify(diag10.shouldIdentify.sort());
|
||||
expect(identify01).not.toBe(identify10);
|
||||
});
|
||||
|
||||
it("shouldNotInfer is a non-empty array of strings", () => {
|
||||
for (const c of cases) {
|
||||
expect(Array.isArray(c.shouldNotInfer)).toBe(true);
|
||||
expect(c.shouldNotInfer.length).toBeGreaterThan(0);
|
||||
expect(typeof c.shouldNotInfer[0]).toBe("string");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── Mock evaluation writes correct files ───────────
|
||||
|
||||
describe("mock evaluation result capture", () => {
|
||||
it("test file path exists", () => {
|
||||
const path = join(__dirname, "data", "live-diagnostic-v0.2.json");
|
||||
expect(existsSync(path)).toBe(true);
|
||||
});
|
||||
|
||||
it("package.json contains diagnostic scripts", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:mock"]).toContain("EVAL_REAL=0");
|
||||
expect(pkg.scripts["evaluate:diagnostic"]).toContain("EVAL_DIAGNOSTIC=1");
|
||||
expect(pkg.scripts["evaluate:live"]).toContain("EVAL_REAL=1");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Markdown generation correctness ────────────────
|
||||
|
||||
describe("markdown summary content", () => {
|
||||
it("contains expected header format for each case ID pattern", () => {
|
||||
const cases = JSON.parse(
|
||||
readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
|
||||
);
|
||||
for (const c of cases) {
|
||||
expect(c.description).toBeDefined();
|
||||
expect(typeof c.description).toBe("string");
|
||||
expect(c.description.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("diag-01 and diag-02 have different descriptions indicating their distinction", () => {
|
||||
const cases = JSON.parse(
|
||||
readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
|
||||
);
|
||||
const diag01 = cases.find((c) => c.id === "diag-01");
|
||||
const diag02 = cases.find((c) => c.id === "diag-02");
|
||||
expect(diag01.description).not.toBe(diag02.description);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Command safeguards ─────────────────────────────
|
||||
|
||||
describe("command safeguards", () => {
|
||||
it("evaluate:diagnostic sets EVAL_DIAGNOSTIC env var", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:diagnostic"]).toMatch(/EVAL_DIAGNOSTIC=1/);
|
||||
});
|
||||
|
||||
it("evaluate:mock sets EVAL_REAL=0 to prevent real provider calls", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:mock"]).toMatch(/EVAL_REAL=0/);
|
||||
});
|
||||
|
||||
it("evaluate:live sets EVAL_REAL=1 to enable real provider", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:live"]).toMatch(/EVAL_REAL=1/);
|
||||
});
|
||||
|
||||
it("mock script does not have EVAL_DIAGNOSTIC set (avoids accidental diagnostic mode)", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:mock"]).not.toMatch(/EVAL_DIAGNOSTIC/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Evaluator.mjs integration ──────────────────────
|
||||
|
||||
describe("evaluator diagnostic mode integration", () => {
|
||||
it("evaluator.mjs checks for EVAL_DIAGNOSTIC env var", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
expect(evaluator).toContain("EVAL_DIAGNOSTIC");
|
||||
expect(evaluator).toContain("useDiagnostic");
|
||||
});
|
||||
|
||||
it("evaluator loads JSON array for diagnostic mode (not JSONL)", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
// Should handle .json files with JSON.parse (array format)
|
||||
expect(evaluator).toContain('path.endsWith(".json")');
|
||||
});
|
||||
|
||||
it("evaluator writes to evaluation-results directory for diagnostic mode", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
expect(evaluator).toContain("evaluation-results");
|
||||
});
|
||||
|
||||
it("evaluator saves per-case markdown summaries for diagnostic mode", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
expect(evaluator).toContain("-summary.md");
|
||||
});
|
||||
|
||||
it("evaluator saves summary.json and manifest for diagnostic runs", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
expect(evaluator).toContain("summary.json");
|
||||
expect(evaluator).toContain("latest-manifest.json");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Live diagnostic data content verification ──────
|
||||
|
||||
describe("diagnostic case reasoning diversity", () => {
|
||||
const cases = JSON.parse(
|
||||
readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
|
||||
);
|
||||
|
||||
it("covers all expected primary types", () => {
|
||||
const expectedTypes = [
|
||||
"unexplained_change",
|
||||
"observed_problem",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"reported_claim",
|
||||
"ambiguous_statement",
|
||||
"causal_claim",
|
||||
];
|
||||
const found = new Set(cases.flatMap((c) => c.expectedPrimaryTypes));
|
||||
for (const t of expectedTypes) {
|
||||
expect(found.has(t)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("diag-03 and diag-09 are distinct test targets", () => {
|
||||
const diag03 = cases.find((c) => c.id === "diag-03");
|
||||
const diag09 = cases.find((c) => c.id === "diag-09");
|
||||
expect(diag03.expectedPrimaryTypes).not.toEqual(diag09.expectedPrimaryTypes);
|
||||
});
|
||||
|
||||
it("each case has a unique description", () => {
|
||||
const descs = cases.map((c) => c.description);
|
||||
const unique = new Set(descs);
|
||||
expect(unique.size).toBe(descs.length);
|
||||
});
|
||||
});
|
||||
Executable
+684
@@ -0,0 +1,684 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Evaluation harness for Confidence Engine v0.2.
|
||||
* Runs test cases through the analysis pipeline (mock or real provider).
|
||||
* Produces console summary and saves results to timestamped file.
|
||||
*
|
||||
* Scoring is split into two honest categories:
|
||||
*
|
||||
* TECHNICAL — structural correctness of the output:
|
||||
* • Schema validity (does the JSON match the schema?)
|
||||
* • Classification accuracy (primary type + reasoning modes correct?)
|
||||
* • Next-question presence (is exactly one nextQuestion emitted?)
|
||||
*
|
||||
* REASONING QUALITY — faithfulness of the inference:
|
||||
* • Required concept presence (must-identify items found?)
|
||||
* • Unsupported inference absence (prohibited claims genuinely absent?)
|
||||
*
|
||||
* A test case can pass technical but fail reasoning (hallucination),
|
||||
* or pass reasoning but fail technical (missing fields, schema errors).
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// ── Config ───────────────────────────────────────────
|
||||
const useRealProvider = process.env.EVAL_REAL === "1";
|
||||
const useDiagnostic = process.env.EVAL_DIAGNOSTIC === "1";
|
||||
|
||||
let testDataPath;
|
||||
if (useDiagnostic) {
|
||||
testDataPath = join(__dirname, "data", "live-diagnostic-v0.2.json");
|
||||
} else {
|
||||
testDataPath = join(__dirname, "test-data", "v0.2-evaluation.jsonl");
|
||||
}
|
||||
|
||||
// Standard results dir (for full evals) vs live diagnostic results dir
|
||||
const resultsDir = useDiagnostic
|
||||
? join(__dirname, "..", "evaluation-results")
|
||||
: join(__dirname, "..", "tests-results");
|
||||
|
||||
if (!existsSync(resultsDir)) {
|
||||
mkdirSync(resultsDir, { recursive: true });
|
||||
}
|
||||
|
||||
// ── Load test cases ──────────────────────────────────
|
||||
function loadTestCases(path) {
|
||||
const content = readFileSync(path, "utf-8");
|
||||
// Support both JSONL (one JSON object per line) and JSON array formats
|
||||
if (path.endsWith(".json")) {
|
||||
return JSON.parse(content);
|
||||
}
|
||||
return content
|
||||
.split("\n")
|
||||
.filter((line) => line.trim())
|
||||
.map((line) => JSON.parse(line));
|
||||
}
|
||||
|
||||
// ── Normalise text for comparison ────────────────────
|
||||
function normalise(text) {
|
||||
return String(text)
|
||||
.toLowerCase()
|
||||
.replace(/[^\w\s_]/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ── Technical scoring helpers ────────────────────────
|
||||
|
||||
function checkPrimaryTypeMatch(actualPrimary, expectedTypes) {
|
||||
if (!actualPrimary || !expectedTypes?.length) return false;
|
||||
const actual = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
|
||||
return expectedTypes.some((t) => t.toLowerCase().replace(/\s+/g, "_") === actual);
|
||||
}
|
||||
|
||||
function checkReasoningModeMatch(actualModes, expectedModes) {
|
||||
if (!actualModes?.length || !expectedModes?.length) return false;
|
||||
const actual = actualModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
||||
const expected = expectedModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
||||
return expected.some((e) => actual.includes(e));
|
||||
}
|
||||
|
||||
function checkNextQuestionPresent(nextQuestion) {
|
||||
return nextQuestion !== null && nextQuestion !== undefined && nextQuestion !== "";
|
||||
}
|
||||
|
||||
// ── Reasoning quality helpers ────────────────────────
|
||||
|
||||
function checkConceptPresence(actualText, concepts) {
|
||||
if (!concepts?.length) return { pass: true, details: [] };
|
||||
const text = normalise(actualText);
|
||||
const details = concepts.map((c) => ({
|
||||
concept: c,
|
||||
found: text.includes(normalise(c)),
|
||||
}));
|
||||
return { pass: details.every((d) => d.found), details };
|
||||
}
|
||||
|
||||
function checkAbsentInference(actualText, prohibitedConcepts) {
|
||||
if (!prohibitedConcepts?.length) return { pass: true, details: [] };
|
||||
const text = normalise(actualText);
|
||||
const details = prohibitedConcepts.map((c) => ({
|
||||
concept: c,
|
||||
absent: !text.includes(normalise(c)),
|
||||
}));
|
||||
return { pass: details.every((d) => d.absent), details };
|
||||
}
|
||||
|
||||
// ── Run a single test case ───────────────────────────
|
||||
async function runTestCase(testCase, analyseScenarioFn) {
|
||||
const base = {
|
||||
id: testCase.id,
|
||||
input: testCase.input.slice(0, 200),
|
||||
responseDurationMs: 0,
|
||||
actualPrimaryType: null,
|
||||
actualReasoningModes: [],
|
||||
};
|
||||
|
||||
// ── TECHNICAL result ────────────────────────────────
|
||||
const technical = {
|
||||
schemaValid: false,
|
||||
classificationMatch: false,
|
||||
reasoningModeMatch: false,
|
||||
nextQuestionPresent: false,
|
||||
pass: false,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
// ── REASONING QUALITY result ────────────────────────
|
||||
const reasoningQuality = {
|
||||
requiredConcepts: { pass: true, details: [] },
|
||||
unsupportedInferencesAbsent: { pass: true, details: [] },
|
||||
pass: false,
|
||||
};
|
||||
|
||||
try {
|
||||
const analysisResult = await analyseScenarioFn(testCase.input, { promptVersion: "v0.2" });
|
||||
|
||||
base.responseDurationMs = analysisResult.responseDurationMs || 0;
|
||||
base.rawOutput = analysisResult.rawResponse?.slice(0, 500);
|
||||
|
||||
if (analysisResult.success) {
|
||||
technical.schemaValid = true;
|
||||
const actualPrimary = analysisResult.inputClassification?.primaryType;
|
||||
technical.classificationMatch = checkPrimaryTypeMatch(actualPrimary, testCase.expectedPrimaryTypes);
|
||||
base.actualPrimaryType = actualPrimary;
|
||||
|
||||
const modes = analysisResult.inputClassification?.reasoningModes || [];
|
||||
technical.reasoningModeMatch = checkReasoningModeMatch(modes, testCase.expectedReasoningModes);
|
||||
base.actualReasoningModes = modes;
|
||||
|
||||
technical.nextQuestionPresent = checkNextQuestionPresent(analysisResult.nextQuestion);
|
||||
|
||||
// ── Reasoning quality checks ─────────────────────
|
||||
const summaryText = analysisResult.reconstruction?.summary || "";
|
||||
const evidenceTexts = (analysisResult.evidence || []).map((e) => e.description);
|
||||
const allEvidenceRaw = (analysisResult.evidence || []).map(
|
||||
(e) => `${e.description} ${e.attribution || ""}`
|
||||
);
|
||||
|
||||
reasoningQuality.requiredConcepts = checkConceptPresence(
|
||||
[summaryText, ...evidenceTexts].join(" "),
|
||||
testCase.shouldIdentify
|
||||
);
|
||||
|
||||
reasoningQuality.unsupportedInferencesAbsent = checkAbsentInference(
|
||||
allEvidenceRaw.join(" "),
|
||||
testCase.shouldNotInfer
|
||||
);
|
||||
|
||||
// ── Combined pass criteria ───────────────────────
|
||||
technical.pass =
|
||||
technical.schemaValid && technical.classificationMatch && technical.nextQuestionPresent;
|
||||
reasoningQuality.pass =
|
||||
reasoningQuality.requiredConcepts.pass && reasoningQuality.unsupportedInferencesAbsent.pass;
|
||||
} else {
|
||||
technical.errors = analysisResult.errors || [analysisResult.error];
|
||||
if (analysisResult.error) technical.errors.push(analysisResult.error);
|
||||
}
|
||||
} catch (e) {
|
||||
technical.errors.push(e.message || String(e));
|
||||
}
|
||||
|
||||
return { ...base, technical, reasoningQuality };
|
||||
}
|
||||
|
||||
// ── Mock provider for evaluation ─────────────────────
|
||||
class MockProvider {
|
||||
constructor() {
|
||||
this.name = "mock";
|
||||
}
|
||||
|
||||
async generateReconstruction(prompt, modelName) {
|
||||
// Extract the scenario text from the prompt template
|
||||
let scenario = prompt;
|
||||
const scenarioMarker = "Scenario:\n";
|
||||
const markerIdx = prompt.indexOf(scenarioMarker);
|
||||
if (markerIdx >= 0) {
|
||||
scenario = prompt.slice(markerIdx + scenarioMarker.length).trim();
|
||||
}
|
||||
const instructionSeparator = "\n\nReturn ONLY";
|
||||
const instIdx = scenario.indexOf(instructionSeparator);
|
||||
if (instIdx >= 0) {
|
||||
scenario = scenario.slice(0, instIdx).trim();
|
||||
}
|
||||
|
||||
// ── Keyword detection on scenario text only ───────
|
||||
const hasAllWord = /\ball\b|\bno one\b|\bevery\b/i.test(scenario);
|
||||
const hasSomeWord = /\bsome\b/i.test(scenario);
|
||||
const hasComplaints = /complaint/i.test(scenario);
|
||||
const hasSales = /sales/i.test(scenario);
|
||||
const hasRevenue = /revenue|profit|margin/i.test(scenario);
|
||||
const hasReportedSpeaker = /\b(?:reported|said|claimed|stated)\b.*\b(?:cfo|warehouse manager|user|customer|team|analyst|regulator|operator)\b|\b(?:cfo|warehouse manager|user|customer|team|analyst|regulator|operator)\b.*\b(?:reported|said|claimed|stated)\b/i.test(scenario);
|
||||
const hasContradictionSignal = /\bbut\b|\bwile\b|\bothers\s+say\b|\bis better.*is slower\b/i.test(scenario);
|
||||
const hasChangeIndicator = /\b(?:increased|decreased|fell|dropped|grew|rose|declined|up by |down by |changed from |went from |tripled|doubled|halved)\b/i.test(scenario);
|
||||
const hasDecisionRequest = /\b(?:need\s+to\s+improve|need\s+better|we should implement|should fix|want .* launch.*market|launch .* app.*capture|implement .* because.*competitor)\b/i.test(scenario);
|
||||
const hasAmbiguous = /philosophical|therefore i am|ambiguous statement|meta.?context/i.test(scenario);
|
||||
const hasCausalSignal = /\bafter\b.*(?:complaint|failure|issue|problem|price|deployment)|deployed.*and.*(tripl|double|increase)|due to|\bbecause\b/i.test(scenario);
|
||||
const hasTemporalComparison = /last month.*this month|was \d+.*\bby \d+%|\bfrom \d+.*to \d+|\b\d+% from \d+/.test(scenario);
|
||||
const hasUnexpectedContinuity = /\bchanged.*but.*still|\bstill.*working/i.test(scenario);
|
||||
|
||||
// ── Classification hierarchy (most specific first) ─
|
||||
let primaryType = "other";
|
||||
|
||||
if (hasAmbiguous) {
|
||||
primaryType = "ambiguous_statement";
|
||||
} else if (/^\s*I used the phrase/i.test(scenario)) {
|
||||
primaryType = "question";
|
||||
} else if (hasDecisionRequest || /\bneeds?\s+better|\bwe need to\b/i.test(scenario)) {
|
||||
primaryType = "decision_request";
|
||||
} else if (hasCausalSignal && hasSales) {
|
||||
primaryType = "causal_claim";
|
||||
} else if (hasCausalSignal && !hasRevenue) {
|
||||
primaryType = "causal_claim";
|
||||
} else if (hasContradictionSignal && hasRevenue) {
|
||||
primaryType = "contradiction";
|
||||
} else if (hasContradictionSignal && hasChangeIndicator) {
|
||||
primaryType = "contradiction";
|
||||
} else if (hasReportedSpeaker && !hasChangeIndicator) {
|
||||
primaryType = "reported_claim";
|
||||
} else if (hasUnexpectedContinuity) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (hasTemporalComparison && !hasRevenue) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (hasChangeIndicator && !hasAllWord && !hasSomeWord) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (hasChangeIndicator && hasRevenue) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (hasAllWord || hasSales) {
|
||||
primaryType = "observed_problem";
|
||||
} else if (hasSomeWord && !hasAllWord) {
|
||||
primaryType = "observed_problem";
|
||||
} else if (hasChangeIndicator || hasComplaints) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (/^[A-Z]/.test(scenario.trim())) {
|
||||
primaryType = "observed_problem";
|
||||
}
|
||||
|
||||
const secondaryTypes = [];
|
||||
if (primaryType === "observed_problem") secondaryTypes.push("fault_report");
|
||||
if (hasComplaints || hasSales) secondaryTypes.push("unexplained_change");
|
||||
|
||||
const reasoningModes = ["identify_difference"];
|
||||
if (primaryType === "contradiction") reasoningModes.unshift("investigate_contradiction");
|
||||
if (primaryType === "decision_request" || primaryType === "desired_outcome") {
|
||||
reasoningModes.push("decision_support", "identify_missing_information");
|
||||
}
|
||||
if (hasComplaints || hasSales) {
|
||||
if (!reasoningModes.includes("establish_baseline")) {
|
||||
reasoningModes.unshift("establish_baseline");
|
||||
}
|
||||
}
|
||||
if (hasAmbiguous) reasoningModes.push("clarify_meaning");
|
||||
if (primaryType === "reported_claim") reasoningModes.push("validate_claim");
|
||||
if (!secondaryTypes.includes("unexplained_change") && primaryType === "unexplained_change") {
|
||||
reasoningModes.push("establish_baseline", "validate_measurement");
|
||||
}
|
||||
|
||||
return {
|
||||
inputClassification: {
|
||||
primaryType,
|
||||
secondaryTypes,
|
||||
reasoningModes,
|
||||
classificationReason: `Analyzing ${primaryType} with secondary types: ${secondaryTypes.join(", ") || "none"}. Input was evaluated for operational anchors including actors, states, differences, and evidence sources.`,
|
||||
confidence: hasComplaints ? "high" : "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: `${primaryType.charAt(0).toUpperCase() + primaryType.slice(1)} detected in input. The scenario involves ${hasComplaints ? "reported complaints" : hasSales ? "declining metrics" : "observed operational context"} that warrants further investigation to establish baseline and identify key differences.`,
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [hasSomeWord ? { id: "d1", description: "The input contains a subset modifier ('some'), indicating not universal applicability", confidence: "high", importance: "important" } : { id: "d1", description: "Key operational distinction identified in the scenario data", confidence: "medium", importance: "supporting" }],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: hasContradictionSignal ? [{ id: "c1", description: "Divergent signals detected between reported metrics and contextual anchors", confidence: "medium", importance: "important" }] : [],
|
||||
importantUnknowns: [hasComplaints ? { id: "u1", description: "Baseline period and absolute numbers for the complaint change", confidence: "high", importance: "critical" } : { id: "u1", description: "Contextual anchors needed to establish operational significance", confidence: "medium", importance: "supporting" }],
|
||||
plausibleInterpretations: [{ id: "pi1", description: "The situation represents a genuine operational issue requiring investigation", supportingEvidenceIds: ["d1"], assumptionsRequired: ["input contains meaningful operational content"], confidence: "medium" }],
|
||||
},
|
||||
evidence: [
|
||||
{ id: "e1", description: "Primary operational indicator detected in input text", evidenceType: "direct_observation", confidence: "high", importance: "supporting" },
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: hasComplaints ? "What is the baseline number of complaints and over what time period?" : "What specific metric or state should be used as the reference point?",
|
||||
targets: ["baseline_context", "measurement_period"],
|
||||
reason: "Establishing a reference point would distinguish whether the reported change is significant or within normal variation.",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "establish_baseline",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Display helpers ──────────────────────────────────
|
||||
|
||||
const CATEGORY_COLORS = {
|
||||
technical: "\x1b[36m", // cyan
|
||||
reasoning: "\x1b[33m", // yellow
|
||||
reset: "\x1b[0m",
|
||||
};
|
||||
|
||||
function categoryLabel(label) {
|
||||
return `${CATEGORY_COLORS.technical}${label}${CATEGORY_COLORS.reset}`;
|
||||
}
|
||||
|
||||
function reasonCategoryLabel() {
|
||||
return `${CATEGORY_COLORS.reasoning}reasoning quality${CATEGORY_COLORS.reset}`;
|
||||
}
|
||||
|
||||
// ── Main evaluation loop ─────────────────────────────
|
||||
async function main() {
|
||||
const testCases = loadTestCases(testDataPath);
|
||||
console.log(`\n⚡ Confidence Engine v0.2 — Evaluation Harness`);
|
||||
console.log(` Provider: ${useRealProvider ? "Ollama (real)" : "Mock"}`);
|
||||
console.log(` Cases loaded: ${testCases.length}\n`);
|
||||
|
||||
// Import or instantiate analysis function
|
||||
let analyseScenarioFn;
|
||||
if (useRealProvider) {
|
||||
const { analyseScenario } = await import("../lib/analysis.js");
|
||||
analyseScenarioFn = analyseScenario;
|
||||
} else {
|
||||
const mockProvider = new MockProvider();
|
||||
const schemaMod = await import("../lib/reconstruction/schema.js");
|
||||
const { reconstructionV2Schema, reconstructionSchema: reconstructionV1Schema } = schemaMod;
|
||||
const { buildPrompt } = await import("../lib/reconstruction/prompt.js");
|
||||
|
||||
analyseScenarioFn = async (scenario, opts = {}) => {
|
||||
const startTime = Date.now();
|
||||
const trimmed = scenario.trim();
|
||||
if (!trimmed) return { success: false, error: "Empty scenario", responseDurationMs: 0 };
|
||||
|
||||
let promptObj;
|
||||
try {
|
||||
promptObj = await buildPrompt(trimmed, opts.promptVersion || "v0.2");
|
||||
} catch {
|
||||
promptObj = { prompt: trimmed, version: "v0.2" };
|
||||
}
|
||||
|
||||
const mockResult = await mockProvider.generateReconstruction(promptObj.prompt, process.env.OLLAMA_MODEL || "mock-model");
|
||||
|
||||
let schemaValid = false;
|
||||
let validatedData = null;
|
||||
if (reconstructionV2Schema.safeParse) {
|
||||
const v2Result = reconstructionV2Schema.safeParse(mockResult);
|
||||
if (v2Result.success) {
|
||||
schemaValid = true;
|
||||
validatedData = v2Result.data;
|
||||
} else {
|
||||
const v1Result = reconstructionV1Schema.safeParse(mockResult);
|
||||
if (v1Result.success) {
|
||||
schemaValid = true;
|
||||
validatedData = v1Result.data;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!schemaValid || !validatedData) {
|
||||
return {
|
||||
success: false,
|
||||
validationStatus: "invalid",
|
||||
modelName: "mock-model",
|
||||
responseDurationMs: Date.now() - startTime,
|
||||
promptVersion: opts.promptVersion || "v0.2",
|
||||
reconstruction: null,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
modelName: "mock-model",
|
||||
responseDurationMs: Date.now() - startTime,
|
||||
promptVersion: opts.promptVersion || "v0.2",
|
||||
inputClassification: validatedData.inputClassification,
|
||||
reconstruction: validatedData.reconstruction,
|
||||
evidence: validatedData.evidence,
|
||||
nextQuestion: validatedData.nextQuestion,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// Run all cases
|
||||
const results = [];
|
||||
for (const tc of testCases) {
|
||||
process.stdout.write(` ${tc.id}: ... `);
|
||||
const r = await runTestCase(tc, analyseScenarioFn);
|
||||
results.push(r);
|
||||
const tStatus = r.technical.pass ? "\x1b[32m✅\x1b[0m" : "\x1b[31m❌\x1b[0m"; // green / red
|
||||
const rqStatus = r.reasoningQuality.pass ? "\x1b[32m✅\x1b[0m" : "\x1b[31m❌\x1b[0m";
|
||||
|
||||
process.stdout.write(`${tStatus} tech ${rqStatus} reason\n`);
|
||||
if (!r.technical.pass && r.technical.errors?.length) {
|
||||
for (const e of r.technical.errors.slice(0, 2)) process.stdout.write(` → [tech] ${e}\n`);
|
||||
} else if (!r.technical.pass) {
|
||||
const reasons = [];
|
||||
if (!r.technical.schemaValid) reasons.push("schema invalid");
|
||||
if (!r.technical.classificationMatch) reasons.push("classification mismatch");
|
||||
if (!r.technical.nextQuestionPresent) reasons.push("no next question");
|
||||
process.stdout.write(` → [tech] ${reasons.join(", ")}\n`);
|
||||
}
|
||||
|
||||
if (!r.reasoningQuality.pass) {
|
||||
const rqReasons = [];
|
||||
if (!r.reasoningQuality.requiredConcepts.pass) {
|
||||
rqReasons.push("missing required concept(s)");
|
||||
}
|
||||
if (!r.reasoningQuality.unsupportedInferencesAbsent.pass) {
|
||||
rqReasons.push("unsupported inference present");
|
||||
}
|
||||
process.stdout.write(` → [reasoning] ${rqReasons.join(", ")}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Compute summary stats ────────────────────────────
|
||||
const total = results.length;
|
||||
const techPassCount = results.filter((r) => r.technical.pass).length;
|
||||
const techSchemaValidCount = results.filter((r) => r.technical.schemaValid).length;
|
||||
const techClassificationMatchCount = results.filter((r) => r.technical.classificationMatch).length;
|
||||
const techNextQuestionPresentCount = results.filter((r) => r.technical.nextQuestionPresent).length;
|
||||
|
||||
const rqPassCount = results.filter((r) => r.reasoningQuality.pass).length;
|
||||
const rqConceptsPassCount = results.filter((r) => r.reasoningQuality.requiredConcepts.pass).length;
|
||||
const rqAbsencePassCount = results.filter((r) => r.reasoningQuality.unsupportedInferencesAbsent.pass).length;
|
||||
|
||||
const anyPassCount = results.filter(
|
||||
(r) => r.technical.pass && r.reasoningQuality.pass
|
||||
).length;
|
||||
|
||||
const avgDuration = total > 0
|
||||
? results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total
|
||||
: 0;
|
||||
|
||||
const failedTechCases = results.filter((r) => !r.technical.pass);
|
||||
const failedRqCases = results.filter((r) => !r.reasoningQuality.pass);
|
||||
const techPassOnly = results.filter(
|
||||
(r) => r.technical.pass && !r.reasoningQuality.pass
|
||||
);
|
||||
const rqPassOnly = results.filter(
|
||||
(r) => !r.technical.pass && r.reasoningQuality.pass
|
||||
);
|
||||
|
||||
// ── Console summary ───────────────────────────────────
|
||||
console.log(`\n${"=".repeat(60)}`);
|
||||
console.log("EVALUATION SUMMARY");
|
||||
console.log(`${"=".repeat(60)}\n`);
|
||||
|
||||
console.log(`Cases run: ${total}\n`);
|
||||
|
||||
// Technical section
|
||||
console.log(categoryLabel("─── TECHNICAL ──────────────────────────────"));
|
||||
console.log(` Schema validity rate: ${techSchemaValidCount}/${total} ${(techSchemaValidCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Classification match: ${techClassificationMatchCount}/${total} ${(techClassificationMatchCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Next-question present: ${techNextQuestionPresentCount}/${total} ${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Technical pass rate: ${techPassCount}/${total} ${(techPassCount / total * 100).toFixed(1)}%\n`);
|
||||
|
||||
// Reasoning quality section
|
||||
console.log(reasonCategoryLabel() + " ─────────────────────────────");
|
||||
console.log(`${CATEGORY_COLORS.reset}`);
|
||||
console.log(` Required concept match: ${rqConceptsPassCount}/${total} ${(rqConceptsPassCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Unsupported inference absent: ${rqAbsencePassCount}/${total} ${(rqAbsencePassCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Reasoning quality pass: ${rqPassCount}/${total} ${(rqPassCount / total * 100).toFixed(1)}%\n`);
|
||||
|
||||
// Combined
|
||||
console.log(`${"─".repeat(60)}`);
|
||||
console.log(` Both technical + reasoning: ${anyPassCount}/${total} ${(anyPassCount / total * 100).toFixed(1)}%`);
|
||||
if (techPassOnly.length > 0) {
|
||||
console.log(` Technical only (hallucinated): ${techPassOnly.length} — IDs: ${techPassOnly.map((r) => r.id).join(", ")}`);
|
||||
}
|
||||
if (rqPassOnly.length > 0) {
|
||||
console.log(` Reasoning only (bad structure): ${rqPassOnly.length} — IDs: ${rqPassOnly.map((r) => r.id).join(", ")}`);
|
||||
}
|
||||
if (failedTechCases.length > 0 && failedRqCases.length > 0) {
|
||||
console.log(` Failed both: ${results.filter((r) => !r.technical.pass && !r.reasoningQuality.pass).length}`);
|
||||
}
|
||||
|
||||
console.log(` Avg response duration: ${avgDuration.toFixed(0)}ms`);
|
||||
console.log(`${"=".repeat(60)}\n`);
|
||||
|
||||
if (failedTechCases.length > 0) {
|
||||
console.log(`Failed technical — case IDs: ${failedTechCases.map((r) => r.id).join(", ")}`);
|
||||
}
|
||||
if (failedRqCases.length > 0) {
|
||||
console.log(`Failed reasoning quality — case IDs: ${failedRqCases.map((r) => r.id).join(", ")}`);
|
||||
}
|
||||
|
||||
// ── Save results ──────────────────────────────────────
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
||||
|
||||
if (useDiagnostic) {
|
||||
// Live diagnostic: save to a dedicated result directory with per-case files + summary
|
||||
const caseResultDir = join(resultsDir, timestamp);
|
||||
mkdirSync(caseResultDir, { recursive: true });
|
||||
|
||||
// Per-case results JSON + Markdown
|
||||
for (const r of results) {
|
||||
const tc = testCases.find((t) => t.id === r.id);
|
||||
const caseFileBase = join(caseResultDir, r.id);
|
||||
|
||||
// Raw case result JSON
|
||||
writeFileSync(
|
||||
`${caseFileBase}-result.json`,
|
||||
JSON.stringify({
|
||||
id: r.id,
|
||||
description: tc?.description || "",
|
||||
input: tc?.input,
|
||||
responseDurationMs: r.responseDurationMs,
|
||||
actualPrimaryType: r.actualPrimaryType,
|
||||
actualReasoningModes: r.actualReasoningModes,
|
||||
rawOutput: r.rawOutput,
|
||||
technical: r.technical,
|
||||
reasoningQuality: r.reasoningQuality,
|
||||
}, null, 2)
|
||||
);
|
||||
|
||||
// Per-case Markdown summary
|
||||
const techStatus = r.technical.pass ? "✅ PASS" : "❌ FAIL";
|
||||
const rqStatus = r.reasoningQuality.pass ? "✅ PASS" : "❌ FAIL";
|
||||
|
||||
let md = `# Diagnostic Case: ${r.id}\n\n`;
|
||||
md += `${tc?.description || ""}\n\n`;
|
||||
md += `## Input\n\n\`\`\`\n${tc?.input || r.input}\n\`\`\`\n\n`;
|
||||
md += `## Result\n\n`;
|
||||
md += `- **Technical**: ${techStatus} (${(r.technical.pass ? 1 : 0)}/${Object.keys(r.technical).filter(k => typeof r.technical[k] === "boolean" && k !== "pass").length} sub-checks pass)\n`;
|
||||
md += `- **Reasoning Quality**: ${rqStatus} (${(r.reasoningQuality.pass ? 1 : 0)}/${2} sub-checks pass)\n`;
|
||||
md += `- **Actual Primary Type**: ${r.actualPrimaryType || "N/A"}\n`;
|
||||
md += `- **Actual Reasoning Modes**: ${(r.actualReasoningModes || []).join(", ") || "N/A"}\n`;
|
||||
md += `- **Response Duration**: ${r.responseDurationMs}ms\n`;
|
||||
|
||||
if (!r.technical.pass) {
|
||||
const reasons = [];
|
||||
if (!r.technical.schemaValid) reasons.push("schema invalid");
|
||||
if (!r.technical.classificationMatch) reasons.push("classification mismatch");
|
||||
if (!r.technical.nextQuestionPresent) reasons.push("no next question");
|
||||
md += `\n### Technical Failures\n\n${reasons.join(", ")}\n`;
|
||||
}
|
||||
|
||||
if (!r.reasoningQuality.pass) {
|
||||
const rqReasons = [];
|
||||
if (!r.reasoningQuality.requiredConcepts.pass) {
|
||||
rqReasons.push("missing required concept(s): " + r.reasoningQuality.requiredConcepts.details.filter(d => !d.found).map(d => d.concept).join(", ") || "unknown");
|
||||
}
|
||||
if (!r.reasoningQuality.unsupportedInferencesAbsent.pass) {
|
||||
rqReasons.push("unsupported inference present: " + r.reasoningQuality.unsupportedInferencesAbsent.details.filter(d => !d.absent).map(d => d.concept).join(", ") || "unknown");
|
||||
}
|
||||
md += `\n### Reasoning Quality Failures\n\n${rqReasons.join("\n")}\n`;
|
||||
}
|
||||
|
||||
writeFileSync(`${caseFileBase}-summary.md`, md);
|
||||
}
|
||||
|
||||
// Directory-level summary JSON
|
||||
const fullResults = {
|
||||
timestamp: new Date().toISOString(),
|
||||
provider: useRealProvider ? "ollama-real" : "mock",
|
||||
promptVersion: "v0.2",
|
||||
casesRun: total,
|
||||
summary: {
|
||||
technical: {
|
||||
schemaValidityRate: `${(techSchemaValidCount / total * 100).toFixed(1)}%`,
|
||||
classificationMatchRate: `${(techClassificationMatchCount / total * 100).toFixed(1)}%`,
|
||||
nextQuestionPresentRate: `${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`,
|
||||
passRate: `${(techPassCount / total * 100).toFixed(1)}%`,
|
||||
},
|
||||
reasoningQuality: {
|
||||
requiredConceptMatchRate: `${(rqConceptsPassCount / total * 100).toFixed(1)}%`,
|
||||
unsupportedInferenceFailures: (total - rqAbsencePassCount).toString(),
|
||||
passRate: `${(rqPassCount / total * 100).toFixed(1)}%`,
|
||||
},
|
||||
combinedPassRate: `${(anyPassCount / total * 100).toFixed(1)}%`,
|
||||
averageResponseDurationMs: avgDuration.toFixed(0),
|
||||
},
|
||||
testCaseResults: results.map((r) => ({
|
||||
id: r.id,
|
||||
input: r.input,
|
||||
responseDurationMs: r.responseDurationMs,
|
||||
actualPrimaryType: r.actualPrimaryType,
|
||||
actualReasoningModes: r.actualReasoningModes,
|
||||
technical: {
|
||||
schemaValid: r.technical.schemaValid,
|
||||
classificationMatch: r.technical.classificationMatch,
|
||||
reasoningModeMatch: r.technical.reasoningModeMatch,
|
||||
nextQuestionPresent: r.technical.nextQuestionPresent,
|
||||
pass: r.technical.pass,
|
||||
errors: r.technical.errors,
|
||||
},
|
||||
reasoningQuality: {
|
||||
requiredConcepts: r.reasoningQuality.requiredConcepts,
|
||||
unsupportedInferencesAbsent: r.reasoningQuality.unsupportedInferencesAbsent,
|
||||
pass: r.reasoningQuality.pass,
|
||||
},
|
||||
})),
|
||||
};
|
||||
|
||||
writeFileSync(join(caseResultDir, "summary.json"), JSON.stringify(fullResults, null, 2));
|
||||
console.log(`Live diagnostic results saved to: ${caseResultDir}/`);
|
||||
|
||||
// Also save a top-level manifest pointing to the latest run
|
||||
const manifestPath = join(resultsDir, "latest-manifest.json");
|
||||
writeFileSync(manifestPath, JSON.stringify({ latestRun: timestamp, caseCount: total }, null, 2));
|
||||
console.log(`Manifest saved to: ${manifestPath}`);
|
||||
|
||||
} else {
|
||||
// Standard (non-diagnostic): single file output
|
||||
const resultsFile = join(resultsDir, `evaluation-${timestamp}.json`);
|
||||
const fullResults = {
|
||||
timestamp: new Date().toISOString(),
|
||||
provider: useRealProvider ? "ollama-real" : "mock",
|
||||
promptVersion: "v0.2",
|
||||
casesRun: total,
|
||||
summary: {
|
||||
technical: {
|
||||
schemaValidityRate: `${(techSchemaValidCount / total * 100).toFixed(1)}%`,
|
||||
classificationMatchRate: `${(techClassificationMatchCount / total * 100).toFixed(1)}%`,
|
||||
nextQuestionPresentRate: `${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`,
|
||||
passRate: `${(techPassCount / total * 100).toFixed(1)}%`,
|
||||
},
|
||||
reasoningQuality: {
|
||||
requiredConceptMatchRate: `${(rqConceptsPassCount / total * 100).toFixed(1)}%`,
|
||||
unsupportedInferenceFailures: (total - rqAbsencePassCount).toString(),
|
||||
passRate: `${(rqPassCount / total * 100).toFixed(1)}%`,
|
||||
},
|
||||
combinedPassRate: `${(anyPassCount / total * 100).toFixed(1)}%`,
|
||||
averageResponseDurationMs: avgDuration.toFixed(0),
|
||||
},
|
||||
testCaseResults: results.map((r) => ({
|
||||
id: r.id,
|
||||
input: r.input,
|
||||
responseDurationMs: r.responseDurationMs,
|
||||
actualPrimaryType: r.actualPrimaryType,
|
||||
actualReasoningModes: r.actualReasoningModes,
|
||||
technical: {
|
||||
schemaValid: r.technical.schemaValid,
|
||||
classificationMatch: r.technical.classificationMatch,
|
||||
reasoningModeMatch: r.technical.reasoningModeMatch,
|
||||
nextQuestionPresent: r.technical.nextQuestionPresent,
|
||||
pass: r.technical.pass,
|
||||
errors: r.technical.errors,
|
||||
},
|
||||
reasoningQuality: {
|
||||
requiredConcepts: r.reasoningQuality.requiredConcepts,
|
||||
unsupportedInferencesAbsent: r.reasoningQuality.unsupportedInferencesAbsent,
|
||||
pass: r.reasoningQuality.pass,
|
||||
},
|
||||
})),
|
||||
};
|
||||
|
||||
writeFileSync(resultsFile, JSON.stringify(fullResults, null, 2));
|
||||
console.log(`Results saved to: ${resultsFile}`);
|
||||
console.log(`${"=".repeat(60)}\n`);
|
||||
}
|
||||
|
||||
// ── Close main() scope if we're in the non-diagnostic branch ──
|
||||
// (The if/else above handles result saving; main closes here)
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error("Evaluator failed:", e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user