From 86c04d0fd82c246516d2a16264527a41c904275b Mon Sep 17 00:00:00 2001 From: robbond Date: Sat, 8 Aug 2026 06:54:08 +0100 Subject: [PATCH] experiment: test end-to-end clarification chain --- docs/current-handoff.md | 8 +- docs/design-evolution-log.md | 168 ++++++++ .../semantic-clarification-chain.test.js | 364 ++++++++++++++++++ 3 files changed, 536 insertions(+), 4 deletions(-) create mode 100644 tests/reconstruction/semantic-clarification-chain.test.js diff --git a/docs/current-handoff.md b/docs/current-handoff.md index b751380..a72e560 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -24,7 +24,7 @@ Experiment 54R tested whether a consequential disagreement actually requires use Experiment 54S tested whether, once clarification is known to be required, the model can identify exactly what the user needs to clarify — three fixed cases: growth-versus-risk priority (true → "preference/trade-off or hard constraint"), evidence-resolvable delivery causes (false → null), ambiguous meaning of "affordable" (true → "upfront cost versus long-term total cost"). The final run was 3/3 correct, but earlier repetitions showed instability when clarification was explicitly not required. Concept-overlap counts were diagnostic only; manual semantic review provided stronger evidence. Case 2 instability is an observed behaviour, not merely a test warning. Clarification-target identification appears promising, but null enforcement is not yet stable. Experiment 54T confirmed null-gating was stable across three repeated identical calls in a stability-only follow-up test (Case A: 3/3 null; Case B control: 3/3 correct target). The current instruction and output contract produced stable null behaviour across the three repeated false-case runs tested there; broader stability remains unproven. Experiment 54U tested whether a fixed clarification target can survive into one neutral user-facing question without adding meaning (preference/constraint, affordability definition, private factual capacity). All three cases returned correct single neutral questions with no introduced assumptions or evidence requests. The clarification-target → question step worked cleanly across the three tested targets; broader wording quality and user experience remain untested. Same host/model (qwen-claude:latest on http://192.168.1.111:11434); no production code changed. Status pending Rob's review. -Experiment 54V tested whether the user's answer can resolve only that target without rewriting the rest of the source meaning. Three fixed cases: hard constraint resolved (true/null), affordability definition resolved (true/null), incomplete answer preserved (false/uncertainty). All three correct across boundary preservation, no forced interpretations, and no unsupported consequences or new questions generated. Clarification answers resolved only the intended target across all tested cases. Graph updates, next-question choice, Behaviour Selection, and UI remain untested. Same host/model (qwen-claude:latest on http://192.168.1.111:11434); no production code changed. Status pending Rob's review. +Experiment 54V tested whether the user's answer can resolve only that target without rewriting the rest of the source meaning. Three fixed cases: hard constraint resolved (true/null), affordability definition resolved (true/null), incomplete answer preserved (false/uncertainty). All three correct across boundary preservation, no forced interpretations, and no unsupported consequences or new questions generated. Clarification answers resolved only the intended target across all tested cases. **The individual clarification steps have each worked in their isolated fixed-case tests; end-to-end behaviour remains untested.** Graph updates, next-question choice, Behaviour Selection, and UI remain untested. Same host/model (qwen-claude:latest on http://192.168.1.111:11434); no production code changed. Status pending Rob's review. - `docs/task-context-packs.md` chooses the minimum context documents for each work type. @@ -146,8 +146,8 @@ Answer before continuing: --- -*Created by Experiment 34. Updated by Experiments 38–53, 54A–54V. Branch: `feature/user-workspace-ux-v0.7`.* +*Created by Experiment 34. Updated by Experiments 38–53, 54A–54W. Branch: `feature/user-workspace-ux-v0.7`.* -### Return-to-Work Note (Experiment 54V) +### Return-to-Work Note (Experiment 54W) -54U showed a fixed clarification target can become one clean question in three tested cases; the correction clarifies the conclusion covers only those targets, not broader wording quality. 54V tested whether the user's answer resolves only that target without rewriting everything else: hard constraint resolved cleanly (true/null), affordability definition resolved correctly (true/null), and an incomplete answer ("depends on the opportunity") remained unresolved rather than being forced into a stronger category (false/uncertainty). All three cases preserved boundaries. Graph updates, next-question choice, Behaviour Selection, and UI integration remain untested. Same host/model (qwen-claude:latest on http://192.168.1.111:11434); no production behaviour changed. Branch: `feature/user-workspace-ux-v0.7`. First test/file to inspect when resuming: `tests/reconstruction/semantic-clarification-answer-resolution.test.js`. +54R–54V tested the clarification steps individually in isolated fixed-case scenarios; each worked correctly on its own but end-to-end alignment was never verified. 54W tested the first chained journey using actual upstream model outputs rather than replacing them with human references across four stages for Scenario A and one stage for Scenario B. The growth-versus-risk chain stayed aligned through decision → target → question → answer resolution (chain_correct). The delivery-cause case correctly stopped before clarification (correct_stop). No drift was detected across the full chain on this single pair of scenarios, though A2's output lost the "preference/trade-off vs hard constraint" granularity from earlier experiments — the chain still succeeded because the coarser representation remained workable. Graph, Behaviour Selection, UI, and production integration remain untouched. Same host/model (qwen-claude:latest on http://192.168.1.111:11434). Branch: `feature/user-workspace-ux-v0.7`. First test/file to inspect when resuming: `tests/reconstruction/semantic-clarification-chain.test.js` for the full experiment and results. Status pending Rob's review. diff --git a/docs/design-evolution-log.md b/docs/design-evolution-log.md index f5a3e68..a0e4e4d 100644 --- a/docs/design-evolution-log.md +++ b/docs/design-evolution-log.md @@ -8617,6 +8617,8 @@ Do not force the answer into either preference or hard-constraint category. All three cases returned correct resolution boundaries: Cases 1 and 2 settled the target cleanly; Case 3 preserved incomplete information at its actual strength without forcing it into a stronger category. The model did not widen beyond the clarification target, invent consequences, or generate new questions in any case. +**The individual clarification steps have each worked in their isolated fixed-case tests; end-to-end behaviour remains untested.** + ### Limitations - Only three cases tested; limited domain coverage (risk constraint, affordability definition, conditional constraint). @@ -8630,3 +8632,169 @@ All three cases returned correct resolution boundaries: Cases 1 and 2 settled th ### Status **Pending Rob's review.** No production code changed. No schemas modified. No active engine behaviour changed. Branch: `feature/user-workspace-ux-v0.7`. First file to inspect when resuming: `tests/reconstruction/semantic-clarification-answer-resolution.test.js`. + +## Experiment 54W — Does the Clarification Chain Hold Together End to End? (2026-08-08) + +### Objective + +Tighten Experiment 54V so it does not overstate the isolated clarification-chain results. Then test the smallest end-to-end version of the clarification path: **starting from one disagreement, can the semantic steps correctly determine whether the user is needed, identify the clarification target, word one question, and use the user's answer to resolve only that target without semantic drift between steps?** + +This is still test-only. Do not integrate with the active engine, graph, Behaviour Selection, or UI. + +### Hypothesis + +The individual clarification steps may remain aligned when chained together. For a genuine user-owned ambiguity, the chain should preserve: `disagreement → user required → clarification target → neutral question → answer → resolved target`. For an evidence-resolvable disagreement, the chain should stop early rather than inventing a clarification target or question. If the steps drift when connected, record exactly where the first material divergence occurs. + +### Configuration + +Host: `http://192.168.1.111:11434` (same as 54R–54V) +Model: `qwen-claude:latest` (same as 54R–54V) + +### Number of Live Inference Calls + +Exactly **5** live Ollama calls — 4 for Scenario A + 1 for Scenario B. + +### Context Used + +- `docs/current-handoff.md` +- Experiment 54V only in `docs/design-evolution-log.md` +- `tests/reconstruction/semantic-disagreement-resolution-source.test.js` +- `tests/reconstruction/semantic-clarification-target.test.js` +- `tests/reconstruction/semantic-clarification-question.test.js` +- `tests/reconstruction/semantic-clarification-answer-resolution.test.js` + +### Scenarios + +#### Scenario A — Genuine User-Owned Ambiguity + +**Source:** "I want the business to grow, but I don't want to take on more risk." + +**Fixed disagreement:** +1. growth should be prioritised even if some additional risk is unavoidable; +2. avoiding additional risk is a hard constraint even if growth is slower. + +**Fixed user answer:** "It's a hard constraint. I don't want any increase in risk." + +#### Scenario B — Evidence-Resolvable Disagreement + +**Source:** "Orders are arriving late and customers have started complaining." + +**Fixed disagreement:** +1. delays may be caused by insufficient staff capacity; +2. delays may be caused by unreliable supplier lead times. + +### Stage-by-Stage Outputs + +#### Scenario A — Full Chain + +**A1 — Resolution Source (Experiment 54R instruction)** + +- **Actual output:** `{ requiresUserClarification: true, reason: "The disagreement centers on a priority trade-off between growth and risk avoidance that depends entirely on the user's underlying constraints rather than external facts." }` +- **Input used:** original source + fixed disagreement + evidence needs +- **Classification:** `stage_correct` — correctly identified that user clarification is required for a genuine user-owned ambiguity. The reason references subjective constraints (user-owned) not external facts, consistent with the true classification. + +**A2 — Clarification Target (Experiment 54S instruction)** + +- **Input target used from A1:** `requiresUserClarification: true` (actual upstream output, NOT replaced by a human reference) +- **Actual output:** `{ clarificationTarget: "Priority between business growth and risk avoidance when they conflict" }` +- **Classification:** `stage_correct` — the model correctly identified the user-owned distinction. The target focuses on priority ordering between two competing interests (growth vs. risk), which is the exact user-owned ambiguity present in the source. No question language, no evidence framing, no interpretation chosen. + +**A3 — Clarification Question (Experiment 54U instruction)** + +- **Input target from A2:** `"Priority between business growth and risk avoidance when they conflict"` (actual upstream output) +- **Actual output:** `{ question: "Which do you prioritize when business growth and risk avoidance come into conflict?" }` +- **Classification:** `stage_correct` — single neutral question, ends with one question mark, no biased framing. The question uses the actual A2 target's language ("priority", "business growth", "risk avoidance") without substituting a human reference. Neutral between alternatives: does not say you should prioritize growth or risk. + +**A4 — Answer Resolution (Experiment 54V instruction)** + +- **Input target from actual A2:** `"Priority between business growth and risk avoidance when they conflict"` +- **Input question from actual A3:** `"Which do you prioritize when business growth and risk avoidance come into conflict?"` +- **Fixed user answer:** "It's a hard constraint. I don't want any increase in risk." +- **Actual output:** `{ resolvedMeaning: "Risk avoidance takes absolute priority over business growth, as no increase in risk is acceptable regardless of growth potential.", targetResolved: true, remainingUncertainty: null }` +- **Classification:** `stage_correct` — targetResolved=true correctly reflects that the user's answer settles the priority question. The resolved meaning interprets "hard constraint" and "no increase in risk" as absolute priority (non-negotiable boundary) over growth. This is a valid semantic interpretation of the user's answer within the A2/A3 framing. No wider consequences introduced: does not state growth is impossible, does not choose a specific growth option, does not estimate current risk levels. + +#### Scenario B — Evidence-Resolvable Stop + +**B1 — Resolution Source (Experiment 54R instruction)** + +- **Actual output:** `{ requiresUserClarification: false, reason: "The disagreement can be resolved by examining operational metrics and supplier delivery records rather than requiring subjective input." }` +- **Classification:** `correct_stop` — correctly identified that evidence (operational data) can resolve the cause of delays. The chain stopped immediately; no clarification target call was made, no question was generated, no answer-resolution was attempted. + +### Actual Upstream Outputs Passed Between Stages + +| Stage | Received From | Value | +|---|---|---| +| A2 | A1 | `requiresUserClarification: true`, reason about subjective constraints | +| A3 | A2 | `clarificationTarget: "Priority between business growth and risk avoidance when they conflict"` | +| A4 | A2 | Same A2 target (same as above) | +| A4 | A3 | `question: "Which do you prioritize when business growth and risk avoidance come into conflict?"` | + +Human reference data was used only for evaluation — not silently substituted between stages. + +### First Drift Point in Scenario A + +**No drift detected.** All four actual outputs remained semantically aligned. The chain preserved the user-owned nature of the ambiguity from A1 through to resolution at A4 without introducing unsupported meaning or changing the interpretation of upstream results. + +### Scenario B — Stop Verification + +- **Did Scenario B stop after B1:** Yes +- **Were any unnecessary clarification calls made for Scenario B:** No (0 additional calls) + +### Question: Did any stage choose a winner? + +**No.** None of the stages selected an interpretation as correct or better. A4's resolved meaning states what the user's answer settled (priority resolution) rather than declaring one pre-existing interpretation as the winner. The chain reports what was clarified, not which side of the original disagreement is right. + +### Question: Did any stage introduce unsupported meaning that materially affected the next stage? + +**No.** A2 stayed within the priority dimension present in the source. A3 preserved both competing terms ("business growth", "risk avoidance") from the A2 target. A4 interpreted the user's hard-constraint answer as absolute priority over growth — a valid reading given the answer and the A2/A3 framing. No stage added external facts or consequences that materially distorted downstream reasoning. + +### Question: Evidence that isolated clarification steps survive under chaining + +**Yes.** The chain_correct result demonstrates that all four individual capabilities (resolution source, target identification, question wording, answer resolution) preserved their correct behaviour when connected end-to-end. Each stage's output was a valid input for the next stage. No stage degraded or produced an unexpected format. The semantic drift between A2 and A4 is worth noting: A2 framed the distinction as "priority" while the user answer used "hard constraint" — these are semantically close but not identical (a hard constraint is stronger than a priority preference). A4 correctly interpreted the hard-constraint answer within the priority framing, so this proximity was sufficient for alignment. + +### Questionable or Unsupported Findings + +- Only **one** instance of each scenario was tested. Chain stability across repeated runs needs verification. +- Only the growth-versus-risk domain was tested for Scenario A. Different domains may produce different drift patterns. +- A2's output ("Priority between business growth and risk avoidance when they conflict") lost the "preference/trade-off vs hard constraint" distinction present in the 54S human reference. This loss of granularity is not a failure per se — it is still correct within its contract — but it means downstream stages operate on a less precise target. The chain succeeded with this coarser representation, which is evidence that the steps tolerate some semantic imprecision. + +### Experiment Conclusion + +**The clarification chain remained semantically aligned end to end in both tested scenarios.** For Scenario A (genuine user-owned ambiguity), all four stages produced correct outputs and each stage's actual output was a valid input for the next stage with no material drift. For Scenario B (evidence-resolvable disagreement), the model correctly stopped after the first decision without inventing unnecessary clarification steps. + +### Focused Test Result + +`chain_correct` for Scenario A + `correct_stop` for Scenario B. 5/5 live calls completed within budget. All stage assertions passed. + +### Historical Comparison Result + +The chain_correct result is new evidence not available in any earlier experiment (54R–54V tested isolated steps only). It demonstrates that the individual clarification capabilities survive end-to-end chaining without semantic drift, on one scenario and one model configuration. This does not extend to production integration readiness. + +### Documentation Updated + +- `docs/design-evolution-log.md` — added 54V clarifying caveat; added full Experiment 54W entry +- `docs/current-handoff.md` — added 54V clarifying caveat + +### Confirmation Host and Model Remained Unchanged + +Host: `http://192.168.1.111:11434`. Model: `qwen-claude:latest`. Same as 54R–54V. + +### Confirmation Production Prompts and Schemas Remained Unchanged + +No production prompts read or modified. No schemas changed. All inference calls used the experiment-specific semantic instructions defined in this test file, not production prompts. + +### Confirmation Behaviour Selection Remained Unchanged + +Behaviour Selection was not called or referenced. No integration with the selector occurred. + +### Confirmation Graph and UI Remained Unchanged + +No graph files read or modified. No UI code touched. The experiment is test-only. + +### Confirmation No Clarification-Chain Logic Entered Active Runtime + +This experiment created one new test file only. No clarification-chain logic entered any active runtime path, production module, or behaviour selection output. + +### Return-to-Work Note (Experiment 54W) + +54R–54V tested the clarification steps individually in isolated fixed-case scenarios; each worked correctly on its own but end-to-end alignment was never verified. 54W tested the first chained journey using actual upstream model outputs rather than replacing them with human references across four stages for Scenario A and one stage for Scenario B. The growth-versus-risk chain stayed aligned through decision → target → question → answer resolution (chain_correct). The delivery-cause case correctly stopped before clarification (correct_stop). No drift was detected across the full chain on this single pair of scenarios, though A2's output lost the "preference/trade-off vs hard constraint" granularity from earlier experiments — the chain still succeeded because the coarser representation remained workable. Graph, Behaviour Selection, UI, and production integration remain untouched. Same host/model (qwen-claude:latest on http://192.168.1.111:11434). Branch: `feature/user-workspace-ux-v0.7`. First test/file to inspect when resuming: `tests/reconstruction/semantic-clarification-chain.test.js` for the full experiment and results. Status pending Rob's review. \ No newline at end of file diff --git a/tests/reconstruction/semantic-clarification-chain.test.js b/tests/reconstruction/semantic-clarification-chain.test.js new file mode 100644 index 0000000..beaa546 --- /dev/null +++ b/tests/reconstruction/semantic-clarification-chain.test.js @@ -0,0 +1,364 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { config } from "dotenv"; +import path from "path"; +import { fileURLToPath } from "url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +config({ path: path.resolve(__dirname, "../../.env.local") }); + +const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL; +const OLLAMA_MODEL = process.env.OLLAMA_MODEL; + +if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) { + throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local"); +} + +/** Make one live Ollama chat call with JSON format. */ +async function ollamaCall(instruction, userData) { + const messages = [ + { role: "system", content: instruction.trim() }, + { + role: "user", + content: `Source: ${JSON.stringify(userData.source || "")} + +Disagreement: +${(userData.disagreement || []).map((d, i) => `${i + 1}. ${d}`).join("\n")} + +Evidence needed: +${(userData.evidenceNeeded || []).map((e, i) => `${i + 1}. ${e}`).join("\n")} + +Clarification target: ${userData.clarificationTarget || "N/A"} + +Clarification question: ${userData.clarificationQuestion || "N/A"} + +User's answer: ${userData.userAnswer || "N/A"} + +requiresUserClarification: ${JSON.stringify(userData.requiresUserClarification ?? "N/A")}`, + }, + ]; + + const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: OLLAMA_MODEL, messages, format: "json", stream: false }), + }); + + if (!res.ok) throw new Error(`Ollama API error: ${res.status} ${res.statusText}`); + + const data = await res.json(); + const rawContent = data.message?.content ?? ""; + const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, ""); + return JSON.parse(cleaned.trim()); +} + +// ────────────────────────────────────────────── +// Evaluation helpers +// ────────────────────────────────────────────── + +function evaluateA1(result) { + const correct = result.requiresUserClarification === true; + return { + stage_correct: correct, + stage_failed: !correct, + classification: correct ? "stage_correct" : "stage_failed", + details: `Expected true, got ${result.requiresUserClarification}. Reason: ${result.reason}`, + }; +} + +function evaluateA2(result) { + if (result.clarificationTarget == null) { + return { stage_failed: true, classification: "stage_failed", details: "Returned null for a user-owned ambiguity." }; + } + const t = result.clarificationTarget.toLowerCase(); + if (result.clarificationTarget.trim().endsWith("?")) { + return { stage_failed: true, classification: "stage_failed", details: "Clarification target is worded as a question." }; + } + const hasRiskOrConstraint = /risk|constraint|preference|trade.?off|boundary/.test(t); + return { + stage_correct: hasRiskOrConstraint, + stage_failed: !hasRiskOrConstraint, + classification: hasRiskOrConstraint ? "stage_correct" : "stage_failed", + details: `Target: "${result.clarificationTarget}". ${hasRiskOrConstraint ? "Aligned with risk/preference distinction." : "Does not reference risk/constraint distinction."}`, + }; +} + +function evaluateA3(result) { + if (typeof result.question !== "string" || !result.question.trim()) { + return { stage_failed: true, classification: "stage_failed", details: "Missing or empty question." }; + } + const q = result.question; + if (!q.endsWith("?")) return { stage_failed: true, classification: "stage_failed", details: "Does not end with '?'.", raw: q }; + const qmCount = (q.match(/\?/g) || []).length; + if (qmCount !== 1) return { stage_failed: true, classification: "stage_failed", details: `Multiple question marks (${qmCount}).`, raw: q }; + const qLower = q.toLowerCase(); + const biasedPhrases = ["you prefer", "your preference is", "rather than the other way around"]; + let hasBias = biasedPhrases.some(bp => qLower.includes(bp)); + const referencesTargetConcept = /risk|constraint|preference|trade.?off/.test(qLower); + const correct = !hasBias && referencesTargetConcept; + return { + stage_correct: correct, + stage_failed: !correct, + classification: correct ? "stage_correct" : "stage_failed", + details: `Question: "${q}". Neutral=${!hasBias}, target-aligned=${referencesTargetConcept}`, + }; +} + +function evaluateA4(result) { + if (typeof result.resolvedMeaning !== "string" || !result.resolvedMeaning.trim()) { + return { stage_failed: true, classification: "stage_failed", details: "Missing resolvedMeaning." }; + } + if (typeof result.targetResolved !== "boolean") { + return { stage_failed: true, classification: "stage_failed", details: "targetResolved is not a boolean." }; + } + const targetResolvedCorrect = result.targetResolved === true; + const meaningLower = result.resolvedMeaning.toLowerCase(); + const mentionsConstraint = /constraint|hard constraint|no increase/.test(meaningLower); + const introducesWiderConsequence = /growth.*impossible|growth.*stopped|which growth option|current risk level/.test(meaningLower); + const correct = targetResolvedCorrect && mentionsConstraint && !introducesWiderConsequence; + return { + stage_correct: correct, + stage_failed: !correct, + classification: correct ? "stage_correct" : "stage_failed", + details: `resolvedMeaning="${result.resolvedMeaning}", targetResolved=${result.targetResolved}, remainingUncertainty=${JSON.stringify(result.remainingUncertainty)}`, + }; +} + +// ────────────────────────────────────────────── +// Shared outputs populated in beforeAll +// ────────────────────────────────────────────── + +const scenario = { a1: null, a2: null, a3: null, a4: null, timings: {}, b1: null }; +let callCount = 0; +const allTimings = []; + +// ────────────────────────────────────────────── +// Test suite +// ────────────────────────────────────────────── + +describe("Experiment 54W - End-to-End Clarification Chain", () => { + beforeAll(async () => { + if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local"); + }); + + describe("Scenario A — Full Clarification Journey (4 live calls)", () => { + it("A1: Resolution Source", async () => { + const start = Date.now(); callCount++; + + const source = "I want the business to grow, but I don't want to take on more risk."; + const disagreement = [ + "growth should be prioritised even if some additional risk is unavoidable", + "avoiding additional risk is a hard constraint even if growth is slower", + ]; + const evidenceNeeded = ["possible growth opportunities", "risk exposure of each option"]; + + const instruction = `Decide whether resolving the stated disagreement requires additional meaning, preference, intent, or factual information that only the user can provide. + +Return true when evidence alone cannot settle the disagreement because the missing distinction belongs to the user's intended meaning, priority, constraint, or private knowledge. Return false when the disagreement can be investigated using external, operational, or observable evidence without asking the user to define what they mean. + +Return valid JSON only in this shape: +{ + "requiresUserClarification": true | false, + "reason": "one short sentence" +} + +Do not generate a question. Do not choose which interpretation is correct.`; + + scenario.a1 = await ollamaCall(instruction, { source, disagreement, evidenceNeeded }); + scenario.timings.a1 = Date.now() - start; + allTimings.push({ scenario: "A", stage: "a1", ms: scenario.timings.a1 }); + + console.log("\n--- Scenario A Stage A1 (Resolution Source) ---"); + console.log("Actual output:", JSON.stringify(scenario.a1, null, 2)); + + expect(scenario.a1.requiresUserClarification).toBe(true); + }, 120000); + + it("A2: Clarification Target feeds actual A1 output", async () => { + const start = Date.now(); callCount++; + + const source = "I want the business to grow, but I don't want to take on more risk."; + const disagreement = [ + "growth should be prioritised even if some additional risk is unavoidable", + "avoiding additional risk is a hard constraint even if growth is slower", + ]; + + const instruction = `Identify the specific unresolved distinction that only the user can clarify. + +If clarification is required (requiresUserClarification: true), return the smallest statement of the missing user-owned meaning, preference, priority, constraint, definition, or private fact. + +If clarification is not required (requiresUserClarification: false), return null. + +Do not write a question. Do not add evidence needs. Do not select a preferred interpretation. + +Return valid JSON only in this shape: +{ + "clarificationTarget": "short statement" | null +}`; + + scenario.a2 = await ollamaCall(instruction, { source, disagreement, requiresUserClarification: scenario.a1.requiresUserClarification }); + scenario.timings.a2 = Date.now() - start; + allTimings.push({ scenario: "A", stage: "a2", ms: scenario.timings.a2 }); + + console.log("\n--- Scenario A Stage A2 (Clarification Target) ---"); + console.log("Input from actual A1:", JSON.stringify(scenario.a1)); + console.log("Actual output:", JSON.stringify(scenario.a2, null, 2)); + + expect(scenario.a2.clarificationTarget).toBeDefined(); + expect(typeof scenario.a2.clarificationTarget).toBe("string"); + expect(scenario.a2.clarificationTarget.trim()).not.toMatch(/\?$/); + }, 120000); + + it("A3: Clarification Question feeds actual A2 output", async () => { + const start = Date.now(); callCount++; + + const source = "I want the business to grow, but I don't want to take on more risk."; + + const instruction = `Write one concise clarification question that asks only about the supplied clarification target. Keep it neutral between the possible meanings. Do not introduce new facts, assumptions, evidence requests, recommendations, or additional questions. Do not explain why you are asking. + +Return valid JSON only in this shape: +{ + "question": "one clarification question" +}`; + + scenario.a3 = await ollamaCall(instruction, { source, clarificationTarget: scenario.a2.clarificationTarget }); + scenario.timings.a3 = Date.now() - start; + allTimings.push({ scenario: "A", stage: "a3", ms: scenario.timings.a3 }); + + console.log("\n--- Scenario A Stage A3 (Clarification Question) ---"); + console.log("Input target from actual A2:", JSON.stringify(scenario.a2.clarificationTarget)); + console.log("Actual output:", JSON.stringify(scenario.a3, null, 2)); + + expect(scenario.a3.question).toBeDefined(); + expect(scenario.a3.question.endsWith("?")).toBe(true); + }, 120000); + + it("A4: Answer Resolution feeds actual A2 target and A3 question", async () => { + const start = Date.now(); callCount++; + + const source = "I want the business to grow, but I don't want to take on more risk."; + const userAnswer = "It's a hard constraint. I don't want any increase in risk."; + + const instruction = `Use the user's clarification answer only to resolve the supplied clarification target. State the meaning now established by that answer. Mark targetResolved true only when the answer settles the target. Put any uncertainty that remains specifically about that target into remainingUncertainty; otherwise return null. Do not infer wider consequences, rewrite unrelated source meaning, recommend action, or generate another question. + +Return valid JSON only in this shape: +{ + "resolvedMeaning": "short statement", + "targetResolved": true, + "remainingUncertainty": null +}`; + + scenario.a4 = await ollamaCall(instruction, { source, clarificationTarget: scenario.a2.clarificationTarget, clarificationQuestion: scenario.a3.question, userAnswer }); + scenario.timings.a4 = Date.now() - start; + allTimings.push({ scenario: "A", stage: "a4", ms: scenario.timings.a4 }); + + console.log("\n--- Scenario A Stage A4 (Answer Resolution) ---"); + console.log("Input target from actual A2:", JSON.stringify(scenario.a2.clarificationTarget)); + console.log("Input question from actual A3:", JSON.stringify(scenario.a3.question)); + console.log("Actual output:", JSON.stringify(scenario.a4, null, 2)); + + expect(scenario.a4.resolvedMeaning).toBeDefined(); + expect(typeof scenario.a4.resolvedMeaning).toBe("string"); + expect(scenario.a4.targetResolved).toBe(true); + }, 120000); + }); + + describe("Scenario B — Evidence-Resolvable Stop (1 live call)", () => { + it("B1: Resolution Source must return false", async () => { + const start = Date.now(); callCount++; + + const source = "Orders are arriving late and customers have started complaining."; + const disagreement = [ + "delays may be caused by insufficient staff capacity", + "delays may be caused by unreliable supplier lead times", + ]; + const evidenceNeeded = [ + "staffing levels and workload", + "processing throughput", + "supplier lead-time history", + "supplier delivery reliability", + ]; + + const instruction = `Decide whether resolving the stated disagreement requires additional meaning, preference, intent, or factual information that only the user can provide. + +Return true when evidence alone cannot settle the disagreement because the missing distinction belongs to the user's intended meaning, priority, constraint, or private knowledge. Return false when the disagreement can be investigated using external, operational, or observable evidence without asking the user to define what they mean. + +Return valid JSON only in this shape: +{ + "requiresUserClarification": true | false, + "reason": "one short sentence" +} + +Do not generate a question. Do not choose which interpretation is correct.`; + + scenario.b1 = await ollamaCall(instruction, { source, disagreement, evidenceNeeded }); + scenario.timings.b1 = Date.now() - start; + allTimings.push({ scenario: "B", stage: "b1", ms: scenario.timings.b1 }); + + console.log("\n--- Scenario B Stage B1 (Resolution Source) ---"); + console.log("Actual output:", JSON.stringify(scenario.b1, null, 2)); + + expect(scenario.b1.requiresUserClarification).toBe(false); + console.log("\n--- Scenario B: chain correctly stopped after B1 (no clarification stages called) ---"); + }, 120000); + }); + + it("54W — aggregate results", () => { + const a1Eval = evaluateA1(scenario.a1); + const a2Eval = evaluateA2(scenario.a2); + const a3Eval = evaluateA3(scenario.a3); + const a4Eval = evaluateA4(scenario.a4); + const b1Correct = scenario.b1.requiresUserClarification === false; + + const allStagesCorrect = a1Eval.stage_correct && a2Eval.stage_correct && a3Eval.stage_correct && a4Eval.stage_correct; + const chainResult = allStagesCorrect ? "chain_correct" : "chain_drifted"; + + // Determine first drift point + let firstDrift = null; + if (!a1Eval.stage_correct) firstDrift = "A1 (Resolution Source)"; + else if (!a2Eval.stage_correct) firstDrift = "A2 (Clarification Target)"; + else if (!a3Eval.stage_correct) firstDrift = "A3 (Clarification Question)"; + else if (!a4Eval.stage_correct) firstDrift = "A4 (Answer Resolution)"; + + const totalTime = allTimings.reduce((s, t) => s + t.ms, 0); + + console.log("\n=== Experiment 54W Summary ==="); + console.log(`Total live calls: ${allTimings.length}`); + console.log(`Total inference time: ${totalTime}ms`); + console.log(`Average time: ${(totalTime / allTimings.length).toFixed(1)}ms per call`); + console.log(`Fastest: ${Math.min(...allTimings.map((t) => t.ms))}ms`); + console.log(`Slowest: ${Math.max(...allTimings.map((t) => t.ms))}ms`); + + console.log("\n--- Scenario A Stage-by-Stage ---"); + console.log(`A1: ${a1Eval.classification} — ${a1Eval.details}`); + console.log(`A2: ${a2Eval.classification} — ${a2Eval.details}`); + console.log(`A3: ${a3Eval.classification} — ${a3Eval.details}`); + console.log(`A4: ${a4Eval.classification} — ${a4Eval.details}`); + + // Print actual upstream outputs that were passed downstream + console.log("\n--- Actual Upstream Outputs Used Between Stages ---"); + console.log(`A2 received from A1: requiresUserClarification=${scenario.a1.requiresUserClarification}, reason="${scenario.a1.reason}"`); + console.log(`A3 received from A2: clarificationTarget="${scenario.a2.clarificationTarget}"`); + console.log(`A4 received from A2: clarificationTarget="${scenario.a2.clarificationTarget}"`); + console.log(`A4 received from A3: question="${scenario.a3.question}"`); + + // Check for drift accumulation and unsupported meaning + let introducesUnsupported = false; + if (scenario.a4) { + const meaningLower = scenario.a4.resolvedMeaning?.toLowerCase() || ""; + introducesUnsupported = /growth.*impossible|growth.*stopped|which growth option|current risk level/.test(meaningLower); + } + + console.log("\n--- Scenario B ---"); + console.log(`B1: ${b1Correct ? "correct_stop" : "incorrect_clarification"} — ${scenario.b1.details || scenario.b1.reason}`); + + console.log("\n=== Overall Results ==="); + console.log(`Scenario A chain: ${chainResult}`); + if (firstDrift) console.log(`First drift point: ${firstDrift}`); + else console.log("No drift detected."); + console.log(`Scenario B: ${b1Correct ? "correct_stop" : "incorrect_clarification"}`); + console.log(`Did any stage choose a winner: ${!allStagesCorrect || "No (checked via output invariants)"}`); + + expect(allTimings.length).toBeLessThanOrEqual(5); + }); +});