From 85ee4bed30cc3419a0c3aa2476708abf03874257 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 9 Aug 2026 14:13:48 +0100 Subject: [PATCH] experiment: probe explicit hard constraint semantic fidelity --- docs/current-handoff.md | 10 ++ docs/experiment-56j.md | 79 +++++++++++ ...ression-d-explicit-hard-constraint.test.js | 131 ++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 docs/experiment-56j.md create mode 100644 tests/reconstruction/semantic-regression-d-explicit-hard-constraint.test.js diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 80f7050..98e4e86 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -201,3 +201,13 @@ Tested whether deterministic derivation refinement from commit `3e78d57` (refine 5. Follow-up question correctly targets the emergent conditional/threshold unknown. **Key observation**: The LLM does not auto-populate `supportCategory` — it is consistently null in `answerMeaning`. The deterministic derivation layer in `readDiagnostics` (and the inline pipeline) is the sole mechanism by which meaning profile category gets determined. This confirms the design: LLM produces raw meaning; deterministic logic categorizes it. No regression detected. Full results in `docs/experiment-56d.md`. Branch: `feature/reasoning-fidelity-v0.8`. Status pending Rob's review. + +### Experiment 56J Summary — Regression D Explicit Hard Constraint Semantic Probe + +Tested whether the configured live Ollama model (`qwen-claude:latest` at `http://192.168.1.111:11434`) preserves explicit hard-constraint meaning from user answer "It's a hard constraint. I don't want any increase in risk." — Regression D from `docs/reasoning-refinement-requirements.md`. + +One live Ollama call (19,343 ms) returned `userSupportedMeaning: "Avoiding additional risk is a hard constraint, and no increase in risk is acceptable."` with `possibleInference: null`. + +**Classification: PASS.** The model preserved the explicit hard-constraint status without weakening it into preference/trade-off language and did not add unsupported interpretation. `possibleInference` is null, which is appropriate for a direct unambiguous answer. + +This experiment does not prove fidelity for other regression cases (E, F), consistency across multiple runs, or behavior in production reasoning paths. Branch: `feature/reasoning-fidelity-v0.8`. Files: `tests/reconstruction/semantic-regression-d-explicit-hard-constraint.test.js` and `docs/experiment-56j.md`. Status pending Rob's review. diff --git a/docs/experiment-56j.md b/docs/experiment-56j.md new file mode 100644 index 0000000..b9317c2 --- /dev/null +++ b/docs/experiment-56j.md @@ -0,0 +1,79 @@ +# Experiment 56J — Explicit Hard Constraint Semantic Fidelity (Regression D) + +## Purpose +Probe whether the configured live Ollama model preserves the user's explicit hard-constraint meaning without weakening it into a preference/trade-off or adding unsupported meaning. + +## Branch / HEAD +- **Branch:** `feature/reasoning-fidelity-v0.8` +- **HEAD:** at time of run, clean working tree on this branch. + +## Historical Live-Call Pattern Reused +Experiment 55D — commit `fcb7218407a2921e9197dbb0a65e4e1282459e4c` +File: `tests/reconstruction/semantic-clarification-stated-vs-inferred.test.js` + +The established mechanism was reused: +- Vitest ESM test; +- `dotenv` loads `.env.local`; +- native `fetch` POST to `${OLLAMA_BASE_URL}/api/chat`; +- `format: "json"`, `stream: false`; +- extract `response.message.content`; +- strip JSON markdown fences; parse structured JSON. + +## Configured Ollama Host / Model +- **Base URL:** `http://192.168.1.111:11434` +- **Model:** `qwen-claude:latest` + +## Call Count +**Exactly 1 real Ollama call.** No retries, no voting, no fallback. + +## Duration +**19,343 ms** (19.3 seconds) + +## Fixed Case — Regression D + +**Source statement:** "I want the business to grow, but I don't want to take on more risk." + +**Clarification target context:** whether avoiding additional risk is a hard constraint or a preference/trade-off + +**Clarification question:** Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off? + +**User's answer (verbatim):** "It's a hard constraint. I don't want any increase in risk." + +## Pre-Written Human Expectation +> Avoiding additional risk is an explicit hard constraint. The user does not accept any increase in risk. + +The answer establishes hard-constraint status. It must **not** be weakened into preference, strong preference, normal tendency, trade-off, or conditionally negotiable language. + +## Raw Parsed Response +```json +{ + "userSupportedMeaning": "Avoiding additional risk is a hard constraint, and no increase in risk is acceptable.", + "possibleInference": null +} +``` + +- **userSupportedMeaning:** "Avoiding additional risk is a hard constraint, and no increase in risk is acceptable." +- **possibleInference:** null (correct — explicit answer does not require inferred implication) + +## Call Duration +19,343 ms + +## Human Semantic Classification: PASS + +### Rationale +`userSupportedMeaning` clearly preserves that avoiding additional risk is an explicit hard constraint with no accepted increase in risk. The output uses the exact phrase "hard constraint" and reinforces it with "no increase in risk is acceptable." No qualification, ambiguity, or extra interpretation weakens fidelity. `possibleInference` is null, which is appropriate for a direct, unambiguous answer. + +### Specific checks +- **Preserves explicit hard-constraint status:** YES — the words "hard constraint" appear directly, reinforced by "no increase in risk is acceptable." +- **Weakened into preference/trade-off language:** NO — no preference, trade-off, or conditional language present. +- **Unsupported interpretation placed in userSupportedMeaning:** NO — `possibleInference` is null; no extra meaning added. + +## What This Experiment Established +For Regression D, the configured live Ollama model (`qwen-claude:latest`) preserves explicit hard-constraint meaning without weakening it. The model did not downgrading the answer into preference/trade-off language, nor did it add unsupported interpretation to `userSupportedMeaning`. + +## What This Experiment Does NOT Prove +- Semantic fidelity for other regression cases (E, F, or others). +- Behavioral fidelity under different prompt framing or system instruction variants. +- Consistency across multiple calls (single-call probe only). +- That the answer would be classified correctly in production reasoning paths (this is not a production-path test). +- That other models or model versions would behave identically. diff --git a/tests/reconstruction/semantic-regression-d-explicit-hard-constraint.test.js b/tests/reconstruction/semantic-regression-d-explicit-hard-constraint.test.js new file mode 100644 index 0000000..980b233 --- /dev/null +++ b/tests/reconstruction/semantic-regression-d-explicit-hard-constraint.test.js @@ -0,0 +1,131 @@ +import { describe, it, expect } from "vitest"; +import { config } from "dotenv"; +import path from "path"; +import { fileURLToPath } from "url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +config({ path: path.resolve(__dirname, "../../.env.local") }); + +const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL; +const OLLAMA_MODEL = process.env.OLLAMA_MODEL; + +if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) { + throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local"); +} + +// Regression D — Explicit hard constraint +const SYSTEM_INSTRUCTION = `You are evaluating whether a user's answer preserves explicit meaning without weakening it. + +State only what the user's answer directly establishes in userSupportedMeaning. Preserve qualification and absoluteness. Do not turn a hard boundary into a preference or trade-off unless the user actually says so. + +If there is a plausible implication that goes beyond what the answer directly establishes, place it only in possibleInference (string or null). + +Do not decide whether clarification is resolved. Do not recommend action.`; + +const CLARIFICATION_TARGET = "whether avoiding additional risk is a hard constraint or a preference/trade-off"; +const CLARIFICATION_QUESTION = "Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off?"; +const USER_ANSWER = "It's a hard constraint. I don't want any increase in risk."; + +const HUMAN_REFERENCE = "Avoiding additional risk is an explicit hard constraint. The user does not accept any increase in risk."; + +async function callRegressionD() { + const messages = [ + { role: "system", content: SYSTEM_INSTRUCTION.trim() }, + { role: "user", content: `Clarification target context: ${CLARIFICATION_TARGET}\n\nClarification question: ${CLARIFICATION_QUESTION}\n\nUser's answer: ${USER_ANSWER}` }, + ]; + + const start = Date.now(); + const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: OLLAMA_MODEL, messages, format: "json", stream: false }), + }); + const duration = Date.now() - start; + + if (!res.ok) throw new Error(`Ollama API error: ${res.status} ${res.statusText}`); + + const data = await res.json(); + const rawContent = data.message?.content ?? ""; + const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, ""); + + return { parsed: JSON.parse(cleaned.trim()), duration }; +} + +describe("Experiment 56J — Explicit Hard Constraint Semantic Fidelity (Regression D)", () => { + it("Regression D: one live call preserves hard-constraint meaning", async () => { + const result = await callRegressionD(); + const { parsed, duration } = result; + const userSupportedMeaning = parsed.userSupportedMeaning ?? ""; + const possibleInference = parsed.possibleInference ?? null; + + expect(userSupportedMeaning).toBeDefined(); + expect(typeof userSupportedMeaning).toBe("string"); + expect(userSupportedMeaning.trim().length).toBeGreaterThan(0); + if (possibleInference !== null) expect(typeof possibleInference).toBe("string"); + + console.log("\n========== Experiment 56J — Regression D =========="); + console.log(`Configured model: ${OLLAMA_MODEL}`); + console.log(`Configured base URL: ${OLLAMA_BASE_URL}`); + console.log(`Call count: 1`); + console.log(`Duration: ${duration}ms`); + console.log(`\n--- Fixed Input ---`); + console.log(`Clarification target: ${CLARIFICATION_TARGET}`); + console.log(`Question: ${CLARIFICATION_QUESTION}`); + console.log(`User answer: "${USER_ANSWER}"`); + console.log(`\n--- Raw Parsed Response ---`); + console.log(JSON.stringify(parsed, null, 2)); + console.log(`\n--- userSupportedMeaning ---`); + console.log(userSupportedMeaning); + console.log(`\n--- possibleInference ---`); + console.log(possibleInference ?? "null"); + console.log(`\n--- Pre-written Human Reference ---`); + console.log(HUMAN_REFERENCE); + + const sm = userSupportedMeaning.toLowerCase().trim(); + const piText = typeof possibleInference === "string" ? possibleInference.toLowerCase().trim() : ""; + const hasHardConstraint = /hard.*constraint|definitively.*not|absolute.*boundary|non.?negotiable|no.*acceptable|won't.*accept.*any|must.*avoid.*any/i.test(sm); + const weakenedToPreference = /^(prefers?|strong.*preference|tends to|would like|should|concerned about|matters more|important|desires)/i.test(sm); + + let classification; + let rationale; + + if (hasHardConstraint) { + if (sm.includes("uncertain") || sm.includes("don't know") || sm.includes("not sure")) { + classification = "FAIL"; + rationale = `Explicit hard constraint replaced with uncertainty: "${userSupportedMeaning}"`; + } else if (weakenedToPreference) { + classification = "FAIL"; + rationale = `Hard constraint meaning weakened into preference/trade-off language: "${userSupportedMeaning}"`; + } else { + if (possibleInference !== null && possibleInference.trim().length > 0) { + classification = "PASS"; + rationale = `Hard constraint preserved clearly. possibleInference present (${possibleInference}) but userSupportedMeaning is clean and unweakened.`; + } else { + classification = "PASS"; + rationale = `Hard constraint preserved clearly with no unnecessary inference: "${userSupportedMeaning}"`; + } + } + } else if (!/uncertain|don't.*know|not.*sure|unsure/i.test(sm)) { + if (weakenedToPreference) { + classification = "FAIL"; + rationale = `Hard constraint weakened into preference/trade-off language: "${userSupportedMeaning}"`; + } else { + classification = "UNRESOLVED"; + rationale = `Output does not establish enough meaning to judge faithfully.`; + } + } else { + classification = "UNRESOLVED"; + rationale = `Output replaced explicit meaning with uncertainty: "${userSupportedMeaning}"`; + } + + console.log(`\n--- Human Semantic Classification ---`); + console.log(classification); + console.log(rationale); + + result._classification = classification; + result._rationale = rationale; + + expect(classification).not.toBe("FAIL"); + }, 120000); +});