test(experiment): checkpoint relationship false-positive apparatus

2026-08-19 17:21:13 +01:00
parent c97f07ba65
commit a73f125f8d
1 changed files with 521 additions and 0 deletions
@@ -0,0 +1,521 @@
+/**
+ * RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus
+ *
+ * Purpose: Test whether semantic relationship discovery correctly returns
+ * "no relationship" when the later fragment does not materially bear on
+ * an earlier uncertainty (false-positive control).
+ *
+ * Design boundary:
+ *   - Standalone experimental runner.
+ *   - Zero production code changes.
+ *   - Inspect-only by default (zero live model calls).
+ *   - --live flag enables exactly one live call.
+ *
+ * Fixed negative-control pair:
+ *   Earlier — RTO.18 Fragment 2 (competitor hiring + conference)
+ *   Later   — negative-control-operational-contract (implementation partner fee)
+ *
+ * Output contract (same as RTO.21):
+ *   { relationshipFound, relationship, remainingQualification }
+ */
+
+import fs from "fs/promises";
+import path from "path";
+import { fileURLToPath } from "url";
+import dotenv from "dotenv";
+
+dotenv.config({ path: ".env.local" });
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const resultsDir = path.resolve(__dirname, "../../tests/experimental/results");
+
+// ─── Fragment paths ──────────────────────────────────────────────────────────
+
+const FRAGMENT_2_PATH = path.resolve(
+  __dirname,
+  "../../tests/experimental/results/rto-granular-fragment-turn2.json"
+);
+
+// ─── Negative-control fragment (EXPERIMENTAL_FIXTURE) ────────────────────────
+
+const NEGATIVE_CONTROL_FRAGMENT = {
+  targetNodeId: "negative-control-operational-contract",
+  question: "When does the implementation partner contract renew and what is the current annual fee?",
+  observations: [
+    "The implementation partner contract renews in November.",
+    "The current annual fee is £42,000."
+  ],
+  uncertainties: [],
+  assumptions: [],
+  relationships: [],
+  possibleFollowUpQuestions: []
+};
+
+// ─── Load fragment 2 ─────────────────────────────────────────────────────────
+
+async function loadFragment2() {
+  const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8");
+  const frag2 = JSON.parse(frag2Raw);
+
+  if (!frag2.structuredResult) {
+    throw new Error("Fragment 2 missing structuredResult");
+  }
+
+  return {
+    fragment: frag2,
+    structuredResult: frag2.structuredResult
+  };
+}
+
+// ─── Negative-control integrity check ────────────────────────────────────────
+
+function verifyNegativeControlClean() {
+  const text = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT);
+
+  const forbiddenTerms = [
+    "competitor",
+    "hiring",
+    "conference",
+    "product development",
+    "patent",
+    "prototype",
+    "launch"
+  ];
+
+  const lowerText = text.toLowerCase();
+  const violations = [];
+
+  for (const term of forbiddenTerms) {
+    if (lowerText.includes(term.toLowerCase())) {
+      violations.push(term);
+    }
+  }
+
+  return {
+    clean: violations.length === 0,
+    violations
+  };
+}
+
+// ─── Build the relationship-discovery prompt ──────────────────────────────────
+
+function buildRelationshipPrompt(srs2, controlSrs) {
+  const parts = [];
+
+  // Instructions — strict semantic discipline (same as RTO.21)
+  parts.push("You are examining two independent reasoning fragments produced during a");
+  parts.push("competitive investigation. Your task is to determine whether the later");
+  parts.push("fragment contains evidence that materially bears on an uncertainty,");
+  parts.push("assumption or observation in the earlier fragment.");
+  parts.push("");
+  parts.push("CRITICAL RULES:");
+  parts.push("- Only consider what is explicitly stated in these two fragments.");
+  parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation.");
+  parts.push("- Do not infer facts not present in either fragment.");
+  parts.push("- Do not rewrite or summarise either fragment.");
+  parts.push("- Do not recommend an action or choose what to investigate next.");
+  parts.push("");
+  parts.push("SEMANTIC DISCIPLINE:");
+  parts.push("- Identify only a material relationship supported by the supplied fragments.");
+  parts.push("- Do not create a relationship merely because both fragments occur in the");
+  parts.push("  same wider case.");
+  parts.push("- Do not infer broad business relevance.");
+  parts.push("- If the later fragment does not materially bear on an earlier item,");
+  parts.push("  return relationshipFound=false.");
+  parts.push("");
+
+  // Fragment 2 (earlier)
+  parts.push("=== Fragment 2 (earlier evidence) ===");
+  parts.push(`Question: ${srs2.question}`);
+  if (srs2.observations && srs2.observations.length) {
+    parts.push("Observations:");
+    for (const o of srs2.observations) {
+      parts.push(`  - ${o}`);
+    }
+  }
+  if (srs2.uncertainties && srs2.uncertainties.length) {
+    parts.push("Uncertainties:");
+    for (const u of srs2.uncertainties) {
+      parts.push(`  - ${u}`);
+    }
+  }
+  if (srs2.assumptions && srs2.assumptions.length) {
+    parts.push("Assumptions:");
+    for (const a of srs2.assumptions) {
+      parts.push(`  - ${a}`);
+    }
+  }
+  if (srs2.relationships && srs2.relationships.length) {
+    parts.push("Relationships (internal to fragment):");
+    for (const r of srs2.relationships) {
+      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
+    }
+  }
+  parts.push("");
+
+  // Negative-control fragment (later)
+  parts.push("=== Fragment (later evidence — negative control) ===");
+  parts.push(`Question: ${controlSrs.question}`);
+  if (controlSrs.observations && controlSrs.observations.length) {
+    parts.push("Observations:");
+    for (const o of controlSrs.observations) {
+      parts.push(`  - ${o}`);
+    }
+  }
+  if (controlSrs.uncertainties && controlSrs.uncertainties.length) {
+    parts.push("Uncertainties:");
+    for (const u of controlSrs.uncertainties) {
+      parts.push(`  - ${u}`);
+    }
+  }
+  if (controlSrs.assumptions && controlSrs.assumptions.length) {
+    parts.push("Assumptions:");
+    for (const a of controlSrs.assumptions) {
+      parts.push(`  - ${a}`);
+    }
+  }
+  if (controlSrs.relationships && controlSrs.relationships.length) {
+    parts.push("Relationships (internal to fragment):");
+    for (const r of controlSrs.relationships) {
+      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
+    }
+  }
+  parts.push("");
+
+  // Core question
+  parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,");
+  parts.push("assumption or observation in the earlier fragment?");
+  parts.push("");
+
+  // Return format — same contract as RTO.21
+  parts.push('Return exactly one JSON object:');
+  parts.push('{ "relationshipFound": <bool>, "relationship": { "laterEvidence": "<string>", "earlierItem": "<string>", "bearing": "<string>" }, "remainingQualification": ["<string>"] }');
+  parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }');
+
+  return parts.join("\n");
+}
+
+// ─── Output contract validation (same as RTO.21) ─────────────────────────────
+
+function validateRelationshipOutput(result) {
+  const errors = [];
+
+  if (typeof result.relationshipFound !== "boolean") {
+    errors.push("relationshipFound must be a boolean");
+  }
+
+  if (!result.relationshipFound) {
+    if (result.relationship !== null && result.relationship !== undefined) {
+      errors.push("When relationshipFound=false, relationship must be null or undefined");
+    }
+    if (!Array.isArray(result.remainingQualification)) {
+      errors.push("remainingQualification must be an array when relationshipFound=false");
+    }
+    return errors;
+  }
+
+  // When true, relationship object is required
+  if (!result.relationship || typeof result.relationship !== "object") {
+    errors.push("When relationshipFound=true, relationship must be an object");
+    return errors;
+  }
+
+  const rel = result.relationship;
+  if (typeof rel.laterEvidence !== "string") {
+    errors.push("relationship.laterEvidence must be a string");
+  }
+  if (typeof rel.earlierItem !== "string") {
+    errors.push("relationship.earlierItem must be a string");
+  }
+  if (typeof rel.bearing !== "string") {
+    errors.push("relationship.bearing must be a string");
+  }
+
+  if (!Array.isArray(result.remainingQualification)) {
+    errors.push("remainingQualification must be an array when relationshipFound=true");
+  } else if (result.remainingQualification.some((s) => typeof s !== "string")) {
+    errors.push("remainingQualification items must all be strings");
+  }
+
+  // No forbidden fields (same as RTO.21)
+  const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
+    "recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"];
+  for (const field of forbidden) {
+    if (field in result && typeof result[field] !== "undefined") {
+      errors.push(`Forbidden field present: ${field}`);
+    }
+  }
+
+  return errors;
+}
+
+// ─── Inspect mode (default: zero model calls) ────────────────────────────────
+
+async function inspectApparatus() {
+  console.log("=== RTO.22A Negative-Control Fragment Relationship-Discovery Apparatus (inspect-only) ===\n");
+
+  // Load and verify fragment 2
+  let structuredResult2;
+  try {
+    const frag2 = await loadFragment2();
+    structuredResult2 = frag2.structuredResult;
+  } catch (e) {
+    console.log("ERROR: Cannot load Fragment 2:", e.message);
+    process.exit(1);
+  }
+
+  // Negative-control integrity check
+  const controlCheck = verifyNegativeControlClean();
+  console.log("--- Negative-control integrity ---");
+  console.log(`Control fragment clean: ${controlCheck.clean ? "YES" : "NO (violations: " + controlCheck.violations.join(", ") + ")"}`);
+
+  if (!controlCheck.clean) {
+    console.log("\nNEGATIVE_CONTROL_CLEAN: NO");
+    console.log("Control fragment contains terms that leak into competitor-development content.");
+    console.log("Cannot proceed with a contaminated negative control.");
+    process.exit(1);
+  }
+
+  // Verify control fragment contents
+  const controlText = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT);
+  const hasContractRenewal = controlText.toLowerCase().includes("renews in november") || controlText.toLowerCase().includes("contract renews");
+  const hasAnnualFee = controlText.includes("£42,000");
+
+  console.log("\n--- Negative-control fixture verification ---");
+  console.log(`Contract renewal observation present: ${hasContractRenewal ? "YES" : "NO"}`);
+  console.log(`£42,000 annual fee observation present: ${hasAnnualFee ? "YES" : "NO"}`);
+  console.log(`Competitor-development content present: NO`);
+
+  // Fragment verification
+  const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
+  console.log("\n--- Fragment verification ---");
+  console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
+  console.log(`Negative-control fragment supplied: YES`);
+
+  // Build prompt for inspection
+  const prompt = buildRelationshipPrompt(structuredResult2, NEGATIVE_CONTROL_FRAGMENT);
+
+  // Measure character sizes
+  const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2"));
+  const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
+  const controlStructuredText = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT, null, 2);
+
+  console.log("\n--- Context-size measurement ---");
+  console.log(`instructionCharacterCount: ${instructionPart.length}`);
+  console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
+  console.log(`negativeControlCharacterCount: ${controlStructuredText.length}`);
+  console.log(`inputCharacterCount: ${prompt.length}`);
+
+  // ── Boundary checks ───────────────────────────────────────────
+
+  console.log("\n--- Input boundary verification ---");
+
+  const knownPositiveRelInPrompt = prompt.includes("Fragment 3's technical evidence bears on Fragment 2") ||
+    prompt.includes("active competing-product development more plausible");
+  console.log(`Known RTO.19/21 positive relationship supplied: ${knownPositiveRelInPrompt ? "YES" : "NO"}`);
+
+  const fragment3Supplied = prompt.includes("Fragment 3") ||
+    NEGATIVE_CONTROL_FRAGMENT.targetNodeId === "fragment-3";
+  console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`);
+
+  const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
+    prompt.includes("whole-case");
+  console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
+
+  const centralCaseInPrompt = prompt.includes("central case") ||
+    prompt.includes("centralStatement");
+  console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
+
+  const currentViewInPrompt = prompt.includes("derived view") ||
+    prompt.includes("focused view") || prompt.includes("current focused");
+  console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
+
+  const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
+    prompt.includes("other fragments") || prompt.includes("additional fragment");
+  console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
+
+  const turnHistoryInPrompt = prompt.includes("turn history") ||
+    prompt.includes("prior turns") || prompt.includes("conversation history");
+  console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
+
+  // ── Output contract validation capability ─────────────────────
+
+  const mockProposal = {
+    relationshipFound: true,
+    relationship: {
+      laterEvidence: "test",
+      earlierItem: "test",
+      bearing: "test",
+    },
+    remainingQualification: ["test"],
+  };
+  const mockNoRel = {
+    relationshipFound: false,
+    relationship: null,
+    remainingQualification: [],
+  };
+  const proposalErrors = validateRelationshipOutput(mockProposal);
+  const noRelErrors = validateRelationshipOutput(mockNoRel);
+  console.log("\n--- Output contract (same as RTO.21) ---");
+  console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
+  console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
+  console.log(`relationshipFound=false allowed: YES`);
+
+  // ── Live route ────────────────────────────────────────────────
+  console.log("\n--- Live route ---");
+  console.log("node scripts/experimental/rto-fragment-relationship-negative-control.mjs --live");
+
+  // ── Environment check ─────────────────────────────────────────
+  const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL;
+  const modelConfigured = !!process.env.OLLAMA_MODEL;
+  console.log("\n--- Environment ---");
+  console.log(`OLLAMA_BASE_URL configured: ${baseUrlConfigured ? "YES" : "NO"}`);
+  console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`);
+
+  // ── Inspect summary ───────────────────────────────────────────
+  const allFragmentsOK = f2Loaded;
+  const noBoundaryLeaks = !knownPositiveRelInPrompt && !fragment3Supplied &&
+    !wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt &&
+    !otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean;
+
+  console.log("\n=== SUMMARY ===");
+  console.log(`Inspect live calls: 0`);
+  console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
+  console.log(`Negative-control fragment supplied: YES`);
+  console.log(`Fragment 3 supplied: NO`);
+  console.log(`Known positive relationship supplied: NO`);
+  console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
+  console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
+  console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
+  console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
+  console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
+  console.log(`NEGATIVE_CONTROL_CLEAN: YES`);
+  console.log(`--live route exposed: YES`);
+  console.log(`maximum live calls per --live: 1`);
+  console.log(`relationshipFound=false allowed: YES`);
+
+  const passed = allFragmentsOK && noBoundaryLeaks && controlCheck.clean;
+  console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);
+
+  if (!passed) {
+    console.log("\nApparatus cannot satisfy boundary without production changes.");
+    process.exit(1);
+  }
+
+  return { prompt };
+}
+
+// ─── Live execution (exactly one model call) ──────────────────────────────────
+
+async function executeLive() {
+  const baseUrl = process.env.OLLAMA_BASE_URL;
+  if (!baseUrl) {
+    console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode.");
+    process.exit(1);
+  }
+  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
+    console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
+    process.exit(1);
+  }
+
+  const modelName = process.env.OLLAMA_MODEL;
+  if (!modelName) {
+    console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode.");
+    process.exit(1);
+  }
+
+  // Load fragment 2 and build prompt
+  const { structuredResult: srs2 } = await loadFragment2();
+  const prompt = buildRelationshipPrompt(srs2, NEGATIVE_CONTROL_FRAGMENT);
+
+  const startedAt = Date.now();
+  console.log("Live mode: sending to model...");
+
+  const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
+  const raw = await provider.generateReconstruction(prompt, modelName);
+  const elapsedMs = Date.now() - startedAt;
+
+  console.log(`Model response received in ${elapsedMs}ms`);
+
+  // Provider returns a parsed JavaScript object via recoverJson().
+  const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => {
+    console.error("Provider returned unexpected type (expected parsed object):", typeof raw);
+    process.exit(1);
+  })();
+
+  // Validate output contract
+  const errors = validateRelationshipOutput(parsedResult);
+  if (errors.length > 0) {
+    console.error("\nLive output contract violations:");
+    for (const err of errors) {
+      console.error(`  - ${err}`);
+    }
+    process.exit(1);
+  }
+
+  // Write artifact
+  await fs.mkdir(resultsDir, { recursive: true });
+
+  const artifactPath = path.resolve(
+    resultsDir,
+    "rto-fragment-relationship-negative-control-live.json"
+  );
+
+  const controlCheck = verifyNegativeControlClean();
+  const payload = {
+    apparatus: "rto-fragment-relationship-negative-control.mjs",
+    experiment: "RTO.22A",
+    artifactType: "LIVE RESULT — Negative-control relationship discovery",
+    modelName: modelName,
+    elapsedMs: elapsedMs,
+    inputCharacterCount: prompt.length,
+    instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length,
+    fragment2CharacterCount: JSON.stringify(srs2, null, 2).length,
+    negativeControlCharacterCount: JSON.stringify(NEGATIVE_CONTROL_FRAGMENT, null, 2).length,
+    fragment2Path: FRAGMENT_2_PATH,
+    controlFixtureType: "EXPERIMENTAL_FIXTURE",
+    negativeControlClean: controlCheck.clean,
+    knownRelationshipSuppliedToModel: false,
+    wholeSituationGraphSupplied: false,
+    centralCaseStatementSupplied: false,
+    turnHistorySupplied: false,
+    proposedRelationship: parsedResult,
+  };
+
+  await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
+  console.log(`\nLive result written to: ${artifactPath}`);
+  console.log("\n=== Proposed Relationship ===");
+  console.log(JSON.stringify(parsedResult, null, 2));
+
+  return payload;
+}
+
+// ─── CLI entry point ──────────────────────────────────────────────────────────
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  // --live flag for exactly one live model call
+  if (args.includes("--live")) {
+    console.log("RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus\n");
+    console.log("WARNING: This will make exactly ONE live model call.\n");
+    const result = await executeLive();
+    return result;
+  }
+
+  // Default: inspect-only, zero model calls
+  console.log("RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus\n");
+  console.log("Run with --live for one live model call.\n");
+
+  const result = await inspectApparatus();
+  if (result === null) {
+    process.exit(1);
+  }
+
+  return result;
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});