test(experiment): checkpoint fragment relationship discovery apparatus

2026-08-19 15:43:06 +01:00
parent 8c52939c02
commit 8fb284c374
1 changed files with 495 additions and 0 deletions
@@ -0,0 +1,495 @@
+/**
+ * RTO.21A — Fragment Relationship-Discovery Apparatus
+ *
+ * Purpose: Test whether an LLM can propose a useful cross-fragment relationship
+ * from two independent granular fragments without whole-case context, without
+ * rewriting either fragment, and without treating the discovered relationship as
+ * automatically authoritative.
+ *
+ * Design boundary:
+ *   - Standalone experimental runner.
+ *   - Zero production code changes.
+ *   - Inspect-only by default (zero live model calls).
+ *   - --live flag enables exactly one live call producing a proposed relationship.
+ *
+ * Fixed record:
+ *   Fragment 2 — competitor hiring + conference signals
+ *   Fragment 3 — technical architecture + prototype evidence
+ *   Known RTO.19 relationship is NEVER supplied to the model; used only for
+ *   evaluation reference after a future live run.
+ *
+ * Output contract (proposal only, never authoritative):
+ *   { relationshipFound, relationship, remainingQualification }
+ */
+
+import fs from "fs/promises";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const resultsDir = path.resolve(__dirname, "../../tests/experimental/results");
+const envLocalPath = path.resolve(__dirname, "../../.env.local");
+
+// ─── Fragment paths ──────────────────────────────────────────────────────────
+
+const FRAGMENT_2_PATH = path.resolve(
+  __dirname,
+  "../../tests/experimental/results/rto-granular-fragment-turn2.json"
+);
+
+const FRAGMENT_3_PATH = path.resolve(
+  __dirname,
+  "../../tests/experimental/results/rto-granular-fragment-turn3.json"
+);
+
+// ─── Known RTO.19 relationship (evaluation reference ONLY) ───────────────────
+// This MUST NOT appear in the prompt sent to the model.
+
+const KNOWN_RELATIONSHIP =
+  "Fragment 3's technical evidence bears on Fragment 2's earlier uncertainty " +
+  "by making active competing-product development more plausible, while " +
+  "commercialisation and other uncertainties remain unresolved.";
+
+// ─── Load fragments ──────────────────────────────────────────────────────────
+
+async function loadFragments() {
+  const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8");
+  const frag3Raw = await fs.readFile(FRAGMENT_3_PATH, "utf-8");
+
+  const frag2 = JSON.parse(frag2Raw);
+  const frag3 = JSON.parse(frag3Raw);
+
+  if (!frag2.structuredResult) {
+    throw new Error("Fragment 2 missing structuredResult");
+  }
+  if (!frag3.structuredResult) {
+    throw new Error("Fragment 3 missing structuredResult");
+  }
+
+  return {
+    fragment2: frag2,
+    structuredResult2: frag2.structuredResult,
+    fragment3: frag3,
+    structuredResult3: frag3.structuredResult,
+  };
+}
+
+// ─── Build the relationship-discovery prompt ──────────────────────────────────
+
+function buildRelationshipPrompt(srs2, srs3) {
+  const parts = [];
+
+  // Instructions
+  parts.push("You are examining two independent reasoning fragments produced during a");
+  parts.push("competitive investigation. Your task is to determine whether the later");
+  parts.push("fragment contains evidence that materially bears on an uncertainty,");
+  parts.push("assumption or observation in the earlier fragment.");
+  parts.push("");
+  parts.push("CRITICAL RULES:");
+  parts.push("- Only consider what is explicitly stated in these two fragments.");
+  parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation.");
+  parts.push("- Do not infer facts not present in either fragment.");
+  parts.push("- Do not rewrite or summarise either fragment.");
+  parts.push("- Do not recommend an action or choose what to investigate next.");
+  parts.push("- If no relationship exists, return relationshipFound=false with empty fields.");
+  parts.push("");
+
+  // Fragment 2 (earlier)
+  parts.push("=== Fragment 2 (earlier evidence) ===");
+  parts.push(`Question: ${srs2.question}`);
+  if (srs2.observations && srs2.observations.length) {
+    parts.push("Observations:");
+    for (const o of srs2.observations) {
+      parts.push(`  - ${o}`);
+    }
+  }
+  if (srs2.uncertainties && srs2.uncertainties.length) {
+    parts.push("Uncertainties:");
+    for (const u of srs2.uncertainties) {
+      parts.push(`  - ${u}`);
+    }
+  }
+  if (srs2.assumptions && srs2.assumptions.length) {
+    parts.push("Assumptions:");
+    for (const a of srs2.assumptions) {
+      parts.push(`  - ${a}`);
+    }
+  }
+  if (srs2.relationships && srs2.relationships.length) {
+    parts.push("Relationships (internal to fragment):");
+    for (const r of srs2.relationships) {
+      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
+    }
+  }
+  parts.push("");
+
+  // Fragment 3 (later)
+  parts.push("=== Fragment 3 (later evidence) ===");
+  parts.push(`Question: ${srs3.question}`);
+  if (srs3.observations && srs3.observations.length) {
+    parts.push("Observations:");
+    for (const o of srs3.observations) {
+      parts.push(`  - ${o}`);
+    }
+  }
+  if (srs3.uncertainties && srs3.uncertainties.length) {
+    parts.push("Uncertainties:");
+    for (const u of srs3.uncertainties) {
+      parts.push(`  - ${u}`);
+    }
+  }
+  if (srs3.assumptions && srs3.assumptions.length) {
+    parts.push("Assumptions:");
+    for (const a of srs3.assumptions) {
+      parts.push(`  - ${a}`);
+    }
+  }
+  if (srs3.relationships && srs3.relationships.length) {
+    parts.push("Relationships (internal to fragment):");
+    for (const r of srs3.relationships) {
+      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
+    }
+  }
+  parts.push("");
+
+  // Core question
+  parts.push("QUESTION: Does anything in Fragment 3 materially bears on an uncertainty,");
+  parts.push("assumption or observation in Fragment 2?");
+  parts.push("");
+  parts.push("If YES, propose exactly one relationship. This is a PROPOSED_RELATIONSHIP");
+  parts.push("--- it must not be treated as authoritative without further verification.");
+  parts.push("");
+
+  // Return format
+  parts.push('Return exactly one JSON object:');
+  parts.push('{ "relationshipFound": <bool>, "relationship": { "laterEvidence": "<string>", "earlierItem": "<string>", "bearing": "<string>" }, "remainingQualification": ["<string>"] }');
+  parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }');
+
+  return parts.join("\n");
+}
+
+// ─── Output contract validation ──────────────────────────────────────────────
+
+function validateRelationshipOutput(result) {
+  const errors = [];
+
+  if (typeof result.relationshipFound !== "boolean") {
+    errors.push("relationshipFound must be a boolean");
+  }
+
+  if (!result.relationshipFound) {
+    if (result.relationship !== null && result.relationship !== undefined) {
+      errors.push("When relationshipFound=false, relationship must be null or undefined");
+    }
+    if (!Array.isArray(result.remainingQualification)) {
+      errors.push("remainingQualification must be an array when relationshipFound=false");
+    }
+    return errors;
+  }
+
+  // When true, relationship object is required
+  if (!result.relationship || typeof result.relationship !== "object") {
+    errors.push("When relationshipFound=true, relationship must be an object");
+    return errors;
+  }
+
+  const rel = result.relationship;
+  if (typeof rel.laterEvidence !== "string") {
+    errors.push("relationship.laterEvidence must be a string");
+  }
+  if (typeof rel.earlierItem !== "string") {
+    errors.push("relationship.earlierItem must be a string");
+  }
+  if (typeof rel.bearing !== "string") {
+    errors.push("relationship.bearing must be a string");
+  }
+
+  if (!Array.isArray(result.remainingQualification)) {
+    errors.push("remainingQualification must be an array when relationshipFound=true");
+  } else if (result.remainingQualification.some((s) => typeof s !== "string")) {
+    errors.push("remainingQualification items must all be strings");
+  }
+
+  // No forbidden fields
+  const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
+    "recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"];
+  for (const field of forbidden) {
+    if (field in result && typeof result[field] !== "undefined") {
+      errors.push(`Forbidden field present: ${field}`);
+    }
+  }
+
+  return errors;
+}
+
+// ─── Inspect mode (default: zero model calls) ─────────────────────────────────
+
+async function inspectApparatus() {
+  console.log("=== RTO.21A Fragment Relationship-Discovery Apparatus (inspect-only) ===\n");
+
+  // Load and verify fragments
+  let fragment2, structuredResult2, fragment3, structuredResult3;
+  try {
+    const frags = await loadFragments();
+    fragment2 = frags.fragment2;
+    structuredResult2 = frags.structuredResult2;
+    fragment3 = frags.fragment3;
+    structuredResult3 = frags.structuredResult3;
+  } catch (e) {
+    console.log("ERROR: Cannot load fragments:", e.message);
+    process.exit(1);
+  }
+
+  // Fragment verification
+  const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
+  const f3Loaded = typeof structuredResult3 === "object" && structuredResult3 !== null;
+  console.log("--- Fragment verification ---");
+  console.log(`Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
+  console.log(`Fragment 3 supplied: ${f3Loaded ? "YES" : "NO"}`);
+
+  // Build prompt for inspection
+  const prompt = buildRelationshipPrompt(structuredResult2, structuredResult3);
+
+  // Measure character sizes precisely
+  const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2"));
+  const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
+  const frag3StructuredText = JSON.stringify(structuredResult3, null, 2);
+
+  console.log("\n--- Context-size measurement ---");
+  console.log(`instructionCharacterCount: ${instructionPart.length}`);
+  console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
+  console.log(`fragment3CharacterCount: ${frag3StructuredText.length}`);
+  console.log(`inputCharacterCount: ${prompt.length}`);
+
+  // ── Boundary checks ───────────────────────────────────────────
+
+  console.log("\n--- Input boundary verification ---");
+
+  const knownRelInPrompt = prompt.includes(KNOWN_RELATIONSHIP) ||
+    prompt.includes("Fragment 3's technical evidence bears on Fragment 2's earlier uncertainty");
+  console.log(`Known relationship supplied: ${knownRelInPrompt ? "YES" : "NO"}`);
+
+  const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
+    prompt.includes("whole-case") || prompt.includes("case statement");
+  console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
+
+  const centralCaseInPrompt = prompt.includes("central case") ||
+    prompt.includes("centralStatement");
+  console.log(`Central case statement supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
+
+  const currentViewInPrompt = prompt.includes("derived view") ||
+    prompt.includes("focused view") || prompt.includes("current focused");
+  console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
+
+  const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
+    prompt.includes("other fragments") || prompt.includes("additional fragment");
+  console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
+
+  const turnHistoryInPrompt = prompt.includes("turn history") ||
+    prompt.includes("prior turns") || prompt.includes("conversation history");
+  console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
+
+  const rto17StateInPrompt = prompt.includes("decisionSignificance") ||
+    prompt.includes("accumulated state") || prompt.includes("focusedUnderstanding");
+  console.log(`RTO.17 accumulated state supplied: ${rto17StateInPrompt ? "YES" : "NO"}`);
+
+  // ── Schema validation capability ──────────────────────────────
+  const mockProposal = {
+    relationshipFound: true,
+    relationship: {
+      laterEvidence: "test",
+      earlierItem: "test",
+      bearing: "test",
+    },
+    remainingQualification: ["test"],
+  };
+  const mockNoRel = {
+    relationshipFound: false,
+    relationship: null,
+    remainingQualification: [],
+  };
+  const proposalErrors = validateRelationshipOutput(mockProposal);
+  const noRelErrors = validateRelationshipOutput(mockNoRel);
+  console.log("\n--- Output schema ---");
+  console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
+  console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
+
+  // ── Live route ────────────────────────────────────────────────
+  console.log("\n--- Live route ---");
+  console.log("node scripts/experimental/rto-fragment-relationship-discovery.mjs --live");
+
+  // ── Inspect summary (must match exact requested format) ───────
+  const allFragmentsOK = f2Loaded && f3Loaded;
+  const knownRelAbsent = !knownRelInPrompt;
+  const noContextLeaks = !(wholeGraphInPrompt || centralCaseInPrompt || currentViewInPrompt || otherFragmentsInPrompt || turnHistoryInPrompt || rto17StateInPrompt);
+
+  console.log("\n=== SUMMARY ===");
+  console.log(`Inspect live calls: 0`);
+  console.log(`Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
+  console.log(`Fragment 3 supplied: ${f3Loaded ? "YES" : "NO"}`);
+  console.log(`Known relationship supplied: ${knownRelInPrompt ? "YES" : "NO"}`);
+  console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
+  console.log(`Central case statement supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
+  console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
+  console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
+  console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
+  console.log(`--live route exposed: YES`);
+  console.log(`maximum live calls per --live: 1`);
+  console.log(`relationshipFound=false allowed: YES`);
+
+  const passed = allFragmentsOK && knownRelAbsent && noContextLeaks;
+  console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);
+
+  if (!passed) {
+    console.log("\nApparatus cannot satisfy boundary without production changes.");
+    process.exit(1);
+  }
+
+  return { prompt };
+}
+
+// ─── Live execution (exactly one model call) ──────────────────────────────────
+
+async function executeLive() {
+  // Load .env.local for authoritative config
+  let envContent = "";
+  try {
+    envContent = await fs.readFile(envLocalPath, "utf-8");
+  } catch {
+    console.error(".env.local not found. Cannot execute live mode.");
+    process.exit(1);
+  }
+
+  const baseUrl = parseEnv(envContent, "OLLAMA_BASE_URL");
+  if (!baseUrl) {
+    console.error("OLLAMA_BASE_URL not set in .env.local. Cannot execute live mode.");
+    process.exit(1);
+  }
+  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
+    console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
+    process.exit(1);
+  }
+
+  const modelName = parseEnv(envContent, "OLLAMA_MODEL");
+  if (!modelName) {
+    console.error("OLLAMA_MODEL not set in .env.local. Cannot execute live mode.");
+    process.exit(1);
+  }
+
+  // Load fragments and build prompt
+  const { structuredResult2, structuredResult3 } = await loadFragments();
+  const prompt = buildRelationshipPrompt(structuredResult2, structuredResult3);
+
+  const startedAt = Date.now();
+  console.log("Live mode: sending to model...");
+
+  const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
+  const raw = await provider.generateReconstruction(prompt, modelName);
+  const elapsedMs = Date.now() - startedAt;
+
+  console.log(`Model response received in ${elapsedMs}ms`);
+
+  // Parse the JSON output
+  let parsedResult;
+  try {
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (jsonMatch) {
+      parsedResult = JSON.parse(jsonMatch[0]);
+    } else {
+      throw new Error("No JSON object found in response");
+    }
+  } catch (e) {
+    console.error("Failed to parse model output as JSON:", e.message);
+    console.error("Raw response:\n", raw);
+    process.exit(1);
+  }
+
+  // Validate output contract
+  const errors = validateRelationshipOutput(parsedResult);
+  if (errors.length > 0) {
+    console.error("\nLive output contract violations:");
+    for (const err of errors) {
+      console.error(`  - ${err}`);
+    }
+    process.exit(1);
+  }
+
+  // Write artifact
+  await fs.mkdir(resultsDir, { recursive: true });
+
+  const artifactPath = path.resolve(
+    resultsDir,
+    "rto-fragment-relationship-discovery-live.json"
+  );
+
+  const payload = {
+    apparatus: "rto-fragment-relationship-discovery.mjs",
+    experiment: "RTO.21A",
+    artifactType: "LIVE RESULT — Proposed relationship discovery",
+    modelName: modelName,
+    elapsedMs: elapsedMs,
+    inputCharacterCount: prompt.length,
+    instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length,
+    fragment2CharacterCount: JSON.stringify(structuredResult2, null, 2).length,
+    fragment3CharacterCount: JSON.stringify(structuredResult3, null, 2).length,
+    fragment2Path: FRAGMENT_2_PATH,
+    fragment3Path: FRAGMENT_3_PATH,
+    knownRelationshipSuppliedToModel: false,
+    wholeSituationGraphSupplied: false,
+    centralCaseStatementSupplied: false,
+    turnHistorySupplied: false,
+    proposedRelationship: parsedResult,
+  };
+
+  await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
+  console.log(`\nLive result written to: ${artifactPath}`);
+  console.log("\n=== Proposed Relationship ===");
+  console.log(JSON.stringify(parsedResult, null, 2));
+
+  return payload;
+}
+
+// ─── Helper: parse .env.local ────────────────────────────────────────────────
+
+function parseEnv(content, key) {
+  const lines = content.split("\n");
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith("#")) continue;
+    const [k, ...vParts] = trimmed.split("=");
+    if (k.trim() === key) {
+      return vParts.join("=").trim().replace(/^["']|["']$/g, "");
+    }
+  }
+  return null;
+}
+
+// ─── CLI entry point ──────────────────────────────────────────────────────────
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  // --live flag for exactly one live model call
+  if (args.includes("--live")) {
+    console.log("RTO.21A — Fragment Relationship-Discovery Apparatus\n");
+    console.log("WARNING: This will make exactly ONE live model call.\n");
+    const result = await executeLive();
+    return result;
+  }
+
+  // Default: inspect-only, zero model calls
+  console.log("RTO.21A — Fragment Relationship-Discovery Apparatus\n");
+  console.log("Run with --live for one live model call.\n");
+
+  const result = await inspectApparatus();
+  if (result === null) {
+    process.exit(1);
+  }
+
+  return result;
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});