2026-09-09 07:58:20 +01:00
1 changed files with 413 additions and 0 deletions
@@ -0,0 +1,413 @@
+/**
+ * RTO.20A — Derived Focused Current-View Apparatus
+ *
+ * Purpose: Test whether an LLM can produce a concise, faithful "where are we
+ * now?" view for one selected investigation using only authoritative granular
+ * fragments and their known relationship, without whole-case reconstruction,
+ * accumulated focused state, or introducing new evidential claims.
+ *
+ * Design boundary:
+ *   - Standalone experimental runner.
+ *   - Zero production code changes.
+ *   - Inspect-only by default (zero live model calls).
+ *   - --live flag enables exactly one live call producing a derived current view.
+ *
+ * Fixed record:
+ *   Fragment 2 — competitor hiring + conference signals
+ *   Fragment 3 — technical architecture + prototype evidence
+ *   One known relationship between them
+ *
+ * Output contract (presentation only, not new reasoning):
+ *   { currentView, whatChanged, remainingUncertainties }
+ */
+
+import fs from "fs/promises";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const resultsDir = path.resolve(__dirname, "../../tests/experimental/results");
+
+// ─── Fragment paths ──────────────────────────────────────────────────────────
+
+const FRAGMENT_2_PATH = path.resolve(
+  __dirname,
+  "../../tests/experimental/results/rto-granular-fragment-turn2.json"
+);
+
+const FRAGMENT_3_PATH = path.resolve(
+  __dirname,
+  "../../tests/experimental/results/rto-granular-fragment-turn3.json"
+);
+
+// ─── Known relationship (supplied, not discovered) ───────────────────────────
+
+const KNOWN_RELATIONSHIP =
+  "Fragment 3's technical evidence bears on Fragment 2's earlier uncertainty " +
+  "by making active competing-product development more plausible, while " +
+  "commercialisation and other uncertainties remain unresolved.";
+
+// ─── Load fragments ──────────────────────────────────────────────────────────
+
+async function loadFragments() {
+  const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8");
+  const frag3Raw = await fs.readFile(FRAGMENT_3_PATH, "utf-8");
+
+  const frag2 = JSON.parse(frag2Raw);
+  const frag3 = JSON.parse(frag3Raw);
+
+  if (!frag2.structuredResult) {
+    throw new Error("Fragment 2 missing structuredResult");
+  }
+  if (!frag3.structuredResult) {
+    throw new Error("Fragment 3 missing structuredResult");
+  }
+
+  return {
+    fragment2: frag2,
+    structuredResult2: frag2.structuredResult,
+    fragment3: frag3,
+    structuredResult3: frag3.structuredResult,
+  };
+}
+
+// ─── Build the derived-current-view prompt ─────────────────────────────────────
+
+function buildCurrentViewPrompt(srs2, srs3) {
+  const parts = [
+    "You are producing a FACILITATOR CURRENT VIEW — a concise summary of where one",
+    "selected investigation stands right now. This is PRESENTATION ONLY.",
+    "",
+    "Return exactly one JSON object with these fields:",
+    "- currentView (string): Where does this investigation stand? What do we understand?",
+    "- whatChanged (string): How has the interpretation shifted from the earlier evidence?",
+    "- remainingUncertainties (array of strings): What is still unresolved?",
+    "",
+    "CRITICAL BOUNDARIES:",
+    "- Every claim must be traceable to Fragment 2, Fragment 3, or the known relationship.",
+    "- Do NOT introduce new evidential claims.",
+    "- Do NOT take action or advise on decisions.",
+    "- Do NOT predict decisions or outcomes.",
+    "- Do NOT select next questions or take ownership from the user.",
+    "- This output is derived presentation, not new authoritative reasoning.",
+    "",
+    "Selected investigation identity:",
+  ];
+
+  // Node label from Fragment 2 (they share the same target)
+  parts.push(`- target node: "${srs2.targetNodeId}"`);
+  parts.push("");
+
+  // ── Fragment 2 ────────────────────────────────────────────────
+  parts.push("=== Fragment 2 (earlier evidence) ===");
+  if (srs2.observations && srs2.observations.length) {
+    parts.push("Observations:");
+    for (const o of srs2.observations) {
+      parts.push(`  - ${o}`);
+    }
+  }
+  if (srs2.uncertainties && srs2.uncertainties.length) {
+    parts.push("Uncertainties:");
+    for (const u of srs2.uncertainties) {
+      parts.push(`  - ${u}`);
+    }
+  }
+  if (srs2.assumptions && srs2.assumptions.length) {
+    parts.push("Assumptions:");
+    for (const a of srs2.assumptions) {
+      parts.push(`  - ${a}`);
+    }
+  }
+  if (srs2.relationships && srs2.relationships.length) {
+    parts.push("Relationships:");
+    for (const r of srs2.relationships) {
+      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
+    }
+  }
+  parts.push("");
+
+  // ── Fragment 3 ────────────────────────────────────────────────
+  parts.push("=== Fragment 3 (newer evidence) ===");
+  if (srs3.observations && srs3.observations.length) {
+    parts.push("Observations:");
+    for (const o of srs3.observations) {
+      parts.push(`  - ${o}`);
+    }
+  }
+  if (srs3.uncertainties && srs3.uncertainties.length) {
+    parts.push("Uncertainties:");
+    for (const u of srs3.uncertainties) {
+      parts.push(`  - ${u}`);
+    }
+  }
+  if (srs3.assumptions && srs3.assumptions.length) {
+    parts.push("Assumptions:");
+    for (const a of srs3.assumptions) {
+      parts.push(`  - ${a}`);
+    }
+  }
+  if (srs3.relationships && srs3.relationships.length) {
+    parts.push("Relationships:");
+    for (const r of srs3.relationships) {
+      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
+    }
+  }
+  parts.push("");
+
+  // ── Known relationship ────────────────────────────────────────
+  parts.push("=== Known relationship (supplied, not discovered) ===");
+  parts.push(KNOWN_RELATIONSHIP);
+  parts.push("");
+
+  // ── Instructions ──────────────────────────────────────────────
+  parts.push("Produce a concise facilitator view of where this selected investigation stands now.");
+  parts.push("Answer: What do we understand now? How has the interpretation shifted? What remains unresolved?");
+  parts.push("Do NOT take action or advise on decisions.");
+  parts.push("Return JSON only. No additional commentary.");
+
+  return parts.join("\n");
+}
+
+// ─── Output schema for live mode ──────────────────────────────────────────────
+
+const CURRENT_VIEW_SCHEMA = {
+  required: ["currentView", "whatChanged", "remainingUncertainties"],
+  forbidden: [
+    "recommendation",
+    "nextQuestion",
+    "confidence",
+    "materiality",
+    "decision",
+    "action",
+    "status",
+    "score",
+    "addedNodes",
+    "updatedNodes",
+    "resolvedNodeIds",
+    "selectedQuestion",
+  ],
+};
+
+function validateOutputSchema(result) {
+  const errors = [];
+
+  for (const field of CURRENT_VIEW_SCHEMA.required) {
+    if (!(field in result)) {
+      errors.push(`Missing required field: ${field}`);
+    }
+  }
+
+  if (!Array.isArray(result.remainingUncertainties)) {
+    errors.push("remainingUncertainties must be an array");
+  } else if (result.remainingUncertainties.some((s) => typeof s !== "string")) {
+    errors.push("remainingUncertainties array items must all be strings");
+  }
+
+  for (const field of CURRENT_VIEW_SCHEMA.forbidden) {
+    if (field in result && typeof result[field] !== "undefined") {
+      errors.push(`Forbidden field present: ${field}`);
+    }
+  }
+
+  return errors;
+}
+
+// ─── Inspect mode (default: zero model calls) ─────────────────────────────────
+
+async function inspectApparatus() {
+  console.log("=== RTO.20A Derived Focused Current-View Apparatus (inspect-only) ===\n");
+
+  // Load and verify fragments
+  let fragment2, structuredResult2, fragment3, structuredResult3;
+  try {
+    const frags = await loadFragments();
+    fragment2 = frags.fragment2;
+    structuredResult2 = frags.structuredResult2;
+    fragment3 = frags.fragment3;
+    structuredResult3 = frags.structuredResult3;
+  } catch (e) {
+    console.log("ERROR: Cannot load fragments:", e.message);
+    process.exit(1);
+  }
+
+  // Fragment verification
+  const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
+  const f3Loaded = typeof structuredResult3 === "object" && structuredResult3 !== null;
+  console.log("--- Fragment verification ---");
+  console.log(`Fragment 2 loaded: ${f2Loaded ? "YES" : "NO"}`);
+  console.log(`Fragment 3 loaded: ${f3Loaded ? "YES" : "NO"}`);
+
+  // Build prompt for inspection
+  const prompt = buildCurrentViewPrompt(structuredResult2, structuredResult3);
+
+  // Known relationship verification
+  const relationshipIncluded = prompt.includes(KNOWN_RELATIONSHIP);
+  console.log("\n--- Relationship ---");
+  console.log(`Known relationship included: ${relationshipIncluded ? "YES" : "NO"}`);
+
+  // Context boundary checks
+  console.log("\n--- Context boundaries ---");
+  const boundaryChecks = {
+    "Whole SituationGraph": !prompt.includes("SituationGraph"),
+    "Central case statement": !prompt.includes("central case") && !prompt.includes("centralStatement"),
+    "Turn history": !prompt.includes("turn history"),
+    "Other fragments": !prompt.includes("Fragment 1"),
+    "Prior focusedUnderstanding": !prompt.includes("focusedUnderstanding"),
+    "Prior decisionSignificance": !prompt.includes("decisionSignificance"),
+    "Graph mutation instructions": !prompt.includes("addedNodes") && !prompt.includes("updatedNodes"),
+    "Recommendation instructions": !prompt.includes("recommend what"),
+  };
+
+  for (const [name, passed] of Object.entries(boundaryChecks)) {
+    const status = passed ? "NO" : "VIOLATION";
+    console.log(`  ${name}: ${status}`);
+  }
+
+  // Schema validation capability
+  const mockOutput = {
+    currentView: "test",
+    whatChanged: "test",
+    remainingUncertainties: ["test"],
+  };
+  const schemaErrors = validateOutputSchema(mockOutput);
+  console.log("\n--- Output schema ---");
+  console.log(`Required fields: ${CURRENT_VIEW_SCHEMA.required.join(", ")}`);
+  console.log(`Validation capability: ${schemaErrors.length === 0 ? "valid" : "errors"}`);
+
+  // Prompt size
+  console.log("\n--- Prompt size ---");
+  console.log(`inputCharacterCount: ${prompt.length}`);
+
+  // Live route
+  console.log("\n--- Live route ---");
+  console.log("node scripts/experimental/rto-derived-focused-current-view.mjs --live");
+
+  // Inspect summary
+  const allFragmentsOK = f2Loaded && f3Loaded;
+  const relationshipOK = relationshipIncluded;
+  const noViolations = Object.values(boundaryChecks).every((v) => v === true);
+
+  console.log("\n=== SUMMARY ===");
+  console.log(`Inspect live calls: 0`);
+  console.log(`Fragments valid: ${allFragmentsOK ? "YES" : "NO"}`);
+  console.log(`Boundary clean: ${noViolations ? "YES" : "VIOLATION DETECTED"}`);
+
+  const passed = allFragmentsOK && relationshipOK && noViolations;
+  console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);
+
+  if (!passed) {
+    console.log("\nApparatus cannot satisfy boundary without production changes.");
+    console.log("F — APPARATUS CANNOT ISOLATE DERIVED VIEW");
+    return null;
+  }
+
+  return { prompt, inputCharacterCount: prompt.length };
+}
+
+// ─── Live execution (exactly one model call) ──────────────────────────────────
+
+async function executeLive() {
+  const baseUrl = process.env.OLLAMA_BASE_URL;
+  if (!baseUrl) {
+    console.error("OLLAMA_BASE_URL not set. Cannot execute live mode.");
+    process.exit(1);
+  }
+  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
+    console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
+    process.exit(1);
+  }
+
+  var z = (await import("zod")).z;
+
+  const { structuredResult2, structuredResult3 } = await loadFragments();
+
+  const prompt = buildCurrentViewPrompt(structuredResult2, structuredResult3);
+
+  // Schema for live output validation
+  const currentViewSchema = z.object({
+    currentView: z.string().min(1),
+    whatChanged: z.string().min(1),
+    remainingUncertainties: z.array(z.string()),
+  }).strict();
+
+  const startedAt = Date.now();
+  console.log("Live mode: sending to model...");
+
+  const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
+  const configEnv = await import(path.resolve(__dirname, "../../lib/config.js"));
+  const modelName = configEnv.assertConfig().OLLAMA_MODEL;
+
+  var raw = await provider.generateReconstruction(prompt, modelName);
+  var elapsedMs = Date.now() - startedAt;
+
+  console.log(`Model response received in ${elapsedMs}ms`);
+
+  const parsedResult = currentViewSchema.parse(raw);
+
+  // Validate no forbidden fields leaked in
+  const schemaErrors = validateOutputSchema(parsedResult);
+  if (schemaErrors.length > 0) {
+    throw new Error(`Live output schema validation errors: ${schemaErrors.join("; ")}`);
+  }
+
+  await fs.mkdir(resultsDir, { recursive: true });
+
+  const artifactPath = path.resolve(
+    resultsDir,
+    "rto-derived-focused-current-view-live.json"
+  );
+
+  const payload = {
+    apparatus: "rto-derived-focused-current-view.mjs",
+    experiment: "RTO.20A",
+    artifactType: "LIVE RESULT — Derived focused current view",
+    modelName: modelName,
+    elapsedMs: elapsedMs,
+    inputCharacterCount: prompt.length,
+    fragment2Path: FRAGMENT_2_PATH,
+    fragment3Path: FRAGMENT_3_PATH,
+    knownRelationshipIncluded: true,
+    wholeSituationGraphSupplied: false,
+    centralCaseStatementSupplied: false,
+    turnHistorySupplied: false,
+    outputSchema: {
+      currentView: typeof parsedResult.currentView === "string",
+      whatChanged: typeof parsedResult.whatChanged === "string",
+      remainingUncertainties: Array.isArray(parsedResult.remainingUncertainties),
+    },
+    derivedCurrentView: parsedResult,
+  };
+
+  await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
+  console.log(`\nLive result written to: ${artifactPath}`);
+  console.log("\n=== Derived Current View ===");
+  console.log(JSON.stringify(parsedResult, null, 2));
+
+  return payload;
+}
+
+// ─── CLI entry point ──────────────────────────────────────────────────────────
+
+async function main() {
+  const args = process.argv.slice(2);
+
+  // --live flag for exactly one live model call
+  if (args.includes("--live")) {
+    const result = await executeLive();
+    return;
+  }
+
+  // Default: inspect-only, zero model calls
+  console.log("RTO.20A — Derived Focused Current View");
+  console.log("Run with --live for one live model call.\n");
+
+  const result = await inspectApparatus();
+  if (result === null) {
+    process.exit(1);
+  }
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});