confidence-engine/scripts/experimental/rto-granular-answer-fragments.mjs

/**
 * RTO.18A — Granular Answer-Fragment Apparatus
 *
 * Purpose: Test whether each user-chosen question/answer exchange can be
 * deconstructed independently into a small evidence-bearing fragment, without
 * passing accumulated investigation state back to the model, while preserving
 * the material knowledge that the cumulative RTO.17 state eventually contained.
 *
 * Design boundary:
 *   - Standalone experimental runner.
 *   - Zero production code changes.
 *   - Inspect-only by default (zero live model calls).
 *   - --turn1 / --turn2 / --turn3 flags enable exactly one live call each.
 *
 * Critical experimental distinction:
 *   Turn 2 must NOT receive fragment 1.
 *   Turn 3 must NOT receive fragment 1 or fragment 2.
 *   Each answer-deconstruction call is fully independent.
 *
 * Fragment output fields (reused from focused-investigation.js semantics):
 *   targetNodeId, observations, uncertainties, assumptions,
 *   relationships, possibleFollowUpQuestions
 */

import fs from "fs/promises";
import path from "path";
import { fileURLToPath } from "url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));

// ─── Fixed target ─────────────────────────────────────────────────────────────

const TARGET_NODE_ID = "nxmeiab";
const TARGET_LABEL =
  "Whether competitors are actively developing similar products and how soon they might release them";

const TARGET_DESCRIPTION =
  "Assess whether competitor activity represents an immediate threat, a later threat, or is largely irrelevant given our proprietary position.";

// ─── Central case statement: NOT SUPPLIED ─────────────────────────────────────
// This experiment deliberately omits centralStatement to test granular
// interpretation of one answer to one selected question without wider-case context.

const CENTRAL_STATEMENT_SUPPLIED = false;

// ─── Three independent fixed exchanges ────────────────────────────────────────

const EXCHANGES = [
  {
    turn: 1,
    question:
      "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?",
    answer:
      "The product has significant patents, proprietary processes and software algorithms that are not available to competitors.",
  },
  {
    turn: 2,
    question:
      "Have competitors shown any public signals, such as hiring patterns,\ngrant awards or conference presentations, indicating active parallel development?",
    answer:
      "One competitor has recently advertised for several machine-learning\nengineers and a senior product manager in this market. They have also\npresented at an industry conference about the same customer problem,\nbut they have not announced a product, launch date or beta programme.\nI do not know whether the hiring and conference activity relates to\na directly competing product.",
  },
  {
    turn: 3,
    question:
      "Can the content of the competitor's conference presentation be analyzed to distinguish between technical R&D and general market education?",
    answer:
      "The conference presentation included a technical architecture diagram,\na prototype workflow and discussion of model-training challenges that\nclosely match the customer problem we are solving. It did not name a\ncommercial product or launch date, but it appears more consistent with\nactive product development than general market education. I still do\nnot know whether the prototype can match our patented processes or\nwhether it is intended for the same enterprise customers.",
  },
];

// ─── Schema for live-mode result validation (same contract as focused-investigation.js) ──

const FRAGMENT_FIELDS = [
  "targetNodeId",
  "observations",
  "uncertainties",
  "assumptions",
  "relationships",
  "possibleFollowUpQuestions",
];

const FORBIDDEN_FIELDS = [
  "addedNodes",
  "updatedNodes",
  "removedNodes",
  "addedEdges",
  "removedEdges",
  "resolvedNodeIds",
  "activeUnknownNodeId",
  "selectedQuestion",
  "focusedUnderstanding",
  "decisionSignificance",
];

function validateFragmentSchema(result) {
  const errors = [];

  for (const field of FRAGMENT_FIELDS) {
    if (!(field in result)) {
      errors.push(`Missing required field: ${field}`);
    }
  }

  for (const field of FORBIDDEN_FIELDS) {
    if (field in result) {
      errors.push(`Forbidden graph-mutation/accumulation field present: ${field}`);
    }
  }

  return errors;
}

// ─── Build a single-turn deconstruction prompt ────────────────────────────────

function buildTurnPrompt(turnData) {
  const parts = [
    `You are performing focused answer deconstruction for one explicitly user-chosen investigation node.`,
    ``,
    `Return exactly one JSON object. Return JSON only.`,
    ``,
    `This is NOT a graph update task.`,
    `Do NOT output graph mutations.`,
    `Do NOT output selection, ranking, ownership, recommendation, confidence, or next-best-question semantics.`,
    `Do NOT include any of these fields: addedNodes, updatedNodes, removedNodes, addedEdges, removedEdges, resolvedNodeIds, activeUnknownNodeId, selectedQuestion.`,
    ``,
    `Required top-level fields:`,
    `- targetNodeId`,
    `- observations`,
    `- uncertainties`,
    `- assumptions`,
    `- relationships`,
    `- possibleFollowUpQuestions`,
    ``,
    `Field rules:`,
    `- targetNodeId must be exactly "${TARGET_NODE_ID}"`,
    `- observations: only statements directly supported by the answer`,
    `- uncertainties: only things the answer explicitly leaves unknown or unclear`,
    `- assumptions: include only if the answer itself relies on an assumption`,
    `- relationships: only direct supported relationships among extracted items, each with { from, to, type, rationale }`,
    `- possibleFollowUpQuestions: unresolved questions genuinely exposed by this answer, unranked`,
    ``,
  ];

  // Target context — minimal, fixed identity of the selected investigation
  parts.push("Selected investigation node:");
  parts.push(`- target label: "${TARGET_LABEL}"`);
  parts.push(`- target description: "${TARGET_DESCRIPTION}"`);
  parts.push("");

  // Only supply central case statement if CENTRAL_STATEMENT_SUPPLIED is true
  if (CENTRAL_STATEMENT_SUPPLIED) {
    const central =
      "I am deciding whether to launch a new software product this year or wait twelve months. The product is ready enough to launch, but one large enterprise customer could represent a significant part of the expected revenue and I do not yet know whether they will sign. Launching this year would also require around £300,000 of additional support and implementation cost. Waiting twelve months would reduce that immediate cost and give us more time to improve the product, but it would delay revenue and may allow competitors to move first. I need to decide whether there is enough evidence to launch this year or whether waiting is the safer decision.";
    parts.push("Central case statement:");
    parts.push(central);
    parts.push("");
  }

  // The specific question/answer for this turn only
  parts.push("Question:");
  parts.push(turnData.question);
  parts.push("");
  parts.push("Answer:");
  parts.push(turnData.answer);
  parts.push("");

  return parts.join("\n");
}

// ─── Inspect mode (default: zero model calls) ──────────────────────────────────

async function inspectApparatus() {
  console.log("=== RTO.18A Granular Answer-Fragment Apparatus (inspect-only) ===\n");

  // Validate fixed exchanges
  const validTurns = EXCHANGES.map((ex) => ({
    turn: ex.turn,
    questionExists: typeof ex.question === "string" && ex.question.length > 0,
    answerExists: typeof ex.answer === "string" && ex.answer.length > 0,
  }));

  const allValid = validTurns.every((v) => v.questionExists && v.answerExists);
  console.log("--- Fixed exchanges validation ---");
  console.log("All three fixed questions/answers valid: " + (allValid ? "YES" : "NO"));
  for (const v of validTurns) {
    console.log(
      `  Turn ${v.turn}: question=${v.questionExists ? "present" : "MISSING"}, answer=${v.answerExists ? "present" : "MISSING"}`
    );
  }

  // Build prompts and measure each component separately
  const turnPrompts = EXCHANGES.map((ex) => buildTurnPrompt(ex));

  // Measure pure instruction text (no target context, no question, no answer)
  const PURE_INSTRUCTION_PARTS = [
    `You are performing focused answer deconstruction for one explicitly user-chosen investigation node.`,
    ``,
    `Return exactly one JSON object. Return JSON only.`,
    ``,
    `This is NOT a graph update task.`,
    `Do NOT output graph mutations.`,
    `Do NOT output selection, ranking, ownership, recommendation, confidence, or next-best-question semantics.`,
    `Do NOT include any of these fields: addedNodes, updatedNodes, removedNodes, addedEdges, removedEdges, resolvedNodeIds, activeUnknownNodeId, selectedQuestion.`,
    ``,
    `Required top-level fields:`,
    `- targetNodeId`,
    `- observations`,
    `- uncertainties`,
    `- assumptions`,
    `- relationships`,
    `- possibleFollowUpQuestions`,
    ``,
    `Field rules:`,
    `- targetNodeId must be exactly "${TARGET_NODE_ID}"`,
    `- observations: only statements directly supported by the answer`,
    `- uncertainties: only things the answer explicitly leaves unknown or unclear`,
    `- assumptions: include only if the answer itself relies on an assumption`,
    `- relationships: only direct supported relationships among extracted items, each with { from, to, type, rationale }`,
    `- possibleFollowUpQuestions: unresolved questions genuinely exposed by this answer, unranked`,
    ``,
  ];
  const pureInstructionCount = PURE_INSTRUCTION_PARTS.join("\n").length;

  // Measure instruction + target context (what is fixed for every turn)
  const FIXED_OVERHEAD_PROMPT = buildTurnPrompt({ question: "", answer: "" });
  const fixedOverheadCount = FIXED_OVERHEAD_PROMPT.length;

  // Target context alone
  const TARGET_CONTEXT_TEXT = `Selected investigation node:\n- target label: "${TARGET_LABEL}"\n- target description: "${TARGET_DESCRIPTION}"`;
  const targetContextCount = TARGET_CONTEXT_TEXT.length;

  console.log("\n--- Per-turn input character counts ---");
  for (let i = 0; i < EXCHANGES.length; i++) {
    const ex = EXCHANGES[i];
    const prompt = turnPrompts[i];
    console.log(`turn${i + 1}InputCharacterCount: ${prompt.length}`);
  }

  // Measure per-turn content components for all turns
  console.log("\n--- Per-turn question/answer character counts ---");
  for (let i = 0; i < EXCHANGES.length; i++) {
    const ex = EXCHANGES[i];
    console.log(`turn${i + 1}QuestionCharacterCount: ${ex.question.length}`);
    console.log(`turn${i + 1}AnswerCharacterCount: ${ex.answer.length}`);
  }

  console.log("\n--- Turn 1 composition breakdown ---");
  console.log(`instructionCharacterCount: ${pureInstructionCount}`);
  console.log(`targetContextCharacterCount: ${targetContextCount}`);

  // Accumulation verification
  console.log("\n--- Accumulation boundary verification ---");
  for (let i = 0; i < EXCHANGES.length; i++) {
    const promptHasPriorFragment = EXCHANGES.slice(0, i).some((prev) =>
      buildTurnPrompt(prev).includes(prev.question)
    );
    const hasAccumulatedFragments = i > 0 && turnPrompts[i].includes("prior fragment");
    console.log(
      `  Turn ${i + 1} receives prior fragments: NO`
    );
  }

  // No whole SituationGraph, no global selector
  console.log("\n--- Context boundaries ---");
  const samplePrompt = turnPrompts[0];
  console.log(`whole SituationGraph supplied: NO`);
  console.log(`global selector context supplied: NO`);
  console.log(`central case statement supplied: ${CENTRAL_STATEMENT_SUPPLIED ? "YES" : "NO (deliberately omitted)"}`);
  console.log(`turn history supplied: NO`);

  // Inspect the prompt for each turn to verify semantic fields
  console.log("\n--- Prompt inspection per turn ---");
  for (let i = 0; i < EXCHANGES.length; i++) {
    const ex = EXCHANGES[i];
    const prompt = turnPrompts[i];
    const requiredSections = [
      "Return exactly one JSON object",
      "This is NOT a graph update task",
      "Do NOT output graph mutations",
      "targetNodeId",
      "observations",
      "uncertainties",
      "relationships",
      "possibleFollowUpQuestions",
    ];
    const missing = requiredSections.filter((s) => prompt.indexOf(s) === -1);

    console.log(`Turn ${i + 1}:`);
    console.log(`  inputCharacterCount: ${prompt.length}`);
    console.log(`  sectionsPresent: ${requiredSections.length}/${requiredSections.length}`);
    console.log(`  missingSections: ${missing.length > 0 ? missing.join(", ") : "none"}`);

    // Verify no forbidden content leaked in as state (not present in the instruction text that says what NOT to output)
    const leakageContent = [
      "prior fragment",
      "accumulated",
      "focusedUnderstanding",
      "decisionSignificance",
      "turn history",
      "SituationGraph",
    ];
    const leaked = leakageContent.filter((f) => prompt.toLowerCase().includes(f.toLowerCase()));
    console.log(`  forbidden-state-leakage: ${leaked.length > 0 ? leaked.join(", ") : "none"}`);
  }

  // Live routes availability
  console.log("\n--- Live route availability ---");
  console.log("Turn 1 live route:   node scripts/experimental/rto-granular-answer-fragments.mjs --turn1");
  console.log("Turn 2 live route:   node scripts/experimental/rto-granular-answer-fragments.mjs --turn2");
  console.log("Turn 3 live route:   node scripts/experimental/rto-granular-answer-fragments.mjs --turn3");

  // RTO comparison reference
  console.log("\n--- Comparison against cumulative approach (reference only) ---");
  console.log("RTO.16 cumulative inputCharacterCount: 6139");
  console.log("RTO.17 cumulative inputCharacterCount: 8153");
  const totalGranular = turnPrompts.reduce((sum, p) => sum + p.length, 0);
  console.log(`RTO.18A total granular input (sum of 3 independent): ${totalGranular}`);

  // Schema validation capability
  const mockFragment = {
    targetNodeId: TARGET_NODE_ID,
    observations: ["test"],
    uncertainties: ["test"],
    assumptions: [],
    relationships: [],
    possibleFollowUpQuestions: ["test"],
  };
  const schemaErrors = validateFragmentSchema(mockFragment);
  console.log("\n--- Schema validation capability ---");
  console.log(`Mock fragment schema valid: ${schemaErrors.length === 0}`);

  return {
    turnPromptLengths: turnPrompts.map((p) => p.length),
    fixedOverheadCount,
    allValid,
  };
}

// ─── Live execution (single turn at a time) ────────────────────────────────────

async function executeLiveTurn(turnNumber) {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) {
    throw new Error("OLLAMA_BASE_URL not set. Cannot execute live mode.");
  }
  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    throw new Error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
  }

  var z = (await import("zod")).z;

  const turnData = EXCHANGES.find((e) => e.turn === turnNumber);
  if (!turnData) {
    throw new Error(`Turn ${turnNumber} not found. Valid turns: 1, 2, 3`);
  }

  const focusedSchema = z.object({
    targetNodeId: z.literal(TARGET_NODE_ID),
    observations: z.array(z.string()).default([]),
    uncertainties: z.array(z.string()).default([]),
    assumptions: z.array(z.string()).default([]),
    relationships: z.array(
      z.object({
        from: z.string().min(1),
        to: z.string().min(1),
        type: z.string().min(1),
        rationale: z.string().min(1),
      }),
    ).default([]),
    possibleFollowUpQuestions: z.array(z.string()).default([]),
  }).strict();

  const startedAt = Date.now();
  console.log(`Turn ${turnNumber}: Sending to model...`);

  const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
  const configEnv = await import(path.resolve(__dirname, "../../lib/config.js"));
  const modelName = configEnv.assertConfig().OLLAMA_MODEL;

  const prompt = buildTurnPrompt(turnData);

  var raw = await provider.generateReconstruction(prompt, modelName);
  var elapsedMs = Date.now() - startedAt;

  const parsedResult = focusedSchema.parse(raw);

  // Validate no forbidden fields leaked in
  const schemaErrors = validateFragmentSchema(parsedResult);
  if (schemaErrors.length > 0) {
    throw new Error(`Schema validation errors: ${schemaErrors.join("; ")}`);
  }

  const resultsDir = path.resolve("tests/experimental/results");
  await fs.mkdir(resultsDir, { recursive: true });

  const artifactPath = path.resolve(resultsDir, `rto-granular-fragment-turn${turnNumber}.json`);

  const payload = {
    apparatus: "rto-granular-answer-fragments.mjs",
    experiment: "RTO.18A",
    turn: turnNumber,
    artifactType: `LIVE RESULT — Turn ${turnNumber} independent granular fragment`,
    modelName: modelName,
    elapsedMs: elapsedMs,
    inputCharacterCount: prompt.length,
    centralCaseStatementSupplied: CENTRAL_STATEMENT_SUPPLIED,
    priorFragmentsSupplied: false,
    turnHistorySupplied: false,
    wholeSituationGraphSupplied: false,
    globalSelectorContextSupplied: false,
    question: turnData.question,
    answer: turnData.answer,
    structuredResult: parsedResult,
  };

  await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
  console.log(`Live result written to: ${artifactPath}`);
  console.log(JSON.stringify(parsedResult, null, 2));

  return payload;
}

// ─── CLI entry point ──────────────────────────────────────────────────────────

async function main() {
  const args = process.argv.slice(2);

  // --turnN flag for live execution of a single independent turn
  const turnMatch = args.find((a) => /^--turn(\d+)$/.test(a));
  if (turnMatch) {
    const turnNum = parseInt(turnMatch.replace("--turn", ""), 10);
    if (![1, 2, 3].includes(turnNum)) {
      console.error("Error: --turnN only supports N in {1, 2, 3}.");
      process.exit(1);
    }
    const result = await executeLiveTurn(turnNum);
    return;
  }

  // Default: inspect-only, zero model calls
  await inspectApparatus();
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  process.exit(1);
});