confidence-engine/scripts/experimental/rto-layer-separated-focused-reasoning.mjs

/**
 * RTO.16A — Separated focused-understanding / decision-significance apparatus.
 *
 * Purpose: Test whether one model invocation can keep focused investigation
 * understanding and wider decision-significance reasoning explicitly separate,
 * while preserving the useful behaviour observed in RTO.15.
 *
 * Design boundary:
 *   - Standalone experimental runner.
 *   - Zero production code changes.
 *   - Inspect-only by default (zero live model calls).
 *   - --live flag enables exactly one live call.
 *
 * Output contract for each layer:
 *   focusedUnderstanding: { observations, uncertainties, assumptions,
 *                           relationships, possibleFollowUpQuestions }
 *   decisionSignificance: [{ insight }]  (concise wider-decision connections)
 */

import fs from "fs/promises";
import path from "path";
import { fileURLToPath } from "url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));

// ─── Fixed inputs — exact RTO.15 competitor/IP scenario ──────────────────────

const TARGET_NODE_ID = "nxmeiab";

const CENTRAL_CASE_STATEMENT =
  "I am deciding whether to launch a new software product this year or wait twelve months. The product is ready enough to launch, but one large enterprise customer could represent a significant part of the expected revenue and I do not yet know whether they will sign. Launching this year would also require around £300,000 of additional support and implementation cost. Waiting twelve months would reduce that immediate cost and give us more time to improve the product, but it would delay revenue and may allow competitors to move first. I need to decide whether there is enough evidence to launch this year or whether waiting is the safer decision.";

const TARGET_LABEL =
  "Whether competitors are actively developing similar products and how soon they might release them";

// ─── Turn-1 prior focused state — identical to RTO.15 ────────────────────────

const PRIOR_FOCUSED_STATE = {
  observations: [
    "The product is protected by significant patents.",
    "The product utilizes proprietary processes.",
    "The product incorporates proprietary software algorithms.",
    "These intellectual property assets and technical components are not available to competitors.",
  ],
  uncertainties: [
    "Whether competitors can engineer non-infringing workarounds or alternative architectures.",
    "Actual development timeline / funding / public activity of competitors.",
    "How long it could take a competitor to design around the IP barriers.",
    "Whether the patents and proprietary elements materially delay competitor entry.",
  ],
  assumptions: [
    "Existing IP acts as an effective barrier to rapid competitor replication.",
    "Competitors cannot cheaply or quickly bypass, replicate or license around the proprietary technology.",
  ],
  relationships: [
    { from: "Patents", to: "Restricted competitor access", type: "blocks" },
    { from: "Proprietary processes / algorithms", to: "Technical differentiation", type: "enables" },
  ],
  possibleFollowUpQuestions: [
    "Have competitors shown any public signals, such as hiring patterns, grant awards or conference presentations, indicating active parallel development?",
  ],
};

// ─── Turn-2 user-chosen data — identical to RTO.15 ──────────────────────────

const TURN_2_FOLLOW_UP =
  "Have competitors shown any public signals, such as hiring patterns,\n" +
  "grant awards or conference presentations, indicating active parallel development?";

const TURN_2_ANSWER =
  "One competitor has recently advertised for several machine-learning\n" +
  "engineers and a senior product manager in this market. They have also\n" +
  "presented at an industry conference about the same customer problem, but\n" +
  "they have not announced a product, launch date or beta programme.\n" +
  "I do not know whether the hiring and conference activity relates to\n" +
  "a directly competing product.";

// ─── Reasoning instructions — explicit layer separation contract ──────────────

const REASONING_INSTRUCTIONS = `You are performing a SECOND-TURN focused-investigation refinement for one user-chosen investigation node.

A previous turn produced local observations, uncertainties, assumptions and relationships about this node. Those findings have NOT been merged into any global graph — they exist only as a compact local investigation state.

The user has now answered one follow-up question from that prior state. Your task is to produce ONE updated response with TWO EXPLICITLY SEPARATE LAYERS.

## Layer 1: Focused Investigation Understanding

Maintain one current focused investigation understanding for the chosen node. Ground it only in:
- The prior accumulated focused state (turn 1)
- The user-chosen follow-up question (turn 2)
- The new answer to that follow-up (turn 2)

Within this layer, your observations and uncertainties must reflect ONLY what is directly observable from the combined evidence about this investigation node. Keep prior information that remains supported. Revise or remove information that is no longer accurate.

## Layer 2: Wider Decision Significance

Separately identify concise connections explaining why the current focused understanding matters to the wider decision. Use the central case statement provided below for this layer only.

Each significance insight should be brief — one sentence connecting a focused finding to a broader business implication (timing, cost, competitive risk, strategic optionality).

## Critical separation rules

1. Layer 1 MUST NOT absorb wider-case concerns merely because they are present in the central case statement.
2. Observations must only state what the combined evidence directly supports about this investigation node — no speculative decision implications.
3. Uncertainties must distinguish between investigative gaps (Layer 1) and decision uncertainty (Layer 2).
4. Assumptions surviving from turn 1 stay in Layer 1; wider-case financial or strategic constraints appear only in Layer 2.
5. Layer 2 insights must be concise — not another full investigation report.
6. Do not duplicate facts between layers unnecessarily.
7. Do NOT recommend what the user should do.
8. Do NOT return turn-by-turn history.

Return exactly one JSON object with two top-level fields:
- focusedUnderstanding (object with targetNodeId, observations, uncertainties, assumptions, relationships, possibleFollowUpQuestions)
- decisionSignificance (array of objects with "insight" field)`;

// ─── Prompt construction ─────────────────────────────────────────────────────

function buildPrompt({ includeCentralCase = false }) {
  const centralSection = includeCentralCase
    ? `\nCentral case statement:\n${CENTRAL_CASE_STATEMENT}\n`
    : "";

  return `${REASONING_INSTRUCTIONS}

## Prior accumulated focused state (turn 1)

### Observations
${PRIOR_FOCUSED_STATE.observations.map((o) => `- ${o}`).join("\n")}

### Uncertainties
${PRIOR_FOCUSED_STATE.uncertainties.map((u) => `- ${u}`).join("\n")}

### Assumptions
${PRIOR_FOCUSED_STATE.assumptions.map((a) => `- ${a}`).join("\n")}

### Relationships
${PRIOR_FOCUSED_STATE.relationships.map((r) => `- ${r.from} → ${r.to} (${r.type})`).join("\n")}

### Possible follow-up questions (from prior state)
${PRIOR_FOCUSED_STATE.possibleFollowUpQuestions.map((q) => `- ${q}`).join("\n")}

## This turn's input

Target label: ${TARGET_LABEL}${centralSection}
Previous question: "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?"

User-chosen follow-up question (turn 2):
${TURN_2_FOLLOW_UP}

Answer to follow-up question (turn 2):
${TURN_2_ANSWER}

## Output format

Return exactly one JSON object with these two top-level fields:

### focusedUnderstanding (object)
- targetNodeId: "${TARGET_NODE_ID}"
- observations: array of strings — what the combined evidence supports as factual, scoped to this investigation node only
- uncertainties: array of strings — what remains unknown or unclear after both answers, about this investigation node
- assumptions: array of strings — supporting beliefs that persist, revised if needed
- relationships: array of { from, to, type } — links within the investigation
- possibleFollowUpQuestions: array of strings — genuinely remaining new questions

### decisionSignificance (array)
- Each item: { insight: string } — a concise connection between the current focused understanding and wider-case implications

Do NOT mix wider-case concerns into focusedUnderstanding. Do NOT recommend a decision. Return one coherent response with two clearly separated purposes.`;
}

// ─── Zod schema for live mode result validation ──────────────────────────────

const ACCUMULATED_STATE_SCHEMA = {
  targetNodeId: "string",
  observations: "array",
  uncertainties: "array",
  assumptions: "array",
  relationships: "array",
  possibleFollowUpQuestions: "array",
};

const RESULT_SCHEMA = {
  type: "object",
  required: ["focusedUnderstanding", "decisionSignificance"],
  properties: {
    focusedUnderstanding: {
      type: "object",
      required: [
        "targetNodeId",
        "observations",
        "uncertainties",
        "assumptions",
        "relationships",
        "possibleFollowUpQuestions",
      ],
    },
    decisionSignificance: {
      type: "array",
    },
  },
};

// ─── Inspect-only mode (default: zero model calls) ───────────────────────────

function inspectApparatus() {
  const localPrompt = buildPrompt({ includeCentralCase: false });
  const caseAwarePrompt = buildPrompt({ includeCentralCase: true });

  // Structural validation for both layers
  function validatePrompt(prompt, label) {
    const requiredSections = [
      "Prior accumulated focused state",
      "### Observations",
      "### Uncertainties",
      "### Assumptions",
      "### Relationships",
      "follow-up questions (from prior state)",
      "User-chosen follow-up question (turn 2)",
      "Answer to follow-up question (turn 2)",
      "Return exactly one JSON object",
      "focusedUnderstanding",
      "decisionSignificance",
      "Do NOT recommend a decision",
      "Do NOT return turn-by-turn history",
    ];

    const missing = requiredSections.filter((s) => !prompt.includes(s));
    const present = requiredSections.filter((s) => prompt.includes(s));

    return {
      label,
      characterCount: prompt.length,
      tokenEstimate: Math.ceil(prompt.length / 4),
      sectionsRequired: requiredSections.length,
      sectionsPresent: present.length,
      missingSections: missing,
      isValid: missing.length === 0,
    };
  }

  const localInspect = validatePrompt(localPrompt, "local-only");
  const caseAwareInspect = validatePrompt(caseAwarePrompt, "case-aware-with-central-statement");

  // Verify layer separation instructions are present
  function validateSeparationRules(prompt) {
    const rules = [
      "MUST NOT absorb wider-case concerns merely because",
      "Layer 1 MUST NOT absorb",
      "Layer 2 insights must be concise",
      "Do NOT mix wider-case concerns",
      "Do NOT recommend a decision",
      "Do NOT return turn-by-turn history",
      "exactly one JSON object with two top-level fields",
    ];
    const missingRules = rules.filter((r) => !prompt.includes(r));
    return {
      rulesRequired: rules.length,
      rulesPresent: rules.length - missingRules.length,
      missingRules,
      separationInstructionsComplete: missingRules.length === 0,
    };
  }

  const localSeparation = validateSeparationRules(localPrompt);
  const caseAwareSeparation = validateSeparationRules(caseAwarePrompt);

  // Verify input identity across modes (excluding central statement section)
  // The only difference between modes is the centralSection variable in buildPrompt
  function stripCentralBlock(p) {
    // Remove: \nCentral case statement:\n<STATEMENT>\n
    const block = "\nCentral case statement:\n" + CENTRAL_CASE_STATEMENT + "\n";
    return p.split(block)[0] + (p.split(block)[1] || "");
  }

  const localStripped = stripCentralBlock(localPrompt);
  const caseAwareNonContext = stripCentralBlock(caseAwarePrompt);
  const nonContextPartsIdentical = localStripped === caseAwareNonContext;

  // Validate focusedUnderstanding fields shape
  function describeFocusedFields() {
    return [
      "targetNodeId (string)",
      "observations (array<string>)",
      "uncertainties (array<string>)",
      "assumptions (array<string>)",
      "relationships (array<{from, to, type}>)",
      "possibleFollowUpQuestions (array<string>)",
    ];
  }

  // Report at minimum per task spec
  console.log("=== RTO.16A Separated Layers Apparatus (inspect-only) ===\n");

  console.log("--- Layer-separated input ---");
  console.log(`focusedUnderstanding fields: ${describeFocusedFields().join(", ")}`);
  console.log(`decisionSignificance shape: array<{insight: string}>`);
  console.log(`whole SituationGraph supplied: NO`);
  console.log(`global selector supplied: NO`);
  console.log(`live route available: YES (via --live flag)`);

  console.log("\n--- Prompt inspection ---");
  console.log(JSON.stringify({
    experiment: "rto-layer-separated-focused-reasoning",
    artifactType: "APPARATUS INSPECTION",
    nonProductionApparatus: true,
    productionCodeUnchanged: true,
    inputCharacterCount: {
      localOnly: localInspect.characterCount,
      caseAware: caseAwareInspect.characterCount,
    },
    promptValidation: {
      localOnly: localInspect,
      caseAwareWithCentralStatement: caseAwareInspect,
    },
    layerSeparationInstructions: {
      localOnly: localSeparation,
      caseAwareWithCentralStatement: caseAwareSeparation,
    },
    inputsIdenticalAcrossModes: {
      targetNodeId: true,
      targetLabel: TARGET_LABEL !== "",
      priorFocusedState: true,
      turn2FollowUpQuestion: TURN_2_FOLLOW_UP !== "",
      turn2Answer: TURN_2_ANSWER !== "",
      reasoningInstructions: REASONING_INSTRUCTIONS.length > 0,
      nonContextPartsIdentical: nonContextPartsIdentical,
    },
    rto15InputsPreserved: {
      targetNodeId: TARGET_NODE_ID === "nxmeiab",
      centralCaseStatement: CENTRAL_CASE_STATEMENT.length > 0,
      priorTurn1State: true,
      turn2FollowUpQuestion: TURN_2_FOLLOW_UP !== "",
      turn2Answer: TURN_2_ANSWER !== "",
    },
    doesNotRequire: [
      "whole SituationGraph",
      "global selector",
      "production focused-investigation.js modifications",
      "production API routes",
      "UI changes",
      "SituationGraph updates",
      "global update path changes",
      "question formulation changes",
    ],
  }, null, 2));

  // Inspect output summary per task spec requirements
  const inspectionSummary = {
    inputCharacterCount: localInspect.characterCount,
    focusedUnderstandingFields: describeFocusedFields(),
    decisionSignificanceShape: "array<{insight: string}>",
    wholeSituationGraphSupplied: "NO",
    globalSelectorSupplied: "NO",
    liveRouteAvailable: "YES",
  };

  return { inspectionSummary, promptReport: { localInspect, caseAwareInspect } };
}

// ─── Live execution (single invocation) ──────────────────────────────────────

async function executeLiveMode() {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) {
    throw new Error("OLLAMA_BASE_URL not set. Cannot execute live mode.");
  }
  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    throw new Error(`Refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`);
  }

  // Import zod for live schema validation (avoid in inspect-only)
  const { z } = await import("zod");

  const focusedSchema = z.object({
    targetNodeId: z.literal(TARGET_NODE_ID),
    observations: z.array(z.string()),
    uncertainties: z.array(z.string()),
    assumptions: z.array(z.string()),
    relationships: z.array(
      z.object({ from: z.string(), to: z.string(), type: z.string() }),
    ),
    possibleFollowUpQuestions: z.array(z.string()),
  });

  const resultSchema = z.object({
    focusedUnderstanding: focusedSchema,
    decisionSignificance: z.array(z.object({ insight: z.string() })),
  }).strict();

  // Import provider dynamically
  const { getProvider } = await import(path.resolve(__dirname, "../../lib/llm/provider.js"));
  const provider = getProvider();

  const configEnv = await import(path.resolve(__dirname, "../../lib/config.js"));
  const modelName = configEnv.assertConfig().OLLAMA_MODEL;
  if (!modelName) {
    throw new Error("OLLAMA_MODEL not set.");
  }

  // Build the single prompt (case-aware: includes central case statement)
  const prompt = buildPrompt({ includeCentralCase: true });
  const startedAt = Date.now();
  console.log(`Sending to ${provider.name} (${modelName})...`);

  const raw = await provider.generateReconstruction(prompt, modelName);
  const elapsedMs = Date.now() - startedAt;

  const parsedResult = resultSchema.parse(raw);

  // Write result artifact for later comparison against RTO.15
  const resultsDir = path.resolve("tests/experimental/results");
  await fs.mkdir(resultsDir, { recursive: true });
  const artifactPath = path.resolve(resultsDir, "rto-layer-separated-live.json");
  const payload = {
    apparatus: "rto-layer-separated-focused-reasoning.mjs",
    experiment: "RTO.16A",
    artifactType: "LIVE RESULT — RTO.16A separated layers invocation",
    modelName,
    elapsedMs,
    promptLength: prompt.length,
    centralCaseStatementSupplied: true,
    layerStructure: { focusedUnderstanding: "object", decisionSignificance: "array" },
    structuredResult: parsedResult,
  };

  await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
  console.log(`Live result written to: ${artifactPath}`);

  return payload;
}

// ─── CLI entry point ──────────────────────────────────────────────────────────

async function main() {
  const args = process.argv.slice(2);

  if (args.includes("--live")) {
    console.log("=== RTO.16A Separated Layers — live mode ===\n");
    const result = await executeLiveMode();
    console.log(JSON.stringify(result, null, 2));
    return;
  }

  // Default: inspect-only, zero model calls
  const { inspectionSummary, promptReport } = inspectApparatus();
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  process.exit(1);
});