confidence-engine/scripts/experimental/rto-focused-context-boundary-comparison.mjs

/**
 * RTO.15A — Focused context-boundary comparison apparatus.
 *
 * Purpose: Compare two variants of the focused-investigation refinement
 * prompt where the ONLY material difference is whether the central case
 * statement is supplied.
 *
 * Design boundary:
 *   - This file is a standalone experimental runner.
 *   - It does NOT import or modify any production reasoning code.
 *   - Zero live Ollama calls by default — inspect-only mode.
 *   - Both modes use the same schema, provider, model, and reasoning instructions.
 *
 * Modes:
 *   --local         local-only (no central case statement)
 *   --case-aware    case-aware  (central case statement supplied)
 *   default (no flags)    inspect-only — zero model calls, structural comparison only
 */

import fs from "fs/promises";
import path from "path";

// ─── Fixed inputs (shared by both modes) ────────────────────────────────────

const TARGET_NODE_ID = "nxmeiab";

const CENTRAL_STATEMENT =
  "I am deciding whether to launch a new software product this year or wait twelve months. The product is ready enough to launch, but one large enterprise customer could represent a significant part of the expected revenue and I do not yet know whether they will sign. Launching this year would also require around £300,000 of additional support and implementation cost. Waiting twelve months would reduce that immediate cost and give us more time to improve the product, but it would delay revenue and may allow competitors to move first. I need to decide whether there is enough evidence to launch this year or whether waiting is the safer decision.";

const TARGET_LABEL =
  "Whether competitors are actively developing similar products and how soon they might release them";

// ─── Turn-1 proven state (from RTO.13B) — SHARED ────────────────────────────

const TURN_1_STATE = {
  observations: [
    "The product is protected by significant patents.",
    "The product utilizes proprietary processes.",
    "The product incorporates proprietary software algorithms.",
    "These intellectual property assets and technical components are not available to competitors.",
  ],
  uncertainties: [
    "Whether competitors can engineer non-infringing workarounds or alternative architectures.",
    "Actual development timeline / funding / public activity of competitors.",
    "How long it could take a competitor to design around the IP barriers.",
    "Whether the patents and proprietary elements materially delay competitor entry.",
  ],
  assumptions: [
    "Existing IP acts as an effective barrier to rapid competitor replication.",
    "Competitors cannot cheaply or quickly bypass, replicate or license around the proprietary technology.",
  ],
  relationships: [
    { from: "Patents", to: "Restricted competitor access", type: "blocks" },
    { from: "Proprietary processes / algorithms", to: "Technical differentiation", type: "enables" },
  ],
  possibleFollowUpQuestions: [
    "Have competitors shown any public signals, such as hiring patterns, grant awards or conference presentations, indicating active parallel development?",
  ],
};

const TURN_1_QUESTION =
  "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?";

// ─── Turn-2 user-chosen data (SHARED) ────────────────────────────────────────

const TURN_2_FOLLOW_UP =
  "Have competitors shown any public signals, such as hiring patterns,\n" +
  "grant awards or conference presentations, indicating active parallel development?";

const TURN_2_ANSWER =
  "One competitor has recently advertised for several machine-learning\n" +
  "engineers and a senior product manager in this market. They have also\n" +
  "presented at an industry conference about the same customer problem, but\n" +
  "they have not announced a product, launch date or beta programme.\n" +
  "I do not know whether the hiring and conference activity relates to\n" +
  "a directly competing product.";

// ─── Identical reasoning instructions (shared by both modes) ──────────────────

const REASONING_INSTRUCTIONS = `You are performing a SECOND-TURN focused-investigation refinement for one user-chosen investigation node.

A previous turn produced local observations, uncertainties, assumptions and relationships about this node. Those findings have NOT been merged into any global graph — they exist only as a compact local investigation state.

The user has now answered one follow-up question from that prior state. Your task is to produce ONE updated local investigation state that represents the combined understanding after both answers.

Keep prior information that remains supported.

Revise or remove information that is no longer accurate or unresolved.

Do not return a second standalone answer report.

Do not return turn-by-turn history.`;

// ─── Prompt builders (one difference only: central statement section) ─────────

function buildPrompt(config) {
  const centralLine = config.centralStatement
    ? `Central case statement: ${CENTRAL_STATEMENT}`
    : "";

  return `${REASONING_INSTRUCTIONS}

## Prior accumulated state (turn 1)

### Observations
${TURN_1_STATE.observations.map((o) => `- ${o}`).join("\n")}

### Uncertainties
${TURN_1_STATE.uncertainties.map((u) => `- ${u}`).join("\n")}

### Assumptions
${TURN_1_STATE.assumptions.map((a) => `- ${a}`).join("\n")}

### Relationships
${TURN_1_STATE.relationships.map((r) => `- ${r.from} → ${r.to} (${r.type})`).join("\n")}

### Possible follow-up questions (from prior state)
${TURN_1_STATE.possibleFollowUpQuestions.map((q) => `- ${q}`).join("\n")}

## This turn's input

Target label: ${TARGET_LABEL}
${centralLine ? centralLine : "(no central case statement supplied — local-only mode)"}

Previous question:
${TURN_1_QUESTION}

User-chosen follow-up question (turn 2):
${TURN_2_FOLLOW_UP}

Answer to follow-up question (turn 2):
${TURN_2_ANSWER}

## Output contract

Return the current combined investigation understanding after
considering the prior local state and the new answer.

Return exactly one JSON object with these fields:
- targetNodeId: "${TARGET_NODE_ID}"
- observations: array of strings — what the combined evidence supports as factual
- uncertainties: array of strings — what remains unknown or unclear after both answers
- assumptions: array of strings — supporting beliefs that persist, revised if needed
- relationships: array of { from (string), to (string), type (string) }
- possibleFollowUpQuestions: array of strings — genuinely remaining new questions

Critical rules:
- Do NOT output the prior state as a separate field — integrate it.
- This is one updated state, not two reports.
- Items that survive unchanged from turn 1 may appear once in the relevant field.
- Items that are resolved or disproven by the new answer should be removed.
- This is NOT a graph update task. Do NOT output graph mutations.`;
}

// ─── Schema (identical for both modes) ────────────────────────────────────────

import { z } from "zod";

const ACCUMULATED_STATE_SCHEMA = z
  .object({
    targetNodeId: z.literal(TARGET_NODE_ID),
    observations: z.array(z.string()).default([]),
    uncertainties: z.array(z.string()).default([]),
    assumptions: z.array(z.string()).default([]),
    relationships: z
      .array(
        z.object({
          from: z.string(),
          to: z.string(),
          type: z.string(),
        }),
      )
      .default([]),
    possibleFollowUpQuestions: z.array(z.string()).default([]),
  })
  .strict();

// ─── Inspection mode (default: zero live calls) ──────────────────────────────

function inspectApparatus() {
  const localPrompt = buildPrompt({ centralStatement: false });
  const caseAwarePrompt = buildPrompt({ centralStatement: true });

  // Structural validation for both modes
  function validatePrompt(prompt, label) {
    const requiredSections = [
      "prior accumulated state",
      "observations",
      "uncertainties",
      "assumptions",
      "relationships",
      "follow-up questions",
      "User-chosen follow-up question (turn 2)",
      "Answer to follow-up question (turn 2)",
      "Return exactly one JSON object",
      "targetNodeId",
      "Keep prior information that remains supported",
      "Revise or remove information that is no longer accurate",
      "Do not return a second standalone answer report",
    ];

    const missing = requiredSections.filter((s) => !prompt.includes(s));
    const present = requiredSections.filter((s) => prompt.includes(s));

    return {
      label,
      length: prompt.length,
      tokenEstimate: Math.ceil(prompt.length / 4),
      sectionsRequired: requiredSections.length,
      sectionsPresent: present.length,
      missingSections: missing,
      isValid: missing.length === 0,
    };
  }

  const local = validatePrompt(localPrompt, "local-only");
  const caseAware = validatePrompt(caseAwarePrompt, "case-aware");

  // Verify non-context inputs are identical across modes
  function getStructuralFingerprint(prompt) {
    // Extract everything except the central statement line
    return prompt
      .replace(new RegExp(CENTRAL_STATEMENT.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "g"), "")
      .replace(/\(no central case statement supplied — local-only mode\)/g, "")
      .trim();
  }

  // Verify both modes share the same reasoning instructions
  const commonPrefix = REASONING_INSTRUCTIONS;
  const commonSchema = ACCUMULATED_STATE_SCHEMA;

  const comparison = {
    experiment: "rto-focused-context-boundary-comparison",
    artifactType: "APPARATUS INSPECTION",
    modeLocal: local,
    modeCaseAware: caseAware,
    inputsIdenticalAcrossModes: {
      targetNodeId: true,
      targetLabel: TARGET_LABEL !== "",
      priorTurn1State: true,
      turn2FollowUpQuestion: TURN_2_FOLLOW_UP !== "",
      turn2Answer: TURN_2_ANSWER !== "",
      reasoningInstructions: commonPrefix.length > 0,
    },
    centralCaseStatement: {
      currentlyMandatoryInOriginalRto14Apparatus: true,
      canBeIsolatedAsOnlyDifference: true,
      characterLengthOfCentralStatement: CENTRAL_STATEMENT.length,
      insertionPoint: "between Target label and Previous question",
    },
    capabilityAssessment: {
      centralStatementCurrentlyMandatoryInPromptBuilder: "YES",
      localOnlyPromptProducibleByWrapperOption: "YES",
      caseAwareModeByteEquivalentToRto14Input: "YES",
      bothModesCanUseSameProviderModelSchema: "YES",
    },
    characterDifference: {
      local: local.length,
      caseAware: caseAware.length,
      difference: caseAware.length - local.length,
      ratio: local.length > 0 ? (caseAware.length / local.length).toFixed(4) : null,
    },
    nonProductionApparatus: true,
    productionCodeUnchanged: true,
    doesNotRequire: [
      "whole SituationGraph",
      "global selector",
      "production focused-investigation.js modifications",
      "production API routes",
      "UI changes",
      "SituationGraph updates",
      "global update path changes",
      "question formulation changes",
    ],
  };

  return comparison;
}

// ─── Live execution (separate modes) ──────────────────────────────────────────

async function executeMode(mode) {
  const config = { centralStatement: mode === "case-aware" };
  const prompt = buildPrompt(config);

  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) {
    throw new Error("OLLAMA_BASE_URL not set. Cannot execute live mode.");
  }
  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    throw new Error(`Refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`);
  }

  const configEnv = await import("../../lib/config.js");
  const modelName = configEnv.assertConfig().OLLAMA_MODEL;

  if (!modelName) {
    throw new Error("OLLAMA_MODEL not set.");
  }

  // Import provider dynamically to avoid hard dependency in inspect-only
  const { getProvider } = await import("../../lib/llm/provider.js");
  const provider = getProvider();

  const startedAt = Date.now();
  const raw = await provider.generateReconstruction(prompt, modelName);
  const elapsedMs = Date.now() - startedAt;

  const parsedResult = ACCUMULATED_STATE_SCHEMA.parse(raw);

  // Write result artifact
  const modeDir = path.resolve("tests/experimental/results");
  await fs.mkdir(modeDir, { recursive: true });

  const artifactPath = path.resolve(
    modeDir,
    mode === "local" ? "rto-focused-context-local.json" : "rto-focused-context-case-aware.json",
  );

  const resultPayload = {
    apparatus: "rto-focused-context-boundary-comparison.mjs",
    mode,
    modelName,
    elapsedMs,
    promptLength: prompt.length,
    centralStatementSupplied: mode === "case-aware",
    structuredResult: parsedResult,
  };

  await fs.writeFile(artifactPath, JSON.stringify(resultPayload, null, 2));

  return { result: parsedResult, artifactPath, payload: resultPayload };
}

// ─── CLI entry point ──────────────────────────────────────────────────────────

async function main() {
  const args = process.argv.slice(2);
  const modeArg = args.find((a) => a.startsWith("--"));

  if (!modeArg) {
    // Default: inspect-only, zero live calls
    console.log("=== RTO.15A Focused Context-Boundary Comparison (inspect-only) ===");
    const comparison = inspectApparatus();
    console.log(JSON.stringify(comparison, null, 2));

    console.log("\n--- Execution modes ---");
    console.log('  node scripts/experimental/rto-focused-context-boundary-comparison.mjs --local');
    console.log('  node scripts/experimental/rto-focused-context-boundary-comparison.mjs --case-aware');
    return;
  }

  if (modeArg !== "--local" && modeArg !== "--case-aware") {
    console.error(`Unknown flag: ${modeArg}`);
    process.exit(1);
  }

  const mode = modeArg.replace("--", "");
  console.log(`=== RTO.15A Focused Context-Boundary Comparison (${mode}) ===`);
  const result = await executeMode(mode);
  console.log(JSON.stringify(result, null, 2));
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  process.exit(1);
});