confidence-engine/scripts/experimental/rto-two-turn-focused-refinement.mjs

/**
 * RTO.14A — Two-turn focused-investigation refinement apparatus.
 *
 * Purpose: Test whether a second user-chosen answer can revise one compact
 * local investigation state instead of producing an appended report.
 *
 * Design boundary:
 *   - This file is the ENTIRE experiment runner for the two-turn case.
 *   - It does NOT import or modify any production reasoning code.
 *   - It defines its own prompt builder, schema, and output apparatus.
 *   - No live Ollama calls are made by default — uncomment makeLiveCall() below.
 *
 * Fixed case data (RTO.13B proven context):
 *   targetNodeId: nxmeiab
 *   centralStatement: product-launch scenario from fixture
 *   turn 1 question + answer: already-proven RTO.13B values
 *   turn 2 follow-up (user-chosen): competitor hiring/conference signals
 *   turn 2 answer: one competitor ML-engineer hiring + conference presentation
 */

import fs from "fs/promises";
import path from "path";
import dotenv from "dotenv";
import { z } from "zod";

import { getProvider } from "../../lib/llm/provider.js";
import { assertConfig } from "../../lib/config.js";

dotenv.config({ path: ".env.local" });

// ─── Fixed case identity ──────────────────────────────────────────────────

const TARGET_NODE_ID = "nxmeiab";

const CENTRAL_STATEMENT =
  "I am deciding whether to launch a new software product this year or wait twelve months. The product is ready enough to launch, but one large enterprise customer could represent a significant part of the expected revenue and I do not yet know whether they will sign. Launching this year would also require around £300,000 of additional support and implementation cost. Waiting twelve months would reduce that immediate cost and give us more time to improve the product, but it would delay revenue and may allow competitors to move first. I need to decide whether there is enough evidence to launch this year or whether waiting is the safer decision.";

const TARGET_LABEL =
  "Whether competitors are actively developing similar products and how soon they might release them";

// ─── Turn-1 proven state (from RTO.13B) ──────────────────────────────────

const TURN_1 = {
  question:
    "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?",
  answer:
    "The product has significant patents, proprietary processes and software algorithms that are not available to competitors.",
};

// The expected turn-1 accumulated local state (what RTO.13B produced)
const TURN_1_STATE = {
  observations: [
    "The product is protected by significant patents.",
    "The product utilizes proprietary processes.",
    "The product incorporates proprietary software algorithms.",
    "These intellectual property assets and technical components are not available to competitors.",
  ],
  uncertainties: [
    "Whether competitors can engineer non-infringing workarounds or alternative architectures.",
    "Actual development timeline / funding / public activity of competitors.",
    "How long it could take a competitor to design around the IP barriers.",
    "Whether the patents and proprietary elements materially delay competitor entry.",
  ],
  assumptions: [
    "Existing IP acts as an effective barrier to rapid competitor replication.",
    "Competitors cannot cheaply or quickly bypass, replicate or license around the proprietary technology.",
  ],
  relationships: [
    { from: "Patents", to: "Restricted competitor access", type: "blocks" },
    { from: "Proprietary processes / algorithms", to: "Technical differentiation", type: "enables" },
  ],
  possibleFollowUpQuestions: [
    // The original question is now partially answered — should be removed or revised.
    "Have competitors shown any public signals, such as hiring patterns, grant awards or conference presentations, indicating active parallel development?",
  ],
};

// ─── Turn-2 user-chosen data ──────────────────────────────────────────────

const TURN_2 = {
  followUpQuestion:
    "Have competitors shown any public signals, such as hiring patterns,\n" +
    "grant awards or conference presentations, indicating active parallel development?",
  answer:
    "One competitor has recently advertised for several machine-learning\n" +
    "engineers and a senior product manager in this market. They have also\n" +
    "presented at an industry conference about the same customer problem, but\n" +
    "they have not announced a product, launch date or beta programme.\n" +
    "I do not know whether the hiring and conference activity relates to\n" +
    "a directly competing product.",
};

// ─── Turn-2 schema (compact revised state after revision) ─────────────────

const ACCUMULATED_STATE_SCHEMA = z
  .object({
    targetNodeId: z.literal(TARGET_NODE_ID),
    observations: z.array(z.string()).default([]),
    uncertainties: z.array(z.string()).default([]),
    assumptions: z.array(z.string()).default([]),
    relationships: z
      .array(
        z.object({
          from: z.string(),
          to: z.string(),
          type: z.string(),
        }),
      )
      .default([]),
    possibleFollowUpQuestions: z.array(z.string()).default([]),
  })
  .strict();

// ─── Turn-2 prompt builder (compact contract) ─────────────────────────────

function buildTwoTurnPrompt() {
  return `You are performing a SECOND-TURN focused-investigation refinement for one user-chosen investigation node.

## Task description

A previous turn produced local observations, uncertainties, assumptions and relationships about this node. Those findings have NOT been merged into any global graph — they exist only as a compact local investigation state.

The user has now answered one follow-up question from that prior state. Your task is to produce ONE updated local investigation state that represents the combined understanding after both answers.

## Prior accumulated state (turn 1)

### Observations
${TURN_1_STATE.observations.map((o) => `- ${o}`).join("\n")}

### Uncertainties
${TURN_1_STATE.uncertainties.map((u) => `- ${u}`).join("\n")}

### Assumptions
${TURN_1_STATE.assumptions.map((a) => `- ${a}`).join("\n")}

### Relationships
${TURN_1_STATE.relationships.map((r) => `- ${r.from} → ${r.to} (${r.type})`).join("\n")}

### Possible follow-up questions (from prior state)
${TURN_1_STATE.possibleFollowUpQuestions.map((q) => `- ${q}`).join("\n")}

## This turn's input

Target label: ${TARGET_LABEL}
Central case statement: ${CENTRAL_STATEMENT}

Previous question:
${TURN_1.question}

User-chosen follow-up question (turn 2):
${TURN_2.followUpQuestion}

Answer to follow-up question (turn 2):
${TURN_2.answer}

## Output contract

Return the current combined investigation understanding after
considering the prior local state and the new answer.

Keep prior information that remains supported.

Revise or remove information that is no longer accurate or unresolved.

Do not return a second standalone answer report.

Do not return turn-by-turn history.

Return exactly one JSON object with these fields:
- targetNodeId: "${TARGET_NODE_ID}"
- observations: array of strings — what the combined evidence supports as factual
- uncertainties: array of strings — what remains unknown or unclear after both answers
- assumptions: array of strings — supporting beliefs that persist, revised if needed
- relationships: array of { from (string), to (string), type (string) }
- possibleFollowUpQuestions: array of strings — genuinely remaining new questions

Critical rules:
- Do NOT output the prior state as a separate field — integrate it.
- This is one updated state, not two reports.
- Items that survive unchanged from turn 1 may appear once in the relevant field.
- Items that are resolved or disproven by the new answer should be removed.
- This is NOT a graph update task. Do NOT output graph mutations.`;
}

// ─── Experimental runner (no live calls by default) ───────────────────────

async function inspectApparatus() {
  const prompt = buildTwoTurnPrompt();

  // Structural inspection: verify the prompt contains all required semantic areas
  const requiredSections = [
    "prior accumulated state",
    "observations",
    "uncertainties",
    "assumptions",
    "relationships",
    "follow-up questions",
    "User-chosen follow-up question (turn 2)",
    "Answer to follow-up question (turn 2)",
    "Return exactly one JSON object",
    "targetNodeId",
  ];

  const missingSections = requiredSections.filter((s) => !prompt.includes(s));

  const result = {
    apparatus: "rto-two-turn-focused-refinement.mjs",
    classification: pendingClassification(),
    artifactType: "APPARATUS DEFINITION",
    promptLength: prompt.length,
    promptTokenEstimate: Math.ceil(prompt.length / 4),
    requiredSectionsPresent: requiredSections.filter((s) => prompt.includes(s)).length,
    requiredSectionsMissing: missingSections,
    minimalContextFields: [
      "target investigation identity (nxmeiab)",
      "central case statement",
      "prior local investigation state (observations/uncertainties/assumptions/relationships/followUps)",
      "chosen follow-up question",
      "new answer",
    ],
    doesNotRequire: [
      "whole SituationGraph",
      "global selector",
      "production focused-investigation.js modifications",
      "production API routes",
      "UI changes",
    ],
  };

  // Write apparatus definition file (no model call)
  const artifactPath = path.resolve(
    "tests/experimental/contract/rto-two-turn-refined-state.json",
  );
  await fs.mkdir(path.dirname(artifactPath), { recursive: true });
  await fs.writeFile(artifactPath, JSON.stringify(result, null, 2));

  return result;
}

// ─── Live call (commented out — requires uncommenting to execute) ──────────

async function makeLiveCall() {
  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    throw new Error(`Refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`);
  }

  const config = assertConfig();
  const modelName = config.OLLAMA_MODEL;

  // Build the prompt (reuse function from above)
  const prompt = buildTwoTurnPrompt();

  // Call the model via existing provider abstraction
  const provider = getProvider();
  const startedAt = Date.now();
  const raw = await provider.generateReconstruction(prompt, modelName);
  const elapsedMs = Date.now() - startedAt;

  // Validate against accumulated state schema
  const parsedResult = ACCUMULATED_STATE_SCHEMA.parse(raw);

  // Write live result artifact (written only when run with makeLiveCall)
  const artifactPath = path.resolve(
    "tests/experimental/results/rto-two-turn-live-result.json",
  );
  await fs.mkdir(path.dirname(artifactPath), { recursive: true });
  await fs.writeFile(
    artifactPath,
    JSON.stringify({
      apparatus: "rto-two-turn-focused-refinement.mjs",
      modelName,
      elapsedMs,
      promptLength: prompt.length,
      structuredResult: parsedResult,
    }, null, 2),
  );

  console.log(
    JSON.stringify(
      {
        targetNodeId: parsedResult.targetNodeId,
        observationCount: parsedResult.observations.length,
        uncertaintyCount: parsedResult.uncertainties.length,
        assumptionCount: parsedResult.assumptions.length,
        relationshipCount: parsedResult.relationships.length,
        followUpCount: parsedResult.possibleFollowUpQuestions.length,
        elapsedMs,
      },
      null,
      2,
    ),
  );

  return { result: parsedResult, artifactPath };
}

// ─── Classification helper ────────────────────────────────────────────────

function pendingClassification() {
  // This will be populated after apparatus inspection completes.
  // The existing focused contract (buildFocusedDeconstructPrompt) CANNOT accept prior state.
  // Therefore a distinct experimental contract IS required.
  return "awaiting-inspection";
}

function assertRequiredEnv(name) {
  const value = process.env[name];
  if (!value) {
    throw new Error(
      `Focused apparatus requires ${name}. Set it in .env.local. Found OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`,
    );
  }
  return value;
}

// ─── Main: inspect by default, live when --live is supplied ────────────────

async function main() {
  if (process.argv.includes("--live")) {
    const result = await makeLiveCall();
    console.log("=== RTO.14A Live Result ===");
    console.log(JSON.stringify(result, null, 2));
    return;
  }

  const result = await inspectApparatus();
  console.log("=== RTO.14A Apparatus Inspection ===");
  console.log(JSON.stringify(result, null, 2));
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  process.exit(1);
});