confidence-engine/scripts/experimental/rto-fragment-relationship-negative-control.mjs

/**
 * RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus
 *
 * Purpose: Test whether semantic relationship discovery correctly returns
 * "no relationship" when the later fragment does not materially bear on
 * an earlier uncertainty (false-positive control).
 *
 * Design boundary:
 *   - Standalone experimental runner.
 *   - Zero production code changes.
 *   - Inspect-only by default (zero live model calls).
 *   - --live flag enables exactly one live call.
 *
 * Fixed negative-control pair:
 *   Earlier — RTO.18 Fragment 2 (competitor hiring + conference)
 *   Later   — negative-control-operational-contract (implementation partner fee)
 *
 * Output contract (same as RTO.21):
 *   { relationshipFound, relationship, remainingQualification }
 */

import fs from "fs/promises";
import path from "path";
import { fileURLToPath } from "url";
import dotenv from "dotenv";

dotenv.config({ path: ".env.local" });

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const resultsDir = path.resolve(__dirname, "../../tests/experimental/results");

// ─── Fragment paths ──────────────────────────────────────────────────────────

const FRAGMENT_2_PATH = path.resolve(
  __dirname,
  "../../tests/experimental/results/rto-granular-fragment-turn2.json"
);

// ─── Negative-control fragment (EXPERIMENTAL_FIXTURE) ────────────────────────

const NEGATIVE_CONTROL_FRAGMENT = {
  targetNodeId: "negative-control-operational-contract",
  question: "When does the implementation partner contract renew and what is the current annual fee?",
  observations: [
    "The implementation partner contract renews in November.",
    "The current annual fee is £42,000."
  ],
  uncertainties: [],
  assumptions: [],
  relationships: [],
  possibleFollowUpQuestions: []
};

// ─── Load fragment 2 ─────────────────────────────────────────────────────────

async function loadFragment2() {
  const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8");
  const frag2 = JSON.parse(frag2Raw);

  if (!frag2.structuredResult) {
    throw new Error("Fragment 2 missing structuredResult");
  }

  return {
    fragment: frag2,
    structuredResult: frag2.structuredResult
  };
}

// ─── Negative-control integrity check ────────────────────────────────────────

function verifyNegativeControlClean() {
  const text = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT);

  const forbiddenTerms = [
    "competitor",
    "hiring",
    "conference",
    "product development",
    "patent",
    "prototype",
    "launch"
  ];

  const lowerText = text.toLowerCase();
  const violations = [];

  for (const term of forbiddenTerms) {
    if (lowerText.includes(term.toLowerCase())) {
      violations.push(term);
    }
  }

  return {
    clean: violations.length === 0,
    violations
  };
}

// ─── Build the relationship-discovery prompt ──────────────────────────────────

function buildRelationshipPrompt(srs2, controlSrs) {
  const parts = [];

  // Instructions — strict semantic discipline (same as RTO.21)
  parts.push("You are examining two independent reasoning fragments produced during a");
  parts.push("competitive investigation. Your task is to determine whether the later");
  parts.push("fragment contains evidence that materially bears on an uncertainty,");
  parts.push("assumption or observation in the earlier fragment.");
  parts.push("");
  parts.push("CRITICAL RULES:");
  parts.push("- Only consider what is explicitly stated in these two fragments.");
  parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation.");
  parts.push("- Do not infer facts not present in either fragment.");
  parts.push("- Do not rewrite or summarise either fragment.");
  parts.push("- Do not recommend an action or choose what to investigate next.");
  parts.push("");
  parts.push("SEMANTIC DISCIPLINE:");
  parts.push("- Identify only a material relationship supported by the supplied fragments.");
  parts.push("- Do not create a relationship merely because both fragments occur in the");
  parts.push("  same wider case.");
  parts.push("- Do not infer broad business relevance.");
  parts.push("- If the later fragment does not materially bear on an earlier item,");
  parts.push("  return relationshipFound=false.");
  parts.push("");

  // Fragment 2 (earlier)
  parts.push("=== Fragment 2 (earlier evidence) ===");
  parts.push(`Question: ${srs2.question}`);
  if (srs2.observations && srs2.observations.length) {
    parts.push("Observations:");
    for (const o of srs2.observations) {
      parts.push(`  - ${o}`);
    }
  }
  if (srs2.uncertainties && srs2.uncertainties.length) {
    parts.push("Uncertainties:");
    for (const u of srs2.uncertainties) {
      parts.push(`  - ${u}`);
    }
  }
  if (srs2.assumptions && srs2.assumptions.length) {
    parts.push("Assumptions:");
    for (const a of srs2.assumptions) {
      parts.push(`  - ${a}`);
    }
  }
  if (srs2.relationships && srs2.relationships.length) {
    parts.push("Relationships (internal to fragment):");
    for (const r of srs2.relationships) {
      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
    }
  }
  parts.push("");

  // Negative-control fragment (later)
  parts.push("=== Fragment (later evidence — negative control) ===");
  parts.push(`Question: ${controlSrs.question}`);
  if (controlSrs.observations && controlSrs.observations.length) {
    parts.push("Observations:");
    for (const o of controlSrs.observations) {
      parts.push(`  - ${o}`);
    }
  }
  if (controlSrs.uncertainties && controlSrs.uncertainties.length) {
    parts.push("Uncertainties:");
    for (const u of controlSrs.uncertainties) {
      parts.push(`  - ${u}`);
    }
  }
  if (controlSrs.assumptions && controlSrs.assumptions.length) {
    parts.push("Assumptions:");
    for (const a of controlSrs.assumptions) {
      parts.push(`  - ${a}`);
    }
  }
  if (controlSrs.relationships && controlSrs.relationships.length) {
    parts.push("Relationships (internal to fragment):");
    for (const r of controlSrs.relationships) {
      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
    }
  }
  parts.push("");

  // Core question
  parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,");
  parts.push("assumption or observation in the earlier fragment?");
  parts.push("");

  // Return format — same contract as RTO.21
  parts.push('Return exactly one JSON object:');
  parts.push('{ "relationshipFound": <bool>, "relationship": { "laterEvidence": "<string>", "earlierItem": "<string>", "bearing": "<string>" }, "remainingQualification": ["<string>"] }');
  parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }');

  return parts.join("\n");
}

// ─── Output contract validation (same as RTO.21) ─────────────────────────────

function validateRelationshipOutput(result) {
  const errors = [];

  if (typeof result.relationshipFound !== "boolean") {
    errors.push("relationshipFound must be a boolean");
  }

  if (!result.relationshipFound) {
    if (result.relationship !== null && result.relationship !== undefined) {
      errors.push("When relationshipFound=false, relationship must be null or undefined");
    }
    if (!Array.isArray(result.remainingQualification)) {
      errors.push("remainingQualification must be an array when relationshipFound=false");
    }
    return errors;
  }

  // When true, relationship object is required
  if (!result.relationship || typeof result.relationship !== "object") {
    errors.push("When relationshipFound=true, relationship must be an object");
    return errors;
  }

  const rel = result.relationship;
  if (typeof rel.laterEvidence !== "string") {
    errors.push("relationship.laterEvidence must be a string");
  }
  if (typeof rel.earlierItem !== "string") {
    errors.push("relationship.earlierItem must be a string");
  }
  if (typeof rel.bearing !== "string") {
    errors.push("relationship.bearing must be a string");
  }

  if (!Array.isArray(result.remainingQualification)) {
    errors.push("remainingQualification must be an array when relationshipFound=true");
  } else if (result.remainingQualification.some((s) => typeof s !== "string")) {
    errors.push("remainingQualification items must all be strings");
  }

  // No forbidden fields (same as RTO.21)
  const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
    "recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"];
  for (const field of forbidden) {
    if (field in result && typeof result[field] !== "undefined") {
      errors.push(`Forbidden field present: ${field}`);
    }
  }

  return errors;
}

// ─── Inspect mode (default: zero model calls) ────────────────────────────────

async function inspectApparatus() {
  console.log("=== RTO.22A Negative-Control Fragment Relationship-Discovery Apparatus (inspect-only) ===\n");

  // Load and verify fragment 2
  let structuredResult2;
  try {
    const frag2 = await loadFragment2();
    structuredResult2 = frag2.structuredResult;
  } catch (e) {
    console.log("ERROR: Cannot load Fragment 2:", e.message);
    process.exit(1);
  }

  // Negative-control integrity check
  const controlCheck = verifyNegativeControlClean();
  console.log("--- Negative-control integrity ---");
  console.log(`Control fragment clean: ${controlCheck.clean ? "YES" : "NO (violations: " + controlCheck.violations.join(", ") + ")"}`);

  if (!controlCheck.clean) {
    console.log("\nNEGATIVE_CONTROL_CLEAN: NO");
    console.log("Control fragment contains terms that leak into competitor-development content.");
    console.log("Cannot proceed with a contaminated negative control.");
    process.exit(1);
  }

  // Verify control fragment contents
  const controlText = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT);
  const hasContractRenewal = controlText.toLowerCase().includes("renews in november") || controlText.toLowerCase().includes("contract renews");
  const hasAnnualFee = controlText.includes("£42,000");

  console.log("\n--- Negative-control fixture verification ---");
  console.log(`Contract renewal observation present: ${hasContractRenewal ? "YES" : "NO"}`);
  console.log(`£42,000 annual fee observation present: ${hasAnnualFee ? "YES" : "NO"}`);
  console.log(`Competitor-development content present: NO`);

  // Fragment verification
  const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
  console.log("\n--- Fragment verification ---");
  console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
  console.log(`Negative-control fragment supplied: YES`);

  // Build prompt for inspection
  const prompt = buildRelationshipPrompt(structuredResult2, NEGATIVE_CONTROL_FRAGMENT);

  // Measure character sizes
  const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2"));
  const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
  const controlStructuredText = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT, null, 2);

  console.log("\n--- Context-size measurement ---");
  console.log(`instructionCharacterCount: ${instructionPart.length}`);
  console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
  console.log(`negativeControlCharacterCount: ${controlStructuredText.length}`);
  console.log(`inputCharacterCount: ${prompt.length}`);

  // ── Boundary checks ───────────────────────────────────────────

  console.log("\n--- Input boundary verification ---");

  const knownPositiveRelInPrompt = prompt.includes("Fragment 3's technical evidence bears on Fragment 2") ||
    prompt.includes("active competing-product development more plausible");
  console.log(`Known RTO.19/21 positive relationship supplied: ${knownPositiveRelInPrompt ? "YES" : "NO"}`);

  const fragment3Supplied = prompt.includes("Fragment 3") ||
    NEGATIVE_CONTROL_FRAGMENT.targetNodeId === "fragment-3";
  console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`);

  const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
    prompt.includes("whole-case");
  console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);

  const centralCaseInPrompt = prompt.includes("central case") ||
    prompt.includes("centralStatement");
  console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);

  const currentViewInPrompt = prompt.includes("derived view") ||
    prompt.includes("focused view") || prompt.includes("current focused");
  console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);

  const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
    prompt.includes("other fragments") || prompt.includes("additional fragment");
  console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);

  const turnHistoryInPrompt = prompt.includes("turn history") ||
    prompt.includes("prior turns") || prompt.includes("conversation history");
  console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);

  // ── Output contract validation capability ─────────────────────

  const mockProposal = {
    relationshipFound: true,
    relationship: {
      laterEvidence: "test",
      earlierItem: "test",
      bearing: "test",
    },
    remainingQualification: ["test"],
  };
  const mockNoRel = {
    relationshipFound: false,
    relationship: null,
    remainingQualification: [],
  };
  const proposalErrors = validateRelationshipOutput(mockProposal);
  const noRelErrors = validateRelationshipOutput(mockNoRel);
  console.log("\n--- Output contract (same as RTO.21) ---");
  console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
  console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
  console.log(`relationshipFound=false allowed: YES`);

  // ── Live route ────────────────────────────────────────────────
  console.log("\n--- Live route ---");
  console.log("node scripts/experimental/rto-fragment-relationship-negative-control.mjs --live");

  // ── Environment check ─────────────────────────────────────────
  const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL;
  const modelConfigured = !!process.env.OLLAMA_MODEL;
  console.log("\n--- Environment ---");
  console.log(`OLLAMA_BASE_URL configured: ${baseUrlConfigured ? "YES" : "NO"}`);
  console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`);

  // ── Inspect summary ───────────────────────────────────────────
  const allFragmentsOK = f2Loaded;
  const noBoundaryLeaks = !knownPositiveRelInPrompt && !fragment3Supplied &&
    !wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt &&
    !otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean;

  console.log("\n=== SUMMARY ===");
  console.log(`Inspect live calls: 0`);
  console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
  console.log(`Negative-control fragment supplied: YES`);
  console.log(`Fragment 3 supplied: NO`);
  console.log(`Known positive relationship supplied: NO`);
  console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
  console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
  console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
  console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
  console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
  console.log(`NEGATIVE_CONTROL_CLEAN: YES`);
  console.log(`--live route exposed: YES`);
  console.log(`maximum live calls per --live: 1`);
  console.log(`relationshipFound=false allowed: YES`);

  const passed = allFragmentsOK && noBoundaryLeaks && controlCheck.clean;
  console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);

  if (!passed) {
    console.log("\nApparatus cannot satisfy boundary without production changes.");
    process.exit(1);
  }

  return { prompt };
}

// ─── Live execution (exactly one model call) ──────────────────────────────────

async function executeLive() {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) {
    console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode.");
    process.exit(1);
  }
  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
    process.exit(1);
  }

  const modelName = process.env.OLLAMA_MODEL;
  if (!modelName) {
    console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode.");
    process.exit(1);
  }

  // Load fragment 2 and build prompt
  const { structuredResult: srs2 } = await loadFragment2();
  const prompt = buildRelationshipPrompt(srs2, NEGATIVE_CONTROL_FRAGMENT);

  const startedAt = Date.now();
  console.log("Live mode: sending to model...");

  const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
  const raw = await provider.generateReconstruction(prompt, modelName);
  const elapsedMs = Date.now() - startedAt;

  console.log(`Model response received in ${elapsedMs}ms`);

  // Provider returns a parsed JavaScript object via recoverJson().
  const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => {
    console.error("Provider returned unexpected type (expected parsed object):", typeof raw);
    process.exit(1);
  })();

  // Validate output contract
  const errors = validateRelationshipOutput(parsedResult);
  if (errors.length > 0) {
    console.error("\nLive output contract violations:");
    for (const err of errors) {
      console.error(`  - ${err}`);
    }
    process.exit(1);
  }

  // Write artifact
  await fs.mkdir(resultsDir, { recursive: true });

  const artifactPath = path.resolve(
    resultsDir,
    "rto-fragment-relationship-negative-control-live.json"
  );

  const controlCheck = verifyNegativeControlClean();
  const payload = {
    apparatus: "rto-fragment-relationship-negative-control.mjs",
    experiment: "RTO.22A",
    artifactType: "LIVE RESULT — Negative-control relationship discovery",
    modelName: modelName,
    elapsedMs: elapsedMs,
    inputCharacterCount: prompt.length,
    instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length,
    fragment2CharacterCount: JSON.stringify(srs2, null, 2).length,
    negativeControlCharacterCount: JSON.stringify(NEGATIVE_CONTROL_FRAGMENT, null, 2).length,
    fragment2Path: FRAGMENT_2_PATH,
    controlFixtureType: "EXPERIMENTAL_FIXTURE",
    negativeControlClean: controlCheck.clean,
    knownRelationshipSuppliedToModel: false,
    wholeSituationGraphSupplied: false,
    centralCaseStatementSupplied: false,
    turnHistorySupplied: false,
    proposedRelationship: parsedResult,
  };

  await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
  console.log(`\nLive result written to: ${artifactPath}`);
  console.log("\n=== Proposed Relationship ===");
  console.log(JSON.stringify(parsedResult, null, 2));

  return payload;
}

// ─── CLI entry point ──────────────────────────────────────────────────────────

async function main() {
  const args = process.argv.slice(2);

  // --live flag for exactly one live model call
  if (args.includes("--live")) {
    console.log("RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus\n");
    console.log("WARNING: This will make exactly ONE live model call.\n");
    const result = await executeLive();
    return result;
  }

  // Default: inspect-only, zero model calls
  console.log("RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus\n");
  console.log("Run with --live for one live model call.\n");

  const result = await inspectApparatus();
  if (result === null) {
    process.exit(1);
  }

  return result;
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  process.exit(1);
});