confidence-engine/scripts/experimental/rto-fragment-relationship-borderline-control.mjs

/**
 * RTO.23C — Simplified Semantic Relationship-Inference Apparatus
 *
 * Purpose: Present two reasoning contributions neutrally and ask the model
 * to infer their meaningful relationship, if any, from meaning alone —
 * without domain dictionaries, keyword rules or expected-result leakage.
 *
 * Design boundary:
 *   - Standalone experimental runner.
 *   - Zero production code changes.
 *   - Inspect-only by default (zero live model calls).
 *   - --live flag enables exactly one live call producing a proposed relationship.
 *
 * Fixed earlier fragment:
 *   RTO.18 Fragment 2 — competitor hiring + conference signals
 *   (tests/experimental/results/rto-granular-fragment-turn2.json)
 *
 * Fixed later contribution:
 *   same-context-public-market-activity — public market activity, same context.
 *
 * Output contract:
 *   { relationshipFound, relationship, evidence, qualification }
 */

import fs from "fs/promises";
import path from "path";
import { fileURLToPath } from "url";
import dotenv from "dotenv";

dotenv.config({ path: ".env.local" });

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const resultsDir = path.resolve(__dirname, "../../tests/experimental/results");

// ─── Fragment paths ──────────────────────────────────────────────────────────

const FRAGMENT_2_PATH = path.resolve(
  __dirname,
  "../../tests/experimental/results/rto-granular-fragment-turn2.json"
);

// ─── Later contribution (fixed experimental fixture) ──────────────────────

const LATER_CONTRIBUTION = {
  targetNodeId: "same-context-public-market-activity",
  question: "What else has the competitor done publicly in this market?",
  observations: [
    "The competitor sponsored the same industry conference.",
    "Its chief executive gave the opening remarks, speaking about growth in the market and the importance of solving this customer problem.",
    "The company also hosted a networking reception for customers and partners attending the event."
  ],
  uncertainties: [],
  assumptions: [],
  relationships: [],
  possibleFollowUpQuestions: []
};

// ─── Load fragment 2 ─────────────────────────────────────────────────────────

async function loadFragment2() {
  const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8");
  const frag2 = JSON.parse(frag2Raw);

  if (!frag2.structuredResult) {
    throw new Error("Fragment 2 missing structuredResult");
  }

  return {
    fragment: frag2,
    structuredResult: frag2.structuredResult
  };
}

// ─── Build the relationship-inference prompt ──────────────────────────────

function buildRelationshipPrompt(srs2, later) {
  const parts = [];

  // Neutral instruction
  parts.push("Consider these two reasoning contributions. What meaningful");
  parts.push("relationship, if any, exists between them? Base the answer only");
  parts.push("on what the contributions actually mean. If neither contribution");
  parts.push("materially changes, qualifies, supports, weakens, contradicts or");
  parts.push("otherwise affects the other, report that no material relationship");
  parts.push("is present.");
  parts.push("");
  parts.push("Preserve uncertainty. Do not invent facts. Do not rewrite either");
  parts.push("contribution. Do not recommend what to do. Do not decide which");
  parts.push("question should be investigated next. A proposed relationship is");
  parts.push("a proposal, not automatically authoritative.");
  parts.push("");

  // Fragment 2 (earlier)
  parts.push("=== Contribution 1 (earlier) ===");
  parts.push(`Question: ${srs2.question}`);
  if (srs2.observations && srs2.observations.length) {
    parts.push("Observations:");
    for (const o of srs2.observations) {
      parts.push(`  - ${o}`);
    }
  }
  if (srs2.uncertainties && srs2.uncertainties.length) {
    parts.push("Uncertainties:");
    for (const u of srs2.uncertainties) {
      parts.push(`  - ${u}`);
    }
  }
  if (srs2.assumptions && srs2.assumptions.length) {
    parts.push("Assumptions:");
    for (const a of srs2.assumptions) {
      parts.push(`  - ${a}`);
    }
  }
  if (srs2.relationships && srs2.relationships.length) {
    parts.push("Relationships (internal to contribution):");
    for (const r of srs2.relationships) {
      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
    }
  }
  parts.push("");

  // Later contribution
  parts.push("=== Contribution 2 (later) ===");
  parts.push(`Question: ${later.question}`);
  if (later.observations && later.observations.length) {
    parts.push("Observations:");
    for (const o of later.observations) {
      parts.push(`  - ${o}`);
    }
  }
  if (later.uncertainties && later.uncertainties.length) {
    parts.push("Uncertainties:");
    for (const u of later.uncertainties) {
      parts.push(`  - ${u}`);
    }
  }
  if (later.assumptions && later.assumptions.length) {
    parts.push("Assumptions:");
    for (const a of later.assumptions) {
      parts.push(`  - ${a}`);
    }
  }
  if (later.relationships && later.relationships.length) {
    parts.push("Relationships (internal to contribution):");
    for (const r of later.relationships) {
      parts.push(`  - ${r.from} -> ${r.to} (${r.type})`);
    }
  }
  parts.push("");

  // Core question
  parts.push("QUESTION: What meaningful relationship, if any, exists between");
  parts.push("these two contributions? If no material relationship is present,");
  parts.push("state that explicitly.");
  parts.push("");

  // Return format
  parts.push("Return exactly one JSON object. If a relationship exists:");
  parts.push('{ "relationshipFound": true, "relationship": "<plain-language description>", "evidence": ["<specific contribution content supporting the relationship>"], "qualification": ["<important uncertainty or limitation>"] }');
  parts.push("");
  parts.push("If no material relationship is present:");
  parts.push('{ "relationshipFound": false, "relationship": null, "evidence": [], "qualification": [] }');

  return parts.join("\n");
}

// ─── Output contract validation ──────────────────────────────────────────────

function validateRelationshipOutput(result) {
  const errors = [];

  if (typeof result.relationshipFound !== "boolean") {
    errors.push("relationshipFound must be a boolean");
  }

  if (!result.relationshipFound) {
    if (result.relationship !== null && result.relationship !== undefined) {
      errors.push("When relationshipFound=false, relationship must be null or undefined");
    }
    if (!Array.isArray(result.evidence)) {
      errors.push("evidence must be an array when relationshipFound=false");
    }
    if (!Array.isArray(result.qualification)) {
      errors.push("qualification must be an array when relationshipFound=false");
    }
    return errors;
  }

  // When true, relationship string is required
  if (typeof result.relationship !== "string" || result.relationship.trim() === "") {
    errors.push("When relationshipFound=true, relationship must be a non-empty plain-language description");
  }
  if (!Array.isArray(result.evidence)) {
    errors.push("evidence must be an array when relationshipFound=true");
  } else if (result.evidence.some((s) => typeof s !== "string")) {
    errors.push("evidence items must all be strings");
  }
  if (!Array.isArray(result.qualification)) {
    errors.push("qualification must be an array when relationshipFound=true");
  } else if (result.qualification.some((s) => typeof s !== "string")) {
    errors.push("qualification items must all be strings");
  }

  // No forbidden fields
  const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
    "keywordScore", "materialityScore", "recommendation", "nextQuestion",
    "decisionSignificance", "graphUpdate", "strengthScore"];
  for (const field of forbidden) {
    if (field in result && typeof result[field] !== "undefined") {
      errors.push(`Forbidden field present: ${field}`);
    }
  }

  return errors;
}

// ─── Prompt-leak check ──────────────────────────────────────────────────────

function checkPromptLeak(prompt) {
  const lowerPrompt = prompt.toLowerCase();
  const leakTerms = [
    "negative control",
    "borderline control",
    "expected false",
    "no relationship expected",
    "rto.22",
    "manual answer"
  ];

  const found = [];
  for (const term of leakTerms) {
    if (lowerPrompt.includes(term.toLowerCase())) {
      found.push(term);
    }
  }

  return {
    leaked: found.length > 0,
    terms: found
  };
}

// ─── Inspect mode (default: zero model calls) ────────────────────────────────

async function inspectApparatus() {
  console.log("=== RTO.23C Simplified Semantic Relationship-Inference Apparatus (inspect-only) ===\n");

  // Load and verify fragment 2
  let structuredResult2;
  try {
    const frag2 = await loadFragment2();
    structuredResult2 = frag2.structuredResult;
  } catch (e) {
    console.log("ERROR: Cannot load Fragment 2:", e.message);
    process.exit(1);
  }

  // Fragment verification
  const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
  const laterLoaded = typeof LATER_CONTRIBUTION === "object" && LATER_CONTRIBUTION !== null;

  console.log("--- Contribution verification ---");
  console.log(`Earlier contribution supplied: ${f2Loaded ? "YES" : "NO"}`);
  console.log(`Later contribution supplied: ${laterLoaded ? "YES" : "NO"}`);

  // Build prompt for inspection
  const prompt = buildRelationshipPrompt(structuredResult2, LATER_CONTRIBUTION);

  // Measure character sizes
  const instructionPart = prompt.substring(0, prompt.indexOf("=== Contribution 1"));
  const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
  const laterContributedText = JSON.stringify(LATER_CONTRIBUTION, null, 2);

  console.log("\n--- Context-size measurement ---");
  console.log(`instructionCharacterCount: ${instructionPart.length}`);
  console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
  console.log(`laterContributionCharacterCount: ${laterContributedText.length}`);
  console.log(`inputCharacterCount: ${prompt.length}`);

  // ── Boundary checks ───────────────────────────────────────────

  console.log("\n--- Input boundary verification ---");

  const fragment3Supplied = prompt.includes("Fragment 3");
  console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`);

  const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
    prompt.includes("whole-case");
  console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);

  const centralCaseInPrompt = prompt.includes("central case") ||
    prompt.includes("centralStatement");
  console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);

  const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
    prompt.includes("other fragments") || prompt.includes("additional fragment");
  console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);

  const turnHistoryInPrompt = prompt.includes("turn history") ||
    prompt.includes("prior turns") || prompt.includes("conversation history");
  console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);

  // Known result / expected relationship check
  const knownResultSupplied = prompt.includes("Fragment 3's technical evidence bears on") ||
    prompt.includes("active competing-product development more plausible");
  console.log(`Known relationship/result supplied: ${knownResultSupplied ? "YES" : "NO"}`);

  // ── Prompt-leak check (mandatory) ─────────────────────────────

  const leakCheck = checkPromptLeak(prompt);
  console.log("\n--- Expected outcome leakage ─────────────────────────────");
  console.log(`Expected outcome disclosed to model:\n${leakCheck.leaked ? "YES" : "NO"}`);
  if (leakCheck.leaked) {
    console.log(`Leaked terms: ${leakCheck.terms.join(", ")}`);
  }

  // ── Output contract validation capability ─────────────────────

  const mockProposal = {
    relationshipFound: true,
    relationship: "test description",
    evidence: ["test"],
    qualification: ["test"]
  };
  const mockNoRel = {
    relationshipFound: false,
    relationship: null,
    evidence: [],
    qualification: []
  };
  const proposalErrors = validateRelationshipOutput(mockProposal);
  const noRelErrors = validateRelationshipOutput(mockNoRel);

  console.log("\n--- Output contract ────────────────────────────────────────");
  console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
  console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
  console.log(`relationshipFound=false allowed: YES`);

  // ── Prompt semantic cleanliness check ────────────────────────

  const lowerPrompt = prompt.toLowerCase();
  const hasDomainDict = !!lowerPrompt.match(/\b(technical\s*architecture|prototype|roadmap|launch\s*timing|development\s*programme)\s*dictionary/);
  const hasKeywordScoring = !!lowerPrompt.match(/keyword.*scor(e|ing)|score.*keyword/);
  const hasFixedTaxonomy = !!lowerPrompt.match(/relationship\s*(type|category|taxonomy)/i) && lowerPrompt.includes("enum");

  console.log("\n--- Semantic cleanliness ───────────────────────────────────");
  console.log(`Keyword/dictionary semantic checks: ${hasDomainDict || hasKeywordScoring ? "PRESENT" : "NONE"}`);
  if (hasDomainDict) console.log("  Domain dictionary used in prompt");
  if (hasKeywordScoring) console.log("  Keyword scoring present in prompt");

  // ── Live route ────────────────────────────────────────────────

  console.log("\n--- Live route ---");
  console.log("node scripts/experimental/rto-fragment-relationship-borderline-control.mjs --live");

  // ── Environment check ─────────────────────────────────────────

  const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL;
  const modelConfigured = !!process.env.OLLAMA_MODEL;
  console.log("\n--- Environment ---");
  console.log(`OLLAMA_BASE_URL configured: ${baseUrlConfigured ? "YES" : "NO"}`);
  console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`);

  // ── Inspect summary ───────────────────────────────────────────

  const noBoundaryLeaks = !fragment3Supplied && !wholeGraphInPrompt &&
    !centralCaseInPrompt && !otherFragmentsInPrompt &&
    !turnHistoryInPrompt && !knownResultSupplied && !leakCheck.leaked;
  const semanticClean = !hasDomainDict && !hasKeywordScoring && !hasFixedTaxonomy;

  console.log("\n=== SUMMARY ===");
  console.log(`Inspect result: ${f2Loaded && laterLoaded && noBoundaryLeaks && semanticClean ? "PASS" : "FAIL"}`);
  console.log(`Inspect live calls: 0`);
  console.log(`Earlier contribution supplied: ${f2Loaded ? "YES" : "NO"}`);
  console.log(`Later contribution supplied: ${laterLoaded ? "YES" : "NO"}`);
  console.log(`Known relationship/result supplied: ${knownResultSupplied ? "YES" : "NO"}`);
  console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
  console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
  console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
  console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
  console.log(`relationshipFound=false allowed: YES`);
  console.log(`maximum live calls per --live: 1`);

  const passed = f2Loaded && laterLoaded && noBoundaryLeaks && semanticClean;

  if (!passed) {
    console.log("\nApparatus cannot satisfy boundary without production changes.");
    process.exit(1);
  }

  return { prompt };
}

// ─── Live execution (exactly one model call) ──────────────────────────────────

async function executeLive() {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) {
    console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode.");
    process.exit(1);
  }
  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
    process.exit(1);
  }

  const modelName = process.env.OLLAMA_MODEL;
  if (!modelName) {
    console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode.");
    process.exit(1);
  }

  // Load fragment 2 and build prompt
  const { structuredResult: srs2 } = await loadFragment2();
  const prompt = buildRelationshipPrompt(srs2, LATER_CONTRIBUTION);

  // Prompt-leak check before sending
  const leakCheck = checkPromptLeak(prompt);
  if (leakCheck.leaked) {
    console.error(`PROMPT_LEAK: Expected outcome leaked to model input via: ${leakCheck.terms.join(", ")}`);
    process.exit(1);
  }

  // Prepare artifact path for live result
  await fs.mkdir(resultsDir, { recursive: true });

  const startedAt = Date.now();
  console.log("Live mode: sending to model...");

  const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
  const raw = await provider.generateReconstruction(prompt, modelName);
  const elapsedMs = Date.now() - startedAt;

  console.log(`Model response received in ${elapsedMs}ms`);

  // Provider returns a parsed JavaScript object via recoverJson().
  const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => {
    console.error("Provider returned unexpected type (expected parsed object):", typeof raw);
    process.exit(1);
  })();

  // Validate output contract
  const errors = validateRelationshipOutput(parsedResult);
  if (errors.length > 0) {
    console.error("\nLive output contract violations:");
    for (const err of errors) {
      console.error(`  - ${err}`);
    }
    process.exit(1);
  }

  // Write artifact
  const artifactPath = path.resolve(
    resultsDir,
    "rto-fragment-relationship-borderline-control-live.json"
  );

  const payload = {
    apparatus: "rto-fragment-relationship-borderline-control.mjs",
    experiment: "RTO.23C",
    artifactType: "LIVE RESULT — Semantic relationship inference",
    modelName: modelName,
    elapsedMs: elapsedMs,
    inputCharacterCount: prompt.length,
    instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Contribution 1")).length,
    fragment2CharacterCount: JSON.stringify(srs2, null, 2).length,
    laterContributionCharacterCount: JSON.stringify(LATER_CONTRIBUTION, null, 2).length,
    fragment2Path: FRAGMENT_2_PATH,
    knownRelationshipSuppliedToModel: false,
    wholeSituationGraphSupplied: false,
    centralCaseStatementSupplied: false,
    turnHistorySupplied: false,
    proposedRelationship: parsedResult,
  };

  await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
  console.log(`\nLive result written to: ${artifactPath}`);
  console.log("\n=== Proposed Relationship ===");
  console.log(JSON.stringify(parsedResult, null, 2));

  return payload;
}

// ─── CLI entry point ──────────────────────────────────────────────────────────

async function main() {
  const args = process.argv.slice(2);

  // --live flag for exactly one live model call
  if (args.includes("--live")) {
    console.log("RTO.23C — Semantic Relationship-Inference Apparatus\n");
    console.log("WARNING: This will make exactly ONE live model call.\n");
    const result = await executeLive();
    return result;
  }

  // Default: inspect-only, zero model calls
  const result = await inspectApparatus();

  return result;
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  process.exit(1);
});