diff --git a/scripts/experimental/rto-fragment-relationship-negative-control.mjs b/scripts/experimental/rto-fragment-relationship-negative-control.mjs new file mode 100644 index 0000000..85f42f9 --- /dev/null +++ b/scripts/experimental/rto-fragment-relationship-negative-control.mjs @@ -0,0 +1,521 @@ +/** + * RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus + * + * Purpose: Test whether semantic relationship discovery correctly returns + * "no relationship" when the later fragment does not materially bear on + * an earlier uncertainty (false-positive control). + * + * Design boundary: + * - Standalone experimental runner. + * - Zero production code changes. + * - Inspect-only by default (zero live model calls). + * - --live flag enables exactly one live call. + * + * Fixed negative-control pair: + * Earlier — RTO.18 Fragment 2 (competitor hiring + conference) + * Later — negative-control-operational-contract (implementation partner fee) + * + * Output contract (same as RTO.21): + * { relationshipFound, relationship, remainingQualification } + */ + +import fs from "fs/promises"; +import path from "path"; +import { fileURLToPath } from "url"; +import dotenv from "dotenv"; + +dotenv.config({ path: ".env.local" }); + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const resultsDir = path.resolve(__dirname, "../../tests/experimental/results"); + +// ─── Fragment paths ────────────────────────────────────────────────────────── + +const FRAGMENT_2_PATH = path.resolve( + __dirname, + "../../tests/experimental/results/rto-granular-fragment-turn2.json" +); + +// ─── Negative-control fragment (EXPERIMENTAL_FIXTURE) ──────────────────────── + +const NEGATIVE_CONTROL_FRAGMENT = { + targetNodeId: "negative-control-operational-contract", + question: "When does the implementation partner contract renew and what is the current annual fee?", + observations: [ + "The implementation partner contract renews in November.", + "The current annual fee is £42,000." + ], + uncertainties: [], + assumptions: [], + relationships: [], + possibleFollowUpQuestions: [] +}; + +// ─── Load fragment 2 ───────────────────────────────────────────────────────── + +async function loadFragment2() { + const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8"); + const frag2 = JSON.parse(frag2Raw); + + if (!frag2.structuredResult) { + throw new Error("Fragment 2 missing structuredResult"); + } + + return { + fragment: frag2, + structuredResult: frag2.structuredResult + }; +} + +// ─── Negative-control integrity check ──────────────────────────────────────── + +function verifyNegativeControlClean() { + const text = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT); + + const forbiddenTerms = [ + "competitor", + "hiring", + "conference", + "product development", + "patent", + "prototype", + "launch" + ]; + + const lowerText = text.toLowerCase(); + const violations = []; + + for (const term of forbiddenTerms) { + if (lowerText.includes(term.toLowerCase())) { + violations.push(term); + } + } + + return { + clean: violations.length === 0, + violations + }; +} + +// ─── Build the relationship-discovery prompt ────────────────────────────────── + +function buildRelationshipPrompt(srs2, controlSrs) { + const parts = []; + + // Instructions — strict semantic discipline (same as RTO.21) + parts.push("You are examining two independent reasoning fragments produced during a"); + parts.push("competitive investigation. Your task is to determine whether the later"); + parts.push("fragment contains evidence that materially bears on an uncertainty,"); + parts.push("assumption or observation in the earlier fragment."); + parts.push(""); + parts.push("CRITICAL RULES:"); + parts.push("- Only consider what is explicitly stated in these two fragments."); + parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation."); + parts.push("- Do not infer facts not present in either fragment."); + parts.push("- Do not rewrite or summarise either fragment."); + parts.push("- Do not recommend an action or choose what to investigate next."); + parts.push(""); + parts.push("SEMANTIC DISCIPLINE:"); + parts.push("- Identify only a material relationship supported by the supplied fragments."); + parts.push("- Do not create a relationship merely because both fragments occur in the"); + parts.push(" same wider case."); + parts.push("- Do not infer broad business relevance."); + parts.push("- If the later fragment does not materially bear on an earlier item,"); + parts.push(" return relationshipFound=false."); + parts.push(""); + + // Fragment 2 (earlier) + parts.push("=== Fragment 2 (earlier evidence) ==="); + parts.push(`Question: ${srs2.question}`); + if (srs2.observations && srs2.observations.length) { + parts.push("Observations:"); + for (const o of srs2.observations) { + parts.push(` - ${o}`); + } + } + if (srs2.uncertainties && srs2.uncertainties.length) { + parts.push("Uncertainties:"); + for (const u of srs2.uncertainties) { + parts.push(` - ${u}`); + } + } + if (srs2.assumptions && srs2.assumptions.length) { + parts.push("Assumptions:"); + for (const a of srs2.assumptions) { + parts.push(` - ${a}`); + } + } + if (srs2.relationships && srs2.relationships.length) { + parts.push("Relationships (internal to fragment):"); + for (const r of srs2.relationships) { + parts.push(` - ${r.from} -> ${r.to} (${r.type})`); + } + } + parts.push(""); + + // Negative-control fragment (later) + parts.push("=== Fragment (later evidence — negative control) ==="); + parts.push(`Question: ${controlSrs.question}`); + if (controlSrs.observations && controlSrs.observations.length) { + parts.push("Observations:"); + for (const o of controlSrs.observations) { + parts.push(` - ${o}`); + } + } + if (controlSrs.uncertainties && controlSrs.uncertainties.length) { + parts.push("Uncertainties:"); + for (const u of controlSrs.uncertainties) { + parts.push(` - ${u}`); + } + } + if (controlSrs.assumptions && controlSrs.assumptions.length) { + parts.push("Assumptions:"); + for (const a of controlSrs.assumptions) { + parts.push(` - ${a}`); + } + } + if (controlSrs.relationships && controlSrs.relationships.length) { + parts.push("Relationships (internal to fragment):"); + for (const r of controlSrs.relationships) { + parts.push(` - ${r.from} -> ${r.to} (${r.type})`); + } + } + parts.push(""); + + // Core question + parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,"); + parts.push("assumption or observation in the earlier fragment?"); + parts.push(""); + + // Return format — same contract as RTO.21 + parts.push('Return exactly one JSON object:'); + parts.push('{ "relationshipFound": , "relationship": { "laterEvidence": "", "earlierItem": "", "bearing": "" }, "remainingQualification": [""] }'); + parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }'); + + return parts.join("\n"); +} + +// ─── Output contract validation (same as RTO.21) ───────────────────────────── + +function validateRelationshipOutput(result) { + const errors = []; + + if (typeof result.relationshipFound !== "boolean") { + errors.push("relationshipFound must be a boolean"); + } + + if (!result.relationshipFound) { + if (result.relationship !== null && result.relationship !== undefined) { + errors.push("When relationshipFound=false, relationship must be null or undefined"); + } + if (!Array.isArray(result.remainingQualification)) { + errors.push("remainingQualification must be an array when relationshipFound=false"); + } + return errors; + } + + // When true, relationship object is required + if (!result.relationship || typeof result.relationship !== "object") { + errors.push("When relationshipFound=true, relationship must be an object"); + return errors; + } + + const rel = result.relationship; + if (typeof rel.laterEvidence !== "string") { + errors.push("relationship.laterEvidence must be a string"); + } + if (typeof rel.earlierItem !== "string") { + errors.push("relationship.earlierItem must be a string"); + } + if (typeof rel.bearing !== "string") { + errors.push("relationship.bearing must be a string"); + } + + if (!Array.isArray(result.remainingQualification)) { + errors.push("remainingQualification must be an array when relationshipFound=true"); + } else if (result.remainingQualification.some((s) => typeof s !== "string")) { + errors.push("remainingQualification items must all be strings"); + } + + // No forbidden fields (same as RTO.21) + const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength", + "recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"]; + for (const field of forbidden) { + if (field in result && typeof result[field] !== "undefined") { + errors.push(`Forbidden field present: ${field}`); + } + } + + return errors; +} + +// ─── Inspect mode (default: zero model calls) ──────────────────────────────── + +async function inspectApparatus() { + console.log("=== RTO.22A Negative-Control Fragment Relationship-Discovery Apparatus (inspect-only) ===\n"); + + // Load and verify fragment 2 + let structuredResult2; + try { + const frag2 = await loadFragment2(); + structuredResult2 = frag2.structuredResult; + } catch (e) { + console.log("ERROR: Cannot load Fragment 2:", e.message); + process.exit(1); + } + + // Negative-control integrity check + const controlCheck = verifyNegativeControlClean(); + console.log("--- Negative-control integrity ---"); + console.log(`Control fragment clean: ${controlCheck.clean ? "YES" : "NO (violations: " + controlCheck.violations.join(", ") + ")"}`); + + if (!controlCheck.clean) { + console.log("\nNEGATIVE_CONTROL_CLEAN: NO"); + console.log("Control fragment contains terms that leak into competitor-development content."); + console.log("Cannot proceed with a contaminated negative control."); + process.exit(1); + } + + // Verify control fragment contents + const controlText = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT); + const hasContractRenewal = controlText.toLowerCase().includes("renews in november") || controlText.toLowerCase().includes("contract renews"); + const hasAnnualFee = controlText.includes("£42,000"); + + console.log("\n--- Negative-control fixture verification ---"); + console.log(`Contract renewal observation present: ${hasContractRenewal ? "YES" : "NO"}`); + console.log(`£42,000 annual fee observation present: ${hasAnnualFee ? "YES" : "NO"}`); + console.log(`Competitor-development content present: NO`); + + // Fragment verification + const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null; + console.log("\n--- Fragment verification ---"); + console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); + console.log(`Negative-control fragment supplied: YES`); + + // Build prompt for inspection + const prompt = buildRelationshipPrompt(structuredResult2, NEGATIVE_CONTROL_FRAGMENT); + + // Measure character sizes + const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2")); + const frag2StructuredText = JSON.stringify(structuredResult2, null, 2); + const controlStructuredText = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT, null, 2); + + console.log("\n--- Context-size measurement ---"); + console.log(`instructionCharacterCount: ${instructionPart.length}`); + console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`); + console.log(`negativeControlCharacterCount: ${controlStructuredText.length}`); + console.log(`inputCharacterCount: ${prompt.length}`); + + // ── Boundary checks ─────────────────────────────────────────── + + console.log("\n--- Input boundary verification ---"); + + const knownPositiveRelInPrompt = prompt.includes("Fragment 3's technical evidence bears on Fragment 2") || + prompt.includes("active competing-product development more plausible"); + console.log(`Known RTO.19/21 positive relationship supplied: ${knownPositiveRelInPrompt ? "YES" : "NO"}`); + + const fragment3Supplied = prompt.includes("Fragment 3") || + NEGATIVE_CONTROL_FRAGMENT.targetNodeId === "fragment-3"; + console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`); + + const wholeGraphInPrompt = prompt.includes("SituationGraph") || + prompt.includes("whole-case"); + console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); + + const centralCaseInPrompt = prompt.includes("central case") || + prompt.includes("centralStatement"); + console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); + + const currentViewInPrompt = prompt.includes("derived view") || + prompt.includes("focused view") || prompt.includes("current focused"); + console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`); + + const otherFragmentsInPrompt = prompt.includes("Fragment 1") || + prompt.includes("other fragments") || prompt.includes("additional fragment"); + console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); + + const turnHistoryInPrompt = prompt.includes("turn history") || + prompt.includes("prior turns") || prompt.includes("conversation history"); + console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); + + // ── Output contract validation capability ───────────────────── + + const mockProposal = { + relationshipFound: true, + relationship: { + laterEvidence: "test", + earlierItem: "test", + bearing: "test", + }, + remainingQualification: ["test"], + }; + const mockNoRel = { + relationshipFound: false, + relationship: null, + remainingQualification: [], + }; + const proposalErrors = validateRelationshipOutput(mockProposal); + const noRelErrors = validateRelationshipOutput(mockNoRel); + console.log("\n--- Output contract (same as RTO.21) ---"); + console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`); + console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`); + console.log(`relationshipFound=false allowed: YES`); + + // ── Live route ──────────────────────────────────────────────── + console.log("\n--- Live route ---"); + console.log("node scripts/experimental/rto-fragment-relationship-negative-control.mjs --live"); + + // ── Environment check ───────────────────────────────────────── + const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL; + const modelConfigured = !!process.env.OLLAMA_MODEL; + console.log("\n--- Environment ---"); + console.log(`OLLAMA_BASE_URL configured: ${baseUrlConfigured ? "YES" : "NO"}`); + console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`); + + // ── Inspect summary ─────────────────────────────────────────── + const allFragmentsOK = f2Loaded; + const noBoundaryLeaks = !knownPositiveRelInPrompt && !fragment3Supplied && + !wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt && + !otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean; + + console.log("\n=== SUMMARY ==="); + console.log(`Inspect live calls: 0`); + console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); + console.log(`Negative-control fragment supplied: YES`); + console.log(`Fragment 3 supplied: NO`); + console.log(`Known positive relationship supplied: NO`); + console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); + console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); + console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`); + console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); + console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); + console.log(`NEGATIVE_CONTROL_CLEAN: YES`); + console.log(`--live route exposed: YES`); + console.log(`maximum live calls per --live: 1`); + console.log(`relationshipFound=false allowed: YES`); + + const passed = allFragmentsOK && noBoundaryLeaks && controlCheck.clean; + console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`); + + if (!passed) { + console.log("\nApparatus cannot satisfy boundary without production changes."); + process.exit(1); + } + + return { prompt }; +} + +// ─── Live execution (exactly one model call) ────────────────────────────────── + +async function executeLive() { + const baseUrl = process.env.OLLAMA_BASE_URL; + if (!baseUrl) { + console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode."); + process.exit(1); + } + if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { + console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl); + process.exit(1); + } + + const modelName = process.env.OLLAMA_MODEL; + if (!modelName) { + console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode."); + process.exit(1); + } + + // Load fragment 2 and build prompt + const { structuredResult: srs2 } = await loadFragment2(); + const prompt = buildRelationshipPrompt(srs2, NEGATIVE_CONTROL_FRAGMENT); + + const startedAt = Date.now(); + console.log("Live mode: sending to model..."); + + const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider(); + const raw = await provider.generateReconstruction(prompt, modelName); + const elapsedMs = Date.now() - startedAt; + + console.log(`Model response received in ${elapsedMs}ms`); + + // Provider returns a parsed JavaScript object via recoverJson(). + const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => { + console.error("Provider returned unexpected type (expected parsed object):", typeof raw); + process.exit(1); + })(); + + // Validate output contract + const errors = validateRelationshipOutput(parsedResult); + if (errors.length > 0) { + console.error("\nLive output contract violations:"); + for (const err of errors) { + console.error(` - ${err}`); + } + process.exit(1); + } + + // Write artifact + await fs.mkdir(resultsDir, { recursive: true }); + + const artifactPath = path.resolve( + resultsDir, + "rto-fragment-relationship-negative-control-live.json" + ); + + const controlCheck = verifyNegativeControlClean(); + const payload = { + apparatus: "rto-fragment-relationship-negative-control.mjs", + experiment: "RTO.22A", + artifactType: "LIVE RESULT — Negative-control relationship discovery", + modelName: modelName, + elapsedMs: elapsedMs, + inputCharacterCount: prompt.length, + instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length, + fragment2CharacterCount: JSON.stringify(srs2, null, 2).length, + negativeControlCharacterCount: JSON.stringify(NEGATIVE_CONTROL_FRAGMENT, null, 2).length, + fragment2Path: FRAGMENT_2_PATH, + controlFixtureType: "EXPERIMENTAL_FIXTURE", + negativeControlClean: controlCheck.clean, + knownRelationshipSuppliedToModel: false, + wholeSituationGraphSupplied: false, + centralCaseStatementSupplied: false, + turnHistorySupplied: false, + proposedRelationship: parsedResult, + }; + + await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2)); + console.log(`\nLive result written to: ${artifactPath}`); + console.log("\n=== Proposed Relationship ==="); + console.log(JSON.stringify(parsedResult, null, 2)); + + return payload; +} + +// ─── CLI entry point ────────────────────────────────────────────────────────── + +async function main() { + const args = process.argv.slice(2); + + // --live flag for exactly one live model call + if (args.includes("--live")) { + console.log("RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus\n"); + console.log("WARNING: This will make exactly ONE live model call.\n"); + const result = await executeLive(); + return result; + } + + // Default: inspect-only, zero model calls + console.log("RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus\n"); + console.log("Run with --live for one live model call.\n"); + + const result = await inspectApparatus(); + if (result === null) { + process.exit(1); + } + + return result; +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +});