/** * RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus * * Purpose: Test whether semantic relationship discovery correctly returns * "no relationship" when the later fragment does not materially bear on * an earlier uncertainty (false-positive control). * * Design boundary: * - Standalone experimental runner. * - Zero production code changes. * - Inspect-only by default (zero live model calls). * - --live flag enables exactly one live call. * * Fixed negative-control pair: * Earlier — RTO.18 Fragment 2 (competitor hiring + conference) * Later — negative-control-operational-contract (implementation partner fee) * * Output contract (same as RTO.21): * { relationshipFound, relationship, remainingQualification } */ import fs from "fs/promises"; import path from "path"; import { fileURLToPath } from "url"; import dotenv from "dotenv"; dotenv.config({ path: ".env.local" }); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const resultsDir = path.resolve(__dirname, "../../tests/experimental/results"); // ─── Fragment paths ────────────────────────────────────────────────────────── const FRAGMENT_2_PATH = path.resolve( __dirname, "../../tests/experimental/results/rto-granular-fragment-turn2.json" ); // ─── Negative-control fragment (EXPERIMENTAL_FIXTURE) ──────────────────────── const NEGATIVE_CONTROL_FRAGMENT = { targetNodeId: "negative-control-operational-contract", question: "When does the implementation partner contract renew and what is the current annual fee?", observations: [ "The implementation partner contract renews in November.", "The current annual fee is £42,000." ], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [] }; // ─── Load fragment 2 ───────────────────────────────────────────────────────── async function loadFragment2() { const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8"); const frag2 = JSON.parse(frag2Raw); if (!frag2.structuredResult) { throw new Error("Fragment 2 missing structuredResult"); } return { fragment: frag2, structuredResult: frag2.structuredResult }; } // ─── Negative-control integrity check ──────────────────────────────────────── function verifyNegativeControlClean() { const text = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT); const forbiddenTerms = [ "competitor", "hiring", "conference", "product development", "patent", "prototype", "launch" ]; const lowerText = text.toLowerCase(); const violations = []; for (const term of forbiddenTerms) { if (lowerText.includes(term.toLowerCase())) { violations.push(term); } } return { clean: violations.length === 0, violations }; } // ─── Build the relationship-discovery prompt ────────────────────────────────── function buildRelationshipPrompt(srs2, controlSrs) { const parts = []; // Instructions — strict semantic discipline (same as RTO.21) parts.push("You are examining two independent reasoning fragments produced during a"); parts.push("competitive investigation. Your task is to determine whether the later"); parts.push("fragment contains evidence that materially bears on an uncertainty,"); parts.push("assumption or observation in the earlier fragment."); parts.push(""); parts.push("CRITICAL RULES:"); parts.push("- Only consider what is explicitly stated in these two fragments."); parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation."); parts.push("- Do not infer facts not present in either fragment."); parts.push("- Do not rewrite or summarise either fragment."); parts.push("- Do not recommend an action or choose what to investigate next."); parts.push(""); parts.push("SEMANTIC DISCIPLINE:"); parts.push("- Identify only a material relationship supported by the supplied fragments."); parts.push("- Do not create a relationship merely because both fragments occur in the"); parts.push(" same wider case."); parts.push("- Do not infer broad business relevance."); parts.push("- If the later fragment does not materially bear on an earlier item,"); parts.push(" return relationshipFound=false."); parts.push(""); // Fragment 2 (earlier) parts.push("=== Fragment 2 (earlier evidence) ==="); parts.push(`Question: ${srs2.question}`); if (srs2.observations && srs2.observations.length) { parts.push("Observations:"); for (const o of srs2.observations) { parts.push(` - ${o}`); } } if (srs2.uncertainties && srs2.uncertainties.length) { parts.push("Uncertainties:"); for (const u of srs2.uncertainties) { parts.push(` - ${u}`); } } if (srs2.assumptions && srs2.assumptions.length) { parts.push("Assumptions:"); for (const a of srs2.assumptions) { parts.push(` - ${a}`); } } if (srs2.relationships && srs2.relationships.length) { parts.push("Relationships (internal to fragment):"); for (const r of srs2.relationships) { parts.push(` - ${r.from} -> ${r.to} (${r.type})`); } } parts.push(""); // Negative-control fragment (later) parts.push("=== Fragment (later evidence — negative control) ==="); parts.push(`Question: ${controlSrs.question}`); if (controlSrs.observations && controlSrs.observations.length) { parts.push("Observations:"); for (const o of controlSrs.observations) { parts.push(` - ${o}`); } } if (controlSrs.uncertainties && controlSrs.uncertainties.length) { parts.push("Uncertainties:"); for (const u of controlSrs.uncertainties) { parts.push(` - ${u}`); } } if (controlSrs.assumptions && controlSrs.assumptions.length) { parts.push("Assumptions:"); for (const a of controlSrs.assumptions) { parts.push(` - ${a}`); } } if (controlSrs.relationships && controlSrs.relationships.length) { parts.push("Relationships (internal to fragment):"); for (const r of controlSrs.relationships) { parts.push(` - ${r.from} -> ${r.to} (${r.type})`); } } parts.push(""); // Core question parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,"); parts.push("assumption or observation in the earlier fragment?"); parts.push(""); // Return format — same contract as RTO.21 parts.push('Return exactly one JSON object:'); parts.push('{ "relationshipFound": , "relationship": { "laterEvidence": "", "earlierItem": "", "bearing": "" }, "remainingQualification": [""] }'); parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }'); return parts.join("\n"); } // ─── Output contract validation (same as RTO.21) ───────────────────────────── function validateRelationshipOutput(result) { const errors = []; if (typeof result.relationshipFound !== "boolean") { errors.push("relationshipFound must be a boolean"); } if (!result.relationshipFound) { if (result.relationship !== null && result.relationship !== undefined) { errors.push("When relationshipFound=false, relationship must be null or undefined"); } if (!Array.isArray(result.remainingQualification)) { errors.push("remainingQualification must be an array when relationshipFound=false"); } return errors; } // When true, relationship object is required if (!result.relationship || typeof result.relationship !== "object") { errors.push("When relationshipFound=true, relationship must be an object"); return errors; } const rel = result.relationship; if (typeof rel.laterEvidence !== "string") { errors.push("relationship.laterEvidence must be a string"); } if (typeof rel.earlierItem !== "string") { errors.push("relationship.earlierItem must be a string"); } if (typeof rel.bearing !== "string") { errors.push("relationship.bearing must be a string"); } if (!Array.isArray(result.remainingQualification)) { errors.push("remainingQualification must be an array when relationshipFound=true"); } else if (result.remainingQualification.some((s) => typeof s !== "string")) { errors.push("remainingQualification items must all be strings"); } // No forbidden fields (same as RTO.21) const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength", "recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"]; for (const field of forbidden) { if (field in result && typeof result[field] !== "undefined") { errors.push(`Forbidden field present: ${field}`); } } return errors; } // ─── Inspect mode (default: zero model calls) ──────────────────────────────── async function inspectApparatus() { console.log("=== RTO.22A Negative-Control Fragment Relationship-Discovery Apparatus (inspect-only) ===\n"); // Load and verify fragment 2 let structuredResult2; try { const frag2 = await loadFragment2(); structuredResult2 = frag2.structuredResult; } catch (e) { console.log("ERROR: Cannot load Fragment 2:", e.message); process.exit(1); } // Negative-control integrity check const controlCheck = verifyNegativeControlClean(); console.log("--- Negative-control integrity ---"); console.log(`Control fragment clean: ${controlCheck.clean ? "YES" : "NO (violations: " + controlCheck.violations.join(", ") + ")"}`); if (!controlCheck.clean) { console.log("\nNEGATIVE_CONTROL_CLEAN: NO"); console.log("Control fragment contains terms that leak into competitor-development content."); console.log("Cannot proceed with a contaminated negative control."); process.exit(1); } // Verify control fragment contents const controlText = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT); const hasContractRenewal = controlText.toLowerCase().includes("renews in november") || controlText.toLowerCase().includes("contract renews"); const hasAnnualFee = controlText.includes("£42,000"); console.log("\n--- Negative-control fixture verification ---"); console.log(`Contract renewal observation present: ${hasContractRenewal ? "YES" : "NO"}`); console.log(`£42,000 annual fee observation present: ${hasAnnualFee ? "YES" : "NO"}`); console.log(`Competitor-development content present: NO`); // Fragment verification const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null; console.log("\n--- Fragment verification ---"); console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); console.log(`Negative-control fragment supplied: YES`); // Build prompt for inspection const prompt = buildRelationshipPrompt(structuredResult2, NEGATIVE_CONTROL_FRAGMENT); // Measure character sizes const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2")); const frag2StructuredText = JSON.stringify(structuredResult2, null, 2); const controlStructuredText = JSON.stringify(NEGATIVE_CONTROL_FRAGMENT, null, 2); console.log("\n--- Context-size measurement ---"); console.log(`instructionCharacterCount: ${instructionPart.length}`); console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`); console.log(`negativeControlCharacterCount: ${controlStructuredText.length}`); console.log(`inputCharacterCount: ${prompt.length}`); // ── Boundary checks ─────────────────────────────────────────── console.log("\n--- Input boundary verification ---"); const knownPositiveRelInPrompt = prompt.includes("Fragment 3's technical evidence bears on Fragment 2") || prompt.includes("active competing-product development more plausible"); console.log(`Known RTO.19/21 positive relationship supplied: ${knownPositiveRelInPrompt ? "YES" : "NO"}`); const fragment3Supplied = prompt.includes("Fragment 3") || NEGATIVE_CONTROL_FRAGMENT.targetNodeId === "fragment-3"; console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`); const wholeGraphInPrompt = prompt.includes("SituationGraph") || prompt.includes("whole-case"); console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); const centralCaseInPrompt = prompt.includes("central case") || prompt.includes("centralStatement"); console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); const currentViewInPrompt = prompt.includes("derived view") || prompt.includes("focused view") || prompt.includes("current focused"); console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`); const otherFragmentsInPrompt = prompt.includes("Fragment 1") || prompt.includes("other fragments") || prompt.includes("additional fragment"); console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); const turnHistoryInPrompt = prompt.includes("turn history") || prompt.includes("prior turns") || prompt.includes("conversation history"); console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); // ── Output contract validation capability ───────────────────── const mockProposal = { relationshipFound: true, relationship: { laterEvidence: "test", earlierItem: "test", bearing: "test", }, remainingQualification: ["test"], }; const mockNoRel = { relationshipFound: false, relationship: null, remainingQualification: [], }; const proposalErrors = validateRelationshipOutput(mockProposal); const noRelErrors = validateRelationshipOutput(mockNoRel); console.log("\n--- Output contract (same as RTO.21) ---"); console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`); console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`); console.log(`relationshipFound=false allowed: YES`); // ── Live route ──────────────────────────────────────────────── console.log("\n--- Live route ---"); console.log("node scripts/experimental/rto-fragment-relationship-negative-control.mjs --live"); // ── Environment check ───────────────────────────────────────── const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL; const modelConfigured = !!process.env.OLLAMA_MODEL; console.log("\n--- Environment ---"); console.log(`OLLAMA_BASE_URL configured: ${baseUrlConfigured ? "YES" : "NO"}`); console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`); // ── Inspect summary ─────────────────────────────────────────── const allFragmentsOK = f2Loaded; const noBoundaryLeaks = !knownPositiveRelInPrompt && !fragment3Supplied && !wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt && !otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean; console.log("\n=== SUMMARY ==="); console.log(`Inspect live calls: 0`); console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); console.log(`Negative-control fragment supplied: YES`); console.log(`Fragment 3 supplied: NO`); console.log(`Known positive relationship supplied: NO`); console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`); console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); console.log(`NEGATIVE_CONTROL_CLEAN: YES`); console.log(`--live route exposed: YES`); console.log(`maximum live calls per --live: 1`); console.log(`relationshipFound=false allowed: YES`); const passed = allFragmentsOK && noBoundaryLeaks && controlCheck.clean; console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`); if (!passed) { console.log("\nApparatus cannot satisfy boundary without production changes."); process.exit(1); } return { prompt }; } // ─── Live execution (exactly one model call) ────────────────────────────────── async function executeLive() { const baseUrl = process.env.OLLAMA_BASE_URL; if (!baseUrl) { console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode."); process.exit(1); } if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl); process.exit(1); } const modelName = process.env.OLLAMA_MODEL; if (!modelName) { console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode."); process.exit(1); } // Load fragment 2 and build prompt const { structuredResult: srs2 } = await loadFragment2(); const prompt = buildRelationshipPrompt(srs2, NEGATIVE_CONTROL_FRAGMENT); const startedAt = Date.now(); console.log("Live mode: sending to model..."); const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider(); const raw = await provider.generateReconstruction(prompt, modelName); const elapsedMs = Date.now() - startedAt; console.log(`Model response received in ${elapsedMs}ms`); // Provider returns a parsed JavaScript object via recoverJson(). const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => { console.error("Provider returned unexpected type (expected parsed object):", typeof raw); process.exit(1); })(); // Validate output contract const errors = validateRelationshipOutput(parsedResult); if (errors.length > 0) { console.error("\nLive output contract violations:"); for (const err of errors) { console.error(` - ${err}`); } process.exit(1); } // Write artifact await fs.mkdir(resultsDir, { recursive: true }); const artifactPath = path.resolve( resultsDir, "rto-fragment-relationship-negative-control-live.json" ); const controlCheck = verifyNegativeControlClean(); const payload = { apparatus: "rto-fragment-relationship-negative-control.mjs", experiment: "RTO.22A", artifactType: "LIVE RESULT — Negative-control relationship discovery", modelName: modelName, elapsedMs: elapsedMs, inputCharacterCount: prompt.length, instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length, fragment2CharacterCount: JSON.stringify(srs2, null, 2).length, negativeControlCharacterCount: JSON.stringify(NEGATIVE_CONTROL_FRAGMENT, null, 2).length, fragment2Path: FRAGMENT_2_PATH, controlFixtureType: "EXPERIMENTAL_FIXTURE", negativeControlClean: controlCheck.clean, knownRelationshipSuppliedToModel: false, wholeSituationGraphSupplied: false, centralCaseStatementSupplied: false, turnHistorySupplied: false, proposedRelationship: parsedResult, }; await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2)); console.log(`\nLive result written to: ${artifactPath}`); console.log("\n=== Proposed Relationship ==="); console.log(JSON.stringify(parsedResult, null, 2)); return payload; } // ─── CLI entry point ────────────────────────────────────────────────────────── async function main() { const args = process.argv.slice(2); // --live flag for exactly one live model call if (args.includes("--live")) { console.log("RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus\n"); console.log("WARNING: This will make exactly ONE live model call.\n"); const result = await executeLive(); return result; } // Default: inspect-only, zero model calls console.log("RTO.22A — Negative-Control Fragment Relationship-Discovery Apparatus\n"); console.log("Run with --live for one live model call.\n"); const result = await inspectApparatus(); if (result === null) { process.exit(1); } return result; } main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); });