/** * RTO.21A — Fragment Relationship-Discovery Apparatus * * Purpose: Test whether an LLM can propose a useful cross-fragment relationship * from two independent granular fragments without whole-case context, without * rewriting either fragment, and without treating the discovered relationship as * automatically authoritative. * * Design boundary: * - Standalone experimental runner. * - Zero production code changes. * - Inspect-only by default (zero live model calls). * - --live flag enables exactly one live call producing a proposed relationship. * * Fixed record: * Fragment 2 — competitor hiring + conference signals * Fragment 3 — technical architecture + prototype evidence * Known RTO.19 relationship is NEVER supplied to the model; used only for * evaluation reference after a future live run. * * Output contract (proposal only, never authoritative): * { relationshipFound, relationship, remainingQualification } */ import fs from "fs/promises"; import path from "path"; import { fileURLToPath } from "url"; import dotenv from "dotenv"; dotenv.config({ path: ".env.local" }); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const resultsDir = path.resolve(__dirname, "../../tests/experimental/results"); // ─── Fragment paths ────────────────────────────────────────────────────────── const FRAGMENT_2_PATH = path.resolve( __dirname, "../../tests/experimental/results/rto-granular-fragment-turn2.json" ); const FRAGMENT_3_PATH = path.resolve( __dirname, "../../tests/experimental/results/rto-granular-fragment-turn3.json" ); // ─── Known RTO.19 relationship (evaluation reference ONLY) ─────────────────── // This MUST NOT appear in the prompt sent to the model. const KNOWN_RELATIONSHIP = "Fragment 3's technical evidence bears on Fragment 2's earlier uncertainty " + "by making active competing-product development more plausible, while " + "commercialisation and other uncertainties remain unresolved."; // ─── Load fragments ────────────────────────────────────────────────────────── async function loadFragments() { const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8"); const frag3Raw = await fs.readFile(FRAGMENT_3_PATH, "utf-8"); const frag2 = JSON.parse(frag2Raw); const frag3 = JSON.parse(frag3Raw); if (!frag2.structuredResult) { throw new Error("Fragment 2 missing structuredResult"); } if (!frag3.structuredResult) { throw new Error("Fragment 3 missing structuredResult"); } return { fragment2: frag2, structuredResult2: frag2.structuredResult, fragment3: frag3, structuredResult3: frag3.structuredResult, }; } // ─── Build the relationship-discovery prompt ────────────────────────────────── function buildRelationshipPrompt(srs2, srs3) { const parts = []; // Instructions parts.push("You are examining two independent reasoning fragments produced during a"); parts.push("competitive investigation. Your task is to determine whether the later"); parts.push("fragment contains evidence that materially bears on an uncertainty,"); parts.push("assumption or observation in the earlier fragment."); parts.push(""); parts.push("CRITICAL RULES:"); parts.push("- Only consider what is explicitly stated in these two fragments."); parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation."); parts.push("- Do not infer facts not present in either fragment."); parts.push("- Do not rewrite or summarise either fragment."); parts.push("- Do not recommend an action or choose what to investigate next."); parts.push("- If no relationship exists, return relationshipFound=false with empty fields."); parts.push(""); // Fragment 2 (earlier) parts.push("=== Fragment 2 (earlier evidence) ==="); parts.push(`Question: ${srs2.question}`); if (srs2.observations && srs2.observations.length) { parts.push("Observations:"); for (const o of srs2.observations) { parts.push(` - ${o}`); } } if (srs2.uncertainties && srs2.uncertainties.length) { parts.push("Uncertainties:"); for (const u of srs2.uncertainties) { parts.push(` - ${u}`); } } if (srs2.assumptions && srs2.assumptions.length) { parts.push("Assumptions:"); for (const a of srs2.assumptions) { parts.push(` - ${a}`); } } if (srs2.relationships && srs2.relationships.length) { parts.push("Relationships (internal to fragment):"); for (const r of srs2.relationships) { parts.push(` - ${r.from} -> ${r.to} (${r.type})`); } } parts.push(""); // Fragment 3 (later) parts.push("=== Fragment 3 (later evidence) ==="); parts.push(`Question: ${srs3.question}`); if (srs3.observations && srs3.observations.length) { parts.push("Observations:"); for (const o of srs3.observations) { parts.push(` - ${o}`); } } if (srs3.uncertainties && srs3.uncertainties.length) { parts.push("Uncertainties:"); for (const u of srs3.uncertainties) { parts.push(` - ${u}`); } } if (srs3.assumptions && srs3.assumptions.length) { parts.push("Assumptions:"); for (const a of srs3.assumptions) { parts.push(` - ${a}`); } } if (srs3.relationships && srs3.relationships.length) { parts.push("Relationships (internal to fragment):"); for (const r of srs3.relationships) { parts.push(` - ${r.from} -> ${r.to} (${r.type})`); } } parts.push(""); // Core question parts.push("QUESTION: Does anything in Fragment 3 materially bears on an uncertainty,"); parts.push("assumption or observation in Fragment 2?"); parts.push(""); parts.push("If YES, propose exactly one relationship. This is a PROPOSED_RELATIONSHIP"); parts.push("--- it must not be treated as authoritative without further verification."); parts.push(""); // Return format parts.push('Return exactly one JSON object:'); parts.push('{ "relationshipFound": , "relationship": { "laterEvidence": "", "earlierItem": "", "bearing": "" }, "remainingQualification": [""] }'); parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }'); return parts.join("\n"); } // ─── Output contract validation ────────────────────────────────────────────── function validateRelationshipOutput(result) { const errors = []; if (typeof result.relationshipFound !== "boolean") { errors.push("relationshipFound must be a boolean"); } if (!result.relationshipFound) { if (result.relationship !== null && result.relationship !== undefined) { errors.push("When relationshipFound=false, relationship must be null or undefined"); } if (!Array.isArray(result.remainingQualification)) { errors.push("remainingQualification must be an array when relationshipFound=false"); } return errors; } // When true, relationship object is required if (!result.relationship || typeof result.relationship !== "object") { errors.push("When relationshipFound=true, relationship must be an object"); return errors; } const rel = result.relationship; if (typeof rel.laterEvidence !== "string") { errors.push("relationship.laterEvidence must be a string"); } if (typeof rel.earlierItem !== "string") { errors.push("relationship.earlierItem must be a string"); } if (typeof rel.bearing !== "string") { errors.push("relationship.bearing must be a string"); } if (!Array.isArray(result.remainingQualification)) { errors.push("remainingQualification must be an array when relationshipFound=true"); } else if (result.remainingQualification.some((s) => typeof s !== "string")) { errors.push("remainingQualification items must all be strings"); } // No forbidden fields const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength", "recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"]; for (const field of forbidden) { if (field in result && typeof result[field] !== "undefined") { errors.push(`Forbidden field present: ${field}`); } } return errors; } // ─── Inspect mode (default: zero model calls) ───────────────────────────────── async function inspectApparatus() { console.log("=== RTO.21A Fragment Relationship-Discovery Apparatus (inspect-only) ===\n"); // Load and verify fragments let fragment2, structuredResult2, fragment3, structuredResult3; try { const frags = await loadFragments(); fragment2 = frags.fragment2; structuredResult2 = frags.structuredResult2; fragment3 = frags.fragment3; structuredResult3 = frags.structuredResult3; } catch (e) { console.log("ERROR: Cannot load fragments:", e.message); process.exit(1); } // Fragment verification const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null; const f3Loaded = typeof structuredResult3 === "object" && structuredResult3 !== null; console.log("--- Fragment verification ---"); console.log(`Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); console.log(`Fragment 3 supplied: ${f3Loaded ? "YES" : "NO"}`); // Build prompt for inspection const prompt = buildRelationshipPrompt(structuredResult2, structuredResult3); // Measure character sizes precisely const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2")); const frag2StructuredText = JSON.stringify(structuredResult2, null, 2); const frag3StructuredText = JSON.stringify(structuredResult3, null, 2); console.log("\n--- Context-size measurement ---"); console.log(`instructionCharacterCount: ${instructionPart.length}`); console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`); console.log(`fragment3CharacterCount: ${frag3StructuredText.length}`); console.log(`inputCharacterCount: ${prompt.length}`); // ── Boundary checks ─────────────────────────────────────────── console.log("\n--- Input boundary verification ---"); const knownRelInPrompt = prompt.includes(KNOWN_RELATIONSHIP) || prompt.includes("Fragment 3's technical evidence bears on Fragment 2's earlier uncertainty"); console.log(`Known relationship supplied: ${knownRelInPrompt ? "YES" : "NO"}`); const wholeGraphInPrompt = prompt.includes("SituationGraph") || prompt.includes("whole-case") || prompt.includes("case statement"); console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); const centralCaseInPrompt = prompt.includes("central case") || prompt.includes("centralStatement"); console.log(`Central case statement supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); const currentViewInPrompt = prompt.includes("derived view") || prompt.includes("focused view") || prompt.includes("current focused"); console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`); const otherFragmentsInPrompt = prompt.includes("Fragment 1") || prompt.includes("other fragments") || prompt.includes("additional fragment"); console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); const turnHistoryInPrompt = prompt.includes("turn history") || prompt.includes("prior turns") || prompt.includes("conversation history"); console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); const rto17StateInPrompt = prompt.includes("decisionSignificance") || prompt.includes("accumulated state") || prompt.includes("focusedUnderstanding"); console.log(`RTO.17 accumulated state supplied: ${rto17StateInPrompt ? "YES" : "NO"}`); // ── Schema validation capability ────────────────────────────── const mockProposal = { relationshipFound: true, relationship: { laterEvidence: "test", earlierItem: "test", bearing: "test", }, remainingQualification: ["test"], }; const mockNoRel = { relationshipFound: false, relationship: null, remainingQualification: [], }; const proposalErrors = validateRelationshipOutput(mockProposal); const noRelErrors = validateRelationshipOutput(mockNoRel); console.log("\n--- Output schema ---"); console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`); console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`); // ── Live route ──────────────────────────────────────────────── console.log("\n--- Live route ---"); console.log("node scripts/experimental/rto-fragment-relationship-discovery.mjs --live"); // ── Inspect summary (must match exact requested format) ─────── const allFragmentsOK = f2Loaded && f3Loaded; const knownRelAbsent = !knownRelInPrompt; const noContextLeaks = !(wholeGraphInPrompt || centralCaseInPrompt || currentViewInPrompt || otherFragmentsInPrompt || turnHistoryInPrompt || rto17StateInPrompt); console.log("\n=== SUMMARY ==="); console.log(`Inspect live calls: 0`); console.log(`Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); console.log(`Fragment 3 supplied: ${f3Loaded ? "YES" : "NO"}`); console.log(`Known relationship supplied: ${knownRelInPrompt ? "YES" : "NO"}`); console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); console.log(`Central case statement supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`); console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); console.log(`--live route exposed: YES`); console.log(`maximum live calls per --live: 1`); console.log(`relationshipFound=false allowed: YES`); const passed = allFragmentsOK && knownRelAbsent && noContextLeaks; console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`); if (!passed) { console.log("\nApparatus cannot satisfy boundary without production changes."); process.exit(1); } return { prompt }; } // ─── Live execution (exactly one model call) ────────────────────────────────── async function executeLive() { const baseUrl = process.env.OLLAMA_BASE_URL; if (!baseUrl) { console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode."); process.exit(1); } if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl); process.exit(1); } const modelName = process.env.OLLAMA_MODEL; if (!modelName) { console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode."); process.exit(1); } // Load fragments and build prompt const { structuredResult2, structuredResult3 } = await loadFragments(); const prompt = buildRelationshipPrompt(structuredResult2, structuredResult3); const startedAt = Date.now(); console.log("Live mode: sending to model..."); const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider(); const raw = await provider.generateReconstruction(prompt, modelName); const elapsedMs = Date.now() - startedAt; console.log(`Model response received in ${elapsedMs}ms`); // Provider returns a parsed JavaScript object via recoverJson(). // Do NOT regex-extract — the result is already structured. const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => { console.error("Provider returned unexpected type (expected parsed object):", typeof raw); process.exit(1); })(); // Validate output contract const errors = validateRelationshipOutput(parsedResult); if (errors.length > 0) { console.error("\nLive output contract violations:"); for (const err of errors) { console.error(` - ${err}`); } process.exit(1); } // Write artifact await fs.mkdir(resultsDir, { recursive: true }); const artifactPath = path.resolve( resultsDir, "rto-fragment-relationship-discovery-live.json" ); const payload = { apparatus: "rto-fragment-relationship-discovery.mjs", experiment: "RTO.21A", artifactType: "LIVE RESULT — Proposed relationship discovery", modelName: modelName, elapsedMs: elapsedMs, inputCharacterCount: prompt.length, instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length, fragment2CharacterCount: JSON.stringify(structuredResult2, null, 2).length, fragment3CharacterCount: JSON.stringify(structuredResult3, null, 2).length, fragment2Path: FRAGMENT_2_PATH, fragment3Path: FRAGMENT_3_PATH, knownRelationshipSuppliedToModel: false, wholeSituationGraphSupplied: false, centralCaseStatementSupplied: false, turnHistorySupplied: false, proposedRelationship: parsedResult, }; await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2)); console.log(`\nLive result written to: ${artifactPath}`); console.log("\n=== Proposed Relationship ==="); console.log(JSON.stringify(parsedResult, null, 2)); return payload; } // ─── CLI entry point ────────────────────────────────────────────────────────── async function main() { const args = process.argv.slice(2); // --live flag for exactly one live model call if (args.includes("--live")) { console.log("RTO.21A — Fragment Relationship-Discovery Apparatus\n"); console.log("WARNING: This will make exactly ONE live model call.\n"); const result = await executeLive(); return result; } // Default: inspect-only, zero model calls console.log("RTO.21A — Fragment Relationship-Discovery Apparatus\n"); console.log("Run with --live for one live model call.\n"); const result = await inspectApparatus(); if (result === null) { process.exit(1); } return result; } main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); });