test(experiment): checkpoint borderline relationship control

This commit is contained in:
2026-08-19 17:35:51 +01:00
parent e1a18e27e7
commit 85204f96ac
@@ -0,0 +1,629 @@
/**
* RTO.23A — Borderline Same-Topic Negative-Control Relationship-Discovery Apparatus
*
* Purpose: Test whether semantic relationship discovery distinguishes
* topical similarity from material evidential bearing when both fragments
* concern the same competitor, market, and industry context but the later
* fragment does not materially advance the earlier uncertainty.
*
* Design boundary:
* - Standalone experimental runner.
* - Zero production code changes.
* - Inspect-only by default (zero live model calls).
* - --live flag enables exactly one live call producing a proposed relationship.
*
* Fixed earlier fragment:
* RTO.18 Fragment 2 — competitor hiring + conference signals
* (tests/experimental/results/rto-granular-fragment-turn2.json)
*
* Fixed borderline-control fragment:
* EXPERIMENTAL_FIXTURE — same competitor, same market, same conference,
* but only general commentary with no technical or product-development evidence.
*
* Output contract (same as RTO.21 / RTO.22):
* { relationshipFound, relationship, remainingQualification }
*/
import fs from "fs/promises";
import path from "path";
import { fileURLToPath } from "url";
import dotenv from "dotenv";
dotenv.config({ path: ".env.local" });
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const resultsDir = path.resolve(__dirname, "../../tests/experimental/results");
// ─── Fragment paths ──────────────────────────────────────────────────────────
const FRAGMENT_2_PATH = path.resolve(
__dirname,
"../../tests/experimental/results/rto-granular-fragment-turn2.json"
);
// ─── Borderline-control fragment (EXPERIMENTAL_FIXTURE) ──────────────────────
// This fixture is deliberately topically close to Fragment 2 but lacks
// any technical architecture, prototype, roadmap, launch timing, development
// programme, or new hiring evidence. Those absences are the experimental
// variable being tested.
const BORDERLINE_CONTROL_FRAGMENT = {
targetNodeId: "borderline-control-competitor-market-presence",
question: "What else has the competitor done publicly in this market?",
observations: [
"The competitor sponsored the same industry conference.",
"Its chief executive gave opening remarks about growth in the market and the importance of solving the customer problem.",
"The remarks contained no technical architecture, prototype, product roadmap, launch timing, development programme, or additional hiring information."
],
uncertainties: [],
assumptions: [],
relationships: [],
possibleFollowUpQuestions: []
};
// ─── Load fragment 2 ─────────────────────────────────────────────────────────
async function loadFragment2() {
const frag2Raw = await fs.readFile(FRAGMENT_2_PATH, "utf-8");
const frag2 = JSON.parse(frag2Raw);
if (!frag2.structuredResult) {
throw new Error("Fragment 2 missing structuredResult");
}
return {
fragment: frag2,
structuredResult: frag2.structuredResult
};
}
// ─── Borderline-control integrity check ──────────────────────────────────────
function hasAffirmativeEvidence(text, terms) {
// Check if any term appears as affirmative evidence (not in a negation).
// A negation pattern is: "contained no X" / "no X present" / "without X".
const lowerText = text.toLowerCase();
// Find all occurrences of each term and check whether they are inside
// a negation scope. We look for the nearest preceding negation marker
// that is closer than the next affirmative context breaker (a period,
// semicolon, or newline).
for (const term of terms) {
let searchPos = 0;
while (true) {
const idx = lowerText.indexOf(term, searchPos);
if (idx === -1) break;
// Find the nearest preceding negation marker
let foundNegation = false;
for (const marker of ["contained no", "not ", "no ", "without", "lacks"]) {
const markerIdx = lowerText.lastIndexOf(marker, idx - 1);
if (markerIdx !== -1) {
// Verify no sentence-breaker between the marker and the term
const between = lowerText.substring(markerIdx + marker.length, idx);
if (!between.match(/[.;\n]/)) {
foundNegation = true;
break;
}
}
}
if (foundNegation) {
// Skip past this occurrence — it's negated
searchPos = idx + term.length;
continue;
}
// Look forward to ensure the term is a complete word (not part of another word)
const after = lowerText.substring(idx);
if (after.match(/[^a-z]/) !== null) {
return true; // Affirmative evidence found
}
searchPos = idx + 1;
}
}
return false;
}
function verifyBorderlineControlClean() {
const text = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT);
// Same competitor / market / conference context should be present
const lowerText = text.toLowerCase();
const sameCompetitorContext = lowerText.includes("competitor");
const sameMarketContext = lowerText.includes("market");
const sameConferenceContext = lowerText.includes("conference");
const customerProblemRef = lowerText.includes("customer problem");
// These must NOT have affirmative evidence (no material advancement of earlier uncertainty)
const technicalArchitecture = hasAffirmativeEvidence(text, ["technical architecture"]);
const prototypeEvidence = hasAffirmativeEvidence(text, ["prototype"]);
const productRoadmap = hasAffirmativeEvidence(text, ["product roadmap"]);
const launchTiming = hasAffirmativeEvidence(text, ["launch timing", "launch date"]);
const developmentProgramme = hasAffirmativeEvidence(text, ["development programme"]);
const newHiringEvidence = hasAffirmativeEvidence(text, ["additional hiring"]);
return {
sameCompetitorContext,
sameMarketContext,
sameConferenceContext,
customerProblemRef,
technicalArchitecture,
prototypeEvidence,
productRoadmap,
launchTiming,
developmentProgramme,
newHiringEvidence,
clean: !technicalArchitecture && !prototypeEvidence && !productRoadmap && !launchTiming && !developmentProgramme && !newHiringEvidence
};
}
// ─── Build the relationship-discovery prompt ──────────────────────────────────
function buildRelationshipPrompt(srs2, controlSrs) {
const parts = [];
// Instructions — strict semantic discipline (same as RTO.21 / RTO.22)
parts.push("You are examining two independent reasoning fragments produced during a");
parts.push("competitive investigation. Your task is to determine whether the later");
parts.push("fragment contains evidence that materially bears on an uncertainty,");
parts.push("assumption or observation in the earlier fragment.");
parts.push("");
parts.push("CRITICAL RULES:");
parts.push("- Only consider what is explicitly stated in these two fragments.");
parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation.");
parts.push("- Do not infer facts not present in either fragment.");
parts.push("- Do not rewrite or summarise either fragment.");
parts.push("- Do not recommend an action or choose what to investigate next.");
parts.push("");
parts.push("SEMANTIC DISCIPLINE:");
parts.push("- Identify only a relationship where the later fragment materially bears");
parts.push(" on an uncertainty, assumption or observation in the earlier fragment.");
parts.push("- Topical similarity alone is not sufficient.");
parts.push("- Shared entities, market, industry or subject matter do not by themselves");
parts.push(" constitute an evidential relationship.");
parts.push("- Do not invent a connection merely because both fragments concern the");
parts.push(" same wider situation.");
parts.push("- If the later fragment does not materially change, qualify, support,");
parts.push(" weaken or contradict an earlier item, return relationshipFound=false.");
parts.push("");
// Fragment 2 (earlier)
parts.push("=== Fragment 2 (earlier evidence) ===");
parts.push(`Question: ${srs2.question}`);
if (srs2.observations && srs2.observations.length) {
parts.push("Observations:");
for (const o of srs2.observations) {
parts.push(` - ${o}`);
}
}
if (srs2.uncertainties && srs2.uncertainties.length) {
parts.push("Uncertainties:");
for (const u of srs2.uncertainties) {
parts.push(` - ${u}`);
}
}
if (srs2.assumptions && srs2.assumptions.length) {
parts.push("Assumptions:");
for (const a of srs2.assumptions) {
parts.push(` - ${a}`);
}
}
if (srs2.relationships && srs2.relationships.length) {
parts.push("Relationships (internal to fragment):");
for (const r of srs2.relationships) {
parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
}
}
parts.push("");
// Borderline-control fragment (later)
parts.push("=== Fragment (later evidence) ===");
parts.push(`Question: ${controlSrs.question}`);
if (controlSrs.observations && controlSrs.observations.length) {
parts.push("Observations:");
for (const o of controlSrs.observations) {
parts.push(` - ${o}`);
}
}
if (controlSrs.uncertainties && controlSrs.uncertainties.length) {
parts.push("Uncertainties:");
for (const u of controlSrs.uncertainties) {
parts.push(` - ${u}`);
}
}
if (controlSrs.assumptions && controlSrs.assumptions.length) {
parts.push("Assumptions:");
for (const a of controlSrs.assumptions) {
parts.push(` - ${a}`);
}
}
if (controlSrs.relationships && controlSrs.relationships.length) {
parts.push("Relationships (internal to fragment):");
for (const r of controlSrs.relationships) {
parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
}
}
parts.push("");
// Core question
parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,");
parts.push("assumption or observation in the earlier fragment?");
parts.push("");
// Return format — same contract as RTO.21 / RTO.22
parts.push('Return exactly one JSON object:');
parts.push('{ "relationshipFound": <bool>, "relationship": { "laterEvidence": "<string>", "earlierItem": "<string>", "bearing": "<string>" }, "remainingQualification": ["<string>"] }');
parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }');
return parts.join("\n");
}
// ─── Output contract validation (same as RTO.21 / RTO.22) ─────────────────────
function validateRelationshipOutput(result) {
const errors = [];
if (typeof result.relationshipFound !== "boolean") {
errors.push("relationshipFound must be a boolean");
}
if (!result.relationshipFound) {
if (result.relationship !== null && result.relationship !== undefined) {
errors.push("When relationshipFound=false, relationship must be null or undefined");
}
if (!Array.isArray(result.remainingQualification)) {
errors.push("remainingQualification must be an array when relationshipFound=false");
}
return errors;
}
// When true, relationship object is required
if (!result.relationship || typeof result.relationship !== "object") {
errors.push("When relationshipFound=true, relationship must be an object");
return errors;
}
const rel = result.relationship;
if (typeof rel.laterEvidence !== "string") {
errors.push("relationship.laterEvidence must be a string");
}
if (typeof rel.earlierItem !== "string") {
errors.push("relationship.earlierItem must be a string");
}
if (typeof rel.bearing !== "string") {
errors.push("relationship.bearing must be a string");
}
if (!Array.isArray(result.remainingQualification)) {
errors.push("remainingQualification must be an array when relationshipFound=true");
} else if (result.remainingQualification.some((s) => typeof s !== "string")) {
errors.push("remainingQualification items must all be strings");
}
// No forbidden fields (same as RTO.21 / RTO.22)
const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
"recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"];
for (const field of forbidden) {
if (field in result && typeof result[field] !== "undefined") {
errors.push(`Forbidden field present: ${field}`);
}
}
return errors;
}
// ─── Prompt-leak check ──────────────────────────────────────────────────────
function checkPromptLeak(prompt) {
const lowerPrompt = prompt.toLowerCase();
const leakTerms = [
"negative control",
"borderline control",
"expected false",
"no relationship expected",
"rto.22",
"manual answer"
];
const found = [];
for (const term of leakTerms) {
if (lowerPrompt.includes(term.toLowerCase())) {
found.push(term);
}
}
return {
leaked: found.length > 0,
terms: found
};
}
// ─── Inspect mode (default: zero model calls) ────────────────────────────────
async function inspectApparatus() {
console.log("=== RTO.23A Borderline Same-Topic Relationship-Control Apparatus (inspect-only) ===\n");
// Load and verify fragment 2
let structuredResult2;
try {
const frag2 = await loadFragment2();
structuredResult2 = frag2.structuredResult;
} catch (e) {
console.log("ERROR: Cannot load Fragment 2:", e.message);
process.exit(1);
}
// Borderline-control integrity check
const controlCheck = verifyBorderlineControlClean();
console.log("--- Borderline-control fixture integrity ---");
console.log(`Same competitor/market context present: ${controlCheck.sameCompetitorContext ? "YES" : "NO"}`);
console.log(`Same conference context present: ${controlCheck.sameConferenceContext ? "YES" : "NO"}`);
console.log(`General customer-problem reference present: ${controlCheck.customerProblemRef ? "YES" : "NO"}`);
console.log(`Technical architecture present: ${controlCheck.technicalArchitecture ? "YES" : "NO"}`);
console.log(`Prototype evidence present: ${controlCheck.prototypeEvidence ? "YES" : "NO"}`);
console.log(`Product roadmap present: ${controlCheck.productRoadmap ? "YES" : "NO"}`);
console.log(`Launch timing evidence present: ${controlCheck.launchTiming ? "YES" : "NO"}`);
console.log(`Development programme present: ${controlCheck.developmentProgramme ? "YES" : "NO"}`);
console.log(`New hiring evidence present: ${controlCheck.newHiringEvidence ? "YES" : "NO"}`);
if (!controlCheck.clean) {
console.log("\nBORDERLINE_CONTROL_CLEAN:\nNO");
process.exit(1);
}
console.log("\nBORDERLINE_CONTROL_CLEAN:\nYES");
// Fragment verification
const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
console.log("\n--- Fragment verification ---");
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
console.log(`Borderline-control fragment supplied: YES`);
// Build prompt for inspection
const prompt = buildRelationshipPrompt(structuredResult2, BORDERLINE_CONTROL_FRAGMENT);
// Measure character sizes
const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2"));
const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
const controlStructuredText = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2);
console.log("\n--- Context-size measurement ---");
console.log(`instructionCharacterCount: ${instructionPart.length}`);
console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
console.log(`borderlineControlCharacterCount: ${controlStructuredText.length}`);
console.log(`inputCharacterCount: ${prompt.length}`);
// ── Boundary checks ───────────────────────────────────────────
console.log("\n--- Input boundary verification ---");
const fragment3Supplied = prompt.includes("Fragment 3");
console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`);
const rto21PositiveResultSupplied = prompt.includes("Fragment 3's technical evidence bears on") ||
prompt.includes("active competing-product development more plausible");
console.log(`RTO.21 positive result supplied: ${rto21PositiveResultSupplied ? "YES" : "NO"}`);
const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
prompt.includes("whole-case");
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
const centralCaseInPrompt = prompt.includes("central case") ||
prompt.includes("centralStatement");
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
const currentViewInPrompt = prompt.includes("derived view") ||
prompt.includes("focused view") || prompt.includes("current focused");
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
prompt.includes("other fragments") || prompt.includes("additional fragment");
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
const turnHistoryInPrompt = prompt.includes("turn history") ||
prompt.includes("prior turns") || prompt.includes("conversation history");
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
// ── Prompt-leak check (mandatory) ─────────────────────────────
const leakCheck = checkPromptLeak(prompt);
console.log("\n--- Prompt-leak check ---");
console.log(`EXPECTED_OUTCOME_LEAKED_TO_MODEL:\n${leakCheck.leaked ? "YES" : "NO"}`);
if (leakCheck.leaked) {
console.log(`Leaked terms: ${leakCheck.terms.join(", ")}`);
}
// ── Output contract validation capability ─────────────────────
const mockProposal = {
relationshipFound: true,
relationship: {
laterEvidence: "test",
earlierItem: "test",
bearing: "test",
},
remainingQualification: ["test"],
};
const mockNoRel = {
relationshipFound: false,
relationship: null,
remainingQualification: [],
};
const proposalErrors = validateRelationshipOutput(mockProposal);
const noRelErrors = validateRelationshipOutput(mockNoRel);
console.log("\n--- Output contract (same as RTO.21) ---");
console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
console.log(`relationshipFound=false allowed: YES`);
console.log(`Topical similarity explicitly insufficient: YES`);
// ── Live route ────────────────────────────────────────────────
console.log("\n--- Live route ---");
console.log("node scripts/experimental/rto-fragment-relationship-borderline-control.mjs --live");
// ── Environment check ─────────────────────────────────────────
const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL;
const modelConfigured = !!process.env.OLLAMA_MODEL;
console.log("\n--- Environment ---");
console.log(`OLLAMA_BASE_URL configured: ${baseUrlConfigured ? "YES" : "NO"}`);
console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`);
// ── Inspect summary ───────────────────────────────────────────
const allFragmentsOK = f2Loaded;
const noBoundaryLeaks = !fragment3Supplied && !rto21PositiveResultSupplied &&
!wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt &&
!otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean && !leakCheck.leaked;
console.log("\n=== SUMMARY ===");
console.log(`Inspect live calls: 0`);
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
console.log(`Borderline-control fragment supplied: YES`);
console.log(`Fragment 3 supplied: NO`);
console.log(`RTO.21 positive result supplied: NO`);
console.log(`RTO.22 negative result supplied: NO`);
console.log(`Known relationship supplied: NO`);
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
console.log(`BORDERLINE_CONTROL_CLEAN:\nYES`);
console.log(`--live route exposed: YES`);
console.log(`maximum live calls per --live: 1`);
console.log(`relationshipFound=false allowed: YES`);
const passed = allFragmentsOK && noBoundaryLeaks;
console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);
if (!passed) {
console.log("\nApparatus cannot satisfy boundary without production changes.");
process.exit(1);
}
return { prompt };
}
// ─── Live execution (exactly one model call) ──────────────────────────────────
async function executeLive() {
const baseUrl = process.env.OLLAMA_BASE_URL;
if (!baseUrl) {
console.error("OLLAMA_BASE_URL not set in environment. Cannot execute live mode.");
process.exit(1);
}
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
console.error("Refuses localhost fallback. OLLAMA_BASE_URL=" + baseUrl);
process.exit(1);
}
const modelName = process.env.OLLAMA_MODEL;
if (!modelName) {
console.error("OLLAMA_MODEL not set in environment. Cannot execute live mode.");
process.exit(1);
}
// Load fragment 2 and build prompt
const { structuredResult: srs2 } = await loadFragment2();
const prompt = buildRelationshipPrompt(srs2, BORDERLINE_CONTROL_FRAGMENT);
// Prompt-leak check before sending
const leakCheck = checkPromptLeak(prompt);
if (leakCheck.leaked) {
console.error(`PROMPT_LEAK: Expected outcome leaked to model input via: ${leakCheck.terms.join(", ")}`);
process.exit(1);
}
// Prepare artifact path for live result
await fs.mkdir(resultsDir, { recursive: true });
const startedAt = Date.now();
console.log("Live mode: sending to model...");
const provider = (await import(path.resolve(__dirname, "../../lib/llm/provider.js"))).getProvider();
const raw = await provider.generateReconstruction(prompt, modelName);
const elapsedMs = Date.now() - startedAt;
console.log(`Model response received in ${elapsedMs}ms`);
// Provider returns a parsed JavaScript object via recoverJson().
const parsedResult = typeof raw === "object" && raw !== null ? raw : (() => {
console.error("Provider returned unexpected type (expected parsed object):", typeof raw);
process.exit(1);
})();
// Validate output contract
const errors = validateRelationshipOutput(parsedResult);
if (errors.length > 0) {
console.error("\nLive output contract violations:");
for (const err of errors) {
console.error(` - ${err}`);
}
process.exit(1);
}
// Write artifact
const artifactPath = path.resolve(
resultsDir,
"rto-fragment-relationship-borderline-control-live.json"
);
const controlCheck = verifyBorderlineControlClean();
const payload = {
apparatus: "rto-fragment-relationship-borderline-control.mjs",
experiment: "RTO.23A",
artifactType: "LIVE RESULT — Borderline same-topic relationship discovery",
modelName: modelName,
elapsedMs: elapsedMs,
inputCharacterCount: prompt.length,
instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length,
fragment2CharacterCount: JSON.stringify(srs2, null, 2).length,
borderlineControlCharacterCount: JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2).length,
fragment2Path: FRAGMENT_2_PATH,
controlFixtureType: "EXPERIMENTAL_FIXTURE",
sameCompetitorContext: controlCheck.sameCompetitorContext,
sameConferenceContext: controlCheck.sameConferenceContext,
borderlineControlClean: controlCheck.clean,
knownRelationshipSuppliedToModel: false,
wholeSituationGraphSupplied: false,
centralCaseStatementSupplied: false,
turnHistorySupplied: false,
proposedRelationship: parsedResult,
};
await fs.writeFile(artifactPath, JSON.stringify(payload, null, 2));
console.log(`\nLive result written to: ${artifactPath}`);
console.log("\n=== Proposed Relationship ===");
console.log(JSON.stringify(parsedResult, null, 2));
return payload;
}
// ─── CLI entry point ──────────────────────────────────────────────────────────
async function main() {
const args = process.argv.slice(2);
// --live flag for exactly one live model call
if (args.includes("--live")) {
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
console.log("WARNING: This will make exactly ONE live model call.\n");
const result = await executeLive();
return result;
}
// Default: inspect-only, zero model calls
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
console.log("Run with --live for one live model call.\n");
const result = await inspectApparatus();
if (result === null) {
process.exit(1);
}
return result;
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});