robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(experiment): simplify relationship inference apparatus

Browse Source
This commit is contained in:
robbond
2026-08-19 18:53:10 +01:00
parent 601e46e4b7
commit a00adfabf4
1 changed files with 127 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/experimental/rto-fragment-relationship-borderline-control.mjs
+127 -250
View File
@@ -1,10 +1,9 @@
/** /**
* RTO.23A — Borderline Same-Topic Negative-Control Relationship-Discovery Apparatus * RTO.23C — Simplified Semantic Relationship-Inference Apparatus
* *
* Purpose: Test whether semantic relationship discovery distinguishes * Purpose: Present two reasoning contributions neutrally and ask the model
* topical similarity from material evidential bearing when both fragments * to infer their meaningful relationship, if any, from meaning alone —
* concern the same competitor, market, and industry context but the later * without domain dictionaries, keyword rules or expected-result leakage.
* fragment does not materially advance the earlier uncertainty.
* *
* Design boundary: * Design boundary:
* - Standalone experimental runner. * - Standalone experimental runner.
@@ -16,12 +15,11 @@
* RTO.18 Fragment 2 — competitor hiring + conference signals * RTO.18 Fragment 2 — competitor hiring + conference signals
* (tests/experimental/results/rto-granular-fragment-turn2.json) * (tests/experimental/results/rto-granular-fragment-turn2.json)
* *
* Fixed borderline-control fragment: * Fixed later contribution:
* EXPERIMENTAL_FIXTURE — same competitor, same market, same conference, * same-context-public-market-activity — public market activity, same context.
* but only general commentary with no technical or product-development evidence.
* *
* Output contract (same as RTO.21 / RTO.22): * Output contract:
* { relationshipFound, relationship, remainingQualification } * { relationshipFound, relationship, evidence, qualification }
*/ */
import fs from "fs/promises"; import fs from "fs/promises";
@@ -41,19 +39,15 @@ const FRAGMENT_2_PATH = path.resolve(
"../../tests/experimental/results/rto-granular-fragment-turn2.json" "../../tests/experimental/results/rto-granular-fragment-turn2.json"
); );
// ─── Borderline-control fragment (EXPERIMENTAL_FIXTURE) ────────────────────── // ─── Later contribution (fixed experimental fixture) ──────────────────────
// This fixture is deliberately topically close to Fragment 2 but lacks
// any technical architecture, prototype, roadmap, launch timing, development
// programme, or new hiring evidence. Those absences are the experimental
// variable being tested.
const BORDERLINE_CONTROL_FRAGMENT = { const LATER_CONTRIBUTION = {
targetNodeId: "borderline-control-competitor-market-presence", targetNodeId: "same-context-public-market-activity",
question: "What else has the competitor done publicly in this market?", question: "What else has the competitor done publicly in this market?",
observations: [ observations: [
"The competitor sponsored the same industry conference.", "The competitor sponsored the same industry conference.",
"Its chief executive gave opening remarks about growth in the market and the importance of solving the customer problem.", "Its chief executive gave the opening remarks, speaking about growth in the market and the importance of solving this customer problem.",
"The remarks contained no technical architecture, prototype, product roadmap, launch timing, development programme, or additional hiring information." "The company also hosted a networking reception for customers and partners attending the event."
], ],
uncertainties: [], uncertainties: [],
assumptions: [], assumptions: [],
@@ -77,119 +71,27 @@ async function loadFragment2() {
}; };
} }
// ─── Borderline-control integrity check ────────────────────────────────────── // ─── Build the relationship-inference prompt ──────────────────────────────
function hasAffirmativeEvidence(text, terms) { function buildRelationshipPrompt(srs2, later) {
// Check if any term appears as affirmative evidence (not in a negation).
// A negation pattern is: "contained no X" / "no X present" / "without X".
const lowerText = text.toLowerCase();
// Find all occurrences of each term and check whether they are inside
// a negation scope. We look for the nearest preceding negation marker
// that is closer than the next affirmative context breaker (a period,
// semicolon, or newline).
for (const term of terms) {
let searchPos = 0;
while (true) {
const idx = lowerText.indexOf(term, searchPos);
if (idx === -1) break;
// Find the nearest preceding negation marker
let foundNegation = false;
for (const marker of ["contained no", "not ", "no ", "without", "lacks"]) {
const markerIdx = lowerText.lastIndexOf(marker, idx - 1);
if (markerIdx !== -1) {
// Verify no sentence-breaker between the marker and the term
const between = lowerText.substring(markerIdx + marker.length, idx);
if (!between.match(/[.;\n]/)) {
foundNegation = true;
break;
}
}
}
if (foundNegation) {
// Skip past this occurrence — it's negated
searchPos = idx + term.length;
continue;
}
// Look forward to ensure the term is a complete word (not part of another word)
const after = lowerText.substring(idx);
if (after.match(/[^a-z]/) !== null) {
return true; // Affirmative evidence found
}
searchPos = idx + 1;
}
}
return false;
}
function verifyBorderlineControlClean() {
const text = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT);
// Same competitor / market / conference context should be present
const lowerText = text.toLowerCase();
const sameCompetitorContext = lowerText.includes("competitor");
const sameMarketContext = lowerText.includes("market");
const sameConferenceContext = lowerText.includes("conference");
const customerProblemRef = lowerText.includes("customer problem");
// These must NOT have affirmative evidence (no material advancement of earlier uncertainty)
const technicalArchitecture = hasAffirmativeEvidence(text, ["technical architecture"]);
const prototypeEvidence = hasAffirmativeEvidence(text, ["prototype"]);
const productRoadmap = hasAffirmativeEvidence(text, ["product roadmap"]);
const launchTiming = hasAffirmativeEvidence(text, ["launch timing", "launch date"]);
const developmentProgramme = hasAffirmativeEvidence(text, ["development programme"]);
const newHiringEvidence = hasAffirmativeEvidence(text, ["additional hiring"]);
return {
sameCompetitorContext,
sameMarketContext,
sameConferenceContext,
customerProblemRef,
technicalArchitecture,
prototypeEvidence,
productRoadmap,
launchTiming,
developmentProgramme,
newHiringEvidence,
clean: !technicalArchitecture && !prototypeEvidence && !productRoadmap && !launchTiming && !developmentProgramme && !newHiringEvidence
};
}
// ─── Build the relationship-discovery prompt ──────────────────────────────────
function buildRelationshipPrompt(srs2, controlSrs) {
const parts = []; const parts = [];
// Instructions — strict semantic discipline (same as RTO.21 / RTO.22) // Neutral instruction
parts.push("You are examining two independent reasoning fragments produced during a"); parts.push("Consider these two reasoning contributions. What meaningful");
parts.push("competitive investigation. Your task is to determine whether the later"); parts.push("relationship, if any, exists between them? Base the answer only");
parts.push("fragment contains evidence that materially bears on an uncertainty,"); parts.push("on what the contributions actually mean. If neither contribution");
parts.push("assumption or observation in the earlier fragment."); parts.push("materially changes, qualifies, supports, weakens, contradicts or");
parts.push("otherwise affects the other, report that no material relationship");
parts.push("is present.");
parts.push(""); parts.push("");
parts.push("CRITICAL RULES:"); parts.push("Preserve uncertainty. Do not invent facts. Do not rewrite either");
parts.push("- Only consider what is explicitly stated in these two fragments."); parts.push("contribution. Do not recommend what to do. Do not decide which");
parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation."); parts.push("question should be investigated next. A proposed relationship is");
parts.push("- Do not infer facts not present in either fragment."); parts.push("a proposal, not automatically authoritative.");
parts.push("- Do not rewrite or summarise either fragment.");
parts.push("- Do not recommend an action or choose what to investigate next.");
parts.push("");
parts.push("SEMANTIC DISCIPLINE:");
parts.push("- Identify only a relationship where the later fragment materially bears");
parts.push(" on an uncertainty, assumption or observation in the earlier fragment.");
parts.push("- Topical similarity alone is not sufficient.");
parts.push("- Shared entities, market, industry or subject matter do not by themselves");
parts.push(" constitute an evidential relationship.");
parts.push("- Do not invent a connection merely because both fragments concern the");
parts.push(" same wider situation.");
parts.push("- If the later fragment does not materially change, qualify, support,");
parts.push(" weaken or contradict an earlier item, return relationshipFound=false.");
parts.push(""); parts.push("");
// Fragment 2 (earlier) // Fragment 2 (earlier)
parts.push("=== Fragment 2 (earlier evidence) ==="); parts.push("=== Contribution 1 (earlier) ===");
parts.push(`Question: ${srs2.question}`); parts.push(`Question: ${srs2.question}`);
if (srs2.observations && srs2.observations.length) { if (srs2.observations && srs2.observations.length) {
parts.push("Observations:"); parts.push("Observations:");
@@ -210,56 +112,59 @@ function buildRelationshipPrompt(srs2, controlSrs) {
} }
} }
if (srs2.relationships && srs2.relationships.length) { if (srs2.relationships && srs2.relationships.length) {
parts.push("Relationships (internal to fragment):"); parts.push("Relationships (internal to contribution):");
for (const r of srs2.relationships) { for (const r of srs2.relationships) {
parts.push(` - ${r.from} -> ${r.to} (${r.type})`); parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
} }
} }
parts.push(""); parts.push("");
// Borderline-control fragment (later) // Later contribution
parts.push("=== Fragment (later evidence) ==="); parts.push("=== Contribution 2 (later) ===");
parts.push(`Question: ${controlSrs.question}`); parts.push(`Question: ${later.question}`);
if (controlSrs.observations && controlSrs.observations.length) { if (later.observations && later.observations.length) {
parts.push("Observations:"); parts.push("Observations:");
for (const o of controlSrs.observations) { for (const o of later.observations) {
parts.push(` - ${o}`); parts.push(` - ${o}`);
} }
} }
if (controlSrs.uncertainties && controlSrs.uncertainties.length) { if (later.uncertainties && later.uncertainties.length) {
parts.push("Uncertainties:"); parts.push("Uncertainties:");
for (const u of controlSrs.uncertainties) { for (const u of later.uncertainties) {
parts.push(` - ${u}`); parts.push(` - ${u}`);
} }
} }
if (controlSrs.assumptions && controlSrs.assumptions.length) { if (later.assumptions && later.assumptions.length) {
parts.push("Assumptions:"); parts.push("Assumptions:");
for (const a of controlSrs.assumptions) { for (const a of later.assumptions) {
parts.push(` - ${a}`); parts.push(` - ${a}`);
} }
} }
if (controlSrs.relationships && controlSrs.relationships.length) { if (later.relationships && later.relationships.length) {
parts.push("Relationships (internal to fragment):"); parts.push("Relationships (internal to contribution):");
for (const r of controlSrs.relationships) { for (const r of later.relationships) {
parts.push(` - ${r.from} -> ${r.to} (${r.type})`); parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
} }
} }
parts.push(""); parts.push("");
// Core question // Core question
parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,"); parts.push("QUESTION: What meaningful relationship, if any, exists between");
parts.push("assumption or observation in the earlier fragment?"); parts.push("these two contributions? If no material relationship is present,");
parts.push("state that explicitly.");
parts.push(""); parts.push("");
// Return format — same contract as RTO.21 / RTO.22 // Return format
parts.push('Return exactly one JSON object:'); parts.push("Return exactly one JSON object. If a relationship exists:");
parts.push('{ "relationshipFound": <bool>, "relationship": { "laterEvidence": "<string>", "earlierItem": "<string>", "bearing": "<string>" }, "remainingQualification": ["<string>"] }'); parts.push('{ "relationshipFound": true, "relationship": "<plain-language description>", "evidence": ["<specific contribution content supporting the relationship>"], "qualification": ["<important uncertainty or limitation>"] }');
parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }'); parts.push("");
parts.push("If no material relationship is present:");
parts.push('{ "relationshipFound": false, "relationship": null, "evidence": [], "qualification": [] }');
return parts.join("\n"); return parts.join("\n");
} }
// ─── Output contract validation (same as RTO.21 / RTO.22) ───────────────────── // ─── Output contract validation ──────────────────────────────────────────────
function validateRelationshipOutput(result) { function validateRelationshipOutput(result) {
const errors = []; const errors = [];
@@ -272,38 +177,34 @@ function validateRelationshipOutput(result) {
if (result.relationship !== null && result.relationship !== undefined) { if (result.relationship !== null && result.relationship !== undefined) {
errors.push("When relationshipFound=false, relationship must be null or undefined"); errors.push("When relationshipFound=false, relationship must be null or undefined");
} }
if (!Array.isArray(result.remainingQualification)) { if (!Array.isArray(result.evidence)) {
errors.push("remainingQualification must be an array when relationshipFound=false"); errors.push("evidence must be an array when relationshipFound=false");
}
if (!Array.isArray(result.qualification)) {
errors.push("qualification must be an array when relationshipFound=false");
} }
return errors; return errors;
} }
// When true, relationship object is required // When true, relationship string is required
if (!result.relationship || typeof result.relationship !== "object") { if (typeof result.relationship !== "string" || result.relationship.trim() === "") {
errors.push("When relationshipFound=true, relationship must be an object"); errors.push("When relationshipFound=true, relationship must be a non-empty plain-language description");
return errors; }
if (!Array.isArray(result.evidence)) {
errors.push("evidence must be an array when relationshipFound=true");
} else if (result.evidence.some((s) => typeof s !== "string")) {
errors.push("evidence items must all be strings");
}
if (!Array.isArray(result.qualification)) {
errors.push("qualification must be an array when relationshipFound=true");
} else if (result.qualification.some((s) => typeof s !== "string")) {
errors.push("qualification items must all be strings");
} }
const rel = result.relationship; // No forbidden fields
if (typeof rel.laterEvidence !== "string") {
errors.push("relationship.laterEvidence must be a string");
}
if (typeof rel.earlierItem !== "string") {
errors.push("relationship.earlierItem must be a string");
}
if (typeof rel.bearing !== "string") {
errors.push("relationship.bearing must be a string");
}
if (!Array.isArray(result.remainingQualification)) {
errors.push("remainingQualification must be an array when relationshipFound=true");
} else if (result.remainingQualification.some((s) => typeof s !== "string")) {
errors.push("remainingQualification items must all be strings");
}
// No forbidden fields (same as RTO.21 / RTO.22)
const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength", const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
"recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"]; "keywordScore", "materialityScore", "recommendation", "nextQuestion",
"decisionSignificance", "graphUpdate", "strengthScore"];
for (const field of forbidden) { for (const field of forbidden) {
if (field in result && typeof result[field] !== "undefined") { if (field in result && typeof result[field] !== "undefined") {
errors.push(`Forbidden field present: ${field}`); errors.push(`Forbidden field present: ${field}`);
@@ -342,7 +243,7 @@ function checkPromptLeak(prompt) {
// ─── Inspect mode (default: zero model calls) ──────────────────────────────── // ─── Inspect mode (default: zero model calls) ────────────────────────────────
async function inspectApparatus() { async function inspectApparatus() {
console.log("=== RTO.23A Borderline Same-Topic Relationship-Control Apparatus (inspect-only) ===\n"); console.log("=== RTO.23C Simplified Semantic Relationship-Inference Apparatus (inspect-only) ===\n");
// Load and verify fragment 2 // Load and verify fragment 2
let structuredResult2; let structuredResult2;
@@ -354,43 +255,26 @@ async function inspectApparatus() {
process.exit(1); process.exit(1);
} }
// Borderline-control integrity check
const controlCheck = verifyBorderlineControlClean();
console.log("--- Borderline-control fixture integrity ---");
console.log(`Same competitor/market context present: ${controlCheck.sameCompetitorContext ? "YES" : "NO"}`);
console.log(`Same conference context present: ${controlCheck.sameConferenceContext ? "YES" : "NO"}`);
console.log(`General customer-problem reference present: ${controlCheck.customerProblemRef ? "YES" : "NO"}`);
console.log(`Technical architecture present: ${controlCheck.technicalArchitecture ? "YES" : "NO"}`);
console.log(`Prototype evidence present: ${controlCheck.prototypeEvidence ? "YES" : "NO"}`);
console.log(`Product roadmap present: ${controlCheck.productRoadmap ? "YES" : "NO"}`);
console.log(`Launch timing evidence present: ${controlCheck.launchTiming ? "YES" : "NO"}`);
console.log(`Development programme present: ${controlCheck.developmentProgramme ? "YES" : "NO"}`);
console.log(`New hiring evidence present: ${controlCheck.newHiringEvidence ? "YES" : "NO"}`);
if (!controlCheck.clean) {
console.log("\nBORDERLINE_CONTROL_CLEAN:\nNO");
process.exit(1);
}
console.log("\nBORDERLINE_CONTROL_CLEAN:\nYES");
// Fragment verification // Fragment verification
const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null; const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
console.log("\n--- Fragment verification ---"); const laterLoaded = typeof LATER_CONTRIBUTION === "object" && LATER_CONTRIBUTION !== null;
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
console.log(`Borderline-control fragment supplied: YES`); console.log("--- Contribution verification ---");
console.log(`Earlier contribution supplied: ${f2Loaded ? "YES" : "NO"}`);
console.log(`Later contribution supplied: ${laterLoaded ? "YES" : "NO"}`);
// Build prompt for inspection // Build prompt for inspection
const prompt = buildRelationshipPrompt(structuredResult2, BORDERLINE_CONTROL_FRAGMENT); const prompt = buildRelationshipPrompt(structuredResult2, LATER_CONTRIBUTION);
// Measure character sizes // Measure character sizes
const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2")); const instructionPart = prompt.substring(0, prompt.indexOf("=== Contribution 1"));
const frag2StructuredText = JSON.stringify(structuredResult2, null, 2); const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
const controlStructuredText = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2); const laterContributedText = JSON.stringify(LATER_CONTRIBUTION, null, 2);
console.log("\n--- Context-size measurement ---"); console.log("\n--- Context-size measurement ---");
console.log(`instructionCharacterCount: ${instructionPart.length}`); console.log(`instructionCharacterCount: ${instructionPart.length}`);
console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`); console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
console.log(`borderlineControlCharacterCount: ${controlStructuredText.length}`); console.log(`laterContributionCharacterCount: ${laterContributedText.length}`);
console.log(`inputCharacterCount: ${prompt.length}`); console.log(`inputCharacterCount: ${prompt.length}`);
// ── Boundary checks ─────────────────────────────────────────── // ── Boundary checks ───────────────────────────────────────────
@@ -400,10 +284,6 @@ async function inspectApparatus() {
const fragment3Supplied = prompt.includes("Fragment 3"); const fragment3Supplied = prompt.includes("Fragment 3");
console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`); console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`);
const rto21PositiveResultSupplied = prompt.includes("Fragment 3's technical evidence bears on") ||
prompt.includes("active competing-product development more plausible");
console.log(`RTO.21 positive result supplied: ${rto21PositiveResultSupplied ? "YES" : "NO"}`);
const wholeGraphInPrompt = prompt.includes("SituationGraph") || const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
prompt.includes("whole-case"); prompt.includes("whole-case");
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
@@ -412,10 +292,6 @@ async function inspectApparatus() {
prompt.includes("centralStatement"); prompt.includes("centralStatement");
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
const currentViewInPrompt = prompt.includes("derived view") ||
prompt.includes("focused view") || prompt.includes("current focused");
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
const otherFragmentsInPrompt = prompt.includes("Fragment 1") || const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
prompt.includes("other fragments") || prompt.includes("additional fragment"); prompt.includes("other fragments") || prompt.includes("additional fragment");
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
@@ -424,11 +300,16 @@ async function inspectApparatus() {
prompt.includes("prior turns") || prompt.includes("conversation history"); prompt.includes("prior turns") || prompt.includes("conversation history");
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`); console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
// Known result / expected relationship check
const knownResultSupplied = prompt.includes("Fragment 3's technical evidence bears on") ||
prompt.includes("active competing-product development more plausible");
console.log(`Known relationship/result supplied: ${knownResultSupplied ? "YES" : "NO"}`);
// ── Prompt-leak check (mandatory) ───────────────────────────── // ── Prompt-leak check (mandatory) ─────────────────────────────
const leakCheck = checkPromptLeak(prompt); const leakCheck = checkPromptLeak(prompt);
console.log("\n--- Prompt-leak check ---"); console.log("\n--- Expected outcome leakage ─────────────────────────────");
console.log(`EXPECTED_OUTCOME_LEAKED_TO_MODEL:\n${leakCheck.leaked ? "YES" : "NO"}`); console.log(`Expected outcome disclosed to model:\n${leakCheck.leaked ? "YES" : "NO"}`);
if (leakCheck.leaked) { if (leakCheck.leaked) {
console.log(`Leaked terms: ${leakCheck.terms.join(", ")}`); console.log(`Leaked terms: ${leakCheck.terms.join(", ")}`);
} }
@@ -437,31 +318,43 @@ async function inspectApparatus() {
const mockProposal = { const mockProposal = {
relationshipFound: true, relationshipFound: true,
relationship: { relationship: "test description",
laterEvidence: "test", evidence: ["test"],
earlierItem: "test", qualification: ["test"]
bearing: "test",
},
remainingQualification: ["test"],
}; };
const mockNoRel = { const mockNoRel = {
relationshipFound: false, relationshipFound: false,
relationship: null, relationship: null,
remainingQualification: [], evidence: [],
qualification: []
}; };
const proposalErrors = validateRelationshipOutput(mockProposal); const proposalErrors = validateRelationshipOutput(mockProposal);
const noRelErrors = validateRelationshipOutput(mockNoRel); const noRelErrors = validateRelationshipOutput(mockNoRel);
console.log("\n--- Output contract (same as RTO.21) ---");
console.log("\n--- Output contract ────────────────────────────────────────");
console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`); console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`); console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
console.log(`relationshipFound=false allowed: YES`); console.log(`relationshipFound=false allowed: YES`);
console.log(`Topical similarity explicitly insufficient: YES`);
// ── Prompt semantic cleanliness check ────────────────────────
const lowerPrompt = prompt.toLowerCase();
const hasDomainDict = !!lowerPrompt.match(/\b(technical\s*architecture|prototype|roadmap|launch\s*timing|development\s*programme)\s*dictionary/);
const hasKeywordScoring = !!lowerPrompt.match(/keyword.*scor(e|ing)|score.*keyword/);
const hasFixedTaxonomy = !!lowerPrompt.match(/relationship\s*(type|category|taxonomy)/i) && lowerPrompt.includes("enum");
console.log("\n--- Semantic cleanliness ───────────────────────────────────");
console.log(`Keyword/dictionary semantic checks: ${hasDomainDict || hasKeywordScoring ? "PRESENT" : "NONE"}`);
if (hasDomainDict) console.log(" Domain dictionary used in prompt");
if (hasKeywordScoring) console.log(" Keyword scoring present in prompt");
// ── Live route ──────────────────────────────────────────────── // ── Live route ────────────────────────────────────────────────
console.log("\n--- Live route ---"); console.log("\n--- Live route ---");
console.log("node scripts/experimental/rto-fragment-relationship-borderline-control.mjs --live"); console.log("node scripts/experimental/rto-fragment-relationship-borderline-control.mjs --live");
// ── Environment check ───────────────────────────────────────── // ── Environment check ─────────────────────────────────────────
const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL; const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL;
const modelConfigured = !!process.env.OLLAMA_MODEL; const modelConfigured = !!process.env.OLLAMA_MODEL;
console.log("\n--- Environment ---"); console.log("\n--- Environment ---");
@@ -469,31 +362,26 @@ async function inspectApparatus() {
console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`); console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`);
// ── Inspect summary ─────────────────────────────────────────── // ── Inspect summary ───────────────────────────────────────────
const allFragmentsOK = f2Loaded;
const noBoundaryLeaks = !fragment3Supplied && !rto21PositiveResultSupplied && const noBoundaryLeaks = !fragment3Supplied && !wholeGraphInPrompt &&
!wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt && !centralCaseInPrompt && !otherFragmentsInPrompt &&
!otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean && !leakCheck.leaked; !turnHistoryInPrompt && !knownResultSupplied && !leakCheck.leaked;
const semanticClean = !hasDomainDict && !hasKeywordScoring && !hasFixedTaxonomy;
console.log("\n=== SUMMARY ==="); console.log("\n=== SUMMARY ===");
console.log(`Inspect result: ${f2Loaded && laterLoaded && noBoundaryLeaks && semanticClean ? "PASS" : "FAIL"}`);
console.log(`Inspect live calls: 0`); console.log(`Inspect live calls: 0`);
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`); console.log(`Earlier contribution supplied: ${f2Loaded ? "YES" : "NO"}`);
console.log(`Borderline-control fragment supplied: YES`); console.log(`Later contribution supplied: ${laterLoaded ? "YES" : "NO"}`);
console.log(`Fragment 3 supplied: NO`); console.log(`Known relationship/result supplied: ${knownResultSupplied ? "YES" : "NO"}`);
console.log(`RTO.21 positive result supplied: NO`);
console.log(`RTO.22 negative result supplied: NO`);
console.log(`Known relationship supplied: NO`);
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`); console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`); console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`); console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
console.log(`BORDERLINE_CONTROL_CLEAN:\nYES`); console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
console.log(`--live route exposed: YES`);
console.log(`maximum live calls per --live: 1`);
console.log(`relationshipFound=false allowed: YES`); console.log(`relationshipFound=false allowed: YES`);
console.log(`maximum live calls per --live: 1`);
const passed = allFragmentsOK && noBoundaryLeaks; const passed = f2Loaded && laterLoaded && noBoundaryLeaks && semanticClean;
console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);
if (!passed) { if (!passed) {
console.log("\nApparatus cannot satisfy boundary without production changes."); console.log("\nApparatus cannot satisfy boundary without production changes.");
@@ -524,7 +412,7 @@ async function executeLive() {
// Load fragment 2 and build prompt // Load fragment 2 and build prompt
const { structuredResult: srs2 } = await loadFragment2(); const { structuredResult: srs2 } = await loadFragment2();
const prompt = buildRelationshipPrompt(srs2, BORDERLINE_CONTROL_FRAGMENT); const prompt = buildRelationshipPrompt(srs2, LATER_CONTRIBUTION);
// Prompt-leak check before sending // Prompt-leak check before sending
const leakCheck = checkPromptLeak(prompt); const leakCheck = checkPromptLeak(prompt);
@@ -567,22 +455,17 @@ async function executeLive() {
"rto-fragment-relationship-borderline-control-live.json" "rto-fragment-relationship-borderline-control-live.json"
); );
const controlCheck = verifyBorderlineControlClean();
const payload = { const payload = {
apparatus: "rto-fragment-relationship-borderline-control.mjs", apparatus: "rto-fragment-relationship-borderline-control.mjs",
experiment: "RTO.23A", experiment: "RTO.23C",
artifactType: "LIVE RESULT — Borderline same-topic relationship discovery", artifactType: "LIVE RESULT — Semantic relationship inference",
modelName: modelName, modelName: modelName,
elapsedMs: elapsedMs, elapsedMs: elapsedMs,
inputCharacterCount: prompt.length, inputCharacterCount: prompt.length,
instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length, instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Contribution 1")).length,
fragment2CharacterCount: JSON.stringify(srs2, null, 2).length, fragment2CharacterCount: JSON.stringify(srs2, null, 2).length,
borderlineControlCharacterCount: JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2).length, laterContributionCharacterCount: JSON.stringify(LATER_CONTRIBUTION, null, 2).length,
fragment2Path: FRAGMENT_2_PATH, fragment2Path: FRAGMENT_2_PATH,
controlFixtureType: "EXPERIMENTAL_FIXTURE",
sameCompetitorContext: controlCheck.sameCompetitorContext,
sameConferenceContext: controlCheck.sameConferenceContext,
borderlineControlClean: controlCheck.clean,
knownRelationshipSuppliedToModel: false, knownRelationshipSuppliedToModel: false,
wholeSituationGraphSupplied: false, wholeSituationGraphSupplied: false,
centralCaseStatementSupplied: false, centralCaseStatementSupplied: false,
@@ -605,20 +488,14 @@ async function main() {
// --live flag for exactly one live model call // --live flag for exactly one live model call
if (args.includes("--live")) { if (args.includes("--live")) {
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n"); console.log("RTO.23C — Semantic Relationship-Inference Apparatus\n");
console.log("WARNING: This will make exactly ONE live model call.\n"); console.log("WARNING: This will make exactly ONE live model call.\n");
const result = await executeLive(); const result = await executeLive();
return result; return result;
} }
// Default: inspect-only, zero model calls // Default: inspect-only, zero model calls
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
console.log("Run with --live for one live model call.\n");
const result = await inspectApparatus(); const result = await inspectApparatus();
if (result === null) {
process.exit(1);
}
return result; return result;
} }