test(experiment): simplify relationship inference apparatus
This commit is contained in:
@@ -1,10 +1,9 @@
|
||||
/**
|
||||
* RTO.23A — Borderline Same-Topic Negative-Control Relationship-Discovery Apparatus
|
||||
* RTO.23C — Simplified Semantic Relationship-Inference Apparatus
|
||||
*
|
||||
* Purpose: Test whether semantic relationship discovery distinguishes
|
||||
* topical similarity from material evidential bearing when both fragments
|
||||
* concern the same competitor, market, and industry context but the later
|
||||
* fragment does not materially advance the earlier uncertainty.
|
||||
* Purpose: Present two reasoning contributions neutrally and ask the model
|
||||
* to infer their meaningful relationship, if any, from meaning alone —
|
||||
* without domain dictionaries, keyword rules or expected-result leakage.
|
||||
*
|
||||
* Design boundary:
|
||||
* - Standalone experimental runner.
|
||||
@@ -16,12 +15,11 @@
|
||||
* RTO.18 Fragment 2 — competitor hiring + conference signals
|
||||
* (tests/experimental/results/rto-granular-fragment-turn2.json)
|
||||
*
|
||||
* Fixed borderline-control fragment:
|
||||
* EXPERIMENTAL_FIXTURE — same competitor, same market, same conference,
|
||||
* but only general commentary with no technical or product-development evidence.
|
||||
* Fixed later contribution:
|
||||
* same-context-public-market-activity — public market activity, same context.
|
||||
*
|
||||
* Output contract (same as RTO.21 / RTO.22):
|
||||
* { relationshipFound, relationship, remainingQualification }
|
||||
* Output contract:
|
||||
* { relationshipFound, relationship, evidence, qualification }
|
||||
*/
|
||||
|
||||
import fs from "fs/promises";
|
||||
@@ -41,19 +39,15 @@ const FRAGMENT_2_PATH = path.resolve(
|
||||
"../../tests/experimental/results/rto-granular-fragment-turn2.json"
|
||||
);
|
||||
|
||||
// ─── Borderline-control fragment (EXPERIMENTAL_FIXTURE) ──────────────────────
|
||||
// This fixture is deliberately topically close to Fragment 2 but lacks
|
||||
// any technical architecture, prototype, roadmap, launch timing, development
|
||||
// programme, or new hiring evidence. Those absences are the experimental
|
||||
// variable being tested.
|
||||
// ─── Later contribution (fixed experimental fixture) ──────────────────────
|
||||
|
||||
const BORDERLINE_CONTROL_FRAGMENT = {
|
||||
targetNodeId: "borderline-control-competitor-market-presence",
|
||||
const LATER_CONTRIBUTION = {
|
||||
targetNodeId: "same-context-public-market-activity",
|
||||
question: "What else has the competitor done publicly in this market?",
|
||||
observations: [
|
||||
"The competitor sponsored the same industry conference.",
|
||||
"Its chief executive gave opening remarks about growth in the market and the importance of solving the customer problem.",
|
||||
"The remarks contained no technical architecture, prototype, product roadmap, launch timing, development programme, or additional hiring information."
|
||||
"Its chief executive gave the opening remarks, speaking about growth in the market and the importance of solving this customer problem.",
|
||||
"The company also hosted a networking reception for customers and partners attending the event."
|
||||
],
|
||||
uncertainties: [],
|
||||
assumptions: [],
|
||||
@@ -77,119 +71,27 @@ async function loadFragment2() {
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Borderline-control integrity check ──────────────────────────────────────
|
||||
// ─── Build the relationship-inference prompt ──────────────────────────────
|
||||
|
||||
function hasAffirmativeEvidence(text, terms) {
|
||||
// Check if any term appears as affirmative evidence (not in a negation).
|
||||
// A negation pattern is: "contained no X" / "no X present" / "without X".
|
||||
const lowerText = text.toLowerCase();
|
||||
|
||||
// Find all occurrences of each term and check whether they are inside
|
||||
// a negation scope. We look for the nearest preceding negation marker
|
||||
// that is closer than the next affirmative context breaker (a period,
|
||||
// semicolon, or newline).
|
||||
for (const term of terms) {
|
||||
let searchPos = 0;
|
||||
while (true) {
|
||||
const idx = lowerText.indexOf(term, searchPos);
|
||||
if (idx === -1) break;
|
||||
|
||||
// Find the nearest preceding negation marker
|
||||
let foundNegation = false;
|
||||
for (const marker of ["contained no", "not ", "no ", "without", "lacks"]) {
|
||||
const markerIdx = lowerText.lastIndexOf(marker, idx - 1);
|
||||
if (markerIdx !== -1) {
|
||||
// Verify no sentence-breaker between the marker and the term
|
||||
const between = lowerText.substring(markerIdx + marker.length, idx);
|
||||
if (!between.match(/[.;\n]/)) {
|
||||
foundNegation = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (foundNegation) {
|
||||
// Skip past this occurrence — it's negated
|
||||
searchPos = idx + term.length;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Look forward to ensure the term is a complete word (not part of another word)
|
||||
const after = lowerText.substring(idx);
|
||||
if (after.match(/[^a-z]/) !== null) {
|
||||
return true; // Affirmative evidence found
|
||||
}
|
||||
searchPos = idx + 1;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function verifyBorderlineControlClean() {
|
||||
const text = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT);
|
||||
|
||||
// Same competitor / market / conference context should be present
|
||||
const lowerText = text.toLowerCase();
|
||||
const sameCompetitorContext = lowerText.includes("competitor");
|
||||
const sameMarketContext = lowerText.includes("market");
|
||||
const sameConferenceContext = lowerText.includes("conference");
|
||||
const customerProblemRef = lowerText.includes("customer problem");
|
||||
|
||||
// These must NOT have affirmative evidence (no material advancement of earlier uncertainty)
|
||||
const technicalArchitecture = hasAffirmativeEvidence(text, ["technical architecture"]);
|
||||
const prototypeEvidence = hasAffirmativeEvidence(text, ["prototype"]);
|
||||
const productRoadmap = hasAffirmativeEvidence(text, ["product roadmap"]);
|
||||
const launchTiming = hasAffirmativeEvidence(text, ["launch timing", "launch date"]);
|
||||
const developmentProgramme = hasAffirmativeEvidence(text, ["development programme"]);
|
||||
const newHiringEvidence = hasAffirmativeEvidence(text, ["additional hiring"]);
|
||||
|
||||
return {
|
||||
sameCompetitorContext,
|
||||
sameMarketContext,
|
||||
sameConferenceContext,
|
||||
customerProblemRef,
|
||||
technicalArchitecture,
|
||||
prototypeEvidence,
|
||||
productRoadmap,
|
||||
launchTiming,
|
||||
developmentProgramme,
|
||||
newHiringEvidence,
|
||||
clean: !technicalArchitecture && !prototypeEvidence && !productRoadmap && !launchTiming && !developmentProgramme && !newHiringEvidence
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Build the relationship-discovery prompt ──────────────────────────────────
|
||||
|
||||
function buildRelationshipPrompt(srs2, controlSrs) {
|
||||
function buildRelationshipPrompt(srs2, later) {
|
||||
const parts = [];
|
||||
|
||||
// Instructions — strict semantic discipline (same as RTO.21 / RTO.22)
|
||||
parts.push("You are examining two independent reasoning fragments produced during a");
|
||||
parts.push("competitive investigation. Your task is to determine whether the later");
|
||||
parts.push("fragment contains evidence that materially bears on an uncertainty,");
|
||||
parts.push("assumption or observation in the earlier fragment.");
|
||||
// Neutral instruction
|
||||
parts.push("Consider these two reasoning contributions. What meaningful");
|
||||
parts.push("relationship, if any, exists between them? Base the answer only");
|
||||
parts.push("on what the contributions actually mean. If neither contribution");
|
||||
parts.push("materially changes, qualifies, supports, weakens, contradicts or");
|
||||
parts.push("otherwise affects the other, report that no material relationship");
|
||||
parts.push("is present.");
|
||||
parts.push("");
|
||||
parts.push("CRITICAL RULES:");
|
||||
parts.push("- Only consider what is explicitly stated in these two fragments.");
|
||||
parts.push("- Preserve uncertainty and qualification. Do not treat correlation as confirmation.");
|
||||
parts.push("- Do not infer facts not present in either fragment.");
|
||||
parts.push("- Do not rewrite or summarise either fragment.");
|
||||
parts.push("- Do not recommend an action or choose what to investigate next.");
|
||||
parts.push("");
|
||||
parts.push("SEMANTIC DISCIPLINE:");
|
||||
parts.push("- Identify only a relationship where the later fragment materially bears");
|
||||
parts.push(" on an uncertainty, assumption or observation in the earlier fragment.");
|
||||
parts.push("- Topical similarity alone is not sufficient.");
|
||||
parts.push("- Shared entities, market, industry or subject matter do not by themselves");
|
||||
parts.push(" constitute an evidential relationship.");
|
||||
parts.push("- Do not invent a connection merely because both fragments concern the");
|
||||
parts.push(" same wider situation.");
|
||||
parts.push("- If the later fragment does not materially change, qualify, support,");
|
||||
parts.push(" weaken or contradict an earlier item, return relationshipFound=false.");
|
||||
parts.push("Preserve uncertainty. Do not invent facts. Do not rewrite either");
|
||||
parts.push("contribution. Do not recommend what to do. Do not decide which");
|
||||
parts.push("question should be investigated next. A proposed relationship is");
|
||||
parts.push("a proposal, not automatically authoritative.");
|
||||
parts.push("");
|
||||
|
||||
// Fragment 2 (earlier)
|
||||
parts.push("=== Fragment 2 (earlier evidence) ===");
|
||||
parts.push("=== Contribution 1 (earlier) ===");
|
||||
parts.push(`Question: ${srs2.question}`);
|
||||
if (srs2.observations && srs2.observations.length) {
|
||||
parts.push("Observations:");
|
||||
@@ -210,56 +112,59 @@ function buildRelationshipPrompt(srs2, controlSrs) {
|
||||
}
|
||||
}
|
||||
if (srs2.relationships && srs2.relationships.length) {
|
||||
parts.push("Relationships (internal to fragment):");
|
||||
parts.push("Relationships (internal to contribution):");
|
||||
for (const r of srs2.relationships) {
|
||||
parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
|
||||
}
|
||||
}
|
||||
parts.push("");
|
||||
|
||||
// Borderline-control fragment (later)
|
||||
parts.push("=== Fragment (later evidence) ===");
|
||||
parts.push(`Question: ${controlSrs.question}`);
|
||||
if (controlSrs.observations && controlSrs.observations.length) {
|
||||
// Later contribution
|
||||
parts.push("=== Contribution 2 (later) ===");
|
||||
parts.push(`Question: ${later.question}`);
|
||||
if (later.observations && later.observations.length) {
|
||||
parts.push("Observations:");
|
||||
for (const o of controlSrs.observations) {
|
||||
for (const o of later.observations) {
|
||||
parts.push(` - ${o}`);
|
||||
}
|
||||
}
|
||||
if (controlSrs.uncertainties && controlSrs.uncertainties.length) {
|
||||
if (later.uncertainties && later.uncertainties.length) {
|
||||
parts.push("Uncertainties:");
|
||||
for (const u of controlSrs.uncertainties) {
|
||||
for (const u of later.uncertainties) {
|
||||
parts.push(` - ${u}`);
|
||||
}
|
||||
}
|
||||
if (controlSrs.assumptions && controlSrs.assumptions.length) {
|
||||
if (later.assumptions && later.assumptions.length) {
|
||||
parts.push("Assumptions:");
|
||||
for (const a of controlSrs.assumptions) {
|
||||
for (const a of later.assumptions) {
|
||||
parts.push(` - ${a}`);
|
||||
}
|
||||
}
|
||||
if (controlSrs.relationships && controlSrs.relationships.length) {
|
||||
parts.push("Relationships (internal to fragment):");
|
||||
for (const r of controlSrs.relationships) {
|
||||
if (later.relationships && later.relationships.length) {
|
||||
parts.push("Relationships (internal to contribution):");
|
||||
for (const r of later.relationships) {
|
||||
parts.push(` - ${r.from} -> ${r.to} (${r.type})`);
|
||||
}
|
||||
}
|
||||
parts.push("");
|
||||
|
||||
// Core question
|
||||
parts.push("QUESTION: Does anything in the later fragment materially bear on an uncertainty,");
|
||||
parts.push("assumption or observation in the earlier fragment?");
|
||||
parts.push("QUESTION: What meaningful relationship, if any, exists between");
|
||||
parts.push("these two contributions? If no material relationship is present,");
|
||||
parts.push("state that explicitly.");
|
||||
parts.push("");
|
||||
|
||||
// Return format — same contract as RTO.21 / RTO.22
|
||||
parts.push('Return exactly one JSON object:');
|
||||
parts.push('{ "relationshipFound": <bool>, "relationship": { "laterEvidence": "<string>", "earlierItem": "<string>", "bearing": "<string>" }, "remainingQualification": ["<string>"] }');
|
||||
parts.push('If no relationship exists: { "relationshipFound": false, "relationship": null, "remainingQualification": [] }');
|
||||
// Return format
|
||||
parts.push("Return exactly one JSON object. If a relationship exists:");
|
||||
parts.push('{ "relationshipFound": true, "relationship": "<plain-language description>", "evidence": ["<specific contribution content supporting the relationship>"], "qualification": ["<important uncertainty or limitation>"] }');
|
||||
parts.push("");
|
||||
parts.push("If no material relationship is present:");
|
||||
parts.push('{ "relationshipFound": false, "relationship": null, "evidence": [], "qualification": [] }');
|
||||
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
||||
// ─── Output contract validation (same as RTO.21 / RTO.22) ─────────────────────
|
||||
// ─── Output contract validation ──────────────────────────────────────────────
|
||||
|
||||
function validateRelationshipOutput(result) {
|
||||
const errors = [];
|
||||
@@ -272,38 +177,34 @@ function validateRelationshipOutput(result) {
|
||||
if (result.relationship !== null && result.relationship !== undefined) {
|
||||
errors.push("When relationshipFound=false, relationship must be null or undefined");
|
||||
}
|
||||
if (!Array.isArray(result.remainingQualification)) {
|
||||
errors.push("remainingQualification must be an array when relationshipFound=false");
|
||||
if (!Array.isArray(result.evidence)) {
|
||||
errors.push("evidence must be an array when relationshipFound=false");
|
||||
}
|
||||
if (!Array.isArray(result.qualification)) {
|
||||
errors.push("qualification must be an array when relationshipFound=false");
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
// When true, relationship object is required
|
||||
if (!result.relationship || typeof result.relationship !== "object") {
|
||||
errors.push("When relationshipFound=true, relationship must be an object");
|
||||
return errors;
|
||||
// When true, relationship string is required
|
||||
if (typeof result.relationship !== "string" || result.relationship.trim() === "") {
|
||||
errors.push("When relationshipFound=true, relationship must be a non-empty plain-language description");
|
||||
}
|
||||
if (!Array.isArray(result.evidence)) {
|
||||
errors.push("evidence must be an array when relationshipFound=true");
|
||||
} else if (result.evidence.some((s) => typeof s !== "string")) {
|
||||
errors.push("evidence items must all be strings");
|
||||
}
|
||||
if (!Array.isArray(result.qualification)) {
|
||||
errors.push("qualification must be an array when relationshipFound=true");
|
||||
} else if (result.qualification.some((s) => typeof s !== "string")) {
|
||||
errors.push("qualification items must all be strings");
|
||||
}
|
||||
|
||||
const rel = result.relationship;
|
||||
if (typeof rel.laterEvidence !== "string") {
|
||||
errors.push("relationship.laterEvidence must be a string");
|
||||
}
|
||||
if (typeof rel.earlierItem !== "string") {
|
||||
errors.push("relationship.earlierItem must be a string");
|
||||
}
|
||||
if (typeof rel.bearing !== "string") {
|
||||
errors.push("relationship.bearing must be a string");
|
||||
}
|
||||
|
||||
if (!Array.isArray(result.remainingQualification)) {
|
||||
errors.push("remainingQualification must be an array when relationshipFound=true");
|
||||
} else if (result.remainingQualification.some((s) => typeof s !== "string")) {
|
||||
errors.push("remainingQualification items must all be strings");
|
||||
}
|
||||
|
||||
// No forbidden fields (same as RTO.21 / RTO.22)
|
||||
// No forbidden fields
|
||||
const forbidden = ["graphEdgeType", "nodeId", "confidenceScore", "numericStrength",
|
||||
"recommendation", "nextQuestion", "decisionSignificance", "graphUpdate"];
|
||||
"keywordScore", "materialityScore", "recommendation", "nextQuestion",
|
||||
"decisionSignificance", "graphUpdate", "strengthScore"];
|
||||
for (const field of forbidden) {
|
||||
if (field in result && typeof result[field] !== "undefined") {
|
||||
errors.push(`Forbidden field present: ${field}`);
|
||||
@@ -342,7 +243,7 @@ function checkPromptLeak(prompt) {
|
||||
// ─── Inspect mode (default: zero model calls) ────────────────────────────────
|
||||
|
||||
async function inspectApparatus() {
|
||||
console.log("=== RTO.23A Borderline Same-Topic Relationship-Control Apparatus (inspect-only) ===\n");
|
||||
console.log("=== RTO.23C Simplified Semantic Relationship-Inference Apparatus (inspect-only) ===\n");
|
||||
|
||||
// Load and verify fragment 2
|
||||
let structuredResult2;
|
||||
@@ -354,43 +255,26 @@ async function inspectApparatus() {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Borderline-control integrity check
|
||||
const controlCheck = verifyBorderlineControlClean();
|
||||
console.log("--- Borderline-control fixture integrity ---");
|
||||
console.log(`Same competitor/market context present: ${controlCheck.sameCompetitorContext ? "YES" : "NO"}`);
|
||||
console.log(`Same conference context present: ${controlCheck.sameConferenceContext ? "YES" : "NO"}`);
|
||||
console.log(`General customer-problem reference present: ${controlCheck.customerProblemRef ? "YES" : "NO"}`);
|
||||
console.log(`Technical architecture present: ${controlCheck.technicalArchitecture ? "YES" : "NO"}`);
|
||||
console.log(`Prototype evidence present: ${controlCheck.prototypeEvidence ? "YES" : "NO"}`);
|
||||
console.log(`Product roadmap present: ${controlCheck.productRoadmap ? "YES" : "NO"}`);
|
||||
console.log(`Launch timing evidence present: ${controlCheck.launchTiming ? "YES" : "NO"}`);
|
||||
console.log(`Development programme present: ${controlCheck.developmentProgramme ? "YES" : "NO"}`);
|
||||
console.log(`New hiring evidence present: ${controlCheck.newHiringEvidence ? "YES" : "NO"}`);
|
||||
|
||||
if (!controlCheck.clean) {
|
||||
console.log("\nBORDERLINE_CONTROL_CLEAN:\nNO");
|
||||
process.exit(1);
|
||||
}
|
||||
console.log("\nBORDERLINE_CONTROL_CLEAN:\nYES");
|
||||
|
||||
// Fragment verification
|
||||
const f2Loaded = typeof structuredResult2 === "object" && structuredResult2 !== null;
|
||||
console.log("\n--- Fragment verification ---");
|
||||
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
|
||||
console.log(`Borderline-control fragment supplied: YES`);
|
||||
const laterLoaded = typeof LATER_CONTRIBUTION === "object" && LATER_CONTRIBUTION !== null;
|
||||
|
||||
console.log("--- Contribution verification ---");
|
||||
console.log(`Earlier contribution supplied: ${f2Loaded ? "YES" : "NO"}`);
|
||||
console.log(`Later contribution supplied: ${laterLoaded ? "YES" : "NO"}`);
|
||||
|
||||
// Build prompt for inspection
|
||||
const prompt = buildRelationshipPrompt(structuredResult2, BORDERLINE_CONTROL_FRAGMENT);
|
||||
const prompt = buildRelationshipPrompt(structuredResult2, LATER_CONTRIBUTION);
|
||||
|
||||
// Measure character sizes
|
||||
const instructionPart = prompt.substring(0, prompt.indexOf("=== Fragment 2"));
|
||||
const instructionPart = prompt.substring(0, prompt.indexOf("=== Contribution 1"));
|
||||
const frag2StructuredText = JSON.stringify(structuredResult2, null, 2);
|
||||
const controlStructuredText = JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2);
|
||||
const laterContributedText = JSON.stringify(LATER_CONTRIBUTION, null, 2);
|
||||
|
||||
console.log("\n--- Context-size measurement ---");
|
||||
console.log(`instructionCharacterCount: ${instructionPart.length}`);
|
||||
console.log(`fragment2CharacterCount: ${frag2StructuredText.length}`);
|
||||
console.log(`borderlineControlCharacterCount: ${controlStructuredText.length}`);
|
||||
console.log(`laterContributionCharacterCount: ${laterContributedText.length}`);
|
||||
console.log(`inputCharacterCount: ${prompt.length}`);
|
||||
|
||||
// ── Boundary checks ───────────────────────────────────────────
|
||||
@@ -400,10 +284,6 @@ async function inspectApparatus() {
|
||||
const fragment3Supplied = prompt.includes("Fragment 3");
|
||||
console.log(`Fragment 3 supplied: ${fragment3Supplied ? "YES" : "NO"}`);
|
||||
|
||||
const rto21PositiveResultSupplied = prompt.includes("Fragment 3's technical evidence bears on") ||
|
||||
prompt.includes("active competing-product development more plausible");
|
||||
console.log(`RTO.21 positive result supplied: ${rto21PositiveResultSupplied ? "YES" : "NO"}`);
|
||||
|
||||
const wholeGraphInPrompt = prompt.includes("SituationGraph") ||
|
||||
prompt.includes("whole-case");
|
||||
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
|
||||
@@ -412,10 +292,6 @@ async function inspectApparatus() {
|
||||
prompt.includes("centralStatement");
|
||||
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
|
||||
|
||||
const currentViewInPrompt = prompt.includes("derived view") ||
|
||||
prompt.includes("focused view") || prompt.includes("current focused");
|
||||
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
|
||||
|
||||
const otherFragmentsInPrompt = prompt.includes("Fragment 1") ||
|
||||
prompt.includes("other fragments") || prompt.includes("additional fragment");
|
||||
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
|
||||
@@ -424,11 +300,16 @@ async function inspectApparatus() {
|
||||
prompt.includes("prior turns") || prompt.includes("conversation history");
|
||||
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
|
||||
|
||||
// Known result / expected relationship check
|
||||
const knownResultSupplied = prompt.includes("Fragment 3's technical evidence bears on") ||
|
||||
prompt.includes("active competing-product development more plausible");
|
||||
console.log(`Known relationship/result supplied: ${knownResultSupplied ? "YES" : "NO"}`);
|
||||
|
||||
// ── Prompt-leak check (mandatory) ─────────────────────────────
|
||||
|
||||
const leakCheck = checkPromptLeak(prompt);
|
||||
console.log("\n--- Prompt-leak check ---");
|
||||
console.log(`EXPECTED_OUTCOME_LEAKED_TO_MODEL:\n${leakCheck.leaked ? "YES" : "NO"}`);
|
||||
console.log("\n--- Expected outcome leakage ─────────────────────────────");
|
||||
console.log(`Expected outcome disclosed to model:\n${leakCheck.leaked ? "YES" : "NO"}`);
|
||||
if (leakCheck.leaked) {
|
||||
console.log(`Leaked terms: ${leakCheck.terms.join(", ")}`);
|
||||
}
|
||||
@@ -437,31 +318,43 @@ async function inspectApparatus() {
|
||||
|
||||
const mockProposal = {
|
||||
relationshipFound: true,
|
||||
relationship: {
|
||||
laterEvidence: "test",
|
||||
earlierItem: "test",
|
||||
bearing: "test",
|
||||
},
|
||||
remainingQualification: ["test"],
|
||||
relationship: "test description",
|
||||
evidence: ["test"],
|
||||
qualification: ["test"]
|
||||
};
|
||||
const mockNoRel = {
|
||||
relationshipFound: false,
|
||||
relationship: null,
|
||||
remainingQualification: [],
|
||||
evidence: [],
|
||||
qualification: []
|
||||
};
|
||||
const proposalErrors = validateRelationshipOutput(mockProposal);
|
||||
const noRelErrors = validateRelationshipOutput(mockNoRel);
|
||||
console.log("\n--- Output contract (same as RTO.21) ---");
|
||||
|
||||
console.log("\n--- Output contract ────────────────────────────────────────");
|
||||
console.log(`proposal path validates: ${proposalErrors.length === 0 ? "YES" : "NO"}`);
|
||||
console.log(`no-relationship path validates: ${noRelErrors.length === 0 ? "YES" : "NO"}`);
|
||||
console.log(`relationshipFound=false allowed: YES`);
|
||||
console.log(`Topical similarity explicitly insufficient: YES`);
|
||||
|
||||
// ── Prompt semantic cleanliness check ────────────────────────
|
||||
|
||||
const lowerPrompt = prompt.toLowerCase();
|
||||
const hasDomainDict = !!lowerPrompt.match(/\b(technical\s*architecture|prototype|roadmap|launch\s*timing|development\s*programme)\s*dictionary/);
|
||||
const hasKeywordScoring = !!lowerPrompt.match(/keyword.*scor(e|ing)|score.*keyword/);
|
||||
const hasFixedTaxonomy = !!lowerPrompt.match(/relationship\s*(type|category|taxonomy)/i) && lowerPrompt.includes("enum");
|
||||
|
||||
console.log("\n--- Semantic cleanliness ───────────────────────────────────");
|
||||
console.log(`Keyword/dictionary semantic checks: ${hasDomainDict || hasKeywordScoring ? "PRESENT" : "NONE"}`);
|
||||
if (hasDomainDict) console.log(" Domain dictionary used in prompt");
|
||||
if (hasKeywordScoring) console.log(" Keyword scoring present in prompt");
|
||||
|
||||
// ── Live route ────────────────────────────────────────────────
|
||||
|
||||
console.log("\n--- Live route ---");
|
||||
console.log("node scripts/experimental/rto-fragment-relationship-borderline-control.mjs --live");
|
||||
|
||||
// ── Environment check ─────────────────────────────────────────
|
||||
|
||||
const baseUrlConfigured = !!process.env.OLLAMA_BASE_URL;
|
||||
const modelConfigured = !!process.env.OLLAMA_MODEL;
|
||||
console.log("\n--- Environment ---");
|
||||
@@ -469,31 +362,26 @@ async function inspectApparatus() {
|
||||
console.log(`OLLAMA_MODEL configured: ${modelConfigured ? "YES" : "NO"}`);
|
||||
|
||||
// ── Inspect summary ───────────────────────────────────────────
|
||||
const allFragmentsOK = f2Loaded;
|
||||
const noBoundaryLeaks = !fragment3Supplied && !rto21PositiveResultSupplied &&
|
||||
!wholeGraphInPrompt && !centralCaseInPrompt && !currentViewInPrompt &&
|
||||
!otherFragmentsInPrompt && !turnHistoryInPrompt && controlCheck.clean && !leakCheck.leaked;
|
||||
|
||||
const noBoundaryLeaks = !fragment3Supplied && !wholeGraphInPrompt &&
|
||||
!centralCaseInPrompt && !otherFragmentsInPrompt &&
|
||||
!turnHistoryInPrompt && !knownResultSupplied && !leakCheck.leaked;
|
||||
const semanticClean = !hasDomainDict && !hasKeywordScoring && !hasFixedTaxonomy;
|
||||
|
||||
console.log("\n=== SUMMARY ===");
|
||||
console.log(`Inspect result: ${f2Loaded && laterLoaded && noBoundaryLeaks && semanticClean ? "PASS" : "FAIL"}`);
|
||||
console.log(`Inspect live calls: 0`);
|
||||
console.log(`Earlier Fragment 2 supplied: ${f2Loaded ? "YES" : "NO"}`);
|
||||
console.log(`Borderline-control fragment supplied: YES`);
|
||||
console.log(`Fragment 3 supplied: NO`);
|
||||
console.log(`RTO.21 positive result supplied: NO`);
|
||||
console.log(`RTO.22 negative result supplied: NO`);
|
||||
console.log(`Known relationship supplied: NO`);
|
||||
console.log(`Earlier contribution supplied: ${f2Loaded ? "YES" : "NO"}`);
|
||||
console.log(`Later contribution supplied: ${laterLoaded ? "YES" : "NO"}`);
|
||||
console.log(`Known relationship/result supplied: ${knownResultSupplied ? "YES" : "NO"}`);
|
||||
console.log(`Whole SituationGraph supplied: ${wholeGraphInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`Central case supplied: ${centralCaseInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`Current derived view supplied: ${currentViewInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`Other fragments supplied: ${otherFragmentsInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`BORDERLINE_CONTROL_CLEAN:\nYES`);
|
||||
console.log(`--live route exposed: YES`);
|
||||
console.log(`maximum live calls per --live: 1`);
|
||||
console.log(`Turn history supplied: ${turnHistoryInPrompt ? "YES" : "NO"}`);
|
||||
console.log(`relationshipFound=false allowed: YES`);
|
||||
console.log(`maximum live calls per --live: 1`);
|
||||
|
||||
const passed = allFragmentsOK && noBoundaryLeaks;
|
||||
console.log(`\nInspect result: ${passed ? "PASS" : "FAIL"}`);
|
||||
const passed = f2Loaded && laterLoaded && noBoundaryLeaks && semanticClean;
|
||||
|
||||
if (!passed) {
|
||||
console.log("\nApparatus cannot satisfy boundary without production changes.");
|
||||
@@ -524,7 +412,7 @@ async function executeLive() {
|
||||
|
||||
// Load fragment 2 and build prompt
|
||||
const { structuredResult: srs2 } = await loadFragment2();
|
||||
const prompt = buildRelationshipPrompt(srs2, BORDERLINE_CONTROL_FRAGMENT);
|
||||
const prompt = buildRelationshipPrompt(srs2, LATER_CONTRIBUTION);
|
||||
|
||||
// Prompt-leak check before sending
|
||||
const leakCheck = checkPromptLeak(prompt);
|
||||
@@ -567,22 +455,17 @@ async function executeLive() {
|
||||
"rto-fragment-relationship-borderline-control-live.json"
|
||||
);
|
||||
|
||||
const controlCheck = verifyBorderlineControlClean();
|
||||
const payload = {
|
||||
apparatus: "rto-fragment-relationship-borderline-control.mjs",
|
||||
experiment: "RTO.23A",
|
||||
artifactType: "LIVE RESULT — Borderline same-topic relationship discovery",
|
||||
experiment: "RTO.23C",
|
||||
artifactType: "LIVE RESULT — Semantic relationship inference",
|
||||
modelName: modelName,
|
||||
elapsedMs: elapsedMs,
|
||||
inputCharacterCount: prompt.length,
|
||||
instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Fragment 2")).length,
|
||||
instructionCharacterCount: prompt.substring(0, prompt.indexOf("=== Contribution 1")).length,
|
||||
fragment2CharacterCount: JSON.stringify(srs2, null, 2).length,
|
||||
borderlineControlCharacterCount: JSON.stringify(BORDERLINE_CONTROL_FRAGMENT, null, 2).length,
|
||||
laterContributionCharacterCount: JSON.stringify(LATER_CONTRIBUTION, null, 2).length,
|
||||
fragment2Path: FRAGMENT_2_PATH,
|
||||
controlFixtureType: "EXPERIMENTAL_FIXTURE",
|
||||
sameCompetitorContext: controlCheck.sameCompetitorContext,
|
||||
sameConferenceContext: controlCheck.sameConferenceContext,
|
||||
borderlineControlClean: controlCheck.clean,
|
||||
knownRelationshipSuppliedToModel: false,
|
||||
wholeSituationGraphSupplied: false,
|
||||
centralCaseStatementSupplied: false,
|
||||
@@ -605,20 +488,14 @@ async function main() {
|
||||
|
||||
// --live flag for exactly one live model call
|
||||
if (args.includes("--live")) {
|
||||
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
|
||||
console.log("RTO.23C — Semantic Relationship-Inference Apparatus\n");
|
||||
console.log("WARNING: This will make exactly ONE live model call.\n");
|
||||
const result = await executeLive();
|
||||
return result;
|
||||
}
|
||||
|
||||
// Default: inspect-only, zero model calls
|
||||
console.log("RTO.23A — Borderline Same-Topic Relationship-Control Apparatus\n");
|
||||
console.log("Run with --live for one live model call.\n");
|
||||
|
||||
const result = await inspectApparatus();
|
||||
if (result === null) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user