feat(confidence-engine): add CE-ULI-08 two-condition uncertainty comparison apparatus
Adds experimental apparatus for testing whether removing the question field from importantUnknowns in reconstruction output improves semantic fidelity. New files: - lib/reconstruction/schema-variant.js (conditionB schema + production passthrough) - prompts/reconstruct-v0.5-b-noreq.md (alt prompt, identical line count to prod) Changed files: - lib/graph/orchestrator.js (pass generationSchema through dependencies) - lib/analysis.js (accept opts.generationSchema fallback to original schema) - scripts/start-case-experiment-helper.cjs (--condition B/A flag support) - tests/scripts/start-case-experiment-helper.test.js (8 new deterministic tests) 43 targeted deterministic tests PASS. Production callers use original prompt/schema unchanged. No live model calls. No full suite evidence available. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
Claude Code
+1
-1
@@ -83,7 +83,7 @@ export async function analyseScenario(scenario, opts = {}) {
|
||||
const providerResult = await provider.generateReconstruction(
|
||||
promptObj.prompt,
|
||||
reconstructionModelName,
|
||||
reconstructionGenerationJsonSchema,
|
||||
opts.generationSchema ?? reconstructionGenerationJsonSchema,
|
||||
{ outputZodSchema: reconstructionGenerationSchema },
|
||||
);
|
||||
if (
|
||||
|
||||
@@ -370,6 +370,9 @@ export async function startCase(body, dependencies = {}) {
|
||||
if (dependencies.reconstructionModelName) {
|
||||
analysisOptions.reconstructionModelName = dependencies.reconstructionModelName;
|
||||
}
|
||||
if (dependencies.generationSchema) {
|
||||
analysisOptions.generationSchema = dependencies.generationSchema;
|
||||
}
|
||||
const analysis = await analyseScenario(scenario, analysisOptions);
|
||||
|
||||
if (!analysis.success) {
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
import { z } from "zod";
|
||||
|
||||
/**
|
||||
* Condition B — uncertainty-description-only variant schema.
|
||||
*
|
||||
* Differs from the production reconstructionGenerationSchema in exactly one way:
|
||||
* importantUnknowns items use the base itemSchemaV1 which has NO `question` property.
|
||||
* Condition A uses the production schema (reconstructionGenerationSchema) unchanged.
|
||||
*/
|
||||
|
||||
// Base item used by both conditions (id, description, confidence only)
|
||||
const confidenceEnumV1 = z.enum(["low", "medium", "high"]);
|
||||
const itemSchemaV1 = z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
confidence: confidenceEnumV1,
|
||||
});
|
||||
|
||||
// Condition B importantUnknowns items: NO question property — just the base item fields
|
||||
const conditionBImportantUnknownSchema = itemSchemaV1;
|
||||
|
||||
// Build the Condition B Zod schema using the tolerant importantUnknownSchema
|
||||
const reconstructionGenerationConditionBSchema = z.object({
|
||||
inputClassification: z.object({
|
||||
primaryType: z.enum([
|
||||
"observed_problem",
|
||||
"unexplained_change",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"causal_claim",
|
||||
"reported_claim",
|
||||
"fault_report",
|
||||
"ambiguous_statement",
|
||||
"question",
|
||||
"desired_outcome",
|
||||
"insufficient_context",
|
||||
"other",
|
||||
]),
|
||||
secondaryTypes: z.array(z.enum([
|
||||
"observed_problem",
|
||||
"unexplained_change",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"causal_claim",
|
||||
"reported_claim",
|
||||
"fault_report",
|
||||
"ambiguous_statement",
|
||||
"question",
|
||||
"desired_outcome",
|
||||
"insufficient_context",
|
||||
"other",
|
||||
])).optional().default([]),
|
||||
reasoningModes: z.array(z.enum([
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"reconstruct_transition",
|
||||
"decompose_aggregate",
|
||||
"validate_measurement",
|
||||
"validate_claim",
|
||||
"investigate_contradiction",
|
||||
"clarify_meaning",
|
||||
"decision_support",
|
||||
"fault_investigation",
|
||||
"identify_missing_information",
|
||||
"test_possible_explanations",
|
||||
"other",
|
||||
])).optional().default([]),
|
||||
classificationReason: z.string().min(1),
|
||||
confidence: z.enum(["low", "medium", "high"]),
|
||||
}),
|
||||
reconstruction: z.object({
|
||||
summary: z.string().min(1),
|
||||
actors: z.array(itemSchemaV1),
|
||||
systemsOrObjects: z.array(itemSchemaV1),
|
||||
expectedStates: z.array(itemSchemaV1),
|
||||
observedStates: z.array(itemSchemaV1),
|
||||
differences: z.array(itemSchemaV1),
|
||||
knownTransitions: z.array(
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1),
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
}),
|
||||
),
|
||||
unexplainedTransitions: z.array(
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1).optional(),
|
||||
previousState: z.string().min(1).optional(),
|
||||
currentState: z.string().min(1).optional(),
|
||||
}),
|
||||
),
|
||||
contradictions: z.array(itemSchemaV1),
|
||||
importantUnknowns: z.array(conditionBImportantUnknownSchema),
|
||||
plausibleInterpretations: z.array(
|
||||
z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
supportingEvidenceIds: z.array(z.string()),
|
||||
assumptionsRequired: z.array(z.string()).optional().default([]),
|
||||
confidence: z.enum(["low", "medium", "high"]),
|
||||
}),
|
||||
),
|
||||
relationships: z.array(z.object({
|
||||
id: z.string().min(1),
|
||||
fromId: z.string().min(1),
|
||||
toId: z.string().min(1),
|
||||
relationship: z.enum([
|
||||
"supports",
|
||||
"weakens",
|
||||
"contradicts",
|
||||
"depends_on",
|
||||
"causes",
|
||||
"may_cause",
|
||||
"compares_with",
|
||||
"other",
|
||||
]),
|
||||
description: z.string().min(1),
|
||||
confidence: z.enum(["low", "medium", "high"]),
|
||||
})).optional().default([]),
|
||||
}),
|
||||
evidence: z.array(z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
evidenceType: z.enum([
|
||||
"direct_observation",
|
||||
"reported_statement",
|
||||
"interpretation",
|
||||
"assumption",
|
||||
"inferred_relationship",
|
||||
]),
|
||||
source: z.string().optional(),
|
||||
attribution: z.string().nullable().optional(),
|
||||
confidence: z.enum(["low", "medium", "high"]),
|
||||
importance: z.enum([
|
||||
"incidental",
|
||||
"supporting",
|
||||
"important",
|
||||
"critical",
|
||||
]),
|
||||
})),
|
||||
nextQuestion: z.object({
|
||||
id: z.string().min(1),
|
||||
question: z.string().min(1),
|
||||
targets: z.array(z.string()),
|
||||
reason: z.string().min(1),
|
||||
expectedInformationValue: z.enum(["low", "medium", "high"]),
|
||||
reasoningMode: z.enum([
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"reconstruct_transition",
|
||||
"decompose_aggregate",
|
||||
"validate_measurement",
|
||||
"validate_claim",
|
||||
"investigate_contradiction",
|
||||
"clarify_meaning",
|
||||
"decision_support",
|
||||
"fault_investigation",
|
||||
"identify_missing_information",
|
||||
"test_possible_explanations",
|
||||
"other",
|
||||
]).optional().default("other"),
|
||||
}),
|
||||
});
|
||||
|
||||
/** JSON Schema derived from Condition B Zod — importantUnknowns has NO question property */
|
||||
export const conditionBJsonSchema = z.toJSONSchema(
|
||||
reconstructionGenerationConditionBSchema,
|
||||
);
|
||||
|
||||
import {
|
||||
reconstructionGenerationSchema as _PROD_SCHEMA,
|
||||
reconstructionGenerationJsonSchema as _PROD_JSON,
|
||||
} from "./schema.js";
|
||||
|
||||
/** Condition A — production schema (importantUnknowns items REQUIRE `question`) */
|
||||
export const conditionASchema = _PROD_SCHEMA;
|
||||
export const conditionAJsonSchema = _PROD_JSON;
|
||||
Reference in New Issue
Block a user