Adds experimental apparatus for testing whether removing the question field from importantUnknowns in reconstruction output improves semantic fidelity. New files: - lib/reconstruction/schema-variant.js (conditionB schema + production passthrough) - prompts/reconstruct-v0.5-b-noreq.md (alt prompt, identical line count to prod) Changed files: - lib/graph/orchestrator.js (pass generationSchema through dependencies) - lib/analysis.js (accept opts.generationSchema fallback to original schema) - scripts/start-case-experiment-helper.cjs (--condition B/A flag support) - tests/scripts/start-case-experiment-helper.test.js (8 new deterministic tests) 43 targeted deterministic tests PASS. Production callers use original prompt/schema unchanged. No live model calls. No full suite evidence available. Co-Authored-By: Claude Code <noreply@anthropic.com>
179 lines
5.4 KiB
JavaScript
179 lines
5.4 KiB
JavaScript
import { z } from "zod";
|
|
|
|
/**
|
|
* Condition B — uncertainty-description-only variant schema.
|
|
*
|
|
* Differs from the production reconstructionGenerationSchema in exactly one way:
|
|
* importantUnknowns items use the base itemSchemaV1 which has NO `question` property.
|
|
* Condition A uses the production schema (reconstructionGenerationSchema) unchanged.
|
|
*/
|
|
|
|
// Base item used by both conditions (id, description, confidence only)
|
|
const confidenceEnumV1 = z.enum(["low", "medium", "high"]);
|
|
const itemSchemaV1 = z.object({
|
|
id: z.string().min(1),
|
|
description: z.string().min(1),
|
|
confidence: confidenceEnumV1,
|
|
});
|
|
|
|
// Condition B importantUnknowns items: NO question property — just the base item fields
|
|
const conditionBImportantUnknownSchema = itemSchemaV1;
|
|
|
|
// Build the Condition B Zod schema using the tolerant importantUnknownSchema
|
|
const reconstructionGenerationConditionBSchema = z.object({
|
|
inputClassification: z.object({
|
|
primaryType: z.enum([
|
|
"observed_problem",
|
|
"unexplained_change",
|
|
"contradiction",
|
|
"decision_request",
|
|
"causal_claim",
|
|
"reported_claim",
|
|
"fault_report",
|
|
"ambiguous_statement",
|
|
"question",
|
|
"desired_outcome",
|
|
"insufficient_context",
|
|
"other",
|
|
]),
|
|
secondaryTypes: z.array(z.enum([
|
|
"observed_problem",
|
|
"unexplained_change",
|
|
"contradiction",
|
|
"decision_request",
|
|
"causal_claim",
|
|
"reported_claim",
|
|
"fault_report",
|
|
"ambiguous_statement",
|
|
"question",
|
|
"desired_outcome",
|
|
"insufficient_context",
|
|
"other",
|
|
])).optional().default([]),
|
|
reasoningModes: z.array(z.enum([
|
|
"establish_baseline",
|
|
"identify_difference",
|
|
"reconstruct_transition",
|
|
"decompose_aggregate",
|
|
"validate_measurement",
|
|
"validate_claim",
|
|
"investigate_contradiction",
|
|
"clarify_meaning",
|
|
"decision_support",
|
|
"fault_investigation",
|
|
"identify_missing_information",
|
|
"test_possible_explanations",
|
|
"other",
|
|
])).optional().default([]),
|
|
classificationReason: z.string().min(1),
|
|
confidence: z.enum(["low", "medium", "high"]),
|
|
}),
|
|
reconstruction: z.object({
|
|
summary: z.string().min(1),
|
|
actors: z.array(itemSchemaV1),
|
|
systemsOrObjects: z.array(itemSchemaV1),
|
|
expectedStates: z.array(itemSchemaV1),
|
|
observedStates: z.array(itemSchemaV1),
|
|
differences: z.array(itemSchemaV1),
|
|
knownTransitions: z.array(
|
|
itemSchemaV1.extend({
|
|
entity: z.string().min(1),
|
|
previousState: z.string().min(1),
|
|
currentState: z.string().min(1),
|
|
explanationStatus: z.string().min(1),
|
|
}),
|
|
),
|
|
unexplainedTransitions: z.array(
|
|
itemSchemaV1.extend({
|
|
entity: z.string().min(1).optional(),
|
|
previousState: z.string().min(1).optional(),
|
|
currentState: z.string().min(1).optional(),
|
|
}),
|
|
),
|
|
contradictions: z.array(itemSchemaV1),
|
|
importantUnknowns: z.array(conditionBImportantUnknownSchema),
|
|
plausibleInterpretations: z.array(
|
|
z.object({
|
|
id: z.string().min(1),
|
|
description: z.string().min(1),
|
|
supportingEvidenceIds: z.array(z.string()),
|
|
assumptionsRequired: z.array(z.string()).optional().default([]),
|
|
confidence: z.enum(["low", "medium", "high"]),
|
|
}),
|
|
),
|
|
relationships: z.array(z.object({
|
|
id: z.string().min(1),
|
|
fromId: z.string().min(1),
|
|
toId: z.string().min(1),
|
|
relationship: z.enum([
|
|
"supports",
|
|
"weakens",
|
|
"contradicts",
|
|
"depends_on",
|
|
"causes",
|
|
"may_cause",
|
|
"compares_with",
|
|
"other",
|
|
]),
|
|
description: z.string().min(1),
|
|
confidence: z.enum(["low", "medium", "high"]),
|
|
})).optional().default([]),
|
|
}),
|
|
evidence: z.array(z.object({
|
|
id: z.string().min(1),
|
|
description: z.string().min(1),
|
|
evidenceType: z.enum([
|
|
"direct_observation",
|
|
"reported_statement",
|
|
"interpretation",
|
|
"assumption",
|
|
"inferred_relationship",
|
|
]),
|
|
source: z.string().optional(),
|
|
attribution: z.string().nullable().optional(),
|
|
confidence: z.enum(["low", "medium", "high"]),
|
|
importance: z.enum([
|
|
"incidental",
|
|
"supporting",
|
|
"important",
|
|
"critical",
|
|
]),
|
|
})),
|
|
nextQuestion: z.object({
|
|
id: z.string().min(1),
|
|
question: z.string().min(1),
|
|
targets: z.array(z.string()),
|
|
reason: z.string().min(1),
|
|
expectedInformationValue: z.enum(["low", "medium", "high"]),
|
|
reasoningMode: z.enum([
|
|
"establish_baseline",
|
|
"identify_difference",
|
|
"reconstruct_transition",
|
|
"decompose_aggregate",
|
|
"validate_measurement",
|
|
"validate_claim",
|
|
"investigate_contradiction",
|
|
"clarify_meaning",
|
|
"decision_support",
|
|
"fault_investigation",
|
|
"identify_missing_information",
|
|
"test_possible_explanations",
|
|
"other",
|
|
]).optional().default("other"),
|
|
}),
|
|
});
|
|
|
|
/** JSON Schema derived from Condition B Zod — importantUnknowns has NO question property */
|
|
export const conditionBJsonSchema = z.toJSONSchema(
|
|
reconstructionGenerationConditionBSchema,
|
|
);
|
|
|
|
import {
|
|
reconstructionGenerationSchema as _PROD_SCHEMA,
|
|
reconstructionGenerationJsonSchema as _PROD_JSON,
|
|
} from "./schema.js";
|
|
|
|
/** Condition A — production schema (importantUnknowns items REQUIRE `question`) */
|
|
export const conditionASchema = _PROD_SCHEMA;
|
|
export const conditionAJsonSchema = _PROD_JSON;
|