test(experiment): checkpoint comparison observability

2026-08-18 19:28:30 +01:00
parent 952cb442b5
commit 913dfec507
5 changed files with 464 additions and 22 deletions
@@ -0,0 +1,48 @@
+{
+  "path": "focused",
+  "targetNodeId": "nxmeiab",
+  "modelName": "qwen-claude:latest",
+  "modelElapsedMs": 55405,
+  "inputCharacterCount": 2405,
+  "structuredResult": {
+    "targetNodeId": "nxmeiab",
+    "observations": [
+      "Two competitors have publicly announced products aimed at the same customer problem.",
+      "One competitor expects a beta release within six months.",
+      "The other competitor has not announced a release date.",
+      "The degree of functional overlap between either competitor's product and ours is currently unknown."
+    ],
+    "uncertainties": [
+      "How closely Competitor 1's product matches our functionality.",
+      "How closely Competitor 2's product matches our functionality.",
+      "When or if Competitor 2 will announce a release date for their product."
+    ],
+    "assumptions": [],
+    "relationships": [
+      {
+        "from": "Competitor 1",
+        "to": "Beta release within six months",
+        "type": "projects",
+        "rationale": "Directly stated by the competitor regarding their product timeline."
+      },
+      {
+        "from": "Competitor 2",
+        "to": "No announced release date",
+        "type": "undisclosed",
+        "rationale": "Explicitly noted as unannounced in public communications."
+      },
+      {
+        "from": "Our product",
+        "to": "Competitors' products",
+        "type": "unknown functional alignment",
+        "rationale": "Answer explicitly states the degree of matching features or capabilities is currently unknown."
+      }
+    ],
+    "possibleFollowUpQuestions": [
+      "What specific features or capabilities in our product are addressed by the competitors' announcements?",
+      "Has Competitor 2 provided any indirect signals about development progress despite lacking a formal release date?",
+      "How does the target customer problem for our product compare in scope or priority to the competitors' stated problem?",
+      "What engineering or market validation metrics would confirm if either competitor's product poses an immediate threat to our launch timeline?"
+    ]
+  }
+}
@@ -0,0 +1,38 @@
+{
+  "path": "global",
+  "targetNodeId": "nxmeiab",
+  "modelName": "qwen-claude:latest",
+  "modelElapsedMs": 88348,
+  "providerGenerateCalls": 1,
+  "providerTimingCaptured": true,
+  "endToEndElapsedMs": 88358,
+  "inputCharacterCount": 22112,
+  "graphUpdateResultSummary": {
+    "userSupportedMeaning": "Two competitors have announced targeted products; one expects a beta in six months, the other has no release date yet, and it is unknown how closely either matches our solution.",
+    "possibleInference": "Competitive pressure is imminent for at least one product, suggesting we cannot wait indefinitely without risk of losing first-mover advantage to the sooner launcher.",
+    "proposalValidation": {
+      "success": false,
+      "errors": []
+    },
+    "selectedQuestion": {
+      "nodeId": "ncomp_fit",
+      "question": "How closely does our product match the capabilities of the two announced competitor offerings?",
+      "reason": "This remaining uncertainty determines whether the imminent competitor launches pose an immediate threat to launching now."
+    }
+  },
+  "answerDerivedComparisonExtraction": {
+    "observations": [
+      "Two competitors have announced targeted products; one expects a beta in six months, the other has no release date yet, and it is unknown how closely either matches our solution."
+    ],
+    "uncertainties": [
+      "Competitive pressure is imminent for at least one product, suggesting we cannot wait indefinitely without risk of losing first-mover advantage to the sooner launcher."
+    ],
+    "assumptions": "not directly exposed",
+    "relationships": "not directly exposed",
+    "newlySurfacedUnknownsOrQuestions": {
+      "nodeId": "ncomp_fit",
+      "question": "How closely does our product match the capabilities of the two announced competitor offerings?",
+      "reason": "This remaining uncertainty determines whether the imminent competitor launches pose an immediate threat to launching now."
+    }
+  }
+}