360 lines
11 KiB
JavaScript
360 lines
11 KiB
JavaScript
import { test, expect } from "@playwright/test";
|
|
import * as fs from "node:fs";
|
|
import * as path from "node:path";
|
|
|
|
const MODEL_RESPONSE_TIMEOUT = 120_000;
|
|
|
|
test("live-capture second-update boundary", async ({ page }) => {
|
|
test.setTimeout(420_000);
|
|
|
|
const timings = {};
|
|
const captures = {
|
|
start: null,
|
|
update1: null,
|
|
update2: null,
|
|
};
|
|
|
|
async function readJson(response) {
|
|
try {
|
|
return await response.json();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function readRequestJson(response) {
|
|
try {
|
|
return response.request().postDataJSON();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function captureResponse(response, startedAt) {
|
|
return {
|
|
url: response.url(),
|
|
status: response.status(),
|
|
elapsedMs: Date.now() - startedAt,
|
|
requestBody: readRequestJson(response),
|
|
responseBody: await readJson(response),
|
|
};
|
|
}
|
|
|
|
// ================================================================
|
|
// AUTHORITATIVE MANUAL JOURNEY
|
|
// Browser actions, selectors, inputs and ordering are kept intact.
|
|
// Instrumentation is additive only. No mocks are enabled here.
|
|
// ================================================================
|
|
|
|
await page.goto("http://localhost:3000/");
|
|
|
|
await page.getByTestId("scenario-textarea").click();
|
|
await page.getByTestId("scenario-textarea").click();
|
|
await page
|
|
.getByTestId("scenario-textarea")
|
|
.fill(
|
|
"I am deciding whether to launch a new software product this year or wait twelve months. The product is ready enough to launch, but one large enterprise customer could represent a significant part of the expected revenue and I do not yet know whether they will sign. Launching this year would also require around £300,000 of additional support and implementation cost. Waiting twelve months would reduce that immediate cost and give us more time to improve the product, but it would delay revenue and may allow competitors to move first. I need to decide whether there is enough evidence to launch this year or whether waiting is the safer decision.",
|
|
);
|
|
|
|
// ===== ANALYSE -> real POST /api/cases/start =====
|
|
const startStartedAt = Date.now();
|
|
|
|
const startResponsePromise = page.waitForResponse(
|
|
(response) =>
|
|
response.request().method() === "POST" &&
|
|
response.url().includes("/api/cases/start"),
|
|
{ timeout: MODEL_RESPONSE_TIMEOUT },
|
|
);
|
|
|
|
await page.getByRole("button", { name: "Analyse" }).click();
|
|
|
|
const startResponse = await startResponsePromise;
|
|
captures.start = await captureResponse(startResponse, startStartedAt);
|
|
timings.startElapsedMs = captures.start.elapsedMs;
|
|
|
|
console.log(
|
|
`[LIVE CAPTURE] start: status=${captures.start.status}, elapsed=${captures.start.elapsedMs}ms`,
|
|
);
|
|
|
|
// Do not move to the first answer until the real app has rendered it.
|
|
const responseTextarea = page.getByTestId("response-textarea");
|
|
|
|
await expect(responseTextarea).toBeVisible({
|
|
timeout: MODEL_RESPONSE_TIMEOUT,
|
|
});
|
|
|
|
await responseTextarea.click();
|
|
|
|
await responseTextarea.fill(
|
|
"Launching this year would need to remain commercially viable without the enterprise customer. I would want to see enough committed or highly probable revenue from other customers to cover the additional £300,000 implementation and support cost and still produce an acceptable return.",
|
|
);
|
|
|
|
// ===== UPDATE 1 -> real POST /api/cases/update =====
|
|
const update1StartedAt = Date.now();
|
|
|
|
const update1ResponsePromise = page.waitForResponse(
|
|
(response) =>
|
|
response.request().method() === "POST" &&
|
|
response.url().includes("/api/cases/update"),
|
|
{ timeout: MODEL_RESPONSE_TIMEOUT },
|
|
);
|
|
|
|
await page.getByRole("button", { name: "Update" }).click();
|
|
|
|
const update1Response = await update1ResponsePromise;
|
|
captures.update1 = await captureResponse(update1Response, update1StartedAt);
|
|
timings.update1ElapsedMs = captures.update1.elapsedMs;
|
|
|
|
console.log(
|
|
`[LIVE CAPTURE] update 1: status=${captures.update1.status}, elapsed=${captures.update1.elapsedMs}ms`,
|
|
);
|
|
|
|
// Do not type the second answer until the real first update has rendered.
|
|
await expect(responseTextarea).toBeVisible({
|
|
timeout: MODEL_RESPONSE_TIMEOUT,
|
|
});
|
|
|
|
await responseTextarea.click();
|
|
|
|
await responseTextarea.fill(
|
|
"We have £450,000 of annual recurring revenue already committed from other customers, plus another £250,000 in late-stage opportunities that I would estimate have about a 70% probability of closing within the next six months.",
|
|
);
|
|
|
|
// ===== UPDATE 2 TARGET -> real POST /api/cases/update =====
|
|
const update2StartedAt = Date.now();
|
|
|
|
const update2ResponsePromise = page.waitForResponse(
|
|
(response) =>
|
|
response.request().method() === "POST" &&
|
|
response.url().includes("/api/cases/update"),
|
|
{ timeout: MODEL_RESPONSE_TIMEOUT },
|
|
);
|
|
|
|
await page.getByRole("button", { name: "Update" }).click();
|
|
|
|
const update2Response = await update2ResponsePromise;
|
|
captures.update2 = await captureResponse(update2Response, update2StartedAt);
|
|
timings.update2ElapsedMs = captures.update2.elapsedMs;
|
|
|
|
console.log(
|
|
`[LIVE CAPTURE] update 2: status=${captures.update2.status}, elapsed=${captures.update2.elapsedMs}ms`,
|
|
);
|
|
|
|
expect(captures.update2.responseBody).not.toBeNull();
|
|
|
|
const resp = captures.update2.responseBody;
|
|
|
|
// ================================================================
|
|
// UPDATE 2 DIAGNOSTICS
|
|
// Use the actual response shape; do not assume top-level aliases.
|
|
// ================================================================
|
|
|
|
const extracted = {
|
|
activeUnknownNodeId:
|
|
resp?.updatedSituationGraph?.activeUnknownNodeId ?? null,
|
|
|
|
selectedQuestionNodeId: resp?.selectedQuestion?.nodeId ?? null,
|
|
|
|
selectedQuestionTemplate:
|
|
resp?.selectedQuestion?.selectedQuestionTemplate ?? null,
|
|
|
|
selectedQuestionText: resp?.selectedQuestion?.question ?? null,
|
|
|
|
diagnosticsSelectedUnknownNodeId:
|
|
resp?.diagnostics?.selectedUnknownNodeId ?? "not exposed",
|
|
|
|
finalGraphBackedQuestion:
|
|
resp?.diagnostics?.finalGraphBackedQuestion ?? null,
|
|
|
|
noQuestionReason: resp?.diagnostics?.noQuestionReason ?? null,
|
|
};
|
|
|
|
// ================================================================
|
|
// ACTIVE NODE LABEL
|
|
// ================================================================
|
|
|
|
let activeNodeLabel = "unknown";
|
|
|
|
const activeId = extracted.activeUnknownNodeId;
|
|
const nodes = resp?.updatedSituationGraph?.nodes;
|
|
|
|
if (activeId && Array.isArray(nodes)) {
|
|
const activeNode = nodes.find((node) => node.id === activeId);
|
|
|
|
if (activeNode?.label) {
|
|
activeNodeLabel = activeNode.label;
|
|
}
|
|
}
|
|
|
|
// ================================================================
|
|
// VISIBLE CURRENT QUESTION
|
|
//
|
|
// response-textarea is the ANSWER field.
|
|
// It must NOT be interpreted as the current question.
|
|
// ================================================================
|
|
|
|
let visibleQuestion = "NOT CAPTURED";
|
|
|
|
const questionTestId = page.getByTestId("question-text");
|
|
|
|
if ((await questionTestId.count()) > 0) {
|
|
const text = (await questionTestId.first().textContent())?.trim();
|
|
|
|
if (text) {
|
|
visibleQuestion = text;
|
|
}
|
|
} else if (extracted.selectedQuestionText) {
|
|
/*
|
|
* If the application has no dedicated question test id,
|
|
* only check whether the exact backend-selected question
|
|
* is visibly rendered somewhere.
|
|
*
|
|
* Do not guess another UI selector.
|
|
*/
|
|
const selectedQuestionOnPage = page.getByText(
|
|
extracted.selectedQuestionText,
|
|
{ exact: true },
|
|
);
|
|
|
|
if ((await selectedQuestionOnPage.count()) > 0) {
|
|
try {
|
|
await expect(selectedQuestionOnPage.first()).toBeVisible({
|
|
timeout: 10_000,
|
|
});
|
|
|
|
visibleQuestion = extracted.selectedQuestionText;
|
|
} catch {
|
|
// Leave as NOT CAPTURED.
|
|
}
|
|
}
|
|
}
|
|
|
|
// ================================================================
|
|
// CLASSIFICATION
|
|
// ================================================================
|
|
|
|
let classification = "A - BACKEND QUESTION AND ACTIVE TARGET AGREE";
|
|
|
|
if (
|
|
extracted.activeUnknownNodeId != null &&
|
|
extracted.selectedQuestionNodeId != null &&
|
|
extracted.activeUnknownNodeId !== extracted.selectedQuestionNodeId
|
|
) {
|
|
classification = "B - BACKEND TARGET MISMATCH";
|
|
}
|
|
|
|
// ================================================================
|
|
// DURABLE CAPTURE
|
|
// ================================================================
|
|
|
|
const artifactDir = path.join(process.cwd(), "tests", "e2e", "artifacts");
|
|
|
|
fs.mkdirSync(artifactDir, {
|
|
recursive: true,
|
|
});
|
|
|
|
const artifactPath = path.join(
|
|
artifactDir,
|
|
"manual-recorded-journey-update2-capture.json",
|
|
);
|
|
|
|
const artifact = {
|
|
capturedAt: new Date().toISOString(),
|
|
|
|
timings,
|
|
|
|
start: {
|
|
status: captures.start.status,
|
|
elapsedMs: captures.start.elapsedMs,
|
|
},
|
|
|
|
update1: {
|
|
status: captures.update1.status,
|
|
elapsedMs: captures.update1.elapsedMs,
|
|
},
|
|
|
|
update2: {
|
|
url: captures.update2.url,
|
|
status: captures.update2.status,
|
|
elapsedMs: captures.update2.elapsedMs,
|
|
|
|
/*
|
|
* Critical evidence for later deterministic replay.
|
|
*/
|
|
requestBody: captures.update2.requestBody,
|
|
|
|
responseBody: captures.update2.responseBody,
|
|
},
|
|
|
|
extracted,
|
|
activeNodeLabel,
|
|
visibleQuestion,
|
|
classification,
|
|
};
|
|
|
|
fs.writeFileSync(artifactPath, JSON.stringify(artifact, null, 2));
|
|
|
|
// ================================================================
|
|
// COMPACT REPORT
|
|
// ================================================================
|
|
|
|
console.log("\n=== LIVE SECOND-UPDATE CAPTURE ===");
|
|
|
|
console.log("Classification:", classification);
|
|
|
|
console.log("Start elapsed ms:", timings.startElapsedMs);
|
|
|
|
console.log("Update 1 elapsed ms:", timings.update1ElapsedMs);
|
|
|
|
console.log("Update 2 elapsed ms:", timings.update2ElapsedMs);
|
|
|
|
console.log(
|
|
"Update 2 request captured:",
|
|
captures.update2.requestBody !== null,
|
|
);
|
|
|
|
console.log("activeUnknownNodeId:", extracted.activeUnknownNodeId);
|
|
|
|
console.log("selectedQuestion.nodeId:", extracted.selectedQuestionNodeId);
|
|
|
|
console.log(
|
|
"diagnostics.selectedUnknownNodeId:",
|
|
extracted.diagnosticsSelectedUnknownNodeId,
|
|
);
|
|
|
|
console.log(
|
|
"selectedQuestion.selectedQuestionTemplate:",
|
|
extracted.selectedQuestionTemplate,
|
|
);
|
|
|
|
console.log("selectedQuestion.question:", extracted.selectedQuestionText);
|
|
|
|
console.log(
|
|
"diagnostics.finalGraphBackedQuestion:",
|
|
extracted.finalGraphBackedQuestion,
|
|
);
|
|
|
|
console.log("diagnostics.noQuestionReason:", extracted.noQuestionReason);
|
|
|
|
console.log("active node label:", activeNodeLabel);
|
|
|
|
console.log("visible current question:", visibleQuestion);
|
|
|
|
console.log("artifact:", artifactPath);
|
|
|
|
// ================================================================
|
|
// ASSERTIONS
|
|
//
|
|
// Assert only that the REAL requests completed and were captured.
|
|
//
|
|
// A target mismatch is diagnostic evidence, not a Playwright
|
|
// test failure.
|
|
// ================================================================
|
|
|
|
expect(captures.start.elapsedMs).toBeGreaterThan(0);
|
|
|
|
expect(captures.update1.elapsedMs).toBeGreaterThan(0);
|
|
|
|
expect(captures.update2.elapsedMs).toBeGreaterThan(0);
|
|
});
|