feature/cl8-instance-health #25

Merged
robbond merged 4 commits from feature/cl8-instance-health into master 2026-03-05 18:04:41 +00:00
7 changed files with 994 additions and 153 deletions

View File

@@ -0,0 +1,38 @@
import { NextResponse } from "next/server";
import { env } from "@/lib/env";
import { runHealthCheckTick } from "@/lib/plesk/health";
import { createSupabaseAdminClient } from "@/lib/supabase/admin";
export async function POST(req: Request) {
const startedAt = Date.now();
try {
const secret = req.headers.get("x-cron-secret");
if (!env.cronSecret || !secret || secret !== env.cronSecret) {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
const supabase = createSupabaseAdminClient() as any;
const summary = await runHealthCheckTick(supabase);
return NextResponse.json({
...summary,
duration_ms: Date.now() - startedAt,
});
} catch (error) {
console.error("[health cron] failed", error);
return NextResponse.json(
{
error: "Health check job failed",
checked: 0,
ok: 0,
degraded: 0,
failed: 0,
duration_ms: Date.now() - startedAt,
},
{ status: 500 },
);
}
}

View File

@@ -0,0 +1,73 @@
import { NextResponse } from "next/server";
import { assertAgencyAdmin, getRouteAgencyContextOrThrow } from "@/lib/agency";
import { assertRateLimit } from "@/lib/api/rate-limit";
import { assertSameOrigin } from "@/lib/api/security";
import { runInstanceHealthCheck } from "@/lib/plesk/health";
import { createSupabaseRouteClient } from "@/lib/supabase/route";
export async function POST(
req: Request,
{ params }: { params: { id: string } },
) {
try {
assertSameOrigin(req);
const context = await getRouteAgencyContextOrThrow();
assertAgencyAdmin(context);
assertRateLimit(`plesk-health-check:${context.userId}`, 20, 60_000);
const supabase = createSupabaseRouteClient() as any;
const { data: instance, error: instanceError } = await supabase
.from("plesk_instances")
.select(
"id, agency_id, base_url, auth_type, encrypted_secret, health_enabled, health_check_frequency_minutes, health_next_check_at, health_lock_until",
)
.eq("id", params.id)
.eq("agency_id", context.agencyId)
.single();
if (instanceError || !instance) {
throw new Error("Plesk instance not found");
}
const result = await runInstanceHealthCheck(supabase, instance, {
actorUserId: context.userId,
force: true,
source: "manual",
});
if (!result.checked) {
return NextResponse.json(
{ error: "Health check is already running for this instance" },
{ status: 409 },
);
}
const { data: updated } = await supabase
.from("plesk_instances")
.select(
"id, health_enabled, health_status, health_last_checked_at, health_last_ok_at, health_last_error, health_last_latency_ms, health_check_frequency_minutes, health_next_check_at",
)
.eq("id", params.id)
.eq("agency_id", context.agencyId)
.single();
return NextResponse.json({
ok: true,
result,
instance: updated,
});
} catch (error) {
return NextResponse.json(
{
error:
error instanceof Error
? error.message
: "Health check execution failed",
},
{ status: 400 },
);
}
}

View File

@@ -17,6 +17,16 @@ type ActionLogSummary = Pick<
Database["public"]["Tables"]["actions_log"]["Row"], Database["public"]["Tables"]["actions_log"]["Row"],
"target_id" | "status" | "action" | "created_at" | "error_message" "target_id" | "status" | "action" | "created_at" | "error_message"
>; >;
type HealthLogSummary = Pick<
Database["public"]["Tables"]["actions_log"]["Row"],
| "id"
| "target_id"
| "status"
| "action"
| "created_at"
| "error_message"
| "metadata"
>;
function normalizeDomainLike(value: string) { function normalizeDomainLike(value: string) {
return value.trim().toLowerCase().replace(/\.$/, ""); return value.trim().toLowerCase().replace(/\.$/, "");
@@ -49,6 +59,7 @@ export default async function DashboardPage() {
{ data: domains }, { data: domains },
{ data: billing }, { data: billing },
{ data: autoSyncLogs }, { data: autoSyncLogs },
{ data: healthLogs },
] = (await Promise.all([ ] = (await Promise.all([
supabase supabase
.from("agencies") .from("agencies")
@@ -58,7 +69,7 @@ export default async function DashboardPage() {
supabase supabase
.from("plesk_instances") .from("plesk_instances")
.select( .select(
"id, name, base_url, auth_type, status, error_message, last_connected_at, last_sync_at, last_sync_status, last_sync_error, last_sync_subscriptions, last_sync_domains, auto_sync_enabled, auto_sync_frequency_minutes, last_auto_sync_at, next_auto_sync_at, auto_sync_lock_until", "id, name, base_url, auth_type, status, error_message, last_connected_at, last_sync_at, last_sync_status, last_sync_error, last_sync_subscriptions, last_sync_domains, auto_sync_enabled, auto_sync_frequency_minutes, last_auto_sync_at, next_auto_sync_at, auto_sync_lock_until, health_enabled, health_status, health_last_checked_at, health_last_ok_at, health_last_error, health_last_latency_ms, health_check_frequency_minutes, health_next_check_at, health_lock_until",
) )
.eq("agency_id", context.agencyId) .eq("agency_id", context.agencyId)
.order("created_at", { ascending: false }), .order("created_at", { ascending: false }),
@@ -89,6 +100,21 @@ export default async function DashboardPage() {
.eq("action", "plesk_sync_auto") .eq("action", "plesk_sync_auto")
.order("created_at", { ascending: false }) .order("created_at", { ascending: false })
.limit(200), .limit(200),
supabase
.from("actions_log")
.select(
"id, target_id, status, action, created_at, error_message, metadata",
)
.eq("agency_id", context.agencyId)
.eq("target_type", "plesk_instance")
.in("action", [
"health_check_started",
"health_check_ok",
"health_check_degraded",
"health_check_failed",
])
.order("created_at", { ascending: false })
.limit(400),
])) as [ ])) as [
{ data: AgencySummary | null }, { data: AgencySummary | null },
{ {
@@ -110,6 +136,15 @@ export default async function DashboardPage() {
last_auto_sync_at: string | null; last_auto_sync_at: string | null;
next_auto_sync_at: string | null; next_auto_sync_at: string | null;
auto_sync_lock_until: string | null; auto_sync_lock_until: string | null;
health_enabled: boolean;
health_status: "ok" | "degraded" | "down" | "unknown";
health_last_checked_at: string | null;
health_last_ok_at: string | null;
health_last_error: string | null;
health_last_latency_ms: number | null;
health_check_frequency_minutes: number;
health_next_check_at: string | null;
health_lock_until: string | null;
}> | null; }> | null;
error: { message: string } | null; error: { message: string } | null;
}, },
@@ -138,6 +173,7 @@ export default async function DashboardPage() {
}, },
{ data: BillingSummary | null }, { data: BillingSummary | null },
{ data: ActionLogSummary[] | null }, { data: ActionLogSummary[] | null },
{ data: HealthLogSummary[] | null },
]; ];
const domainInstanceIds = Array.from( const domainInstanceIds = Array.from(
@@ -230,6 +266,33 @@ export default async function DashboardPage() {
}); });
} }
const healthEventsByInstance = new Map<
string,
Array<{
id: string;
action: string;
status: string;
created_at: string;
error_message: string | null;
metadata: unknown;
}>
>();
for (const log of healthLogs ?? []) {
if (!log.target_id) continue;
const existing = healthEventsByInstance.get(log.target_id) ?? [];
if (existing.length >= 10) continue;
existing.push({
id: log.id,
action: log.action,
status: log.status,
created_at: log.created_at,
error_message: log.error_message,
metadata: log.metadata,
});
healthEventsByInstance.set(log.target_id, existing);
}
return ( return (
<main className="mx-auto flex w-full max-w-6xl flex-col gap-6 px-6 py-8"> <main className="mx-auto flex w-full max-w-6xl flex-col gap-6 px-6 py-8">
<header className="flex items-center justify-between"> <header className="flex items-center justify-between">
@@ -312,6 +375,9 @@ export default async function DashboardPage() {
}; };
})} })}
autoSyncByInstance={Object.fromEntries(autoSyncByInstance.entries())} autoSyncByInstance={Object.fromEntries(autoSyncByInstance.entries())}
healthEventsByInstance={Object.fromEntries(
healthEventsByInstance.entries(),
)}
/> />
</main> </main>
); );

View File

@@ -23,6 +23,15 @@ type Instance = {
last_auto_sync_at: string | null; last_auto_sync_at: string | null;
next_auto_sync_at: string | null; next_auto_sync_at: string | null;
auto_sync_lock_until: string | null; auto_sync_lock_until: string | null;
health_enabled: boolean;
health_status: "ok" | "degraded" | "down" | "unknown";
health_last_checked_at: string | null;
health_last_ok_at: string | null;
health_last_error: string | null;
health_last_latency_ms: number | null;
health_check_frequency_minutes: number;
health_next_check_at: string | null;
health_lock_until: string | null;
}; };
type Subscription = { type Subscription = {
@@ -54,8 +63,44 @@ type Props = {
string, string,
{ status: string; created_at: string; error_message: string | null } { status: string; created_at: string; error_message: string | null }
>; >;
healthEventsByInstance: Record<
string,
Array<{
id: string;
action: string;
status: string;
created_at: string;
error_message: string | null;
metadata: unknown;
}>
>;
}; };
function getHealthBadge(status: string | null) {
const normalized = (status ?? "unknown").toLowerCase();
if (normalized === "ok") {
return { label: "OK", className: "bg-emerald-100 text-emerald-700" };
}
if (normalized === "degraded") {
return { label: "Degraded", className: "bg-amber-100 text-amber-700" };
}
if (normalized === "down") {
return { label: "Down", className: "bg-rose-100 text-rose-700" };
}
return { label: "Unknown", className: "bg-slate-100 text-slate-600" };
}
function mapHealthResult(action: string) {
if (action === "health_check_ok") return "ok";
if (action === "health_check_degraded") return "degraded";
if (action === "health_check_failed") return "failed";
return "started";
}
function getDomainStatusBadge(status: string | null) { function getDomainStatusBadge(status: string | null) {
const normalized = (status ?? "unknown").toLowerCase(); const normalized = (status ?? "unknown").toLowerCase();
@@ -120,6 +165,7 @@ export function DashboardControls({
subscriptions, subscriptions,
domains, domains,
autoSyncByInstance, autoSyncByInstance,
healthEventsByInstance,
}: Props) { }: Props) {
const autoSyncFrequencyOptions = [15, 30, 60, 180, 360, 1440]; const autoSyncFrequencyOptions = [15, 30, 60, 180, 360, 1440];
const [instanceName, setInstanceName] = useState(""); const [instanceName, setInstanceName] = useState("");
@@ -245,6 +291,22 @@ export function DashboardControls({
} }
} }
async function onRunHealthCheck(instanceId: string) {
setLoading(true);
setMessage(null);
try {
await runRequest(`/api/plesk/instances/${instanceId}/health`);
setMessage("Health check completed.");
window.location.reload();
} catch (error) {
setMessage(
error instanceof Error ? error.message : "Health check failed",
);
} finally {
setLoading(false);
}
}
async function onUpdateAutoSync( async function onUpdateAutoSync(
instanceId: string, instanceId: string,
input: { auto_sync_enabled: boolean; auto_sync_frequency_minutes: number }, input: { auto_sync_enabled: boolean; auto_sync_frequency_minutes: number },
@@ -442,159 +504,308 @@ export function DashboardControls({
key={instance.id} key={instance.id}
className="rounded border border-slate-200 p-3" className="rounded border border-slate-200 p-3"
> >
<div className="flex items-start justify-between gap-2"> {(() => {
<div> const health = getHealthBadge(instance.health_status);
{autoSyncByInstance[instance.id] ? (
<span return (
className={`inline-flex rounded px-2 py-0.5 text-[11px] font-medium ${ <div className="flex items-start justify-between gap-2">
autoSyncByInstance[instance.id].status === "success" <div>
? "bg-emerald-100 text-emerald-700" {autoSyncByInstance[instance.id] ? (
: "bg-rose-100 text-rose-700" <span
}`} className={`inline-flex rounded px-2 py-0.5 text-[11px] font-medium ${
> autoSyncByInstance[instance.id].status ===
Auto-sync {autoSyncByInstance[instance.id].status} "success"
</span> ? "bg-emerald-100 text-emerald-700"
) : null} : "bg-rose-100 text-rose-700"
<p className="text-sm font-semibold text-slate-900"> }`}
{instance.name} >
</p> Auto-sync {autoSyncByInstance[instance.id].status}
<p className="text-xs text-slate-500"> </span>
{instance.base_url} ) : null}
</p> <p className="text-sm font-semibold text-slate-900">
<p className="mt-1 text-xs text-slate-600"> {instance.name}
Status:{" "} </p>
<span className="font-medium">{instance.status}</span> <p className="text-xs text-slate-500">
</p> {instance.base_url}
<p className="text-xs text-slate-500"> </p>
Subscriptions: {instance.last_sync_subscriptions ?? 0} <p className="mt-1 text-xs text-slate-600">
</p> Status:{" "}
<p className="text-xs text-slate-500"> <span className="font-medium">
Domains: {instance.last_sync_domains ?? 0} {instance.status}
</p> </span>
<p className="text-xs text-slate-500"> </p>
Last sync status:{" "} <p className="text-xs text-slate-500">
<span Health:{" "}
className={`inline-flex rounded px-2 py-0.5 text-[11px] font-medium ${ <span
instance.last_sync_status === "success" title={instance.health_last_error ?? undefined}
? "bg-emerald-100 text-emerald-700" className={`inline-flex rounded px-2 py-0.5 text-[11px] font-medium ${health.className}`}
: instance.last_sync_status === "failed" || >
instance.last_sync_status === "error" {health.label}
? "bg-rose-100 text-rose-700" </span>
: "bg-slate-100 text-slate-600" </p>
}`} <p className="text-xs text-slate-500">
> Health last checked:{" "}
{instance.last_sync_status ?? "unknown"} {formatDateTime(instance.health_last_checked_at)}
</span> </p>
</p> <p className="text-xs text-slate-500">
<p className="text-xs text-slate-500"> Health latency:{" "}
Last sync time: {formatDateTime(instance.last_sync_at)} {instance.health_last_latency_ms != null
</p> ? `${instance.health_last_latency_ms} ms`
<p className="text-xs text-slate-500"> : "—"}
Last connected:{" "} </p>
{instance.last_connected_at <p className="text-xs text-slate-500">
? formatDateTime(instance.last_connected_at) Subscriptions:{" "}
: "never"} {instance.last_sync_subscriptions ?? 0}
</p> </p>
<p className="text-xs text-slate-500"> <p className="text-xs text-slate-500">
Auto-sync:{" "} Domains: {instance.last_sync_domains ?? 0}
{instance.auto_sync_enabled ? "Enabled" : "Disabled"} </p>
</p> <p className="text-xs text-slate-500">
<p className="text-xs text-slate-500"> Last sync status:{" "}
Auto-sync frequency:{" "} <span
{instance.auto_sync_frequency_minutes ?? 60} min className={`inline-flex rounded px-2 py-0.5 text-[11px] font-medium ${
</p> instance.last_sync_status === "success"
<p className="text-xs text-slate-500"> ? "bg-emerald-100 text-emerald-700"
Last auto-sync:{" "} : instance.last_sync_status === "failed" ||
{formatDateTime(instance.last_auto_sync_at)} instance.last_sync_status === "error"
</p> ? "bg-rose-100 text-rose-700"
<p className="text-xs text-slate-500"> : "bg-slate-100 text-slate-600"
Next auto-sync:{" "} }`}
{formatDateTime(instance.next_auto_sync_at)} >
</p> {instance.last_sync_status ?? "unknown"}
<p className="text-xs text-slate-500"> </span>
Lock status:{" "} </p>
{instance.auto_sync_lock_until && <p className="text-xs text-slate-500">
new Date(instance.auto_sync_lock_until).getTime() > Last sync time:{" "}
Date.now() {formatDateTime(instance.last_sync_at)}
? "Sync in progress" </p>
: "Idle"} <p className="text-xs text-slate-500">
</p> Last connected:{" "}
{autoSyncByInstance[instance.id] ? ( {instance.last_connected_at
<p className="text-xs text-slate-500"> ? formatDateTime(instance.last_connected_at)
Last auto-sync:{" "} : "never"}
{formatDateTime( </p>
autoSyncByInstance[instance.id].created_at, <p className="text-xs text-slate-500">
)} Auto-sync:{" "}
</p> {instance.auto_sync_enabled
) : null} ? "Enabled"
{instance.error_message ? ( : "Disabled"}
<p className="mt-1 text-xs text-rose-600"> </p>
{instance.error_message} <p className="text-xs text-slate-500">
</p> Auto-sync frequency:{" "}
) : null} {instance.auto_sync_frequency_minutes ?? 60} min
{instance.last_sync_error ? ( </p>
<p className="mt-1 text-xs text-rose-600"> <p className="text-xs text-slate-500">
Last sync error: {instance.last_sync_error} Last auto-sync:{" "}
</p> {formatDateTime(instance.last_auto_sync_at)}
) : null} </p>
{autoSyncByInstance[instance.id]?.error_message ? ( <p className="text-xs text-slate-500">
<p className="mt-1 text-xs text-rose-600"> Next auto-sync:{" "}
Auto-sync error:{" "} {formatDateTime(instance.next_auto_sync_at)}
{autoSyncByInstance[instance.id].error_message} </p>
</p> <p className="text-xs text-slate-500">
) : null} Lock status:{" "}
</div> {instance.auto_sync_lock_until &&
<div className="flex flex-col gap-2"> new Date(instance.auto_sync_lock_until).getTime() >
<div className="flex items-center gap-2"> Date.now()
<label className="text-xs text-slate-600"> ? "Sync in progress"
Auto-sync : "Idle"}
</label> </p>
<input {autoSyncByInstance[instance.id] ? (
type="checkbox" <p className="text-xs text-slate-500">
checked={instance.auto_sync_enabled} Last auto-sync:{" "}
onChange={(e: ChangeEvent<HTMLInputElement>) => {formatDateTime(
onUpdateAutoSync(instance.id, { autoSyncByInstance[instance.id].created_at,
auto_sync_enabled: e.target.checked, )}
auto_sync_frequency_minutes: </p>
instance.auto_sync_frequency_minutes ?? 60, ) : null}
}) {instance.error_message ? (
} <p className="mt-1 text-xs text-rose-600">
disabled={loading} {instance.error_message}
/> </p>
) : null}
{instance.last_sync_error ? (
<p className="mt-1 text-xs text-rose-600">
Last sync error: {instance.last_sync_error}
</p>
) : null}
{autoSyncByInstance[instance.id]?.error_message ? (
<p className="mt-1 text-xs text-rose-600">
Auto-sync error:{" "}
{autoSyncByInstance[instance.id].error_message}
</p>
) : null}
{instance.health_last_error ? (
<p
className="mt-1 text-xs text-rose-600"
title={instance.health_last_error}
>
Health error: {instance.health_last_error}
</p>
) : null}
</div>
<div className="flex flex-col gap-2">
<div className="flex items-center gap-2">
<label className="text-xs text-slate-600">
Auto-sync
</label>
<input
type="checkbox"
checked={instance.auto_sync_enabled}
onChange={(e: ChangeEvent<HTMLInputElement>) =>
onUpdateAutoSync(instance.id, {
auto_sync_enabled: e.target.checked,
auto_sync_frequency_minutes:
instance.auto_sync_frequency_minutes ?? 60,
})
}
disabled={loading}
/>
</div>
<select
value={String(
instance.auto_sync_frequency_minutes ?? 60,
)}
onChange={(e: ChangeEvent<HTMLSelectElement>) =>
onUpdateAutoSync(instance.id, {
auto_sync_enabled: instance.auto_sync_enabled,
auto_sync_frequency_minutes: Number(
e.target.value,
),
})
}
disabled={loading}
className="rounded border px-2 py-1 text-xs"
>
{autoSyncFrequencyOptions.map((minutes) => (
<option key={minutes} value={minutes}>
Every {minutes} min
</option>
))}
</select>
<button
onClick={() => onTestConnection(instance.id)}
disabled={loading}
className="rounded border px-3 py-1.5 text-xs"
>
Test Connection
</button>
<button
onClick={() => onRunHealthCheck(instance.id)}
disabled={loading}
className="rounded border px-3 py-1.5 text-xs"
>
Run Health Check
</button>
<button
onClick={() => onSync(instance.id)}
disabled={loading}
className="rounded border px-3 py-1.5 text-xs"
>
Sync Now
</button>
</div>
</div> </div>
<select );
value={String( })()}
instance.auto_sync_frequency_minutes ?? 60,
)} <div className="mt-3 rounded border border-slate-100 bg-slate-50 p-3">
onChange={(e: ChangeEvent<HTMLSelectElement>) => <p className="text-xs font-semibold uppercase tracking-wide text-slate-600">
onUpdateAutoSync(instance.id, { Health Panel
auto_sync_enabled: instance.auto_sync_enabled, </p>
auto_sync_frequency_minutes: Number(e.target.value), <div className="mt-2 grid gap-1 text-xs text-slate-600 sm:grid-cols-2">
}) <p>
} Current Status:{" "}
disabled={loading} <span className="font-medium">
className="rounded border px-2 py-1 text-xs" {getHealthBadge(instance.health_status).label}
> </span>
{autoSyncFrequencyOptions.map((minutes) => ( </p>
<option key={minutes} value={minutes}> <p>
Every {minutes} min Last Checked:{" "}
</option> <span className="font-medium">
))} {formatDateTime(instance.health_last_checked_at)}
</select> </span>
<button </p>
onClick={() => onTestConnection(instance.id)} <p>
disabled={loading} Last Successful Check:{" "}
className="rounded border px-3 py-1.5 text-xs" <span className="font-medium">
> {formatDateTime(instance.health_last_ok_at)}
Test Connection </span>
</button> </p>
<button <p>
onClick={() => onSync(instance.id)} Latency:{" "}
disabled={loading} <span className="font-medium">
className="rounded border px-3 py-1.5 text-xs" {instance.health_last_latency_ms != null
> ? `${instance.health_last_latency_ms} ms`
Sync Now : "—"}
</button> </span>
</p>
<p className="sm:col-span-2">
Last Error:{" "}
<span className="font-medium">
{instance.health_last_error ?? "—"}
</span>
</p>
</div>
<div className="mt-3 overflow-x-auto">
<table className="min-w-full text-left text-xs">
<thead>
<tr className="border-b border-slate-200 text-slate-500">
<th className="px-2 py-1">Time</th>
<th className="px-2 py-1">Event</th>
<th className="px-2 py-1">Result</th>
<th className="px-2 py-1">Message</th>
</tr>
</thead>
<tbody>
{(healthEventsByInstance[instance.id] ?? [])
.length === 0 ? (
<tr>
<td
className="px-2 py-2 text-slate-500"
colSpan={4}
>
No health events yet.
</td>
</tr>
) : (
(healthEventsByInstance[instance.id] ?? []).map(
(event) => (
<tr
key={event.id}
className="border-b border-slate-100 align-top"
>
<td className="px-2 py-1">
{formatDateTime(event.created_at)}
</td>
<td className="px-2 py-1">{event.action}</td>
<td className="px-2 py-1">
{mapHealthResult(event.action)}
</td>
<td className="px-2 py-1">
{event.error_message ?? "—"}
{event.metadata ? (
<details>
<summary className="cursor-pointer text-[11px] text-brand-700">
metadata
</summary>
<pre className="mt-1 max-w-[440px] overflow-auto rounded bg-white p-2 text-[10px] text-slate-700">
{JSON.stringify(
event.metadata,
null,
2,
)}
</pre>
</details>
) : null}
</td>
</tr>
),
)
)}
</tbody>
</table>
</div> </div>
</div> </div>
</div> </div>

345
lib/plesk/health.ts Normal file
View File

@@ -0,0 +1,345 @@
import { randomUUID } from "crypto";
import { testPleskConnection } from "@/lib/plesk/client";
type SupabaseLike = any;
type HealthStatus = "ok" | "degraded" | "down" | "unknown";
type HealthInstance = {
id: string;
agency_id: string;
base_url: string;
auth_type: "api_key" | "basic";
encrypted_secret: string;
health_enabled: boolean;
health_check_frequency_minutes: number | null;
health_next_check_at: string | null;
health_lock_until: string | null;
};
type HealthCheckResult = {
status: Exclude<HealthStatus, "unknown">;
latency_ms: number;
error_message: string | null;
};
type RunSingleHealthCheckResult = {
checked: boolean;
status?: Exclude<HealthStatus, "unknown">;
latency_ms?: number;
error_message?: string | null;
};
export type HealthTickSummary = {
checked: number;
ok: number;
degraded: number;
failed: number;
duration_ms: number;
};
const MAX_INSTANCES_PER_TICK = 10;
const MAX_CONCURRENCY = 3;
const LOCK_LEASE_SECONDS = 5 * 60;
const HEALTH_TIMEOUT_MS = 15_000;
const DEGRADED_LATENCY_THRESHOLD_MS = 2_500;
function isDue(instance: HealthInstance, now: number) {
if (!instance.health_next_check_at) return true;
const dueAt = new Date(instance.health_next_check_at).getTime();
return Number.isFinite(dueAt) && dueAt <= now;
}
function isUnlocked(instance: HealthInstance, now: number) {
if (!instance.health_lock_until) return true;
const lockUntil = new Date(instance.health_lock_until).getTime();
return !Number.isFinite(lockUntil) || lockUntil <= now;
}
async function insertActionLogBestEffort(
supabase: SupabaseLike,
payload: Record<string, unknown>,
) {
try {
const { error } = await supabase.from("actions_log").insert(payload);
if (error) {
console.error("[health] actions_log insert failed", error.message);
}
} catch (error) {
console.error("[health] actions_log insert threw", error);
}
}
async function withTimeout<T>(
promise: Promise<T>,
timeoutMs: number,
): Promise<T> {
let timeout: ReturnType<typeof setTimeout> | null = null;
const timeoutPromise = new Promise<never>((_, reject) => {
timeout = setTimeout(() => {
reject(new Error("Health check timed out"));
}, timeoutMs);
});
try {
return await Promise.race([promise, timeoutPromise]);
} finally {
if (timeout) clearTimeout(timeout);
}
}
async function checkPleskInstanceHealth(
instance: Pick<HealthInstance, "base_url" | "auth_type" | "encrypted_secret">,
): Promise<HealthCheckResult> {
const startedAt = Date.now();
try {
await withTimeout(
testPleskConnection({
baseUrl: instance.base_url,
authType: instance.auth_type,
encryptedSecret: instance.encrypted_secret,
}),
HEALTH_TIMEOUT_MS,
);
const latencyMs = Date.now() - startedAt;
const status: "ok" | "degraded" =
latencyMs > DEGRADED_LATENCY_THRESHOLD_MS ? "degraded" : "ok";
return {
status,
latency_ms: latencyMs,
error_message: null,
};
} catch (error) {
return {
status: "down",
latency_ms: Date.now() - startedAt,
error_message:
error instanceof Error
? error.message.slice(0, 1000)
: "health_check_failed",
};
}
}
export async function runInstanceHealthCheck(
supabase: SupabaseLike,
instance: HealthInstance,
input?: {
actorUserId?: string | null;
force?: boolean;
source?: "cron" | "manual";
},
): Promise<RunSingleHealthCheckResult> {
const lockToken = randomUUID();
const actorUserId = input?.actorUserId ?? null;
const source = input?.source ?? "cron";
const { data: lockAcquired, error: lockError } = await supabase.rpc(
"acquire_instance_health_lock",
{
p_instance_id: instance.id,
p_lock_token: lockToken,
p_ttl_seconds: LOCK_LEASE_SECONDS,
p_force: Boolean(input?.force),
},
);
if (lockError || !lockAcquired) {
return { checked: false };
}
const startedAt = Date.now();
const nowIso = new Date().toISOString();
const frequencyMinutes = Math.max(
instance.health_check_frequency_minutes ?? 30,
5,
);
const nextCheckAtIso = new Date(
Date.now() + frequencyMinutes * 60 * 1000,
).toISOString();
await insertActionLogBestEffort(supabase, {
agency_id: instance.agency_id,
actor_user_id: actorUserId,
target_type: "plesk_instance",
target_id: instance.id,
action: "health_check_started",
status: "pending",
metadata: {
instance_id: instance.id,
lock_token: lockToken,
source,
},
});
try {
const result = await checkPleskInstanceHealth(instance);
const updates: Record<string, unknown> = {
health_status: result.status,
health_last_checked_at: nowIso,
health_last_latency_ms: result.latency_ms,
health_last_error: result.error_message,
health_next_check_at: nextCheckAtIso,
health_lock_until: null,
health_lock_token: null,
};
if (result.status === "ok") {
updates.health_last_ok_at = nowIso;
}
const { error: updateError } = await supabase
.from("plesk_instances")
.update(updates)
.eq("id", instance.id)
.eq("health_lock_token", lockToken);
if (updateError) {
throw new Error(updateError.message);
}
const action =
result.status === "ok"
? "health_check_ok"
: result.status === "degraded"
? "health_check_degraded"
: "health_check_failed";
await insertActionLogBestEffort(supabase, {
agency_id: instance.agency_id,
actor_user_id: actorUserId,
target_type: "plesk_instance",
target_id: instance.id,
action,
status: result.status === "down" ? "failed" : "success",
error_message: result.error_message,
metadata: {
instance_id: instance.id,
lock_token: lockToken,
source,
latency_ms: result.latency_ms,
duration_ms: Date.now() - startedAt,
},
});
return {
checked: true,
status: result.status,
latency_ms: result.latency_ms,
error_message: result.error_message,
};
} catch (error) {
const errorMessage =
error instanceof Error
? error.message.slice(0, 1000)
: "health_check_failed";
await supabase
.from("plesk_instances")
.update({
health_status: "down",
health_last_checked_at: nowIso,
health_last_error: errorMessage,
health_next_check_at: nextCheckAtIso,
})
.eq("id", instance.id);
await insertActionLogBestEffort(supabase, {
agency_id: instance.agency_id,
actor_user_id: actorUserId,
target_type: "plesk_instance",
target_id: instance.id,
action: "health_check_failed",
status: "failed",
error_message: errorMessage,
metadata: {
instance_id: instance.id,
lock_token: lockToken,
source,
duration_ms: Date.now() - startedAt,
},
});
return {
checked: true,
status: "down",
latency_ms: Date.now() - startedAt,
error_message: errorMessage,
};
} finally {
await supabase.rpc("release_instance_health_lock", {
p_instance_id: instance.id,
p_lock_token: lockToken,
});
}
}
export async function runHealthCheckTick(
supabase: SupabaseLike,
): Promise<HealthTickSummary> {
const startedAt = Date.now();
const now = Date.now();
const { data: rows, error } = await supabase
.from("plesk_instances")
.select(
"id, agency_id, base_url, auth_type, encrypted_secret, health_enabled, health_check_frequency_minutes, health_next_check_at, health_lock_until",
)
.eq("health_enabled", true)
.order("health_next_check_at", { ascending: true, nullsFirst: true })
.limit(50);
if (error) {
throw new Error(error.message);
}
const eligible = ((rows ?? []) as HealthInstance[])
.filter((instance) => isDue(instance, now) && isUnlocked(instance, now))
.slice(0, MAX_INSTANCES_PER_TICK);
let checked = 0;
let ok = 0;
let degraded = 0;
let failed = 0;
const queue = [...eligible];
const workers = Array.from(
{ length: Math.min(MAX_CONCURRENCY, queue.length) },
async () => {
while (queue.length > 0) {
const instance = queue.shift();
if (!instance) break;
const result = await runInstanceHealthCheck(supabase, instance, {
actorUserId: null,
force: false,
source: "cron",
});
if (!result.checked || !result.status) continue;
checked += 1;
if (result.status === "ok") ok += 1;
else if (result.status === "degraded") degraded += 1;
else failed += 1;
}
},
);
await Promise.all(workers);
return {
checked,
ok,
degraded,
failed,
duration_ms: Date.now() - startedAt,
};
}

View File

@@ -89,6 +89,16 @@ export type Database = {
next_auto_sync_at: string | null; next_auto_sync_at: string | null;
auto_sync_lock_until: string | null; auto_sync_lock_until: string | null;
auto_sync_lock_token: string | null; auto_sync_lock_token: string | null;
health_enabled: boolean;
health_status: "ok" | "degraded" | "down" | "unknown";
health_last_checked_at: string | null;
health_last_ok_at: string | null;
health_last_error: string | null;
health_last_latency_ms: number | null;
health_check_frequency_minutes: number;
health_next_check_at: string | null;
health_lock_until: string | null;
health_lock_token: string | null;
created_at: string; created_at: string;
}; };
Insert: { Insert: {
@@ -113,6 +123,16 @@ export type Database = {
next_auto_sync_at?: string | null; next_auto_sync_at?: string | null;
auto_sync_lock_until?: string | null; auto_sync_lock_until?: string | null;
auto_sync_lock_token?: string | null; auto_sync_lock_token?: string | null;
health_enabled?: boolean;
health_status?: "ok" | "degraded" | "down" | "unknown";
health_last_checked_at?: string | null;
health_last_ok_at?: string | null;
health_last_error?: string | null;
health_last_latency_ms?: number | null;
health_check_frequency_minutes?: number;
health_next_check_at?: string | null;
health_lock_until?: string | null;
health_lock_token?: string | null;
created_at?: string; created_at?: string;
}; };
Update: { Update: {
@@ -137,6 +157,16 @@ export type Database = {
next_auto_sync_at?: string | null; next_auto_sync_at?: string | null;
auto_sync_lock_until?: string | null; auto_sync_lock_until?: string | null;
auto_sync_lock_token?: string | null; auto_sync_lock_token?: string | null;
health_enabled?: boolean;
health_status?: "ok" | "degraded" | "down" | "unknown";
health_last_checked_at?: string | null;
health_last_ok_at?: string | null;
health_last_error?: string | null;
health_last_latency_ms?: number | null;
health_check_frequency_minutes?: number;
health_next_check_at?: string | null;
health_lock_until?: string | null;
health_lock_token?: string | null;
created_at?: string; created_at?: string;
}; };
}; };

View File

@@ -0,0 +1,78 @@
-- CL8: instance health monitoring fields + lock helpers
alter table public.plesk_instances
add column if not exists health_enabled boolean not null default true,
add column if not exists health_status text not null default 'unknown',
add column if not exists health_last_checked_at timestamptz,
add column if not exists health_last_ok_at timestamptz,
add column if not exists health_last_error text,
add column if not exists health_last_latency_ms integer,
add column if not exists health_check_frequency_minutes integer not null default 30,
add column if not exists health_next_check_at timestamptz,
add column if not exists health_lock_until timestamptz,
add column if not exists health_lock_token text;
alter table public.plesk_instances
drop constraint if exists plesk_instances_health_status_check;
alter table public.plesk_instances
add constraint plesk_instances_health_status_check
check (health_status in ('ok', 'degraded', 'down', 'unknown'));
alter table public.plesk_instances
drop constraint if exists plesk_instances_health_check_frequency_minutes_check;
alter table public.plesk_instances
add constraint plesk_instances_health_check_frequency_minutes_check
check (health_check_frequency_minutes between 5 and 10080);
create index if not exists idx_plesk_instances_health_due
on public.plesk_instances (health_enabled, health_next_check_at);
create or replace function public.acquire_instance_health_lock(
p_instance_id uuid,
p_lock_token text,
p_ttl_seconds integer default 300,
p_force boolean default false
)
returns boolean
language plpgsql
security definer
set search_path = public
as $$
declare
v_acquired boolean := false;
begin
update public.plesk_instances
set
health_lock_until = now() + make_interval(secs => p_ttl_seconds),
health_lock_token = p_lock_token
where id = p_instance_id
and health_enabled = true
and (
p_force = true
or health_next_check_at is null
or health_next_check_at <= now()
)
and (health_lock_until is null or health_lock_until < now());
get diagnostics v_acquired = row_count;
return v_acquired;
end;
$$;
create or replace function public.release_instance_health_lock(
p_instance_id uuid,
p_lock_token text
)
returns void
language sql
security definer
set search_path = public
as $$
update public.plesk_instances
set health_lock_until = null,
health_lock_token = null
where id = p_instance_id
and health_lock_token = p_lock_token;
$$;