Commit f33b060cafc96d7c3229744315c230a3b8d1f631 | the-knife

apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx+59-15

...	...	@@ -4,7 +4,10 @@ import type {
4	4	ScheduledTask,
5	5	ScheduledTaskUpsertInput,
6	6	} from "@t3tools/contracts";
7		import { resolveEnvironmentMachineKind } from "@t3tools/contracts";
	7	import {
	8	MAX_WEBHOOK_DELIVERY_AGE_MINUTES,
	9	resolveEnvironmentMachineKind,
	10	} from "@t3tools/contracts";
8	11	import type { MenuAction } from "@react-native-menu/menu";
9	12	import { DateTimePicker } from "@expo/ui/community/datetime-picker";
10	13	import {
...	...	@@ -12,6 +15,10 @@ import {
12	15	squashAtomCommandFailure,
13	16	type AtomCommandResult,
14	17	} from "@t3tools/client-runtime/state/runtime";
	18	import {
	19	DEFAULT_WEBHOOK_PROMPT,
	20	parseMaxDeliveryAge,
	21	} from "@t3tools/client-runtime/scheduled-task-webhook";
15	22	import {
16	23	useCallback,
17	24	useEffect,
...	...	@@ -59,7 +66,6 @@ import { SettingsSection } from "./components/SettingsSection";
59	66	import { useSettingsEnvironmentFilter, type SettingsTarget } from "./settings-environment-filter";
60	67	import {
61	68	editDraft,
62		DEFAULT_WEBHOOK_PROMPT,
63	69	scheduledTaskDefaultModel,
64	70	scheduleFromDraft,
65	71	type ScheduledTaskDraft as Draft,
...	...	@@ -132,7 +138,7 @@ function FormField(props: {
132	138	readonly label: string;
133	139	readonly value: string;
134	140	readonly onChange: (value: string) => void;
135		readonly keyboardType?: "decimal-pad";
	141	readonly keyboardType?: "decimal-pad" \| "number-pad";
136	142	readonly disabled?: boolean;
137	143	readonly placeholder?: string;
138	144	readonly borderTop?: boolean;
...	...	@@ -611,6 +617,16 @@ function TaskForm({
611	617	? { ...draft.schedule, signature: liveTask.schedule.signature }
612	618	: draft.schedule,
613	619	);
	620	if (
	621	draft.schedule.mode === "webhook" &&
	622	parseMaxDeliveryAge(draft.schedule.maxDeliveryAgeMinutes) === undefined
	623	) {
	624	Alert.alert(
	625	"Invalid age limit",
	626	`Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`,
	627	);
	628	return;
	629	}
614	630	if (
615	631	!draft.title.trim() \|\|
616	632	!draft.prompt.trim() \|\|
...	...	@@ -823,7 +839,7 @@ function TaskForm({
823	839	options={[
824	840	{ value: "fixed_time", label: "At a time" },
825	841	{ value: "interval", label: "Interval" },
826		{ value: "webhook", label: "Webhook" },
	842	{ value: "webhook", label: "On webhook" },
827	843	]}
828	844	selected={draft.schedule.mode}
829	845	onSelect={(mode) => {
...	...	@@ -910,13 +926,31 @@ function TaskForm({
910	926	/>
911	927	</>
912	928	) : draft.schedule.mode === "webhook" ? (
913		<WebhookScheduleDetails
914		environmentId={environmentId}
915		task={
916		tasks.data?.tasks.find((task) => task.id === draft.task?.id) ?? draft.task ?? null
917		}
918		signatureConfigured={draft.schedule.signature !== null}
919		/>
	929	<>
	930	<WebhookScheduleDetails
	931	environmentId={environmentId}
	932	// The live row, so a rotated URL shows up without reopening the form.
	933	// Once the list has loaded, a missing task is gone; don't keep showing its URL.
	934	task={
	935	tasks.data
	936	? (tasks.data.tasks.find((task) => task.id === draft.task?.id) ?? null)
	937	: draft.task
	938	}
	939	signatureConfigured={draft.schedule.signature !== null}
	940	disabled={saving \|\| environmentUnavailable}
	941	/>
	942	<FormField
	943	label="Skip requests older than (minutes)"
	944	value={draft.schedule.maxDeliveryAgeMinutes}
	945	placeholder="Run every request"
	946	keyboardType="number-pad"
	947	disabled={saving}
	948	borderTop
	949	onChange={(maxDeliveryAgeMinutes) =>
	950	setDraft({ ...draft, schedule: { ...draft.schedule, maxDeliveryAgeMinutes } })
	951	}
	952	/>
	953	</>
920	954	) : (
921	955	<>
922	956	<FormField
...	...	@@ -976,11 +1010,14 @@ function WebhookScheduleDetails({
976	1010	environmentId,
977	1011	task,
978	1012	signatureConfigured,
	1013	disabled,
979	1014	}: {
980	1015	readonly environmentId: EnvironmentId;
981	1016	readonly task: ScheduledTask \| null;
982	1017	readonly signatureConfigured: boolean;
	1018	readonly disabled: boolean;
983	1019	}) {
	1020	const [rotating, setRotating] = useState(false);
984	1021	const rotate = useAtomCommand(serverEnvironment.rotateScheduledTaskWebhookToken, {
985	1022	label: "scheduled task rotate webhook token",
986	1023	reportFailure: false,
...	...	@@ -1023,27 +1060,34 @@ function WebhookScheduleDetails({
1023	1060	) : null}
1024	1061	<Pressable
1025	1062	accessibilityRole="button"
	1063	accessibilityState={{ disabled: disabled \|\| rotating }}
	1064	disabled={disabled \|\| rotating}
1026	1065	onPress={() =>
1027	1066	Alert.alert("Rotate URL?", "The current URL stops working immediately.", [
1028	1067	{ text: "Cancel", style: "cancel" },
1029	1068	{
1030	1069	text: "Rotate",
1031	1070	style: "destructive",
1032		onPress: () =>
	1071	onPress: () => {
	1072	setRotating(true);
1033	1073	void rotate({ environmentId, input: { id: task.id } }).then((result) => {
	1074	setRotating(false);
1034	1075	if (result._tag === "Failure" && !isAtomCommandInterrupted(result)) {
1035	1076	Alert.alert(
1036	1077	"Could not rotate URL",
1037	1078	String(squashAtomCommandFailure(result)),
1038	1079	);
1039	1080	}
1040		}),
	1081	});
	1082	},
1041	1083	},
1042	1084	])
1043	1085	}
1044		className="min-h-11 justify-center active:opacity-70"
	1086	className="min-h-11 justify-center active:opacity-70 disabled:opacity-50"
1045	1087	>
1046		<Text className="text-base text-danger-foreground">Rotate URL</Text>
	1088	<Text className="text-base text-danger-foreground">
	1089	{rotating ? "Rotating…" : "Rotate URL"}
	1090	</Text>
1047	1091	</Pressable>
1048	1092	</>
1049	1093	)}

apps/mobile/src/features/settings/scheduledTaskDraft.test.ts+20-2

...	...	@@ -32,7 +32,25 @@ describe("scheduleDraftForTask", () => {
32	32	it("round-trips a webhook schedule without a signature", () => {
33	33	const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } });
34	34	expect(draft.mode).toBe("webhook");
35		expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null });
	35	expect(scheduleFromDraft(draft)).toEqual({
	36	type: "webhook",
	37	signature: null,
	38	maxDeliveryAgeMinutes: null,
	39	});
	40	});
	41
	42	it("round-trips a webhook max age and treats blank input as no limit", () => {
	43	const draft = scheduleDraftForTask({
	44	schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 45 },
	45	});
	46	expect(draft.maxDeliveryAgeMinutes).toBe("45");
	47	expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 45 });
	48	// An invalid limit is an invalid schedule, never a silently removed one.
	49	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "1.5" })).toBeNull();
	50	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "0" })).toBeNull();
	51	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({
	52	maxDeliveryAgeMinutes: null,
	53	});
36	54	});
37	55
38	56	it("keeps a webhook signature on save without sending a secret", () => {
...	...	@@ -44,7 +62,7 @@ describe("scheduleDraftForTask", () => {
44	62	const saved = scheduleFromDraft(
45	63	scheduleDraftForTask({ schedule: { type: "webhook", signature } }),
46	64	);
47		expect(saved).toEqual({ type: "webhook", signature });
	65	expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null });
48	66	expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false);
49	67	});
50	68	});

apps/mobile/src/features/settings/scheduledTaskDraft.ts+17-4

...	...	@@ -9,6 +9,7 @@ import type {
9	9	} from "@t3tools/contracts";
10	10
11	11	import { DEFAULT_SERVER_SETTINGS } from "@t3tools/contracts";
	12	import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook";
12	13	import {
13	14	resolveProjectSettings,
14	15	type LegacyProjectSettingsFields,
...	...	@@ -44,6 +45,8 @@ export type ScheduleDraft = {
44	45	readonly intervalMinutes: string;
45	46	/** A webhook signature check configured elsewhere; mobile keeps it but does not edit it. */
46	47	readonly signature: ScheduledTaskWebhookSignature \| null;
	48	/** Minutes as typed; empty runs every held request regardless of age. */
	49	readonly maxDeliveryAgeMinutes: string;
47	50	};
48	51
49	52	export const DEFAULT_SCHEDULE: ScheduleDraft = {
...	...	@@ -52,11 +55,9 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = {
52	55	weekdays: [1, 2, 3, 4, 5],
53	56	intervalMinutes: "15",
54	57	signature: null,
	58	maxDeliveryAgeMinutes: "",
55	59	};
56	60
57		/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */
58		export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}";
59
60	61	export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): ScheduleDraft {
61	62	switch (task.schedule.type) {
62	63	case "fixed_time":
...	...	@@ -74,12 +75,22 @@ export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): Sch
74	75	intervalMinutes: String(Math.max(1, task.schedule.everyMs / 60_000)),
75	76	};
76	77	case "webhook":
77		return { ...DEFAULT_SCHEDULE, mode: "webhook", signature: task.schedule.signature };
	78	return {
	79	...DEFAULT_SCHEDULE,
	80	mode: "webhook",
	81	signature: task.schedule.signature,
	82	maxDeliveryAgeMinutes:
	83	task.schedule.maxDeliveryAgeMinutes == null
	84	? ""
	85	: String(task.schedule.maxDeliveryAgeMinutes),
	86	};
78	87	}
79	88	}
80	89
81	90	export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSchedule \| null {
82	91	if (draft.mode === "webhook") {
	92	const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes);
	93	if (maxDeliveryAgeMinutes === undefined) return null;
83	94	// No secret is sent, so the server keeps the stored one.
84	95	return {
85	96	type: "webhook",
...	...	@@ -91,6 +102,7 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
91	102	encoding: draft.signature.encoding,
92	103	prefix: draft.signature.prefix,
93	104	},
	105	maxDeliveryAgeMinutes,
94	106	};
95	107	}
96	108	if (draft.mode === "interval") {
...	...	@@ -145,6 +157,7 @@ function draftSignature(draft: ScheduledTaskDraft): string {
145	157	draft.schedule.timeOfDay,
146	158	[...draft.schedule.weekdays].sort((a, b) => a - b),
147	159	draft.schedule.intervalMinutes,
	160	draft.schedule.maxDeliveryAgeMinutes,
148	161	draft.workspace,
149	162	draft.baseRef,
150	163	draft.checkoutPath,

apps/server/src/cloud/ManagedEndpointRuntime.test.ts+30

...	...	@@ -209,6 +209,36 @@ describe("CloudManagedEndpointRuntime", () => {
209	209	}),
210	210	);
211	211
	212	it.effect("signals each registered tunnel connection", () =>
	213	Effect.gen(function* () {
	214	const output = yield* Queue.unbounded<Uint8Array>();
	215	const spawner = ChildProcessSpawner.make(() =>
	216	Effect.gen(function* () {
	217	const handle = makeHandle({
	218	pid: 700,
	219	onKill: () => {},
	220	output: Stream.fromQueue(output),
	221	});
	222	yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
	223	return handle;
	224	}),
	225	);
	226	const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
	227	yield* runtime.applyConfig({
	228	providerKind: "cloudflare_tunnel",
	229	connectorToken: "token",
	230	tunnelId: "tunnel-1",
	231	});
	232	yield* Queue.offer(
	233	output,
	234	new TextEncoder().encode(
	235	"2026-10-04T06:30:43Z INF Registered tunnel connection connIndex=0 event=0\n",
	236	),
	237	);
	238	expect(Option.isSome(yield* Stream.runHead(runtime.tunnelConnected))).toBe(true);
	239	}),
	240	);
	241
212	242	it.effect("recovers a rejected tunnel without waiting for the connector to exit", () =>
213	243	Effect.gen(function* () {
214	244	const output = yield* Queue.unbounded<Uint8Array>();

apps/server/src/cloud/ManagedEndpointRuntime.ts+8-1

...	...	@@ -47,6 +47,8 @@ export class CloudManagedEndpointRuntime extends Context.Service<
47	47	) => Effect.Effect<CloudManagedEndpointRuntimeStatus>;
48	48	readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
49	49	readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
	50	/** Emits when the connector registers a tunnel connection, i.e. the relay can reach us again. */
	51	readonly tunnelConnected: Stream.Stream<void>;
50	52	readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
51	53	}
52	54	>()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {}
...	...	@@ -134,6 +136,7 @@ export const make = Effect.gen(function* () {
134	136	const activeRef = yield* Ref.make<ActiveConnector \| null>(null);
135	137	const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig \| null>(null);
136	138	const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1);
	139	const tunnelConnections = yield* Queue.sliding<void>(1);
137	140	const reconcileSemaphore = yield* Semaphore.make(1);
138	141	const restartDelayRef = yield* Ref.make(0);
139	142	const linkStateSemaphore = yield* Semaphore.make(1);
...	...	@@ -236,7 +239,10 @@ export const make = Effect.gen(function* () {
236	239	switch (classifyRelayClientOutput(line)) {
237	240	case "connected":
238	241	rejectedRegistrations = 0;
239		return Effect.logInfo("Relay client tunnel connection registered", attributes);
	242	return Effect.logInfo("Relay client tunnel connection registered", attributes).pipe(
	243	Effect.andThen(Queue.offer(tunnelConnections, undefined)),
	244	Effect.asVoid,
	245	);
240	246	case "warning":
241	247	if (isRejectedRelayClientTunnelOutput(line)) {
242	248	rejectedRegistrations += 1;
...	...	@@ -412,6 +418,7 @@ export const make = Effect.gen(function* () {
412	418	applyConfig,
413	419	recoveryRequests: Stream.fromQueue(recoveryRequests),
414	420	requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid),
	421	tunnelConnected: Stream.fromQueue(tunnelConnections),
415	422	withLinkStateLock: linkStateSemaphore.withPermits(1),
416	423	});
417	424

apps/server/src/cloud/config.ts+31

...	...	@@ -16,6 +16,7 @@ export const RELAY_URL_SECRET = "cloud-relay-url";
16	16	export const RELAY_ISSUER_SECRET = "cloud-relay-issuer";
17	17	export const RELAY_ENVIRONMENT_CREDENTIAL_SECRET = "cloud-relay-environment-credential";
18	18	export const PUBLISH_AGENT_ACTIVITY_SECRET = "cloud-publish-agent-activity";
	19	export const HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET = "cloud-hold-webhooks-while-offline";
19	20
20	21	export const encodeEndpointRuntimeConfigJson = Schema.encodeEffect(
21	22	Schema.fromJsonString(RelayManagedEndpointRuntimeConfig),
...	...	@@ -73,3 +74,33 @@ export const readAgentActivityPublishingActive = (
73	74	environmentCredential !== ""
74	75	);
75	76	}).pipe(Effect.orElseSucceed(() => false));
	77
	78	const readSecretString = (
	79	secrets: ServerSecretStore.ServerSecretStore["Service"],
	80	name: string,
	81	): Effect.Effect<string \| null> =>
	82	secrets.get(name).pipe(
	83	Effect.map((bytes) =>
	84	Option.isSome(bytes) && bytes.value.length > 0 ? new TextDecoder().decode(bytes.value) : null,
	85	),
	86	Effect.orElseSucceed(() => null),
	87	);
	88
	89	/** The relay URL and environment credential, or null when not linked to T3 Connect. */
	90	export const readRelayConnection = (secrets: ServerSecretStore.ServerSecretStore["Service"]) =>
	91	Effect.all([
	92	readSecretString(secrets, RELAY_URL_SECRET),
	93	readSecretString(secrets, RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
	94	]).pipe(
	95	Effect.map(([url, environmentCredential]) =>
	96	url && environmentCredential ? { url, environmentCredential } : null,
	97	),
	98	);
	99
	100	/** Whether this environment opted in to T3 Connect holding webhooks while it is offline. */
	101	export const readHoldWebhooksWhileOffline = (
	102	secrets: ServerSecretStore.ServerSecretStore["Service"],
	103	) =>
	104	readSecretString(secrets, HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET).pipe(
	105	Effect.map((value) => value === "true"),
	106	);

apps/server/src/cloud/http.test.ts+2

...	...	@@ -249,6 +249,7 @@ describe("reconcileDesiredCloudLink", () => {
249	249	applyConfig: unusedSecretStoreOperation,
250	250	recoveryRequests: Stream.empty,
251	251	requestRecovery: () => Effect.void,
	252	tunnelConnected: Stream.empty,
252	253	withLinkStateLock: (effect) => effect,
253	254	} satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]),
254	255	),
...	...	@@ -410,6 +411,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
410	411	}),
411	412	recoveryRequests: Stream.empty,
412	413	requestRecovery: () => Effect.void,
	414	tunnelConnected: Stream.empty,
413	415	withLinkStateLock: (effect) => effect,
414	416	}),
415	417	),

apps/server/src/cloud/http.ts+71-16

...	...	@@ -8,6 +8,7 @@ import {
8	8	EnvironmentCloudRelayConfigResult,
9	9	EnvironmentHttpApi,
10	10	EnvironmentHttpBadRequestError,
	11	type EnvironmentCloudPreferencesRequest,
11	12	EnvironmentHttpConflictError,
12	13	EnvironmentHttpInternalServerError,
13	14	EnvironmentHttpUnauthorizedError,
...	...	@@ -69,6 +70,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
69	70	import { requireEnvironmentScope } from "../auth/http.ts";
70	71	import * as ServerConfig from "../config.ts";
71	72	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
	73	import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts";
72	74	import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
73	75	import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
74	76	import {
...	...	@@ -86,6 +88,9 @@ import {
86	88	encodeEndpointRuntimeConfigJson,
87	89	encodeConfirmedOriginJson,
88	90	PUBLISH_AGENT_ACTIVITY_SECRET,
	91	HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
	92	readHoldWebhooksWhileOffline,
	93	readRelayConnection,
89	94	RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
90	95	RELAY_ISSUER_SECRET,
91	96	RELAY_URL_SECRET,
...	...	@@ -1277,17 +1282,24 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
1277	1282	const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* (
1278	1283	dependencies: CloudHttpDependencies,
1279	1284	) {
1280		const [cloudUserId, relayUrl, relayIssuer, endpointRuntimeConfig, publishAgentActivity] =
1281		yield* Effect.all(
1282		[
1283		dependencies.secrets.get(CLOUD_LINKED_USER_ID),
1284		dependencies.secrets.get(RELAY_URL_SECRET),
1285		dependencies.secrets.get(RELAY_ISSUER_SECRET),
1286		dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1287		dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
1288		],
1289		{ concurrency: 5 },
1290		);
	1285	const [
	1286	cloudUserId,
	1287	relayUrl,
	1288	relayIssuer,
	1289	endpointRuntimeConfig,
	1290	publishAgentActivity,
	1291	holdWebhooks,
	1292	] = yield* Effect.all(
	1293	[
	1294	dependencies.secrets.get(CLOUD_LINKED_USER_ID),
	1295	dependencies.secrets.get(RELAY_URL_SECRET),
	1296	dependencies.secrets.get(RELAY_ISSUER_SECRET),
	1297	dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
	1298	dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
	1299	dependencies.secrets.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
	1300	],
	1301	{ concurrency: 6 },
	1302	);
1291	1303	return {
1292	1304	linked: Option.isSome(cloudUserId),
1293	1305	cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null,
...	...	@@ -1299,6 +1311,8 @@ const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function
1299	1311	publishAgentActivity: Option.isSome(publishAgentActivity)
1300	1312	? bytesToString(publishAgentActivity.value) === "true"
1301	1313	: false,
	1314	holdWebhooksWhileOffline:
	1315	Option.isSome(holdWebhooks) && bytesToString(holdWebhooks.value) === "true",
1302	1316	} satisfies EnvironmentCloudLinkStateResult;
1303	1317	});
1304	1318
...	...	@@ -1329,8 +1343,9 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
1329	1343	dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1330	1344	dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
1331	1345	dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
	1346	dependencies.secrets.remove(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
1332	1347	],
1333		{ concurrency: 8 },
	1348	{ concurrency: 9 },
1334	1349	);
1335	1350	yield* setCliDesiredCloudLink(false);
1336	1351	return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
...	...	@@ -1343,12 +1358,52 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
1343	1358	),
1344	1359	);
1345	1360
	1361	const pushHoldWebhooksWhileOffline = Effect.fn("environment.cloud.pushHoldWebhooksWhileOffline")(
	1362	function* (dependencies: CloudHttpDependencies, holdWebhooksWhileOffline: boolean) {
	1363	const connection = yield* readRelayConnection(dependencies.secrets);
	1364	if (connection === null) {
	1365	return yield* new EnvironmentHttpBadRequestError({
	1366	message: "Link this environment to T3 Connect first.",
	1367	});
	1368	}
	1369	const environmentId = yield* dependencies.environment.getEnvironmentId;
	1370	const client = yield* makeRelayEnvironmentClient(connection);
	1371	yield* client.server
	1372	.updateLinkPreferences({
	1373	params: { environmentId },
	1374	payload: { holdWebhooksWhileOffline },
	1375	})
	1376	.pipe(
	1377	Effect.timeout("10 seconds"),
	1378	Effect.catch(
	1379	failEnvironmentCloudInternalError("Could not update T3 Connect webhook settings."),
	1380	),
	1381	);
	1382	},
	1383	);
	1384
1346	1385	const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")(
1347		function* (
1348		dependencies: CloudHttpDependencies,
1349		payload: { readonly publishAgentActivity: boolean },
1350		) {
	1386	function* (dependencies: CloudHttpDependencies, payload: EnvironmentCloudPreferencesRequest) {
1351	1387	yield* requireEnvironmentScope(AuthRelayWriteScope);
	1388	if (payload.holdWebhooksWhileOffline !== undefined) {
	1389	// The relay decides whether to hold a request, so it is told first; the
	1390	// local copy is only saved once the relay has the same value, and the
	1391	// relay is put back if that save fails.
	1392	const previous = yield* readHoldWebhooksWhileOffline(dependencies.secrets);
	1393	yield* pushHoldWebhooksWhileOffline(dependencies, payload.holdWebhooksWhileOffline);
	1394	yield* dependencies.secrets
	1395	.set(
	1396	HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
	1397	stringToBytes(String(payload.holdWebhooksWhileOffline)),
	1398	)
	1399	.pipe(
	1400	Effect.tapError(() =>
	1401	previous === payload.holdWebhooksWhileOffline
	1402	? Effect.void
	1403	: pushHoldWebhooksWhileOffline(dependencies, previous).pipe(Effect.ignore),
	1404	),
	1405	);
	1406	}
1352	1407	yield* dependencies.secrets.set(
1353	1408	PUBLISH_AGENT_ACTIVITY_SECRET,
1354	1409	stringToBytes(String(payload.publishAgentActivity)),

apps/server/src/observability/Metrics.ts+24

...	...	@@ -70,6 +70,30 @@ export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total"
70	70	description: "Total terminal restart requests handled.",
71	71	});
72	72
	73	/**
	74	* One per webhook request that reached a task, by `outcome` (accepted,
	75	* not_found, rejected_signature, disabled, rate_limited, queue_full, expired,
	76	* prompt_too_long, error) and `source` (relay or direct).
	77	*/
	78	export const webhookDeliveriesTotal = Metric.counter("t3_webhook_deliveries_total", {
	79	description: "Webhook requests handled, by outcome and source.",
	80	});
	81
	82	export const webhookDeliveryDuration = Metric.timer("t3_webhook_delivery_duration", {
	83	description: "Time to verify, log, and enqueue one webhook request.",
	84	});
	85
	86	/** How long a relay-held request waited before this environment got it. */
	87	export const webhookHeldDelay = Metric.timer("t3_webhook_held_delay", {
	88	description:
	89	"Time between the relay receiving a webhook request and the environment handling it.",
	90	});
	91
	92	/** Runs started by webhook deliveries, by `outcome` (started, skipped, failed). */
	93	export const webhookRunsTotal = Metric.counter("t3_webhook_runs_total", {
	94	description: "Runs started from webhook deliveries, by outcome.",
	95	});
	96
73	97	export const metricAttributes = (
74	98	attributes: Readonly<Record<string, unknown>>,
75	99	): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes));

apps/server/src/persistence/Migrations.ts+2

...	...	@@ -71,6 +71,7 @@ import Migration0054 from "./Migrations/054_ProjectionThreadsAutoSettleDisabledA
71	71	import Migration0055 from "./Migrations/055_OrchestrationV2.ts";
72	72	import Migration0056 from "./Migrations/056_RemoveRedundantProjectionIndexes.ts";
73	73	import Migration0057 from "./Migrations/057_ScheduledTaskWebhooks.ts";
	74	import Migration0058 from "./Migrations/058_WebhookRelayDeliveries.ts";
74	75
75	76	/**
76	77	* Migration loader with all migrations defined inline.
...	...	@@ -142,6 +143,7 @@ export const migrationEntries = [
142	143	[55, "OrchestrationV2", Migration0055],
143	144	[56, "RemoveRedundantProjectionIndexes", Migration0056],
144	145	[57, "ScheduledTaskWebhooks", Migration0057],
	146	[58, "WebhookRelayDeliveries", Migration0058],
145	147	] as const;
146	148
147	149	export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);

apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts+3-1

...	...	@@ -13,7 +13,7 @@ layer("055_OrchestrationV2", (it) => {
13	13	Effect.sync(() => {
14	14	assert.deepStrictEqual(
15	15	migrationEntries.map(([id]) => id),
16		Array.from({ length: 57 }, (_, index) => index + 1),
	16	Array.from({ length: 58 }, (_, index) => index + 1),
17	17	);
18	18	}),
19	19	);
...	...	@@ -29,6 +29,7 @@ layer("055_OrchestrationV2", (it) => {
29	29	[55, "OrchestrationV2"],
30	30	[56, "RemoveRedundantProjectionIndexes"],
31	31	[57, "ScheduledTaskWebhooks"],
	32	[58, "WebhookRelayDeliveries"],
32	33	]);
33	34	assert.deepStrictEqual(yield* runMigrations(), []);
34	35
...	...	@@ -52,6 +53,7 @@ layer("055_OrchestrationV2", (it) => {
52	53	{ migration_id: 55, name: "OrchestrationV2" },
53	54	{ migration_id: 56, name: "RemoveRedundantProjectionIndexes" },
54	55	{ migration_id: 57, name: "ScheduledTaskWebhooks" },
	56	{ migration_id: 58, name: "WebhookRelayDeliveries" },
55	57	]);
56	58
57	59	const tables = yield* sql<{ readonly name: string }>`

apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts created+22

...	...	@@ -0,0 +1,22 @@
	1	import * as Effect from "effect/Effect";
	2	import * as SqlClient from "effect/sql/SqlClient";
	3
	4	export default Effect.gen(function* () {
	5	const sql = yield* SqlClient.SqlClient;
	6
	7	// Relay delivery ids already handled, kept longer than the relay holds a
	8	// request so a replay can never run twice. The delivery log keeps only the
	9	// newest 50 rows per task, which is too short for that.
	10	yield* sql`
	11	CREATE TABLE IF NOT EXISTS scheduled_task_webhook_relay_deliveries (
	12	relay_delivery_id TEXT PRIMARY KEY,
	13	task_id TEXT NOT NULL,
	14	seen_at TEXT NOT NULL
	15	)
	16	`;
	17
	18	yield* sql`
	19	CREATE INDEX IF NOT EXISTS idx_scheduled_task_webhook_relay_deliveries_seen
	20	ON scheduled_task_webhook_relay_deliveries(seen_at)
	21	`;
	22	});

apps/server/src/persistence/reconcileV2PreviewMigration.test.ts+2

...	...	@@ -38,6 +38,7 @@ describe("V2 preview upgrade", () => {
38	38	[54, "ProjectionThreadsAutoSettleDisabledAt"],
39	39	[56, "RemoveRedundantProjectionIndexes"],
40	40	[57, "ScheduledTaskWebhooks"],
	41	[58, "WebhookRelayDeliveries"],
41	42	]);
42	43	assert.deepStrictEqual(yield* runMigrations(), []);
43	44	assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
...	...	@@ -118,6 +119,7 @@ describe("V2 preview upgrade", () => {
118	119	[54, "ProjectionThreadsAutoSettleDisabledAt"],
119	120	[56, "RemoveRedundantProjectionIndexes"],
120	121	[57, "ScheduledTaskWebhooks"],
	122	[58, "WebhookRelayDeliveries"],
121	123	]);
122	124	}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
123	125	);

apps/server/src/relay/HeldHooksWaker.ts created+52

...	...	@@ -0,0 +1,52 @@
	1	import * as Effect from "effect/Effect";
	2	import * as Layer from "effect/Layer";
	3	import * as Schedule from "effect/Schedule";
	4	import * as Stream from "effect/Stream";
	5
	6	import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
	7	import * as CloudManagedEndpointRuntime from "../cloud/ManagedEndpointRuntime.ts";
	8	import { readHoldWebhooksWhileOffline, readRelayConnection } from "../cloud/config.ts";
	9	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
	10	import { makeRelayEnvironmentClient } from "./relayEnvironmentClient.ts";
	11
	12	/**
	13	* Tells T3 Connect this environment is reachable again, so the relay delivers
	14	* the webhook requests it held while we were offline now rather than at its
	15	* next backoff step. Nothing happens unless the environment opted in.
	16	*/
	17	const wakeHeldHooks = Effect.fn("HeldHooksWaker.wake")(function* () {
	18	const secrets = yield* ServerSecretStore.ServerSecretStore;
	19	if (!(yield* readHoldWebhooksWhileOffline(secrets))) return false;
	20	const connection = yield* readRelayConnection(secrets);
	21	if (connection === null) return false;
	22	const environmentId = yield* (yield* ServerEnvironment.ServerEnvironment).getEnvironmentId;
	23	const client = yield* makeRelayEnvironmentClient(connection);
	24	const { pending } = yield* client.server.wakeHeldHooks({ params: { environmentId } });
	25	yield* Effect.annotateCurrentSpan({ "relay.inbox.pending": pending });
	26	return pending;
	27	});
	28
	29	/** Wakes held webhooks each time the managed tunnel connects. */
	30	export const layer = Layer.effectDiscard(
	31	Effect.gen(function* () {
	32	const runtime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime;
	33	const wake = wakeHeldHooks().pipe(
	34	Effect.timeout("10 seconds"),
	35	// The relay retries on its own schedule too, so a few tries are enough.
	36	Effect.retry({ schedule: Schedule.exponential("2 seconds"), times: 3 }),
	37	Effect.tap((pending) =>
	38	pending ? Effect.logInfo("T3 Connect is delivering held webhook requests") : Effect.void,
	39	),
	40	Effect.catchCause((cause) =>
	41	Effect.logWarning("Could not ask T3 Connect to deliver held webhook requests", { cause }),
	42	),
	43	);
	44	yield* runtime.tunnelConnected.pipe(
	45	// cloudflared registers several connections per (re)connect within a
	46	// few seconds; they are one wake.
	47	Stream.debounce("3 seconds"),
	48	Stream.runForEach(() => wake),
	49	Effect.forkScoped,
	50	);
	51	}),
	52	);

apps/server/src/relay/relayEnvironmentClient.ts created+21

...	...	@@ -0,0 +1,21 @@
	1	import { RelayApi } from "@t3tools/contracts/relay";
	2	import * as Effect from "effect/Effect";
	3	import * as FetchHttpClient from "effect/http/FetchHttpClient";
	4	import * as HttpClient from "effect/http/HttpClient";
	5	import * as HttpClientRequest from "effect/http/HttpClientRequest";
	6	import * as HttpApiClient from "effect/http-api/HttpApiClient";
	7
	8	/**
	9	* A typed RelayApi client that authenticates as this environment, for the
	10	* environment-credential endpoints (link preferences, held webhooks).
	11	*/
	12	export const makeRelayEnvironmentClient = (connection: {
	13	readonly url: string;
	14	readonly environmentCredential: string;
	15	}) =>
	16	HttpApiClient.make(RelayApi, {
	17	baseUrl: connection.url,
	18	transformClient: HttpClient.mapRequest(
	19	HttpClientRequest.setHeader("authorization", `Bearer ${connection.environmentCredential}`),
	20	),
	21	}).pipe(Effect.provide(FetchHttpClient.layer));

apps/server/src/scheduledTasks/ScheduledTaskService.ts+345-125

...	...	@@ -1,6 +1,7 @@
1	1	import {
2	2	CommandId,
3	3	MessageId,
	4	PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
4	5	ScheduledTask,
5	6	ScheduledTaskError,
6	7	ScheduledTaskId,
...	...	@@ -27,6 +28,8 @@ import * as Crypto from "effect/Crypto";
27	28	import * as Data from "effect/Data";
28	29	import * as DateTime from "effect/DateTime";
29	30	import * as Effect from "effect/Effect";
	31	import * as Duration from "effect/Duration";
	32	import * as Metric from "effect/Metric";
30	33	import * as Layer from "effect/Layer";
31	34	import * as Option from "effect/Option";
32	35	import * as PubSub from "effect/PubSub";
...	...	@@ -38,10 +41,16 @@ import * as Stream from "effect/Stream";
38	41	import * as SqlClient from "effect/sql/SqlClient";
39	42
40	43	import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
	44	import * as Metrics from "../observability/Metrics.ts";
41	45	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
42	46	import * as Scheduler from "../scheduling/Scheduler.ts";
43	47	import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";
44		import { renderWebhookPrompt, type WebhookRequest } from "./webhookTemplate.ts";
	48	import {
	49	redactHeaders,
	50	redactQuery,
	51	renderWebhookPrompt,
	52	type WebhookRequest,
	53	} from "./webhookTemplate.ts";
45	54	import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts";
46	55
47	56	/** Path prefix of the environment route that receives webhook requests. */
...	...	@@ -50,21 +59,44 @@ export const WEBHOOK_ROUTE_PREFIX = "/api/hooks";
50	59	const WEBHOOK_DELIVERY_RETENTION = 50;
51	60	/** Body text kept in the delivery log. Larger bodies are cut and flagged. */
52	61	const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
	62	/** Rendered prompt text kept in the delivery log. */
	63	const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024;
53	64	/** Deliveries one task may hold at once, running or waiting their turn. */
54	65	const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
55	66	/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */
56	67	const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;
57	68
58		/** Where a webhook task's public URL points. `relayUrl` is null when the environment is not linked to T3 Connect. */
	69	/**
	70	* Where a webhook task's public URL points: `${relayHookBaseUrl}/${taskId}/${token}`.
	71	* Null when the environment has no managed tunnel on T3 Connect; clients then show the path.
	72	*/
59	73	interface WebhookOrigin {
60		readonly environmentId: string;
61		readonly relayUrl: string \| null;
	74	readonly relayHookBaseUrl: string \| null;
	75	}
	76
	77	const ENDPOINT_KEY = /^[0-9a-f]{16}$/;
	78
	79	/**
	80	* The relay's hook URL prefix for this environment. The relay finds the
	81	* environment by its managed tunnel's key (the tunnel name's last segment),
	82	* so the URL never reveals the environment id.
	83	*/
	84	export function relayHookBaseUrl(input: {
	85	readonly relayUrl: string;
	86	readonly tunnelName: string \| undefined;
	87	}): string \| null {
	88	const endpointKey = input.tunnelName?.split("-").at(-1);
	89	const relayUrl = input.relayUrl.replace(/\/+$/, "");
	90	if (relayUrl === "" \|\| endpointKey === undefined \|\| !ENDPOINT_KEY.test(endpointKey)) {
	91	return null;
	92	}
	93	return `${relayUrl}/v1/hooks/${endpointKey}`;
62	94	}
63	95
64	96	export class ScheduledTaskWebhookOrigin extends Context.Reference<Effect.Effect<WebhookOrigin>>(
65	97	"t3/scheduledTasks/ScheduledTaskWebhookOrigin",
66	98	{
67		defaultValue: () => Effect.succeed({ environmentId: "local", relayUrl: null }),
	99	defaultValue: () => Effect.succeed({ relayHookBaseUrl: null }),
68	100	},
69	101	) {}
70	102
...	...	@@ -83,15 +115,45 @@ export interface WebhookTriggerRequest extends WebhookRequest {
83	115	readonly hookId: string;
84	116	readonly token: string;
85	117	readonly body: Uint8Array;
	118	/** Set by T3 Connect; the same id is never dispatched twice. */
	119	readonly relayDeliveryId?: string;
	120	/** When the relay received a held request; defaults to now. */
	121	readonly receivedAt?: string;
86	122	}
87	123
88	124	/** What the HTTP route should answer. `not_found` covers unknown hooks and wrong tokens alike. */
	125	/**
	126	* What happened to one webhook request, as recorded in metrics and spans.
	127	* Sent back to the relay as `x-t3-hook-outcome`; never request contents.
	128	*/
	129	export type WebhookDeliveryOutcome =
	130	\| "accepted"
	131	\| "duplicate"
	132	\| "prompt_too_long"
	133	\| "queue_full"
	134	\| "rate_limited"
	135	\| "disabled"
	136	\| "rejected_signature"
	137	\| "expired"
	138	\| "not_found"
	139	\| "error";
	140
89	141	export type WebhookTriggerResult =
90		\| { readonly _tag: "accepted"; readonly deliveryId: ScheduledTaskWebhookDeliveryId }
	142	\| {
	143	readonly _tag: "accepted";
	144	readonly deliveryId: ScheduledTaskWebhookDeliveryId;
	145	/** A 202 covers more than a started run; this says which. */
	146	readonly outcome: "accepted" \| "duplicate" \| "prompt_too_long";
	147	}
91	148	\| { readonly _tag: "not_found" }
92	149	\| { readonly _tag: "rejected_signature" }
93	150	\| { readonly _tag: "disabled" }
94		\| { readonly _tag: "rate_limited" };
	151	\| {
	152	readonly _tag: "rate_limited";
	153	/** Too many requests to this hook, or too many runs already waiting. */
	154	readonly outcome: "rate_limited" \| "queue_full";
	155	}
	156	\| { readonly _tag: "expired" };
95	157
96	158	const decodeTask = Schema.decodeUnknownEffect(ScheduledTask);
97	159	const decodeTaskId = Schema.decodeUnknownOption(ScheduledTaskId);
...	...	@@ -228,19 +290,6 @@ function errorMessage(error: unknown): string {
228	290	return String(error);
229	291	}
230	292
231		/** Headers kept out of the delivery log because they commonly carry credentials. */
232		const REDACTED_HEADER =
233		/^(authorization\|proxy-authorization\|cookie\|set-cookie)$\|token\|secret\|signature\|key\|password\|auth/i;
234
235		function redactHeaders(headers: Readonly<Record<string, string>>): Record<string, string> {
236		return Object.fromEntries(
237		Object.entries(headers).map(([name, value]) => [
238		name,
239		REDACTED_HEADER.test(name) ? "[redacted]" : value,
240		]),
241		);
242		}
243
244	293	function webhookPath(taskId: string, token: string): string {
245	294	return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`;
246	295	}
...	...	@@ -250,13 +299,10 @@ function webhookEndpoint(
250	299	origin: WebhookOrigin \| null,
251	300	): ScheduledTask["webhook"] {
252	301	if (row.webhook_token === null) return undefined;
253		const relayUrl = origin?.relayUrl?.replace(/\/+$/, "") ?? null;
	302	const base = origin?.relayHookBaseUrl ?? null;
254	303	return {
255	304	path: webhookPath(row.task_id, row.webhook_token),
256		url:
257		relayUrl === null \|\| origin === null
258		? null
259		: `${relayUrl}/v1/hooks/${encodeURIComponent(origin.environmentId)}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`,
	305	url: base === null ? null : `${base}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`,
260	306	hasSecret: row.webhook_secret !== null,
261	307	};
262	308	}
...	...	@@ -985,6 +1031,7 @@ export const layer = Layer.effect(
985	1031	encoding: input.schedule.signature.encoding,
986	1032	prefix: input.schedule.signature.prefix,
987	1033	},
	1034	maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null,
988	1035	}
989	1036	: input.schedule;
990	1037	const webhook =
...	...	@@ -1197,6 +1244,7 @@ export const layer = Layer.effect(
1197	1244	readonly signatureVerified: boolean;
1198	1245	readonly missing: ReadonlyArray<string>;
1199	1246	readonly renderedPrompt: string \| null;
	1247	readonly error?: string;
1200	1248	}) => {
1201	1249	const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1202	1250	const loggedBody = truncated
...	...	@@ -1215,12 +1263,22 @@ export const layer = Layer.effect(
1215	1263	)
1216	1264	SELECT
1217	1265	${input.id}, ${input.taskId}, ${input.receivedAt}, ${input.request.method},
1218		${input.request.query}, ${encodeHeadersJson(redactHeaders(input.request.headers))},
	1266	${redactQuery(input.request.query)},
	1267	${encodeHeadersJson(redactHeaders(input.request.headers))},
1219	1268	${loggedBody},
1220	1269	${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome},
1221	1270	${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)},
1222		${input.renderedPrompt}, NULL
	1271	${input.renderedPrompt?.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT) ?? null},
	1272	${input.error ?? null}
1223	1273	WHERE EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.taskId})
	1274	-- A held request retried after a rate limit reuses its relay
	1275	-- delivery id; the newer attempt replaces the logged one.
	1276	ON CONFLICT (delivery_id) DO UPDATE SET
	1277	outcome = excluded.outcome,
	1278	signature_verified = excluded.signature_verified,
	1279	missing_fields_json = excluded.missing_fields_json,
	1280	rendered_prompt = excluded.rendered_prompt,
	1281	error = excluded.error
1224	1282	`;
1225	1283	yield* sql`
1226	1284	DELETE FROM scheduled_task_webhook_deliveries
...	...	@@ -1295,10 +1353,55 @@ export const layer = Layer.effect(
1295	1353	// run; scoped to the service so shutdown interrupts it.
1296	1354	const serviceScope = yield* Effect.scope;
1297	1355
	1356	/**
	1357	* Records one handled request on the span and in metrics. `outcome` is
	1358	* finer than the result tag: it separates a full queue and an oversized
	1359	* prompt from the rest, which is what an operator needs to act on.
	1360	*/
	1361	const observeDelivery = (
	1362	request: WebhookTriggerRequest,
	1363	outcome: WebhookDeliveryOutcome,
	1364	receivedAt: DateTime.DateTime \| undefined,
	1365	now: DateTime.DateTime \| undefined,
	1366	) =>
	1367	Effect.gen(function* () {
	1368	const source = request.relayDeliveryId === undefined ? "direct" : "relay";
	1369	yield* Effect.annotateCurrentSpan({
	1370	"scheduled_task.webhook.outcome": outcome,
	1371	"scheduled_task.webhook.source": source,
	1372	});
	1373	yield* Metrics.increment(Metrics.webhookDeliveriesTotal, { outcome, source });
	1374	if (request.receivedAt !== undefined && receivedAt !== undefined && now !== undefined) {
	1375	const heldMs = Math.max(
	1376	0,
	1377	DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt),
	1378	);
	1379	yield* Effect.annotateCurrentSpan({ "scheduled_task.webhook.held_ms": heldMs });
	1380	yield* Metric.update(Metrics.webhookHeldDelay, Duration.millis(heldMs));
	1381	}
	1382	});
	1383
1298	1384	const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) =>
	1385	triggerWebhookUnobserved(request).pipe(
	1386	Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)),
	1387	Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }),
	1388	Effect.withSpan("ScheduledTaskService.triggerWebhook", {
	1389	attributes: {
	1390	"scheduled_task.webhook.method": request.method,
	1391	"scheduled_task.webhook.body_bytes": request.body.byteLength,
	1392	"scheduled_task.webhook.source":
	1393	request.relayDeliveryId === undefined ? "direct" : "relay",
	1394	},
	1395	}),
	1396	);
	1397
	1398	const triggerWebhookUnobserved = (request: WebhookTriggerRequest) =>
1299	1399	Effect.gen(function* () {
1300	1400	const taskId = decodeTaskId(request.hookId);
1301		if (Option.isNone(taskId)) return { _tag: "not_found" as const };
	1401	const notFound = observeDelivery(request, "not_found", undefined, undefined).pipe(
	1402	Effect.as({ _tag: "not_found" as const }),
	1403	);
	1404	if (Option.isNone(taskId)) return yield* notFound;
1302	1405	const rows = yield* getRows(taskId.value).pipe(
1303	1406	Effect.mapError((cause) =>
1304	1407	taskError("Could not load schedule task.", { taskId: taskId.value, cause }),
...	...	@@ -1312,110 +1415,227 @@ export const layer = Layer.effect(
1312	1415	row.webhook_token === null \|\|
1313	1416	!constantTimeEquals(request.token, row.webhook_token)
1314	1417	) {
1315		return { _tag: "not_found" as const };
	1418	return yield* notFound;
1316	1419	}
1317	1420	const task = yield* decodeRow(row);
1318		if (task.schedule.type !== "webhook") return { _tag: "not_found" as const };
	1421	yield* Effect.annotateCurrentSpan({ "scheduled_task.id": task.id });
	1422	const schedule = task.schedule;
	1423	if (schedule.type !== "webhook") return yield* notFound;
1319	1424
1320		const receivedAt = yield* localNow;
1321		const deliveryUuid = yield* crypto.randomUUIDv4.pipe(
1322		Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })),
	1425	const now = yield* localNow;
	1426	// Anyone can reach the tunnel directly and set the relay's header, so
	1427	// a receive time is never later than now: a future one would pin the
	1428	// delivery in the log and slip past the task's max age.
	1429	const receivedAt =
	1430	request.receivedAt === undefined
	1431	? now
	1432	: DateTime.min(
	1433	Option.getOrElse(DateTime.make(request.receivedAt), () => now),
	1434	now,
	1435	);
	1436	const observe = (outcome: WebhookDeliveryOutcome) =>
	1437	observeDelivery(request, outcome, receivedAt, now);
	1438	const deliveryId = ScheduledTaskWebhookDeliveryId.make(
	1439	request.relayDeliveryId === undefined
	1440	? `delivery:${yield* crypto.randomUUIDv4.pipe(
	1441	Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })),
	1442	)}`
	1443	: `delivery:relay:${request.relayDeliveryId}`,
1323	1444	);
1324		const deliveryId = ScheduledTaskWebhookDeliveryId.make(`delivery:${deliveryUuid}`);
1325		const log = (
1326		outcome: ScheduledTaskWebhookDeliveryOutcome,
1327		details: {
1328		readonly signatureVerified?: boolean;
1329		readonly missing?: ReadonlyArray<string>;
1330		readonly renderedPrompt?: string;
1331		} = {},
1332		) =>
1333		recordDelivery({
1334		id: deliveryId,
1335		taskId: task.id,
1336		receivedAt: iso(receivedAt),
1337		request,
1338		outcome,
1339		signatureVerified: details.signatureVerified ?? false,
1340		missing: details.missing ?? [],
1341		renderedPrompt: details.renderedPrompt ?? null,
1342		});
	1445	// A held request may already have reached this environment directly
	1446	// before a timeout; it runs once. Claimed in its own table, kept longer
	1447	// than the relay holds a request, because the delivery log is trimmed.
	1448	// A rate-limited delivery stays held on the relay, so it must give up
	1449	// its claim or the next pass would treat it as already delivered.
	1450	const releaseClaim = <A>(result: A) =>
	1451	request.relayDeliveryId === undefined
	1452	? Effect.succeed(result)
	1453	: sql`
	1454	DELETE FROM scheduled_task_webhook_relay_deliveries
	1455	WHERE relay_delivery_id = ${request.relayDeliveryId}
	1456	`.pipe(Effect.ignore, Effect.as(result));
	1457	// From the claim until the run is forked nothing may interrupt: a
	1458	// request dropped in between (the relay hangs up after its timeout)
	1459	// would leave a claimed delivery that never runs, or a queue slot
	1460	// that is never released. Everything in here is local and quick.
	1461	return yield* Effect.uninterruptible(
	1462	Effect.gen(function* () {
	1463	if (request.relayDeliveryId !== undefined) {
	1464	const claimed = yield* sql<{ relay_delivery_id: string }>`
	1465	INSERT INTO scheduled_task_webhook_relay_deliveries
	1466	(relay_delivery_id, task_id, seen_at)
	1467	VALUES (${request.relayDeliveryId}, ${task.id}, ${iso(now)})
	1468	ON CONFLICT (relay_delivery_id) DO NOTHING
	1469	RETURNING relay_delivery_id
	1470	`.pipe(
	1471	Effect.mapError((cause) =>
	1472	taskError("Could not record webhook delivery.", { taskId: task.id, cause }),
	1473	),
	1474	);
	1475	if (claimed.length === 0) {
	1476	// A held request this environment already ran: accepted, not run twice.
	1477	yield* observe("duplicate");
	1478	return { _tag: "accepted" as const, deliveryId, outcome: "duplicate" as const };
	1479	}
	1480	// Older claims can no longer be replayed by the relay.
	1481	yield* sql`
	1482	DELETE FROM scheduled_task_webhook_relay_deliveries
	1483	WHERE seen_at < ${iso(DateTime.subtract(now, { hours: 48 }))}
	1484	`.pipe(Effect.ignore);
	1485	}
	1486	const log = (
	1487	outcome: ScheduledTaskWebhookDeliveryOutcome,
	1488	details: {
	1489	readonly signatureVerified?: boolean;
	1490	readonly missing?: ReadonlyArray<string>;
	1491	readonly renderedPrompt?: string;
	1492	readonly error?: string;
	1493	} = {},
	1494	) =>
	1495	recordDelivery({
	1496	id: deliveryId,
	1497	taskId: task.id,
	1498	receivedAt: iso(receivedAt),
	1499	request,
	1500	outcome,
	1501	signatureVerified: details.signatureVerified ?? false,
	1502	missing: details.missing ?? [],
	1503	renderedPrompt: details.renderedPrompt ?? null,
	1504	...(details.error === undefined ? {} : { error: details.error }),
	1505	});
	1506
	1507	// Only the first rejected request in a window is logged, so a flood
	1508	// cannot write rows or push the real deliveries out of the log.
	1509	const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now));
	1510	if (slot !== "allowed") {
	1511	if (slot === "first_rejected") yield* log("rate_limited");
	1512	yield* observe("rate_limited");
	1513	return yield* releaseClaim({
	1514	_tag: "rate_limited" as const,
	1515	outcome: "rate_limited" as const,
	1516	});
	1517	}
	1518	if (!task.enabled) {
	1519	yield* log("disabled");
	1520	yield* observe("disabled");
	1521	return { _tag: "disabled" as const };
	1522	}
	1523	const signature = schedule.signature;
	1524	if (signature !== null) {
	1525	const verified =
	1526	row.webhook_secret !== null &&
	1527	verifyWebhookSignature({
	1528	signature,
	1529	secret: row.webhook_secret,
	1530	headers: request.headers,
	1531	body: request.body,
	1532	});
	1533	if (!verified) {
	1534	yield* log("rejected_signature");
	1535	yield* observe("rejected_signature");
	1536	return { _tag: "rejected_signature" as const };
	1537	}
	1538	}
	1539	const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null;
	1540	if (
	1541	maxAgeMinutes !== null &&
	1542	DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) >
	1543	maxAgeMinutes * 60_000
	1544	) {
	1545	yield* log("expired", { signatureVerified: signature !== null });
	1546	yield* observe("expired");
	1547	return { _tag: "expired" as const };
	1548	}
1343	1549
1344		// Only the first rejected request in a window is logged, so a flood
1345		// cannot write rows or push the real deliveries out of the log.
1346		const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(receivedAt));
1347		if (slot !== "allowed") {
1348		if (slot === "first_rejected") yield* log("rate_limited");
1349		return { _tag: "rate_limited" as const };
1350		}
1351		if (!task.enabled) {
1352		yield* log("disabled");
1353		return { _tag: "disabled" as const };
1354		}
1355		const signature = task.schedule.signature;
1356		if (signature !== null) {
1357		const verified =
1358		row.webhook_secret !== null &&
1359		verifyWebhookSignature({
1360		signature,
1361		secret: row.webhook_secret,
1362		headers: request.headers,
1363		body: request.body,
	1550	const rendered = renderWebhookPrompt(task.prompt, request);
	1551	// A provider refuses a turn this long, so it is not started. The
	1552	// delivery is not retryable, so the claim is kept. Providers trim
	1553	// the prompt before checking, so whitespace around it is free.
	1554	if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
	1555	yield* log("dispatch_failed", {
	1556	signatureVerified: signature !== null,
	1557	missing: rendered.missing,
	1558	renderedPrompt: rendered.prompt,
	1559	error: "The filled-in prompt is too long.",
	1560	});
	1561	yield* observe("prompt_too_long");
	1562	return { _tag: "accepted" as const, deliveryId, outcome: "prompt_too_long" as const };
	1563	}
	1564	// Bound the deliveries one task holds, so steady traffic to a stuck
	1565	// task cannot pile up parked fibers. A refused request is not logged,
	1566	// so it cannot push real deliveries out of the log.
	1567	const queueKey = `${task.id}\u0000${task.createdAt}`;
	1568	const queued = yield* Ref.modify(webhookQueued, (counts) => {
	1569	const count = counts.get(queueKey) ?? 0;
	1570	return count >= WEBHOOK_MAX_QUEUED_PER_TASK
	1571	? ([false, counts] as const)
	1572	: ([true, new Map(counts).set(queueKey, count + 1)] as const);
1364	1573	});
1365		if (!verified) {
1366		yield* log("rejected_signature");
1367		return { _tag: "rejected_signature" as const };
1368		}
1369		}
1370
1371		const rendered = renderWebhookPrompt(task.prompt, request);
1372		// Bound the deliveries one task holds, so steady traffic to a stuck
1373		// task cannot pile up parked fibers. A refused request is not logged,
1374		// so it cannot push real deliveries out of the log.
1375		const queueKey = `${task.id}\u0000${task.createdAt}`;
1376		const queued = yield* Ref.modify(webhookQueued, (counts) => {
1377		const count = counts.get(queueKey) ?? 0;
1378		return count >= WEBHOOK_MAX_QUEUED_PER_TASK
1379		? ([false, counts] as const)
1380		: ([true, new Map(counts).set(queueKey, count + 1)] as const);
1381		});
1382		if (!queued) return { _tag: "rate_limited" as const };
1383		// Entries leave the map when their count reaches zero, so a deleted
1384		// task's key does not linger once its last delivery finishes.
1385		const release = Ref.update(webhookQueued, (counts) => {
1386		const next = new Map(counts);
1387		const count = (next.get(queueKey) ?? 1) - 1;
1388		if (count <= 0) next.delete(queueKey);
1389		else next.set(queueKey, count);
1390		return next;
1391		});
1392		yield* log("accepted", {
1393		signatureVerified: signature !== null,
1394		missing: rendered.missing,
1395		renderedPrompt: rendered.prompt,
1396		}).pipe(Effect.onError(() => release));
1397		const permit = yield* webhookPermit(task.id);
1398		yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe(
1399		Effect.flatMap((completed) =>
1400		completed.lastRunStatus === "failed"
1401		? markDeliveryFailed(deliveryId, "The run failed to start.")
1402		: Effect.void,
1403		),
1404		Effect.catchTag("WebhookDeliverySkipped", (skipped) =>
1405		markDeliveryFailed(deliveryId, skipped.reason),
1406		),
1407		// The log is readable over RPC, so it gets a fixed reason; the
1408		// cause, which can carry request data, stays in the server log.
1409		Effect.catchCause((cause) =>
1410		Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe(
1411		Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1412		),
1413		),
1414		permit.withPermits(1),
1415		Effect.ensuring(release),
1416		Effect.forkIn(serviceScope),
	1574	if (!queued) {
	1575	// The task is busy with WEBHOOK_MAX_QUEUED_PER_TASK deliveries already.
	1576	yield* observe("queue_full");
	1577	return yield* releaseClaim({
	1578	_tag: "rate_limited" as const,
	1579	outcome: "queue_full" as const,
	1580	});
	1581	}
	1582	// Entries leave the map when their count reaches zero, so a deleted
	1583	// task's key does not linger once its last delivery finishes.
	1584	const release = Ref.update(webhookQueued, (counts) => {
	1585	const next = new Map(counts);
	1586	const count = (next.get(queueKey) ?? 1) - 1;
	1587	if (count <= 0) next.delete(queueKey);
	1588	else next.set(queueKey, count);
	1589	return next;
	1590	});
	1591	yield* log("accepted", {
	1592	signatureVerified: signature !== null,
	1593	missing: rendered.missing,
	1594	renderedPrompt: rendered.prompt,
	1595	}).pipe(Effect.onError(() => release));
	1596	yield* observe("accepted");
	1597	const permit = yield* webhookPermit(task.id);
	1598	const runOutcome = (outcome: "started" \| "skipped" \| "failed") =>
	1599	Effect.all([
	1600	Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }),
	1601	Metrics.increment(Metrics.webhookRunsTotal, { outcome }),
	1602	]);
	1603	yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe(
	1604	Effect.flatMap((completed) =>
	1605	completed.lastRunStatus === "failed"
	1606	? runOutcome("failed").pipe(
	1607	Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
	1608	)
	1609	: runOutcome("started"),
	1610	),
	1611	Effect.catchTag("WebhookDeliverySkipped", (skipped) =>
	1612	runOutcome("skipped").pipe(
	1613	Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)),
	1614	),
	1615	),
	1616	// The log is readable over RPC, so it gets a fixed reason; the
	1617	// cause, which can carry request data, stays in the server log.
	1618	Effect.catchCause((cause) =>
	1619	Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe(
	1620	Effect.andThen(runOutcome("failed")),
	1621	Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
	1622	),
	1623	),
	1624	permit.withPermits(1),
	1625	// Its own trace: the request that triggered it has already been answered.
	1626	Effect.withSpan("ScheduledTaskService.runWebhookDelivery", {
	1627	root: true,
	1628	attributes: {
	1629	"scheduled_task.id": task.id,
	1630	"scheduled_task.webhook.delivery_id": deliveryId,
	1631	},
	1632	}),
	1633	Effect.ensuring(release),
	1634	Effect.forkIn(serviceScope),
	1635	);
	1636	return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const };
	1637	}),
1417	1638	);
1418		return { _tag: "accepted" as const, deliveryId };
1419	1639	});
1420	1640
1421	1641	return ScheduledTaskService.of({

apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts+308-2

...	...	@@ -3,10 +3,14 @@ import * as NodeCrypto from "node:crypto";
3	3	import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
4	4	import { assert, it } from "@effect/vitest";
5	5	import { ScheduledTaskUpsertInput } from "@t3tools/contracts";
	6	import * as DateTime from "effect/DateTime";
6	7	import * as Deferred from "effect/Deferred";
7	8	import * as Effect from "effect/Effect";
	9	import * as Fiber from "effect/Fiber";
8	10	import * as Layer from "effect/Layer";
	11	import * as Metric from "effect/Metric";
9	12	import * as Queue from "effect/Queue";
	13	import * as EffectScheduler from "effect/Scheduler";
10	14	import * as Schema from "effect/Schema";
11	15	import * as TestClock from "effect/testing/TestClock";
12	16
...	...	@@ -63,7 +67,7 @@ const withService = <A, E>(
63	67	readonly service: ScheduledTaskService.ScheduledTaskService["Service"];
64	68	readonly launches: Queue.Queue<LaunchInput>;
65	69	}) => Effect.Effect<A, E, never>,
66		options: { readonly gate?: Deferred.Deferred<void> } = {},
	70	options: { readonly gate?: Deferred.Deferred<void>; readonly relayHookBaseUrl?: string } = {},
67	71	) =>
68	72	Effect.gen(function* () {
69	73	const launches = yield* Queue.unbounded<LaunchInput>();
...	...	@@ -78,6 +82,10 @@ const withService = <A, E>(
78	82	),
79	83	}),
80	84	Layer.mock(ThreadManagementService.ThreadManagementService)({}),
	85	Layer.succeed(
	86	ScheduledTaskService.ScheduledTaskWebhookOrigin,
	87	Effect.succeed({ relayHookBaseUrl: options.relayHookBaseUrl ?? null }),
	88	),
81	89	);
82	90	return yield* Effect.gen(function* () {
83	91	const service = yield* ScheduledTaskService.ScheduledTaskService;
...	...	@@ -120,6 +128,35 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
120	128	),
121	129	);
122	130
	131	it("builds the relay hook URL from the managed tunnel's key, never the environment id", () => {
	132	const relayUrl = "https://relay.example.com/";
	133	assert.equal(
	134	ScheduledTaskService.relayHookBaseUrl({
	135	relayUrl,
	136	tunnelName: "t3coderelay-managedendpoint-dev-julius-0123456789abcdef",
	137	}),
	138	"https://relay.example.com/v1/hooks/0123456789abcdef",
	139	);
	140	for (const tunnelName of [undefined, "t3coderelay-managedendpoint", "x-0123456789ABCDEF"]) {
	141	assert.isNull(ScheduledTaskService.relayHookBaseUrl({ relayUrl, tunnelName }));
	142	}
	143	});
	144
	145	it.effect("gives webhook tasks a relay URL when the environment has a managed tunnel", () =>
	146	withService(
	147	({ service }) =>
	148	Effect.gen(function* () {
	149	const { task } = yield* service.upsert(yield* webhookTaskInput());
	150	const token = task.webhook!.path.split("/").at(-1);
	151	assert.equal(
	152	task.webhook?.url,
	153	`https://relay.example.com/v1/hooks/0123456789abcdef/scheduled-task%3Ahook/${token}`,
	154	);
	155	}),
	156	{ relayHookBaseUrl: "https://relay.example.com/v1/hooks/0123456789abcdef" },
	157	),
	158	);
	159
123	160	it.effect("answers not found for a wrong token or unknown hook without logging", () =>
124	161	withService(({ service }) =>
125	162	Effect.gen(function* () {
...	...	@@ -168,6 +205,7 @@ it.effect("checks the configured signature and keeps the secret write-only", ()
168	205	assert.deepEqual(task.schedule, {
169	206	type: "webhook",
170	207	signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
	208	maxDeliveryAgeMinutes: null,
171	209	});
172	210	assert.isTrue(task.webhook?.hasSecret);
173	211
...	...	@@ -298,12 +336,13 @@ it.effect("keeps the newest 50 deliveries when they share a timestamp", () =>
298	336	),
299	337	);
300	338
301		it.effect("keeps credential headers out of the delivery log", () =>
	339	it.effect("keeps credential headers and query values out of the delivery log", () =>
302	340	withService(({ service }) =>
303	341	Effect.gen(function* () {
304	342	const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
305	343	yield* service.triggerWebhook(
306	344	requestFor(task, {
	345	query: "page=2&api_key=k",
307	346	headers: {
308	347	"content-type": "application/json",
309	348	authorization: "Bearer sender-token",
...	...	@@ -317,6 +356,7 @@ it.effect("keeps credential headers out of the delivery log", () =>
317	356	id: task.id,
318	357	deliveryId: summary!.id,
319	358	});
	359	assert.equal(delivery.query, "page=2&api_key=[redacted]");
320	360	assert.equal(delivery.headers.authorization, "[redacted]");
321	361	assert.equal(delivery.headers["x-webhook-key"], "[redacted]");
322	362	assert.equal(delivery.headers["x-github-event"], "push");
...	...	@@ -454,6 +494,209 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () =>
454	494	),
455	495	);
456	496
	497	it.effect("counts the prompt as the provider does, without surrounding whitespace", () =>
	498	withService(({ service, launches }) =>
	499	Effect.gen(function* () {
	500	const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body.text}}" }));
	501	// Over the limit as sent, within it once the padding is trimmed.
	502	const text = `{"text":"${" ".repeat(1_000)}${"x".repeat(119_990)}${"\\n".repeat(1_000)}"}`;
	503	const result = yield* service.triggerWebhook(
	504	requestFor(task, { body: new TextEncoder().encode(text), bodyText: text }),
	505	);
	506	assert.equal(result._tag, "accepted");
	507	yield* Queue.take(launches);
	508	}),
	509	),
	510	);
	511
	512	it.effect("does not start a run when the filled-in prompt is too long", () =>
	513	withService(({ service, launches }) =>
	514	Effect.gen(function* () {
	515	const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body}}" }));
	516	const text = "x".repeat(200_000);
	517	const body = new TextEncoder().encode(text);
	518	const result = yield* service.triggerWebhook(requestFor(task, { body, bodyText: text }));
	519	assert.equal(result._tag, "accepted");
	520	assert.equal(yield* Queue.size(launches), 0);
	521	const { delivery } = yield* service.getWebhookDelivery({
	522	id: task.id,
	523	deliveryId: result._tag === "accepted" ? result.deliveryId : ("" as never),
	524	});
	525	assert.equal(delivery.outcome, "dispatch_failed");
	526	assert.equal(delivery.error, "The filled-in prompt is too long.");
	527	assert.equal(delivery.renderedPrompt?.length, 64 * 1024);
	528	// The queue slot was never taken: a normal delivery still runs.
	529	const ok = yield* service.triggerWebhook(requestFor(task));
	530	assert.equal(ok._tag, "accepted");
	531	yield* Queue.take(launches);
	532	}),
	533	),
	534	);
	535
	536	it.effect("a held request already delivered directly runs only once", () =>
	537	withService(({ service, launches }) =>
	538	Effect.gen(function* () {
	539	const { task } = yield* service.upsert(yield* webhookTaskInput());
	540	const direct = yield* service.triggerWebhook(
	541	requestFor(task, { relayDeliveryId: "relay-1" }),
	542	);
	543	const replayed = yield* service.triggerWebhook(
	544	requestFor(task, { relayDeliveryId: "relay-1", receivedAt: "2026-10-04T10:00:00.000Z" }),
	545	);
	546	assert.equal(direct._tag, "accepted");
	547	// Same delivery, answered the same way, but recorded as a duplicate.
	548	assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed);
	549	yield* Queue.take(launches);
	550	const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
	551	assert.equal(logged.length, 1);
	552	}),
	553	),
	554	);
	555
	556	it.effect("a held request whose sender hung up mid-request still runs", () =>
	557	withService(({ service, launches }) =>
	558	Effect.gen(function* () {
	559	const { task } = yield* service.upsert(yield* webhookTaskInput());
	560	// The relay times out and the request fiber is interrupted. A small
	561	// operation budget makes the request yield often, so stepping the
	562	// interrupt one yield later each time lands it at every point in the
	563	// request (one takes about 40 yields), then the relay retries.
	564	for (let step = 0; step < 60; step++) {
	565	const request = requestFor(task, { relayDeliveryId: `hung-up-${step}` });
	566	const fiber = yield* service
	567	.triggerWebhook(request)
	568	.pipe(Effect.provideService(EffectScheduler.MaxOpsBeforeYield, 8), Effect.forkChild);
	569	for (let yields = 0; yields < step; yields++) yield* Effect.yieldNow;
	570	yield* Fiber.interrupt(fiber);
	571	const retried = yield* service.triggerWebhook(request);
	572	assert.equal(retried._tag, "accepted");
	573	const deliveryId = retried._tag === "accepted" ? retried.deliveryId : undefined;
	574	// Logged and run exactly once, whether or not the first attempt got through.
	575	yield* service.getWebhookDelivery({ id: task.id, deliveryId: deliveryId! });
	576	const launch = yield* Queue.take(launches);
	577	assert.include(launch.commandId, `hung-up-${step}`);
	578	// Keep each step in a fresh rate-limit window.
	579	yield* TestClock.adjust("61 seconds");
	580	}
	581	assert.equal(yield* Queue.size(launches), 0);
	582	}),
	583	),
	584	);
	585
	586	it.effect("a held request runs once even after the log has trimmed it", () =>
	587	withService(({ service, launches }) =>
	588	Effect.gen(function* () {
	589	const { task } = yield* service.upsert(yield* webhookTaskInput());
	590	const first = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
	591	assert.equal(first._tag, "accepted");
	592	yield* Queue.take(launches);
	593	// Push the original row out of the 50-row delivery log.
	594	const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
	595	yield* Effect.forEach(Array.from({ length: 55 }), () =>
	596	service.triggerWebhook(requestFor(paused.task)),
	597	);
	598	yield* service.upsert(yield* webhookTaskInput());
	599	const replay = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
	600	assert.equal(replay._tag, "accepted");
	601	assert.equal(yield* Queue.size(launches), 0);
	602	}),
	603	),
	604	);
	605
	606	it.effect("a rate-limited held request can run on a later pass", () =>
	607	withService(({ service, launches }) =>
	608	Effect.gen(function* () {
	609	const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
	610	// Spend the task's 60-a-minute budget.
	611	yield* Effect.forEach(Array.from({ length: 60 }), () =>
	612	service.triggerWebhook(requestFor(task)),
	613	);
	614	const limited = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" }));
	615	assert.equal(limited._tag, "rate_limited");
	616	yield* service.upsert(yield* webhookTaskInput());
	617	yield* TestClock.adjust("61 seconds");
	618	const retried = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" }));
	619	assert.equal(retried._tag, "accepted");
	620	yield* Queue.take(launches);
	621	}),
	622	),
	623	);
	624
	625	it.effect("logs a held request at the time the relay received it", () =>
	626	withService(({ service }) =>
	627	Effect.gen(function* () {
	628	const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
	629	const receivedAt = DateTime.formatIso(DateTime.subtract(yield* DateTime.now, { minutes: 5 }));
	630	yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "relay-2", receivedAt }));
	631	const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
	632	assert.equal(delivery?.receivedAt, receivedAt);
	633	}),
	634	),
	635	);
	636
	637	it.effect("a receive time in the future counts as now", () =>
	638	withService(({ service, launches }) =>
	639	Effect.gen(function* () {
	640	const { task } = yield* service.upsert(
	641	yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }),
	642	);
	643	const now = yield* DateTime.now;
	644	const result = yield* service.triggerWebhook(
	645	requestFor(task, {
	646	relayDeliveryId: "future",
	647	receivedAt: DateTime.formatIso(DateTime.add(now, { days: 365 })),
	648	}),
	649	);
	650	assert.equal(result._tag, "accepted");
	651	yield* Queue.take(launches);
	652	const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
	653	assert.equal(delivery?.receivedAt, DateTime.formatIso(now));
	654	}),
	655	),
	656	);
	657
	658	it.effect("skips a held request older than the task's max age", () =>
	659	withService(({ service, launches }) =>
	660	Effect.gen(function* () {
	661	const { task } = yield* service.upsert(
	662	yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }),
	663	);
	664	const now = yield* DateTime.now;
	665	const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
	666	const fresh = DateTime.formatIso(DateTime.subtract(now, { minutes: 5 }));
	667	const tooOld = yield* service.triggerWebhook(
	668	requestFor(task, { relayDeliveryId: "old", receivedAt: old }),
	669	);
	670	assert.equal(tooOld._tag, "expired");
	671	assert.equal(yield* Queue.size(launches), 0);
	672	const ok = yield* service.triggerWebhook(
	673	requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }),
	674	);
	675	assert.equal(ok._tag, "accepted");
	676	const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
	677	(delivery) => delivery.outcome,
	678	);
	679	assert.includeMembers(outcomes, ["expired", "accepted"]);
	680	}),
	681	),
	682	);
	683
	684	it.effect("runs a held request of any age when no max age is set", () =>
	685	withService(({ service }) =>
	686	Effect.gen(function* () {
	687	const { task } = yield* service.upsert(yield* webhookTaskInput());
	688	const now = yield* DateTime.now;
	689	const result = yield* service.triggerWebhook(
	690	requestFor(task, {
	691	relayDeliveryId: "ancient",
	692	receivedAt: DateTime.formatIso(DateTime.subtract(now, { hours: 23 })),
	693	}),
	694	);
	695	assert.equal(result._tag, "accepted");
	696	}),
	697	),
	698	);
	699
457	700	it.effect("deleting a task removes its delivery log", () =>
458	701	withService(({ service }) =>
459	702	Effect.gen(function* () {
...	...	@@ -464,3 +707,66 @@ it.effect("deleting a task removes its delivery log", () =>
464	707	}),
465	708	),
466	709	);
	710
	711	const signatureFor = (secret: string) =>
	712	`sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;
	713
	714	/** Count recorded by `t3_webhook_deliveries_total` for one outcome and source. */
	715	const deliveriesCounted = (outcome: string, source: "relay" \| "direct") =>
	716	Metric.snapshot.pipe(
	717	Effect.map((snapshots) => {
	718	const found = snapshots.find(
	719	(snapshot) =>
	720	snapshot.id === "t3_webhook_deliveries_total" &&
	721	snapshot.attributes?.outcome === outcome &&
	722	snapshot.attributes?.source === source,
	723	);
	724	return found?.type === "Counter" ? Number(found.state.count) : 0;
	725	}),
	726	);
	727
	728	it.effect("counts each handled request by what happened to it", () =>
	729	withService(({ service, launches }) =>
	730	Effect.gen(function* () {
	731	const { task } = yield* service.upsert(
	732	yield* webhookTaskInput({
	733	schedule: {
	734	type: "webhook",
	735	signature: {
	736	header: "x-hub-signature-256",
	737	encoding: "hex",
	738	prefix: "sha256=",
	739	secret: "github-secret",
	740	},
	741	},
	742	}),
	743	);
	744	const before = {
	745	accepted: yield* deliveriesCounted("accepted", "direct"),
	746	rejected: yield* deliveriesCounted("rejected_signature", "direct"),
	747	notFound: yield* deliveriesCounted("not_found", "direct"),
	748	relayAccepted: yield* deliveriesCounted("accepted", "relay"),
	749	duplicate: yield* deliveriesCounted("duplicate", "relay"),
	750	};
	751	const signed = {
	752	"content-type": "application/json",
	753	"x-hub-signature-256": signatureFor("github-secret"),
	754	};
	755	yield* service.triggerWebhook(requestFor(task, { headers: signed }));
	756	yield* Queue.take(launches);
	757	yield* service.triggerWebhook(requestFor(task));
	758	yield* service.triggerWebhook(requestFor(task, { token: "wrong" }));
	759	// A request the relay held, then the same request again.
	760	const relayed = requestFor(task, { headers: signed, relayDeliveryId: "relay-1" });
	761	yield* service.triggerWebhook(relayed);
	762	yield* Queue.take(launches);
	763	yield* service.triggerWebhook(relayed);
	764
	765	assert.equal((yield* deliveriesCounted("accepted", "direct")) - before.accepted, 1);
	766	assert.equal((yield* deliveriesCounted("rejected_signature", "direct")) - before.rejected, 1);
	767	assert.equal((yield* deliveriesCounted("not_found", "direct")) - before.notFound, 1);
	768	assert.equal((yield* deliveriesCounted("accepted", "relay")) - before.relayAccepted, 1);
	769	assert.equal((yield* deliveriesCounted("duplicate", "relay")) - before.duplicate, 1);
	770	}),
	771	),
	772	);

apps/server/src/scheduledTasks/relayDeliveryProof.ts created+74

...	...	@@ -0,0 +1,74 @@
	1	/**
	2	* Whether a webhook request really came through this environment's relay.
	3	*
	4	* The relay's delivery id, receive time and trace context arrive as plain
	5	* headers, and the webhook URL can also be called directly, so they are only
	6	* trusted alongside a proof the relay signed with its mint key. A request
	7	* without a valid proof is handled as a direct request.
	8	*/
	9	import { RelayHookDeliveryProofPayload } from "@t3tools/contracts/relay";
	10	import {
	11	normalizeRelayIssuer,
	12	RELAY_HOOK_DELIVERY_HEADER,
	13	RELAY_HOOK_DELIVERY_TYP,
	14	verifyRelayJwt,
	15	} from "@t3tools/shared/relayJwt";
	16	import * as Clock from "effect/Clock";
	17	import * as Effect from "effect/Effect";
	18	import * as Option from "effect/Option";
	19	import * as Schema from "effect/Schema";
	20
	21	import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
	22	import { CLOUD_MINT_PUBLIC_KEY, RELAY_ISSUER_SECRET, RELAY_URL_SECRET } from "../cloud/config.ts";
	23	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
	24
	25	/** Covers the relay's 24-hour hold plus a margin. */
	26	const MAX_PROOF_AGE_SECONDS = 25 * 60 * 60;
	27
	28	const decodePayload = Schema.decodeUnknownOption(RelayHookDeliveryProofPayload);
	29	const text = (bytes: Option.Option<Uint8Array>) =>
	30	Option.map(bytes, (value) => new TextDecoder().decode(value));
	31
	32	/** The relay's own delivery id and receive time, when the request proves it came from the relay. */
	33	export const makeRelayDeliveryVerifier = Effect.gen(function* () {
	34	const secrets = yield* ServerSecretStore.ServerSecretStore;
	35	const environment = yield* ServerEnvironment.ServerEnvironment;
	36	return (input: { readonly headers: Readonly<Record<string, string>>; readonly hookId: string }) =>
	37	Effect.gen(function* () {
	38	const proof = input.headers[RELAY_HOOK_DELIVERY_HEADER];
	39	const deliveryId = input.headers["x-t3-relay-delivery-id"];
	40	const receivedAt = input.headers["x-t3-relay-received-at"];
	41	if (proof === undefined \|\| deliveryId === undefined \|\| receivedAt === undefined) {
	42	return Option.none();
	43	}
	44	const publicKey = text(yield* secrets.get(CLOUD_MINT_PUBLIC_KEY));
	45	const issuer = Option.orElse(text(yield* secrets.get(RELAY_ISSUER_SECRET)), () =>
	46	Option.none<string>(),
	47	);
	48	const relayUrl = text(yield* secrets.get(RELAY_URL_SECRET));
	49	const relayIssuer = Option.isSome(issuer) ? issuer : relayUrl;
	50	// Not linked to T3 Connect: no relay can be delivering to us.
	51	if (Option.isNone(publicKey) \|\| Option.isNone(relayIssuer)) return Option.none();
	52	const environmentId = yield* environment.getEnvironmentId;
	53	const payload = yield* verifyRelayJwt({
	54	publicKey: publicKey.value,
	55	token: proof,
	56	typ: RELAY_HOOK_DELIVERY_TYP,
	57	issuer: normalizeRelayIssuer(relayIssuer.value),
	58	audience: `t3-env:${environmentId}`,
	59	nowEpochSeconds: Math.floor((yield* Clock.currentTimeMillis) / 1_000),
	60	maxTokenAge: MAX_PROOF_AGE_SECONDS,
	61	}).pipe(Effect.map(decodePayload), Effect.orElseSucceed(Option.none));
	62	// The proof must name exactly this delivery, so it cannot be lifted onto another one.
	63	if (
	64	Option.isNone(payload) \|\|
	65	payload.value.environmentId !== environmentId \|\|
	66	payload.value.deliveryId !== deliveryId \|\|
	67	payload.value.receivedAt !== receivedAt \|\|
	68	payload.value.hookId !== input.hookId
	69	) {
	70	return Option.none();
	71	}
	72	return Option.some({ deliveryId, receivedAt });
	73	}).pipe(Effect.orElseSucceed(Option.none), Effect.withSpan("webhook.verifyRelayDelivery"));
	74	});

apps/server/src/scheduledTasks/webhookRoute.test.ts+204-20

...	...	@@ -1,6 +1,9 @@
1	1	import { describe, expect, it } from "@effect/vitest";
	2	import * as NodeCrypto from "node:crypto";
	3	import * as Clock from "effect/Clock";
2	4	import * as Effect from "effect/Effect";
3	5	import * as Layer from "effect/Layer";
	6	import * as Option from "effect/Option";
4	7	import * as NodeServices from "@effect/platform-node/NodeServices";
5	8	import * as Etag from "effect/http/Etag";
6	9	import * as HttpPlatform from "effect/http/HttpPlatform";
...	...	@@ -8,8 +11,10 @@ import * as HttpRouter from "effect/http/HttpRouter";
8	11	import * as HttpApi from "effect/http-api/HttpApi";
9	12	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
10	13
	14	import { RELAY_HOOK_DELIVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt";
11	15	import {
12	16	EnvironmentHttpApi,
	17	EnvironmentId,
13	18	ScheduledTaskWebhookDeliveryId,
14	19	ScheduledTaskError,
15	20	} from "@t3tools/contracts";
...	...	@@ -18,19 +23,76 @@ import {
18	23	type WebhookTriggerRequest,
19	24	type WebhookTriggerResult,
20	25	} from "./ScheduledTaskService.ts";
	26	import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
	27	import { CLOUD_MINT_PUBLIC_KEY, RELAY_ISSUER_SECRET } from "../cloud/config.ts";
	28	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
21	29	import { WEBHOOK_MAX_BODY_BYTES, webhookHttpApiLayer } from "./webhookRoute.ts";
22	30
23	31	class WebhookTestApi extends HttpApi.make("environment").add(EnvironmentHttpApi.groups.webhooks) {}
24	32
	33	const environmentId = EnvironmentId.make("env-1");
	34	const relayIssuer = "https://relay.example.test";
	35	const mintKeys = NodeCrypto.generateKeyPairSync("ed25519", {
	36	privateKeyEncoding: { format: "pem", type: "pkcs8" },
	37	publicKeyEncoding: { format: "pem", type: "spki" },
	38	});
	39	/** The secrets a T3 Connect-linked environment holds, keyed by name. */
	40	const linkedSecrets: ReadonlyMap<string, string> = new Map([
	41	[CLOUD_MINT_PUBLIC_KEY, mintKeys.publicKey],
	42	[RELAY_ISSUER_SECRET, relayIssuer],
	43	]);
	44
	45	/** A proof as the relay signs it for one delivery. */
	46	const relayProof = Effect.fn("relayProof")(function* (claims: {
	47	readonly deliveryId: string;
	48	readonly receivedAt: string;
	49	readonly hookId: string;
	50	readonly privateKey?: string;
	51	}) {
	52	const now = Math.floor((yield* Clock.currentTimeMillis) / 1_000);
	53	return yield* signRelayJwt({
	54	privateKey: claims.privateKey ?? mintKeys.privateKey,
	55	typ: RELAY_HOOK_DELIVERY_TYP,
	56	payload: {
	57	iss: relayIssuer,
	58	aud: `t3-env:${environmentId}`,
	59	sub: environmentId,
	60	jti: `proof:${claims.deliveryId}`,
	61	iat: now,
	62	exp: now + 3_600,
	63	environmentId,
	64	deliveryId: claims.deliveryId,
	65	receivedAt: claims.receivedAt,
	66	hookId: claims.hookId,
	67	},
	68	});
	69	});
	70
25	71	const handlerFor = (
26	72	trigger: (
27	73	request: WebhookTriggerRequest,
28	74	) => Effect.Effect<WebhookTriggerResult, ScheduledTaskError>,
	75	secrets: ReadonlyMap<string, string> = linkedSecrets,
29	76	) =>
30	77	HttpRouter.toWebHandler(
31	78	HttpApiBuilder.layer(WebhookTestApi).pipe(
32	79	Layer.provide(webhookHttpApiLayer),
33	80	Layer.provide(Layer.mock(ScheduledTaskService)({ triggerWebhook: trigger })),
	81	Layer.provide(
	82	Layer.mock(ServerSecretStore.ServerSecretStore)({
	83	get: (name) =>
	84	Effect.succeed(
	85	Option.map(Option.fromUndefinedOr(secrets.get(name)), (value) =>
	86	new TextEncoder().encode(value),
	87	),
	88	),
	89	}),
	90	),
	91	Layer.provide(
	92	Layer.mock(ServerEnvironment.ServerEnvironment)({
	93	getEnvironmentId: Effect.succeed(environmentId),
	94	}),
	95	),
34	96	Layer.provide(
35	97	HttpPlatform.layer.pipe(
36	98	Layer.provideMerge(NodeServices.layer),
...	...	@@ -56,6 +118,7 @@ describe("webhook route", () => {
56	118	return Effect.succeed({
57	119	_tag: "accepted",
58	120	deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"),
	121	outcome: "accepted",
59	122	});
60	123	});
61	124	try {
...	...	@@ -77,25 +140,142 @@ describe("webhook route", () => {
77	140	}
78	141	});
79	142
80		it("maps service outcomes to status codes", async () => {
81		const cases: ReadonlyArray<[WebhookTriggerResult["_tag"], number]> = [
82		["not_found", 404],
83		["rejected_signature", 401],
84		["disabled", 409],
85		["rate_limited", 429],
86		];
87		for (const [tag, status] of cases) {
88		const { handler, dispose } = handlerFor(() =>
89		Effect.succeed({ _tag: tag } as WebhookTriggerResult),
	143	// Live clock: the server checks proofs against real time.
	144	it.live("trusts the relay's delivery id and receive time only with its signed proof", () =>
	145	Effect.gen(function* () {
	146	const received: Array<WebhookTriggerRequest> = [];
	147	const { handler, dispose } = handlerFor((request) => {
	148	received.push(request);
	149	return Effect.succeed({
	150	_tag: "accepted",
	151	deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"),
	152	outcome: "accepted",
	153	});
	154	});
	155	const send = (path: string, headers: Record<string, string>) =>
	156	Effect.promise(() => handler(post(path, "{}", headers)));
	157	const receivedAt = "2026-10-04T10:00:00.000Z";
	158	const relayHeaders = {
	159	"x-t3-relay-delivery-id": "relay-1",
	160	"x-t3-relay-received-at": receivedAt,
	161	};
	162	const forged = NodeCrypto.generateKeyPairSync("ed25519", {
	163	privateKeyEncoding: { format: "pem", type: "pkcs8" },
	164	publicKeyEncoding: { format: "pem", type: "spki" },
	165	});
	166	const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" });
	167	const forgedProof = yield* relayProof({
	168	deliveryId: "relay-1",
	169	receivedAt,
	170	hookId: "id",
	171	privateKey: forged.privateKey,
	172	});
	173	// 0: signed by the relay for exactly this delivery.
	174	yield* send("/api/hooks/id/tok", { ...relayHeaders, "x-t3-relay-delivery": proof });
	175	// 1: a direct caller claiming to be the relay, without a proof.
	176	yield* send("/api/hooks/id/tok", relayHeaders);
	177	// 2: a proof signed with any other key.
	178	yield* send("/api/hooks/id/tok", { ...relayHeaders, "x-t3-relay-delivery": forgedProof });
	179	// 3: a real proof lifted onto another delivery id.
	180	yield* send("/api/hooks/id/tok", {
	181	...relayHeaders,
	182	"x-t3-relay-delivery-id": "relay-2",
	183	"x-t3-relay-delivery": proof,
	184	});
	185	// 4: a real proof for another hook.
	186	yield* send("/api/hooks/other/tok", { ...relayHeaders, "x-t3-relay-delivery": proof });
	187	yield* Effect.promise(() => dispose());
	188	expect(received.map((request) => request.relayDeliveryId)).toEqual([
	189	"relay-1",
	190	undefined,
	191	undefined,
	192	undefined,
	193	undefined,
	194	]);
	195	expect(received[0]?.receivedAt).toBe(receivedAt);
	196	expect(received.slice(1).every((request) => request.receivedAt === undefined)).toBe(true);
	197	}),
	198	);
	199
	200	// Live clock: the server checks proofs against real time.
	201	it.live("trusts no relay headers on an environment not linked to T3 Connect", () =>
	202	Effect.gen(function* () {
	203	const received: Array<WebhookTriggerRequest> = [];
	204	const { handler, dispose } = handlerFor((request) => {
	205	received.push(request);
	206	return Effect.succeed({ _tag: "not_found" });
	207	}, new Map());
	208	const receivedAt = "2026-10-04T10:00:00.000Z";
	209	const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" });
	210	yield* Effect.promise(() =>
	211	handler(
	212	post("/api/hooks/id/tok", "{}", {
	213	"x-t3-relay-delivery-id": "relay-1",
	214	"x-t3-relay-received-at": receivedAt,
	215	"x-t3-relay-delivery": proof,
	216	}),
	217	),
90	218	);
	219	yield* Effect.promise(() => dispose());
	220	expect(received[0]?.relayDeliveryId).toBeUndefined();
	221	}),
	222	);
	223
	224	it("maps service outcomes to status codes and names each outcome for the relay", async () => {
	225	const deliveryId = ScheduledTaskWebhookDeliveryId.make("delivery:1");
	226	const cases: ReadonlyArray<[WebhookTriggerResult, number, string]> = [
	227	[{ _tag: "accepted", deliveryId, outcome: "accepted" }, 202, "accepted"],
	228	// Same status as a started run; only the header tells them apart.
	229	[{ _tag: "accepted", deliveryId, outcome: "duplicate" }, 202, "duplicate"],
	230	[{ _tag: "accepted", deliveryId, outcome: "prompt_too_long" }, 202, "prompt_too_long"],
	231	[{ _tag: "not_found" }, 404, "not_found"],
	232	[{ _tag: "rejected_signature" }, 401, "rejected_signature"],
	233	[{ _tag: "disabled" }, 409, "disabled"],
	234	[{ _tag: "rate_limited", outcome: "rate_limited" }, 429, "rate_limited"],
	235	[{ _tag: "rate_limited", outcome: "queue_full" }, 429, "queue_full"],
	236	[{ _tag: "expired" }, 410, "expired"],
	237	];
	238	for (const [result, status, outcome] of cases) {
	239	const { handler, dispose } = handlerFor(() => Effect.succeed(result));
91	240	try {
92		expect((await handler(post("/api/hooks/id/tok", "{}"))).status).toBe(status);
	241	const response = await handler(post("/api/hooks/id/tok", "{}"));
	242	expect(response.status).toBe(status);
	243	expect(response.headers.get("x-t3-hook-outcome")).toBe(outcome);
93	244	} finally {
94	245	await dispose();
95	246	}
96	247	}
97	248	});
98	249
	250	// Live clock: the server checks proofs against real time.
	251	it.live("joins the relay's trace only for requests the relay forwarded", () =>
	252	Effect.gen(function* () {
	253	const parents: Array<string \| undefined> = [];
	254	const { handler, dispose } = handlerFor(() =>
	255	Effect.gen(function* () {
	256	const span = yield* Effect.currentParentSpan.pipe(Effect.option);
	257	parents.push(span._tag === "Some" ? span.value.traceId : undefined);
	258	return { _tag: "not_found" } as const;
	259	}),
	260	);
	261	const send = (headers: Record<string, string>) =>
	262	Effect.promise(() => handler(post("/api/hooks/id/tok", "{}", headers)));
	263	const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01";
	264	const receivedAt = "2026-10-04T10:00:00.000Z";
	265	const relayHeaders = {
	266	"x-t3-relay-delivery-id": "relay-1",
	267	"x-t3-relay-received-at": receivedAt,
	268	};
	269	const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" });
	270	yield* send({ ...relayHeaders, traceparent, "x-t3-relay-delivery": proof });
	271	// Claiming to be the relay without its proof cannot attach to our traces.
	272	yield* send({ ...relayHeaders, traceparent });
	273	yield* send({ traceparent });
	274	yield* Effect.promise(() => dispose());
	275	expect(parents).toEqual(["0af7651916cd43dd8448eb211c80319c", undefined, undefined]);
	276	}),
	277	);
	278
99	279	it("rejects oversized bodies and malformed paths before reaching the service", async () => {
100	280	let calls = 0;
101	281	const { handler, dispose } = handlerFor(() => {
...	...	@@ -132,16 +312,20 @@ describe("webhook route", () => {
132	312	}
133	313	});
134	314
135		it("hides service failures behind a 500", async () => {
136		const { handler, dispose } = handlerFor(() =>
	315	it("hides service failures and defects behind a fixed 500", async () => {
	316	const failures = [
137	317	Effect.fail(new ScheduledTaskError({ message: "database locked" })),
138		);
139		try {
140		const response = await handler(post("/api/hooks/id/tok", "{}"));
141		expect(response.status).toBe(500);
142		expect(await response.text()).not.toContain("database");
143		} finally {
144		await dispose();
	318	Effect.die(new Error("database exploded")),
	319	];
	320	for (const failure of failures) {
	321	const { handler, dispose } = handlerFor(() => failure);
	322	try {
	323	const response = await handler(post("/api/hooks/id/tok", "{}"));
	324	expect(response.status).toBe(500);
	325	expect(await response.json()).toEqual({ error: "internal_error" });
	326	} finally {
	327	await dispose();
	328	}
145	329	}
146	330	});
147	331	});

apps/server/src/scheduledTasks/webhookRoute.ts+56-15

...	...	@@ -2,27 +2,39 @@ import { EnvironmentHttpApi } from "@t3tools/contracts";
2	2	import * as ByteSize from "effect/ByteSize";
3	3	import * as Effect from "effect/Effect";
4	4	import * as Option from "effect/Option";
	5	import * as Tracer from "effect/Tracer";
5	6	import * as HttpIncomingMessage from "effect/http/HttpIncomingMessage";
6	7	import type * as HttpServerRequest from "effect/http/HttpServerRequest";
7	8	import * as HttpServerResponse from "effect/http/HttpServerResponse";
	9	import * as HttpTraceContext from "effect/http/HttpTraceContext";
	10	import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
8	11	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
9	12
	13	import * as Metrics from "../observability/Metrics.ts";
	14	import { makeRelayDeliveryVerifier } from "./relayDeliveryProof.ts";
10	15	import * as ScheduledTaskService from "./ScheduledTaskService.ts";
11	16
12	17	/** Largest request body a webhook accepts. The relay enforces the same cap. */
13	18	export const WEBHOOK_MAX_BODY_BYTES = 1024 * 1024;
14	19
15		const json = (status: number, body: Record<string, string>) =>
16		HttpServerResponse.jsonUnsafe(body, { status });
	20	/** Response header naming what happened to the request; the relay records it. */
	21	const WEBHOOK_OUTCOME_HEADER = "x-t3-hook-outcome";
	22
	23	const json = (status: number, body: Record<string, string>, outcome: string) =>
	24	HttpServerResponse.jsonUnsafe(body, { status, headers: { [WEBHOOK_OUTCOME_HEADER]: outcome } });
17	25
18	26	/**
19	27	* Handles `/api/hooks/:hookId/:token` for every accepted method. The endpoint
20	28	* is raw so the signature is checked over the exact body bytes; the service
21	29	* checks the token and signature. It is reachable directly, over the managed
22		* tunnel, or through the relay's stable `/v1/hooks/...` URL.
	30	* tunnel, or through the relay's stable `/v1/hooks/:endpointKey/:hookId/:token`
	31	* URL, where the endpoint key is this environment's managed tunnel key.
23	32	*/
24	33	const handleWebhook =
25		(scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"]) =>
	34	(
	35	scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"],
	36	verifyRelayDelivery: Effect.Success<typeof makeRelayDeliveryVerifier>,
	37	) =>
26	38	({
27	39	params,
28	40	request,
...	...	@@ -33,7 +45,7 @@ const handleWebhook =
33	45	Effect.gen(function* () {
34	46	const contentLength = Number(request.headers["content-length"] ?? "0");
35	47	if (!Number.isFinite(contentLength) \|\| contentLength > WEBHOOK_MAX_BODY_BYTES) {
36		return json(413, { error: "body_too_large" });
	48	return json(413, { error: "body_too_large" }, "body_too_large");
37	49	}
38	50	// Chunked requests carry no content-length, so the reader itself is capped.
39	51	const body = yield* request.arrayBuffer.pipe(
...	...	@@ -44,9 +56,15 @@ const handleWebhook =
44	56	),
45	57	Effect.option,
46	58	);
47		if (Option.isNone(body)) return json(413, { error: "body_too_large_or_unreadable" });
	59	// Refused before a task is looked up, so the service never sees them.
	60	const tooLarge = (error: string) =>
	61	Metrics.increment(Metrics.webhookDeliveriesTotal, {
	62	outcome: "body_too_large",
	63	source: request.headers["x-t3-relay-delivery-id"] ? "relay" : "direct",
	64	}).pipe(Effect.as(json(413, { error }, "body_too_large")));
	65	if (Option.isNone(body)) return yield* tooLarge("body_too_large_or_unreadable");
48	66	if (body.value.byteLength > WEBHOOK_MAX_BODY_BYTES) {
49		return json(413, { error: "body_too_large" });
	67	return yield* tooLarge("body_too_large");
50	68	}
51	69
52	70	const headers: Record<string, string> = {};
...	...	@@ -54,7 +72,18 @@ const handleWebhook =
54	72	if (typeof value === "string") headers[name.toLowerCase()] = value;
55	73	}
56	74	const queryIndex = request.url.indexOf("?");
	75	// The relay's delivery id and receive time count only with its signed
	76	// proof; the URL can also be called directly. The receive time matters
	77	// for requests the relay held while we were offline.
	78	const relay = yield* verifyRelayDelivery({ headers, hookId: params.hookId });
	79	const relayDeliveryId = Option.isSome(relay) ? relay.value.deliveryId : undefined;
	80	const relayReceivedAt = Option.isSome(relay) ? relay.value.receivedAt : undefined;
57	81
	82	// A relay delivery joins the relay's trace, and goes to the T3 Connect
	83	// tracer with it. Anyone else's traceparent is never trusted.
	84	const relayParent = Option.isSome(relay)
	85	? HttpTraceContext.fromHeaders(request.headers)
	86	: Option.none<Tracer.ExternalSpan>();
58	87	const result = yield* scheduledTasks
59	88	.triggerWebhook({
60	89	hookId: params.hookId,
...	...	@@ -65,9 +94,16 @@ const handleWebhook =
65	94	headers,
66	95	body: body.value,
67	96	bodyText: new TextDecoder().decode(body.value),
	97	...(relayDeliveryId ? { relayDeliveryId } : {}),
	98	...(relayReceivedAt ? { receivedAt: relayReceivedAt } : {}),
68	99	})
69	100	.pipe(
70		Effect.catch((cause) =>
	101	Option.isSome(relayParent)
	102	? (effect) =>
	103	effect.pipe(Effect.withParentSpan(relayParent.value), withRelayClientTracing)
	104	: (effect) => effect,
	105	// Defects too, so the sender only ever sees the fixed error body.
	106	Effect.catchCause((cause) =>
71	107	Effect.logWarning("Webhook delivery failed").pipe(
72	108	Effect.annotateLogs({ hookId: params.hookId }),
73	109	Effect.andThen(Effect.logDebug("Webhook delivery failure cause", { cause })),
...	...	@@ -78,17 +114,19 @@ const handleWebhook =
78	114
79	115	switch (result._tag) {
80	116	case "accepted":
81		return json(202, { deliveryId: result.deliveryId });
	117	return json(202, { deliveryId: result.deliveryId }, result.outcome);
82	118	case "not_found":
83		return json(404, { error: "hook_not_found" });
	119	return json(404, { error: "hook_not_found" }, "not_found");
84	120	case "rejected_signature":
85		return json(401, { error: "invalid_signature" });
	121	return json(401, { error: "invalid_signature" }, "rejected_signature");
86	122	case "disabled":
87		return json(409, { error: "hook_disabled" });
	123	return json(409, { error: "hook_disabled" }, "disabled");
88	124	case "rate_limited":
89		return json(429, { error: "rate_limited" });
	125	return json(429, { error: "rate_limited" }, result.outcome);
	126	case "expired":
	127	return json(410, { error: "delivery_too_old" }, "expired");
90	128	case "error":
91		return json(500, { error: "internal_error" });
	129	return json(500, { error: "internal_error" }, "error");
92	130	}
93	131	});
94	132
...	...	@@ -96,7 +134,10 @@ export const webhookHttpApiLayer = HttpApiBuilder.group(
96	134	EnvironmentHttpApi,
97	135	"webhooks",
98	136	Effect.fnUntraced(function* (handlers) {
99		const handler = handleWebhook(yield* ScheduledTaskService.ScheduledTaskService);
	137	const handler = handleWebhook(
	138	yield* ScheduledTaskService.ScheduledTaskService,
	139	yield* makeRelayDeliveryVerifier,
	140	);
100	141	return handlers
101	142	.handleRaw("webhookPost", handler)
102	143	.handleRaw("webhookPut", handler)

apps/server/src/scheduledTasks/webhookTemplate.test.ts+25

...	...	@@ -69,6 +69,31 @@ describe("renderWebhookPrompt", () => {
69	69	assert.deepEqual(plain.missing, ["body.field"]);
70	70	});
71	71
	72	it("redacts credentials in whole-request placeholders but not named ones", () => {
	73	const request: WebhookRequest = {
	74	...githubPullRequest,
	75	query: "source=github&access_token=q-secret",
	76	headers: {
	77	...githubPullRequest.headers,
	78	authorization: "Bearer h-secret",
	79	"x-hub-signature-256": "sha256=abc",
	80	},
	81	};
	82	const whole = renderWebhookPrompt("{{request}}\|{{headers}}\|{{query}}", request).prompt;
	83	for (const secret of ["q-secret", "h-secret", "sha256=abc"]) {
	84	assert.notInclude(whole, secret);
	85	}
	86	assert.include(whole, "?source=github&access_token=[redacted]\n");
	87	assert.include(whole, "authorization: [redacted]");
	88	assert.include(whole, "x-github-event: pull_request");
	89
	90	const named = renderWebhookPrompt(
	91	"{{headers.authorization}} {{query.access_token}}",
	92	request,
	93	).prompt;
	94	assert.equal(named, "Bearer h-secret q-secret");
	95	});
	96
72	97	it("does not resolve inherited object properties", () => {
73	98	const rendered = renderWebhookPrompt(
74	99	"{{body.constructor}}{{body.__proto__}}",

apps/server/src/scheduledTasks/webhookTemplate.ts+48-6

...	...	@@ -14,6 +14,10 @@
14	14	*
15	15	* Strings and numbers render as text, objects and arrays as JSON. A path with
16	16	* no value renders empty and is reported in `missing`.
	17	*
	18	* Credential-looking headers and query parameters are redacted wherever the
	19	* whole set renders (`{{request}}`, `{{headers}}`, `{{query}}`), as in the
	20	* delivery log. Naming one (`{{headers.authorization}}`) gives its raw value.
17	21	*/
18	22
19	23	export interface WebhookRequest {
...	...	@@ -31,6 +35,40 @@ export interface RenderedWebhookPrompt {
31	35	readonly missing: ReadonlyArray<string>;
32	36	}
33	37
	38	/** Header and query parameter names that commonly carry credentials. */
	39	const CREDENTIAL_NAME =
	40	/^(authorization\|proxy-authorization\|cookie\|set-cookie)$\|token\|secret\|signature\|key\|password\|auth/i;
	41	const REDACTED = "[redacted]";
	42
	43	export function redactHeaders(headers: Readonly<Record<string, string>>): Record<string, string> {
	44	return Object.fromEntries(
	45	Object.entries(headers).map(([name, value]) => [
	46	name,
	47	CREDENTIAL_NAME.test(name) ? REDACTED : value,
	48	]),
	49	);
	50	}
	51
	52	/** Redacts credential-named values in a raw query string, keeping the rest as sent. */
	53	export function redactQuery(query: string): string {
	54	if (query === "") return query;
	55	return query
	56	.split("&")
	57	.map((part) => {
	58	const separator = part.indexOf("=");
	59	if (separator === -1) return part;
	60	const name = part.slice(0, separator);
	61	let decoded = name;
	62	try {
	63	decoded = decodeURIComponent(name.replaceAll("+", " "));
	64	} catch {
	65	// A malformed escape is matched as sent.
	66	}
	67	return CREDENTIAL_NAME.test(decoded) ? `${name}=${REDACTED}` : part;
	68	})
	69	.join("&");
	70	}
	71
34	72	const PLACEHOLDER = /\{\{\s([^{}]?)\s*\}\}/g;
35	73
36	74	function parseBody(request: WebhookRequest): unknown {
...	...	@@ -64,9 +102,12 @@ function stringify(value: unknown): string {
64	102	}
65	103
66	104	function formatWebhookRequest(request: WebhookRequest): string {
67		const headerLines = Object.entries(request.headers).map(([name, value]) => `${name}: ${value}`);
	105	const headerLines = Object.entries(redactHeaders(request.headers)).map(
	106	([name, value]) => `${name}: ${value}`,
	107	);
	108	const query = redactQuery(request.query);
68	109	return [
69		`${request.method} ${request.path}${request.query ? `?${request.query}` : ""}`,
	110	`${request.method} ${request.path}${query ? `?${query}` : ""}`,
70	111	...headerLines,
71	112	"",
72	113	request.bodyText,
...	...	@@ -90,12 +131,13 @@ export function renderWebhookPrompt(
90	131	return segments.length === 0 ? request.bodyText : lookup(parsedBody(), segments);
91	132	case "headers":
92	133	return segments.length === 0
93		? request.headers
	134	? redactHeaders(request.headers)
94	135	: request.headers[segments.join(".").toLowerCase()];
95	136	case "query": {
96		const params = new URLSearchParams(request.query);
97		if (segments.length === 0) return Object.fromEntries(params);
98		return params.get(segments.join(".")) ?? undefined;
	137	if (segments.length === 0) {
	138	return Object.fromEntries(new URLSearchParams(redactQuery(request.query)));
	139	}
	140	return new URLSearchParams(request.query).get(segments.join(".")) ?? undefined;
99	141	}
100	142	default:
101	143	return undefined;

apps/server/src/server.ts+32-20

...	...	@@ -119,8 +119,16 @@ import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http
119	119	import * as ReplayMarkers from "./auth/replayMarkers.ts";
120	120	import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
121	121	import { webhookHttpApiLayer } from "./scheduledTasks/webhookRoute.ts";
122		import { ScheduledTaskWebhookOrigin } from "./scheduledTasks/ScheduledTaskService.ts";
123		import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./cloud/config.ts";
	122	import * as HeldHooksWaker from "./relay/HeldHooksWaker.ts";
	123	import {
	124	relayHookBaseUrl,
	125	ScheduledTaskWebhookOrigin,
	126	} from "./scheduledTasks/ScheduledTaskService.ts";
	127	import {
	128	CLOUD_ENDPOINT_RUNTIME_CONFIG,
	129	decodeRuntimeConfig,
	130	RELAY_URL_SECRET,
	131	} from "./cloud/config.ts";
124	132	import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
125	133	import {
126	134	connectHttpApiLayer,
...	...	@@ -452,24 +460,26 @@ const CloudManagedEndpointRuntimeLive = Layer.mergeAll(
452	460	// to is configured; otherwise clients show the environment-relative path.
453	461	const ScheduledTaskWebhookOriginLive = Layer.effect(
454	462	ScheduledTaskWebhookOrigin,
455		Effect.map(
456		Effect.all([ServerEnvironment.ServerEnvironment, ServerSecretStore.ServerSecretStore]),
457		([environment, secrets]) =>
458		// The reference holds an effect so each read sees the current link state.
459		Effect.gen(function* () {
460		const [relayUrl, tunnelConfig] = yield* Effect.all([
461		secrets.get(RELAY_URL_SECRET),
462		secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
463		]).pipe(Effect.orElseSucceed(() => [Option.none(), Option.none()] as const));
464		return {
465		environmentId: yield* environment.getEnvironmentId,
466		relayUrl:
467		Option.isSome(relayUrl) && Option.isSome(tunnelConfig)
468		? new TextDecoder().decode(relayUrl.value) \|\| null
469		: null,
470		};
471		}),
472		),
	463	Effect.gen(function* () {
	464	const secrets = yield* ServerSecretStore.ServerSecretStore;
	465	// The reference holds an effect so each read sees the current link state.
	466	return Effect.gen(function* () {
	467	const [relayUrl, tunnelConfig] = yield* Effect.all([
	468	secrets.get(RELAY_URL_SECRET),
	469	secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
	470	]).pipe(Effect.orElseSucceed(() => [Option.none(), Option.none()] as const));
	471	if (Option.isNone(relayUrl) \|\| Option.isNone(tunnelConfig)) {
	472	return { relayHookBaseUrl: null };
	473	}
	474	const config = decodeRuntimeConfig(new TextDecoder().decode(tunnelConfig.value));
	475	return {
	476	relayHookBaseUrl: relayHookBaseUrl({
	477	relayUrl: new TextDecoder().decode(relayUrl.value),
	478	tunnelName: Option.isSome(config) ? config.value.tunnelName : undefined,
	479	}),
	480	};
	481	});
	482	}),
473	483	);
474	484
475	485	const OrchestrationV2RuntimeLayerLive = OrchestrationV2ProductionLayerLive.pipe(
...	...	@@ -537,6 +547,8 @@ const ProviderInstallationRefreshLive = Layer.effectDiscard(
537	547
538	548	const RuntimeCoreDependenciesBaseLive = Layer.mergeAll(
539	549	AgentAwarenessRelay.layer,
	550	// Asks T3 Connect to deliver webhooks it held while this environment was offline.
	551	HeldHooksWaker.layer,
540	552	ThreadSettlementWorkerLive,
541	553	Layer.effectDiscard(StorageCleanup.make.pipe(Effect.flatMap((service) => service.start()))).pipe(
542	554	Layer.provide(ProjectionStoreV2.layer),

apps/web/src/cloud/linkEnvironment.ts+3-1

...	...	@@ -199,13 +199,15 @@ export function readPrimaryCloudLinkState(input: {
199	199	export function updatePrimaryCloudPreferences(input: {
200	200	readonly target: CloudLinkTarget;
201	201	readonly publishAgentActivity: boolean;
	202	readonly holdWebhooksWhileOffline?: boolean;
202	203	}): Effect.Effect<CloudLinkState, CloudEnvironmentLinkError, HttpClient.HttpClient> {
203	204	return Effect.gen(function* () {
204	205	const client = yield* makeEnvironmentHttpApiClient(input.target.httpBaseUrl);
	206	const { target: _target, ...payload } = input;
205	207	return yield* client.connect
206	208	.preferences({
207	209	headers: {},
208		payload: input,
	210	payload,
209	211	})
210	212	.pipe(
211	213	Effect.mapError(environmentApiError("Could not update environment cloud preferences.")),

apps/web/src/cloud/linkEnvironmentAtoms.ts+5-2

...	...	@@ -41,6 +41,9 @@ export const updatePrimaryEnvironmentPreferences = createRuntimeCommand(connecti
41	41	label: "web:cloud:update-primary-environment-preferences",
42	42	scheduler: cloudLinkScheduler,
43	43	concurrency: cloudLinkConcurrency,
44		execute: (input: { readonly target: CloudLinkTarget; readonly publishAgentActivity: boolean }) =>
45		updatePrimaryCloudPreferences(input),
	44	execute: (input: {
	45	readonly target: CloudLinkTarget;
	46	readonly publishAgentActivity: boolean;
	47	readonly holdWebhooksWhileOffline?: boolean;
	48	}) => updatePrimaryCloudPreferences(input),
46	49	});

apps/web/src/cloud/primaryCloudLinkState.ts+6-3

...	...	@@ -13,6 +13,7 @@ import { usePrimaryEnvironment } from "../state/environments";
13	13	import { runtime } from "../lib/runtime";
14	14	import { appAtomRegistry } from "../rpc/atomRegistry";
15	15	import { readPrimaryCloudLinkState, type CloudLinkTarget } from "./linkEnvironment";
	16	import { hasCloudPublicConfig } from "./publicConfig";
16	17
17	18	const primaryCloudLinkAtomRuntime = Atom.runtime(
18	19	Layer.effect(
...	...	@@ -62,9 +63,11 @@ export function usePrimaryCloudLinkState() {
62	63	: null,
63	64	[primary],
64	65	);
65		const atom = target
66		? primaryCloudLinkStateAtom(targetKey(target))
67		: EMPTY_PRIMARY_CLOUD_LINK_STATE_ATOM;
	66	// Builds without T3 Connect have no link to read; skip the request.
	67	const atom =
	68	target && hasCloudPublicConfig()
	69	? primaryCloudLinkStateAtom(targetKey(target))
	70	: EMPTY_PRIMARY_CLOUD_LINK_STATE_ATOM;
68	71	const result = useAtomValue(atom);
69	72	const refresh = useCallback(() => {
70	73	refreshPrimaryCloudLinkState(target);

apps/web/src/cloud/useCloudLinkController.ts+7

...	...	@@ -21,6 +21,8 @@ import { resolveRelayClerkTokenOptions } from "./publicConfig";
21	21	export interface CloudLinkDesiredState {
22	22	readonly managedTunnel: boolean;
23	23	readonly publish: boolean;
	24	/** Omit to leave the webhook-hold setting as it is. */
	25	readonly holdWebhooksWhileOffline?: boolean;
24	26	}
25	27
26	28	/**
...	...	@@ -75,6 +77,7 @@ export function useCloudLinkController() {
75	77	const managedTunnelActive =
76	78	primaryCloudLinkState.data?.managedTunnelActive ?? primaryCloudLinkState.data?.linked ?? false;
77	79	const publishAgentActivity = primaryCloudLinkState.data?.publishAgentActivity ?? false;
	80	const holdWebhooksWhileOffline = primaryCloudLinkState.data?.holdWebhooksWhileOffline ?? false;
78	81	const linked = primaryCloudLinkState.data?.linked ?? false;
79	82
80	83	const reconcileCloudState = async (desired: CloudLinkDesiredState): Promise<boolean> => {
...	...	@@ -132,6 +135,9 @@ export function useCloudLinkController() {
132	135	const prefResult = await updatePrimaryEnvironmentPreferences({
133	136	target,
134	137	publishAgentActivity: desired.publish,
	138	...(desired.holdWebhooksWhileOffline === undefined
	139	? {}
	140	: { holdWebhooksWhileOffline: desired.holdWebhooksWhileOffline }),
135	141	});
136	142	if (prefResult._tag === "Failure") {
137	143	if (!isAtomCommandInterrupted(prefResult)) {
...	...	@@ -157,6 +163,7 @@ export function useCloudLinkController() {
157	163	linked,
158	164	managedTunnelActive,
159	165	publishAgentActivity,
	166	holdWebhooksWhileOffline,
160	167	operationError,
161	168	reconcileCloudState,
162	169	};

apps/web/src/components/chat/ThreadAutomationsPanel.tsx+20-22

...	...	@@ -186,28 +186,26 @@ export function ThreadAutomationsPanel(props: {
186	186	/>
187	187	<TooltipPopup>Edit automation</TooltipPopup>
188	188	</Tooltip>
189		<Tooltip>
190		<TooltipTrigger
191		render={
192		<ThreadDetailsControl
193		size="icon-xs"
194		variant="ghost"
195		part="icon"
196		aria-label={`Run ${task.title} now`}
197		// A webhook task runs from its URL; there is no request to run it with.
198		disabled={
199		busyTaskId !== null \|\|
200		task.lastRunStatus === "running" \|\|
201		task.schedule.type === "webhook"
202		}
203		onClick={() => void runNow(task)}
204		>
205		<PlayIcon className="size-3.5" />
206		</ThreadDetailsControl>
207		}
208		/>
209		<TooltipPopup>Run now</TooltipPopup>
210		</Tooltip>
	189	{/* A webhook task runs from its URL; there is no request to run it with. */}
	190	{task.schedule.type === "webhook" ? null : (
	191	<Tooltip>
	192	<TooltipTrigger
	193	render={
	194	<ThreadDetailsControl
	195	size="icon-xs"
	196	variant="ghost"
	197	part="icon"
	198	aria-label={`Run ${task.title} now`}
	199	disabled={busyTaskId !== null \|\| task.lastRunStatus === "running"}
	200	onClick={() => void runNow(task)}
	201	>
	202	<PlayIcon className="size-3.5" />
	203	</ThreadDetailsControl>
	204	}
	205	/>
	206	<TooltipPopup>Run now</TooltipPopup>
	207	</Tooltip>
	208	)}
211	209	<Switch
212	210	checked={task.enabled}
213	211	disabled={busyTaskId !== null}

apps/web/src/components/settings/ConnectionsSettings.tsx+35

...	...	@@ -1761,6 +1761,7 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b
1761	1761	linkState: primaryCloudLinkState,
1762	1762	managedTunnelActive,
1763	1763	publishAgentActivity,
	1764	holdWebhooksWhileOffline,
1764	1765	operationError,
1765	1766	reconcileCloudState,
1766	1767	} = useCloudLinkController();
...	...	@@ -1812,6 +1813,25 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b
1812	1813	setIsUpdatingPreference(false);
1813	1814	};
1814	1815
	1816	const updateHoldWebhooks = async (enabled: boolean) => {
	1817	setIsUpdatingPreference(true);
	1818	const ok = await reconcileCloudState({
	1819	managedTunnel: managedTunnelActive,
	1820	publish: publishAgentActivity,
	1821	holdWebhooksWhileOffline: enabled,
	1822	});
	1823	if (ok) {
	1824	toastManager.add({
	1825	type: "success",
	1826	title: enabled ? "Webhooks held while offline" : "Webhooks no longer held",
	1827	description: enabled
	1828	? "T3 Connect keeps webhook requests for up to 24 hours while this environment is offline."
	1829	: "Requests to an offline environment now fail. Anything already held is still delivered.",
	1830	});
	1831	}
	1832	setIsUpdatingPreference(false);
	1833	};
	1834
1815	1835	return (
1816	1836	<>
1817	1837	{window.desktopBridge ? (
...	...	@@ -1846,6 +1866,21 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b
1846	1866	/>
1847	1867	}
1848	1868	/>
	1869	{managedTunnelActive ? (
	1870	<SettingsRow
	1871	title={searchableSetting("hold-webhooks-while-offline").title}
	1872	description="Keep webhook requests for up to 24 hours while this environment is offline, then deliver them. Off: T3 Connect only forwards requests and stores nothing."
	1873	control={
	1874	<CloudLinkSwitch
	1875	ariaLabel="Hold webhook requests while this environment is offline"
	1876	checked={holdWebhooksWhileOffline}
	1877	disabled={!canManageRelay \|\| !isSignedIn \|\| primaryCloudLinkState.isPending \|\| isBusy}
	1878	disabledReason={disabledReason}
	1879	onCheckedChange={(enabled) => void updateHoldWebhooks(enabled)}
	1880	/>
	1881	}
	1882	/>
	1883	) : null}
1849	1884	</>
1850	1885	);
1851	1886	}

apps/web/src/components/settings/ScheduledTasksSettings.tsx+58-2

...	...	@@ -23,7 +23,9 @@ import type {
23	23	ScheduledTaskWebhookDeliverySummary,
24	24	ThreadId,
25	25	} from "@t3tools/contracts";
	26	import { DEFAULT_WEBHOOK_PROMPT } from "@t3tools/client-runtime/scheduled-task-webhook";
26	27	import {
	28	MAX_WEBHOOK_DELIVERY_AGE_MINUTES,
27	29	MIN_SCHEDULED_TASK_INTERVAL_MS,
28	30	ProviderInstanceId,
29	31	resolveEnvironmentMachineKind,
...	...	@@ -41,8 +43,10 @@ import {
41	43	deriveProviderInstanceEntries,
42	44	sortProviderInstanceEntries,
43	45	} from "../../providerInstances";
	46	import { usePrimaryCloudLinkState } from "../../cloud/primaryCloudLinkState";
44	47	import { requestConfirmDialog } from "../../confirmDialog";
45	48	import { webhookAddress } from "@t3tools/client-runtime/webhook-address";
	49	import { Link } from "@tanstack/react-router";
46	50	import { useCopyToClipboard } from "../../hooks/useCopyToClipboard";
47	51	import {
48	52	useEnvironment,
...	...	@@ -57,7 +61,6 @@ import { WorktreeBaseBranchPicker } from "../WorktreeBaseBranchPicker";
57	61	import { EnvironmentMachineIcon } from "../EnvironmentMachineIcon";
58	62	import { useSettingsScope } from "./SettingsScopeContext";
59	63	import {
60		DEFAULT_WEBHOOK_PROMPT,
61	64	WEBHOOK_SIGNATURE_DEFAULTS,
62	65	matchesScheduledTaskScope,
63	66	scheduleFromDraft,
...	...	@@ -130,6 +133,7 @@ const EMPTY_DRAFT: DraftState = {
130	133	signatureEnabled: false,
131	134	...WEBHOOK_SIGNATURE_DEFAULTS,
132	135	signatureSecret: "",
	136	maxDeliveryAgeMinutes: "",
133	137	};
134	138
135	139	/** Labelled field: a caption sitting above its control. */
...	...	@@ -211,11 +215,14 @@ const DELIVERY_OUTCOME_LABELS: Record<ScheduledTaskWebhookDeliveryOutcome, strin
211	215	rejected_signature: "Bad signature",
212	216	disabled: "Task paused",
213	217	rate_limited: "Rate limited",
	218	expired: "Too old",
214	219	};
215	220
216	221	function deliveryOutcomeVariant(outcome: ScheduledTaskWebhookDeliveryOutcome) {
217	222	if (outcome === "accepted") return "success";
218		if (outcome === "disabled" \|\| outcome === "rate_limited") return "warning";
	223	if (outcome === "disabled" \|\| outcome === "rate_limited" \|\| outcome === "expired") {
	224	return "warning";
	225	}
219	226	return "error";
220	227	}
221	228
...	...	@@ -725,10 +732,31 @@ function WebhookEndpointField({
725	732	</Button>
726	733	</div>
727	734	{note !== null ? <p className="text-xs text-muted-foreground">{note}</p> : null}
	735	{endpoint.url !== null ? <WebhookDeliveryMode environmentId={environmentId} /> : null}
728	736	</div>
729	737	);
730	738	}
731	739
	740	/**
	741	* Whether T3 Connect forwards requests live or holds them while the
	742	* environment is offline. The setting is per environment and only readable
	743	* for this machine's own environment, so other environments show nothing.
	744	*/
	745	function WebhookDeliveryMode({ environmentId }: { readonly environmentId: EnvironmentId }) {
	746	const cloudLink = usePrimaryCloudLinkState();
	747	if (cloudLink.target?.environmentId !== environmentId \|\| cloudLink.data === null) return null;
	748	return (
	749	<p className="text-xs text-muted-foreground">
	750	{cloudLink.data.holdWebhooksWhileOffline
	751	? "Held for up to 24 hours while this environment is offline. "
	752	: "Forwarded live. Requests fail while this environment is offline. "}
	753	<Link to="/settings/connections" className="underline underline-offset-2">
	754	Change in Connections
	755	</Link>
	756	</p>
	757	);
	758	}
	759
732	760	function ScheduledTaskEditorDialog({
733	761	initialEnvironmentId,
734	762	task,
...	...	@@ -830,6 +858,13 @@ function ScheduledTaskEditorDialog({
830	858	return;
831	859	}
832	860	const schedule = scheduleFromDraft(draft);
	861	if (schedule === null) {
	862	reportFailure(
	863	"Invalid age limit",
	864	`Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`,
	865	);
	866	return;
	867	}
833	868	if (
834	869	schedule.type === "webhook" &&
835	870	schedule.signature &&
...	...	@@ -1126,6 +1161,27 @@ function ScheduledTaskEditorDialog({
1126	1161	"Each request runs the prompt. Use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} or {{request}} in the prompt; only what it names reaches the agent."
1127	1162	}
1128	1163	</p>
	1164	<Field
	1165	label="Skip requests older than"
	1166	hint="minutes, optional"
	1167	htmlFor="scheduled-task-max-age"
	1168	>
	1169	<Input
	1170	id="scheduled-task-max-age"
	1171	type="number"
	1172	nativeInput
	1173	min={1}
	1174	max={MAX_WEBHOOK_DELIVERY_AGE_MINUTES}
	1175	placeholder="Run every request"
	1176	value={draft.maxDeliveryAgeMinutes}
	1177	onChange={(event) =>
	1178	setDraft((current) => ({
	1179	...current,
	1180	maxDeliveryAgeMinutes: event.target.value,
	1181	}))
	1182	}
	1183	/>
	1184	</Field>
1129	1185	<div className="flex items-center justify-between gap-4">
1130	1186	<div className="min-w-0 space-y-1">
1131	1187	<Label htmlFor="scheduled-task-signature">Require signature</Label>

apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts+28-2

...	...	@@ -181,10 +181,15 @@ describe("webhook scheduled tasks", () => {
181	181	const draft = taskToDraft(webhookTask);
182	182	expect(draft.scheduleMode).toBe("webhook");
183	183	expect(draft.signatureSecret).toBe("");
184		expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature });
	184	expect(scheduleFromDraft(draft)).toEqual({
	185	type: "webhook",
	186	signature,
	187	maxDeliveryAgeMinutes: null,
	188	});
185	189	expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({
186	190	type: "webhook",
187	191	signature: { ...signature, secret: "new" },
	192	maxDeliveryAgeMinutes: null,
188	193	});
189	194	});
190	195
...	...	@@ -192,7 +197,28 @@ describe("webhook scheduled tasks", () => {
192	197	const draft = taskToDraft({ ...webhookTask, schedule: { type: "webhook", signature: null } });
193	198	expect(draft.signatureEnabled).toBe(false);
194	199	expect(draft.signatureHeader).toBe("x-hub-signature-256");
195		expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null });
	200	expect(scheduleFromDraft(draft)).toEqual({
	201	type: "webhook",
	202	signature: null,
	203	maxDeliveryAgeMinutes: null,
	204	});
	205	});
	206
	207	it("round-trips the max age, treats blank as no limit, and rejects an invalid limit", () => {
	208	const draft = taskToDraft({
	209	...webhookTask,
	210	schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 90 },
	211	});
	212	expect(draft.maxDeliveryAgeMinutes).toBe("90");
	213	expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 90 });
	214	for (const blank of ["", " "]) {
	215	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: blank })).toMatchObject({
	216	maxDeliveryAgeMinutes: null,
	217	});
	218	}
	219	for (const invalid of ["0", "-5", "1.5", "abc", "1441"]) {
	220	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: invalid })).toBeNull();
	221	}
196	222	});
197	223	});
198	224

apps/web/src/components/settings/scheduledTasksSettings.logic.ts+12-4

...	...	@@ -9,6 +9,7 @@ import {
9	9	type ProviderInteractionMode,
10	10	type ServerSettings,
11	11	} from "@t3tools/contracts";
	12	import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook";
12	13
13	14	import {
14	15	resolveProjectSettings,
...	...	@@ -78,6 +79,8 @@ export interface DraftState {
78	79	readonly signaturePrefix: string;
79	80	/** Write-only: empty keeps the secret already stored on the server. */
80	81	readonly signatureSecret: string;
	82	/** Minutes as typed; empty runs every held request regardless of age. */
	83	readonly maxDeliveryAgeMinutes: string;
81	84	}
82	85
83	86	/** GitHub's signature settings, the most common sender. */
...	...	@@ -87,11 +90,11 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = {
87	90	signaturePrefix: "sha256=",
88	91	} as const;
89	92
90		/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */
91		export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}";
92
93		export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule {
	93	/** Null when the draft's webhook age limit is invalid; the caller reports it and does not save. */
	94	export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule \| null {
94	95	if (draft.scheduleMode === "webhook") {
	96	const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes);
	97	if (maxDeliveryAgeMinutes === undefined) return null;
95	98	const secret = draft.signatureSecret.trim();
96	99	return {
97	100	type: "webhook",
...	...	@@ -103,6 +106,7 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul
103	106	...(secret ? { secret } : {}),
104	107	}
105	108	: null,
	109	maxDeliveryAgeMinutes,
106	110	};
107	111	}
108	112	if (draft.scheduleMode === "interval") {
...	...	@@ -159,6 +163,10 @@ export function taskToDraft(task: ScheduledTask): DraftState {
159	163	}
160	164	: { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS }),
161	165	signatureSecret: "",
	166	maxDeliveryAgeMinutes:
	167	schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null
	168	? String(schedule.maxDeliveryAgeMinutes)
	169	: "",
162	170	};
163	171	}
164	172

apps/web/src/components/settings/settingsSearch.test.ts+19

...	...	@@ -170,6 +170,7 @@ describe("searchSettings", () => {
170	170	"source-control-writer-model",
171	171	"source-control-writing-style",
172	172	"t3-connect",
	173	"hold-webhooks-while-offline",
173	174	"tailscale-https",
174	175	"wsl-backend",
175	176	"auto-settle-inactive-threads",
...	...	@@ -221,6 +222,24 @@ describe("searchSettings", () => {
221	222	expect(browser).toContain("publish-agent-activity");
222	223	});
223	224
	225	it("offers webhook holding only while the managed tunnel is on, like its row", () => {
	226	const availability = {
	227	hasCloudPublicConfig: true,
	228	hasEnvironment: true,
	229	hasProviderSettingsEnvironment: true,
	230	hasMacProviderSettingsEnvironment: false,
	231	canManageLocalBackend: true,
	232	isWslSettingsRowVisible: false,
	233	hasThreadAutoSettlement: false,
	234	};
	235	const itemIds = (managedTunnelActive: boolean) =>
	236	filterAvailableSettingsSearchItems({ ...availability, managedTunnelActive }).map(
	237	(item) => item.id,
	238	);
	239	expect(itemIds(false)).not.toContain("hold-webhooks-while-offline");
	240	expect(itemIds(true)).toContain("hold-webhooks-while-offline");
	241	});
	242
224	243	it("shows automatic settlement settings when the server supports them", () => {
225	244	const available = filterAvailableSettingsSearchItems({
226	245	hasCloudPublicConfig: false,

apps/web/src/components/settings/settingsSearch.ts+15-1

...	...	@@ -60,6 +60,8 @@ export interface SettingsSearchItem {
60	60	readonly localBackendManagementOnly?: boolean;
61	61	readonly localEnvironmentOnly?: boolean;
62	62	readonly wslAvailableOnly?: boolean;
	63	// Its row only renders while this environment's T3 Connect managed tunnel is on.
	64	readonly managedTunnelOnly?: boolean;
63	65	/**
64	66	* Sorts after every other match. Keybinding commands mirror rows on other
65	67	* surfaces, so "model" must still lead with Default model, not Model Picker.
...	...	@@ -77,6 +79,7 @@ export interface SettingsSearchAvailability {
77	79	readonly canManageLocalBackend: boolean;
78	80	readonly isWslSettingsRowVisible: boolean;
79	81	readonly hasThreadAutoSettlement: boolean;
	82	readonly managedTunnelActive?: boolean;
80	83	}
81	84
82	85	/**
...	...	@@ -836,6 +839,16 @@ export const SETTINGS_SEARCH_ITEMS = [
836	839	desktopOnly: true,
837	840	cloudOnly: true,
838	841	},
	842	{
	843	id: "hold-webhooks-while-offline",
	844	localEnvironmentOnly: true,
	845	title: "Hold webhooks while offline",
	846	to: "/settings/connections",
	847	targetId: "connections-environment",
	848	searchTerms: ["webhook automations offline queue mailbox t3 connect"],
	849	cloudOnly: true,
	850	managedTunnelOnly: true,
	851	},
839	852	{
840	853	id: "publish-agent-activity",
841	854	localEnvironmentOnly: true,
...	...	@@ -1018,7 +1031,8 @@ export function filterAvailableSettingsSearchItems(
1018	1031	(!item.localBackendManagementOnly \|\| availability.canManageLocalBackend) &&
1019	1032	(!item.localEnvironmentOnly \|\| !availability.localEnvironmentDisabled) &&
1020	1033	(!item.wslAvailableOnly \|\| availability.isWslSettingsRowVisible) &&
1021		(!item.requiresThreadAutoSettlement \|\| availability.hasThreadAutoSettlement),
	1034	(!item.requiresThreadAutoSettlement \|\| availability.hasThreadAutoSettlement) &&
	1035	(!item.managedTunnelOnly \|\| availability.managedTunnelActive === true),
1022	1036	);
1023	1037	}
1024	1038

apps/web/src/components/settings/useAvailableSettingsSearchItems.ts+7

...	...	@@ -1,6 +1,7 @@
1	1	import { useMemo } from "react";
2	2	import { AuthAccessWriteScope } from "@t3tools/contracts";
3	3
	4	import { usePrimaryCloudLinkState } from "~/cloud/primaryCloudLinkState";
4	5	import { hasCloudPublicConfig } from "~/cloud/publicConfig";
5	6	import { isElectron } from "~/env";
6	7	import { isLocalEnvironmentDisabled } from "~/localEnvironment";
...	...	@@ -23,6 +24,10 @@ export function useAvailableSettingsSearchItems(scopeSearch: SettingsScopeSearch
23	24	const desktopWsl = useEnvironmentQuery(
24	25	isElectron && !localEnvironmentDisabled ? desktopWslStateAtom : null,
25	26	);
	27	const cloudLinkState = usePrimaryCloudLinkState().data;
	28	// Same fallback as the Connections row: older servers imply a tunnel from `linked`.
	29	const managedTunnelActive =
	30	cloudLinkState?.managedTunnelActive ?? cloudLinkState?.linked ?? false;
26	31	const canManageLocalBackend =
27	32	!localEnvironmentDisabled &&
28	33	(isElectron \|\|
...	...	@@ -59,8 +64,10 @@ export function useAvailableSettingsSearchItems(scopeSearch: SettingsScopeSearch
59	64	}),
60	65	hasThreadAutoSettlement:
61	66	getThreadAutoSettlementSearchAvailability(environments).eligibleEnvironmentIds.length > 0,
	67	managedTunnelActive,
62	68	}),
63	69	[
	70	managedTunnelActive,
64	71	canManageLocalBackend,
65	72	desktopWsl.data,
66	73	desktopWsl.error,

docs/internals/t3-connect.md+22-4

...	...	@@ -5,12 +5,30 @@ credentials for reaching environments, and managed tunnel allocations. After
5	5	bootstrap, clients send application traffic through the environment's tunnel
6	6	hostname; the relay Worker does not proxy their HTTP or WebSocket sessions.
7	7	The one exception is automation webhooks: the relay forwards
8		`/v1/hooks/:environmentId/:hookId/:token` statelessly to the environment's
9		tunnel so senders get a stable URL. It stores nothing, keeps bodies and tokens
10		out of its traces, and leaves token and signature verification to the
11		environment
	8	`/v1/hooks/:environmentId/:hookId/:token` to the environment's tunnel so
	9	senders get a stable URL. It keeps bodies and tokens out of its traces and
	10	leaves token and signature verification to the environment
12	11	([forwarder](../../infra/relay/src/hooks/HookForwarder.ts)).
13	12
	13	By default the forwarder stores nothing. An environment can opt in to having
	14	the relay hold requests while it is offline
	15	(`hold_webhooks_while_offline` on its link). Only then does the relay store the
	16	raw request, including the hook token in the path, in a Durable Object for
	17	that environment, with SQLite storage. The object pushes held requests back
	18	through the tunnel from its alarm, oldest first, and backs off while the
	19	environment stays away. When the tunnel reconnects, the environment asks the
	20	relay to deliver right away. Requests are deleted once the environment
	21	answers, after 24 hours, or when no user has the environment linked. The relay
	22	still never checks the token; delivery goes through the same environment route.
	23	Every forward carries `x-t3-relay-delivery-id`, so a request that reached the
	24	environment before a timeout and is delivered again later runs once
	25	([inbox object](../../infra/relay/src/hooks/HookInboxObject.ts)).
	26
	27	A Durable Object, not Postgres or Queues, because held requests are write-once,
	28	read-once bodies of up to 1 MiB that need per-environment order, caps, and
	29	retry timing. Queues cap messages at 128 KB and cannot hold one environment's
	30	requests back while it is away.
	31
14	32	Clerk, deployment, and native authentication setup live in the
15	33	[Connect setup runbook](../operations/connect-setup.md).
16	34

docs/operations/observability.md+15

...	...	@@ -386,6 +386,21 @@ Counters tell you volume and failure rate:
386	386	- `t3_provider_turns_total`
387	387	- `t3_git_commands_total`
388	388
	389	Webhooks have their own families:
	390
	391	- `t3_webhook_deliveries_total` by `outcome` and `source` (`relay` or `direct`). Beyond what the
	392	sender sees, `queue_full` means a task already had its limit of deliveries waiting,
	393	`prompt_too_long` means the filled-in prompt passed the provider limit, and `duplicate` means
	394	the relay delivered a request this environment had already run.
	395	- `t3_webhook_runs_total` by `outcome` (`started`, `skipped`, `failed`) for the runs those
	396	deliveries start, which happen after the sender has its answer.
	397	- `t3_webhook_held_delay` for how long requests the relay held waited before arriving.
	398
	399	`ScheduledTaskService.triggerWebhook` spans carry the same outcome per request, and each run
	400	started from a delivery is its own `ScheduledTaskService.runWebhookDelivery` trace. For a request
	401	the relay forwarded, the span also goes to the T3 Connect trace export as a child of the relay's
	402	span; requests that reach the environment directly never join a sender's trace.
	403
389	404	Use metrics when the question is:
390	405
391	406	- "is this always slow?"

docs/operations/relay-observability.md+51

...	...	@@ -57,3 +57,54 @@ failure means that a signed proof was too old or too far in the future for the r
57	57	window. It can point to a date or time problem on either device, but it can also result from a
58	58	delayed request. The client uses this category, and the absence of a category from an older relay,
59	59	to decide whether clock skew is confirmed or only one possible cause.
	60
	61	## Webhooks
	62
	63	A public webhook request is one `relay.hooks.forward` span. Its `relay.hook.outcome` says what
	64	happened: `forwarded`, `held`, `rate_limited`, `inbox_full`, `not_found`, `payload_too_large`,
	65	`environment_unavailable`, or `environment_timeout`. `relay.hook.endpoint_key` identifies the
	66	managed endpoint, and with it the environment. On a forward, `relay.hook.upstream_status` or
	67	`relay.hook.upstream_error` records the environment's answer, and
	68	`relay.hook.upstream_outcome` what it did with the request (`accepted`, `duplicate`,
	69	`prompt_too_long`, `queue_full`, `rejected_signature`, `expired`, `disabled`, ...), from its
	70	`x-t3-hook-outcome` response header. A held request's delivery records the same on its
	71	`relay.inbox.deliver` span. The relay sends its own `traceparent` with each forward and drops any a
	72	sender supplied, and signs each forward (`x-t3-relay-delivery`) so the environment trusts the
	73	relay's delivery id, receive time and trace context only from the relay, so on environments that export to T3 Connect, the environment's
	74	`ScheduledTaskService.triggerWebhook` span lands in the same trace. `relay.hook.rate_limit` says which
	75	budget ran out: `endpoint` or `hook`. `relay.hook.rate_limiter_failed_open` is set when the
	76	Cloudflare rate limiter was unavailable and the request went through unlimited.
	77
	78	Held requests are handled in the endpoint's `HookInboxObject`, and each call into it is its own
	79	trace, not a child of the forward span: `relay.inbox.hold`, `relay.inbox.wake`, and
	80	`relay.inbox.deliver` for each alarm run. Each carries `relay.hook.endpoint_key`, so they join to
	81	forwards. The details are on the child spans: `HookInboxStore.hold` carries
	82	`relay.inbox.refused` when the inbox refused a request (`max_per_hook`, `max_requests`,
	83	`max_bytes`, or `already_held`). `HookInboxStore.deliverDue` carries `relay.inbox.run_result`
	84	(`drained`, `more_pending`, `busy`, or `unreachable`), how many requests it sent and delivered,
	85	the consecutive failures behind the retry delay, the longest time a delivered request waited, and
	86	the backlog left. A run that errored outright has `relay.inbox.run_result = failed` on the root
	87	`relay.inbox.deliver` span instead.
	88
	89	Questions these answer:
	90
	91	```apl
	92	// Webhook outcomes per hour
	93	['t3-code-relay-traces-prod']
	94	\| where name == 'relay.hooks.forward'
	95	\| summarize count() by bin(_time, 1h), outcome = tostring(['attributes.custom']['relay.hook.outcome'])
	96
	97	// Inboxes stuck behind an unreachable environment, by endpoint
	98	['t3-code-relay-traces-prod']
	99	\| where name == 'HookInboxStore.deliverDue'
	100	\| extend c = ['attributes.custom']
	101	\| where tostring(c['relay.inbox.run_result']) == 'unreachable'
	102	\| project trace_id, failures = toint(c['relay.inbox.consecutive_failures']),
	103	backlog = toint(c['relay.inbox.held_count'])
	104	\| join kind=inner (
	105	['t3-code-relay-traces-prod']
	106	\| where name == 'relay.inbox.deliver'
	107	\| project trace_id, endpoint = tostring(['attributes.custom']['relay.hook.endpoint_key'])
	108	) on trace_id
	109	\| summarize runs = count(), failures = max(failures), backlog = max(backlog) by endpoint
	110	```

docs/user/project-settings.md+21-10

...	...	@@ -61,16 +61,18 @@ using its project, model, and workspace settings. Fixed-time schedules use that
61	61	environment's time zone, which may differ from your phone's.
62	62
63	63	You can edit, pause, resume, run immediately, or delete a task from the list.
	64	Webhook tasks only run when their URL is called, so they can't be run
	65	immediately.
64	66	Leaving an edited form asks before discarding unsaved changes.
65	67
66	68	## Webhook automations
67	69
68		In Settings → Scheduled tasks, choose On webhook as a task's schedule to run it whenever another service
69		calls its URL, such as GitHub on a new pull request or a CI job that failed.
70		After you save the task, copy its URL from the editor. If the environment uses a
71		[T3 Connect](remote-access.md) managed tunnel, the URL is public; otherwise it
72		works anywhere the environment itself is reachable. Rotate replaces the URL and
73		the old one stops working.
	70	In Settings → Scheduled tasks, choose On webhook
	71	as a task's schedule to run it whenever another service calls its URL, such as
	72	GitHub on a new pull request or a CI job that failed. A public URL needs a
	73	[T3 Connect](remote-access.md) managed tunnel; after you save the task, copy
	74	its URL from the editor. Without one, the editor shows only the URL's path.
	75	Rotate replaces the URL and the old one stops working.
74	76
75	77	The prompt decides what the agent sees. Placeholders pull values out of the
76	78	request: `{{body.path}}` for a JSON or form field, `{{headers.name}}`,
...	...	@@ -81,12 +83,21 @@ pull request link. A placeholder with no value is left empty.
81	83	For GitHub, turn on Require signature, keep the header
82	84	`x-hub-signature-256`, hex encoding and the `sha256=` prefix, and enter the
83	85	same secret in the repository's webhook settings with content type
84		`application/json`. Requests without a valid signature are rejected.
	86	`application/json`. Requests without a valid signature are rejected. Set this
	87	up on desktop or web; mobile keeps an existing signature check but can't turn
	88	one on.
85	89
86	90	On desktop and web, pick Deliveries from a task's menu to see recent
87		requests and the prompt each one produced. If the environment is offline, the sender gets an error and
88		nothing runs; redeliver from the sender, such as GitHub's Recent Deliveries,
89		once it is back.
	91	requests and the prompt each one produced.
	92
	93	If the environment is offline, the sender gets an error and nothing runs;
	94	redeliver from the sender, such as GitHub's Recent Deliveries, once it is
	95	back. To have T3 Connect keep requests instead, turn on **Hold webhooks while
	96	offline in Settings → Connections**. T3 Connect then stores requests to a
	97	T3 Connect URL for up to 24 hours and delivers them when the environment
	98	returns. Leave it off if you don't want request bodies stored outside your
	99	machine. To skip requests that waited too long, set **Skip requests older
	100	than** on the task.
90	101
91	102	## Defaults and inheritance
92	103

infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql created+1

...	...	@@ -0,0 +1 @@
	1	ALTER TABLE "relay_environment_links" ADD COLUMN "hold_webhooks_while_offline" boolean DEFAULT false NOT NULL;
		\ No newline at end of file

infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/snapshot.json created+1581

...	...	@@ -0,0 +1,1581 @@
	1	{
	2	"version": "8",
	3	"dialect": "postgres",
	4	"id": "13972659-45db-4ce9-9818-523b450bceca",
	5	"prevIds": ["3809c51e-3821-4a08-818d-e5d28dd3e9b4"],
	6	"ddl": [
	7	{
	8	"isRlsEnabled": false,
	9	"name": "relay_agent_activity_rows",
	10	"entityType": "tables",
	11	"schema": "public"
	12	},
	13	{
	14	"isRlsEnabled": false,
	15	"name": "relay_delivery_attempts",
	16	"entityType": "tables",
	17	"schema": "public"
	18	},
	19	{
	20	"isRlsEnabled": false,
	21	"name": "relay_dpop_proofs",
	22	"entityType": "tables",
	23	"schema": "public"
	24	},
	25	{
	26	"isRlsEnabled": false,
	27	"name": "relay_environment_credentials",
	28	"entityType": "tables",
	29	"schema": "public"
	30	},
	31	{
	32	"isRlsEnabled": false,
	33	"name": "relay_environment_links",
	34	"entityType": "tables",
	35	"schema": "public"
	36	},
	37	{
	38	"isRlsEnabled": false,
	39	"name": "relay_live_activities",
	40	"entityType": "tables",
	41	"schema": "public"
	42	},
	43	{
	44	"isRlsEnabled": false,
	45	"name": "relay_managed_endpoint_allocations",
	46	"entityType": "tables",
	47	"schema": "public"
	48	},
	49	{
	50	"isRlsEnabled": false,
	51	"name": "relay_managed_tunnel_limits",
	52	"entityType": "tables",
	53	"schema": "public"
	54	},
	55	{
	56	"isRlsEnabled": false,
	57	"name": "relay_mobile_devices",
	58	"entityType": "tables",
	59	"schema": "public"
	60	},
	61	{
	62	"type": "varchar(191)",
	63	"typeSchema": null,
	64	"notNull": true,
	65	"dimensions": 0,
	66	"default": null,
	67	"generated": null,
	68	"identity": null,
	69	"name": "environment_id",
	70	"entityType": "columns",
	71	"schema": "public",
	72	"table": "relay_agent_activity_rows"
	73	},
	74	{
	75	"type": "text",
	76	"typeSchema": null,
	77	"notNull": true,
	78	"dimensions": 0,
	79	"default": null,
	80	"generated": null,
	81	"identity": null,
	82	"name": "environment_public_key",
	83	"entityType": "columns",
	84	"schema": "public",
	85	"table": "relay_agent_activity_rows"
	86	},
	87	{
	88	"type": "varchar(512)",
	89	"typeSchema": null,
	90	"notNull": true,
	91	"dimensions": 0,
	92	"default": null,
	93	"generated": null,
	94	"identity": null,
	95	"name": "thread_id",
	96	"entityType": "columns",
	97	"schema": "public",
	98	"table": "relay_agent_activity_rows"
	99	},
	100	{
	101	"type": "jsonb",
	102	"typeSchema": null,
	103	"notNull": true,
	104	"dimensions": 0,
	105	"default": null,
	106	"generated": null,
	107	"identity": null,
	108	"name": "state_json",
	109	"entityType": "columns",
	110	"schema": "public",
	111	"table": "relay_agent_activity_rows"
	112	},
	113	{
	114	"type": "varchar(64)",
	115	"typeSchema": null,
	116	"notNull": true,
	117	"dimensions": 0,
	118	"default": null,
	119	"generated": null,
	120	"identity": null,
	121	"name": "updated_at",
	122	"entityType": "columns",
	123	"schema": "public",
	124	"table": "relay_agent_activity_rows"
	125	},
	126	{
	127	"type": "varchar(64)",
	128	"typeSchema": null,
	129	"notNull": true,
	130	"dimensions": 0,
	131	"default": null,
	132	"generated": null,
	133	"identity": null,
	134	"name": "created_at",
	135	"entityType": "columns",
	136	"schema": "public",
	137	"table": "relay_agent_activity_rows"
	138	},
	139	{
	140	"type": "varchar(36)",
	141	"typeSchema": null,
	142	"notNull": true,
	143	"dimensions": 0,
	144	"default": null,
	145	"generated": null,
	146	"identity": null,
	147	"name": "id",
	148	"entityType": "columns",
	149	"schema": "public",
	150	"table": "relay_delivery_attempts"
	151	},
	152	{
	153	"type": "varchar(64)",
	154	"typeSchema": null,
	155	"notNull": true,
	156	"dimensions": 0,
	157	"default": null,
	158	"generated": null,
	159	"identity": null,
	160	"name": "created_at",
	161	"entityType": "columns",
	162	"schema": "public",
	163	"table": "relay_delivery_attempts"
	164	},
	165	{
	166	"type": "varchar(255)",
	167	"typeSchema": null,
	168	"notNull": false,
	169	"dimensions": 0,
	170	"default": null,
	171	"generated": null,
	172	"identity": null,
	173	"name": "user_id",
	174	"entityType": "columns",
	175	"schema": "public",
	176	"table": "relay_delivery_attempts"
	177	},
	178	{
	179	"type": "varchar(191)",
	180	"typeSchema": null,
	181	"notNull": false,
	182	"dimensions": 0,
	183	"default": null,
	184	"generated": null,
	185	"identity": null,
	186	"name": "environment_id",
	187	"entityType": "columns",
	188	"schema": "public",
	189	"table": "relay_delivery_attempts"
	190	},
	191	{
	192	"type": "varchar(512)",
	193	"typeSchema": null,
	194	"notNull": false,
	195	"dimensions": 0,
	196	"default": null,
	197	"generated": null,
	198	"identity": null,
	199	"name": "thread_id",
	200	"entityType": "columns",
	201	"schema": "public",
	202	"table": "relay_delivery_attempts"
	203	},
	204	{
	205	"type": "varchar(255)",
	206	"typeSchema": null,
	207	"notNull": false,
	208	"dimensions": 0,
	209	"default": null,
	210	"generated": null,
	211	"identity": null,
	212	"name": "device_id",
	213	"entityType": "columns",
	214	"schema": "public",
	215	"table": "relay_delivery_attempts"
	216	},
	217	{
	218	"type": "varchar(64)",
	219	"typeSchema": null,
	220	"notNull": true,
	221	"dimensions": 0,
	222	"default": null,
	223	"generated": null,
	224	"identity": null,
	225	"name": "kind",
	226	"entityType": "columns",
	227	"schema": "public",
	228	"table": "relay_delivery_attempts"
	229	},
	230	{
	231	"type": "varchar(64)",
	232	"typeSchema": null,
	233	"notNull": false,
	234	"dimensions": 0,
	235	"default": null,
	236	"generated": null,
	237	"identity": null,
	238	"name": "source_job_id",
	239	"entityType": "columns",
	240	"schema": "public",
	241	"table": "relay_delivery_attempts"
	242	},
	243	{
	244	"type": "varchar(16)",
	245	"typeSchema": null,
	246	"notNull": false,
	247	"dimensions": 0,
	248	"default": null,
	249	"generated": null,
	250	"identity": null,
	251	"name": "token_suffix",
	252	"entityType": "columns",
	253	"schema": "public",
	254	"table": "relay_delivery_attempts"
	255	},
	256	{
	257	"type": "integer",
	258	"typeSchema": null,
	259	"notNull": false,
	260	"dimensions": 0,
	261	"default": null,
	262	"generated": null,
	263	"identity": null,
	264	"name": "apns_status",
	265	"entityType": "columns",
	266	"schema": "public",
	267	"table": "relay_delivery_attempts"
	268	},
	269	{
	270	"type": "text",
	271	"typeSchema": null,
	272	"notNull": false,
	273	"dimensions": 0,
	274	"default": null,
	275	"generated": null,
	276	"identity": null,
	277	"name": "apns_reason",
	278	"entityType": "columns",
	279	"schema": "public",
	280	"table": "relay_delivery_attempts"
	281	},
	282	{
	283	"type": "varchar(128)",
	284	"typeSchema": null,
	285	"notNull": false,
	286	"dimensions": 0,
	287	"default": null,
	288	"generated": null,
	289	"identity": null,
	290	"name": "apns_id",
	291	"entityType": "columns",
	292	"schema": "public",
	293	"table": "relay_delivery_attempts"
	294	},
	295	{
	296	"type": "text",
	297	"typeSchema": null,
	298	"notNull": false,
	299	"dimensions": 0,
	300	"default": null,
	301	"generated": null,
	302	"identity": null,
	303	"name": "transport_error",
	304	"entityType": "columns",
	305	"schema": "public",
	306	"table": "relay_delivery_attempts"
	307	},
	308	{
	309	"type": "varchar(128)",
	310	"typeSchema": null,
	311	"notNull": true,
	312	"dimensions": 0,
	313	"default": null,
	314	"generated": null,
	315	"identity": null,
	316	"name": "thumbprint",
	317	"entityType": "columns",
	318	"schema": "public",
	319	"table": "relay_dpop_proofs"
	320	},
	321	{
	322	"type": "varchar(255)",
	323	"typeSchema": null,
	324	"notNull": true,
	325	"dimensions": 0,
	326	"default": null,
	327	"generated": null,
	328	"identity": null,
	329	"name": "jti",
	330	"entityType": "columns",
	331	"schema": "public",
	332	"table": "relay_dpop_proofs"
	333	},
	334	{
	335	"type": "integer",
	336	"typeSchema": null,
	337	"notNull": true,
	338	"dimensions": 0,
	339	"default": null,
	340	"generated": null,
	341	"identity": null,
	342	"name": "iat",
	343	"entityType": "columns",
	344	"schema": "public",
	345	"table": "relay_dpop_proofs"
	346	},
	347	{
	348	"type": "varchar(64)",
	349	"typeSchema": null,
	350	"notNull": true,
	351	"dimensions": 0,
	352	"default": null,
	353	"generated": null,
	354	"identity": null,
	355	"name": "expires_at",
	356	"entityType": "columns",
	357	"schema": "public",
	358	"table": "relay_dpop_proofs"
	359	},
	360	{
	361	"type": "varchar(64)",
	362	"typeSchema": null,
	363	"notNull": true,
	364	"dimensions": 0,
	365	"default": null,
	366	"generated": null,
	367	"identity": null,
	368	"name": "created_at",
	369	"entityType": "columns",
	370	"schema": "public",
	371	"table": "relay_dpop_proofs"
	372	},
	373	{
	374	"type": "varchar(64)",
	375	"typeSchema": null,
	376	"notNull": true,
	377	"dimensions": 0,
	378	"default": null,
	379	"generated": null,
	380	"identity": null,
	381	"name": "credential_id",
	382	"entityType": "columns",
	383	"schema": "public",
	384	"table": "relay_environment_credentials"
	385	},
	386	{
	387	"type": "varchar(191)",
	388	"typeSchema": null,
	389	"notNull": true,
	390	"dimensions": 0,
	391	"default": null,
	392	"generated": null,
	393	"identity": null,
	394	"name": "environment_id",
	395	"entityType": "columns",
	396	"schema": "public",
	397	"table": "relay_environment_credentials"
	398	},
	399	{
	400	"type": "text",
	401	"typeSchema": null,
	402	"notNull": true,
	403	"dimensions": 0,
	404	"default": null,
	405	"generated": null,
	406	"identity": null,
	407	"name": "environment_public_key",
	408	"entityType": "columns",
	409	"schema": "public",
	410	"table": "relay_environment_credentials"
	411	},
	412	{
	413	"type": "varchar(191)",
	414	"typeSchema": null,
	415	"notNull": true,
	416	"dimensions": 0,
	417	"default": null,
	418	"generated": null,
	419	"identity": null,
	420	"name": "credential_hash",
	421	"entityType": "columns",
	422	"schema": "public",
	423	"table": "relay_environment_credentials"
	424	},
	425	{
	426	"type": "varchar(64)",
	427	"typeSchema": null,
	428	"notNull": false,
	429	"dimensions": 0,
	430	"default": null,
	431	"generated": null,
	432	"identity": null,
	433	"name": "revoked_at",
	434	"entityType": "columns",
	435	"schema": "public",
	436	"table": "relay_environment_credentials"
	437	},
	438	{
	439	"type": "varchar(64)",
	440	"typeSchema": null,
	441	"notNull": true,
	442	"dimensions": 0,
	443	"default": null,
	444	"generated": null,
	445	"identity": null,
	446	"name": "created_at",
	447	"entityType": "columns",
	448	"schema": "public",
	449	"table": "relay_environment_credentials"
	450	},
	451	{
	452	"type": "varchar(64)",
	453	"typeSchema": null,
	454	"notNull": true,
	455	"dimensions": 0,
	456	"default": null,
	457	"generated": null,
	458	"identity": null,
	459	"name": "updated_at",
	460	"entityType": "columns",
	461	"schema": "public",
	462	"table": "relay_environment_credentials"
	463	},
	464	{
	465	"type": "varchar(191)",
	466	"typeSchema": null,
	467	"notNull": true,
	468	"dimensions": 0,
	469	"default": null,
	470	"generated": null,
	471	"identity": null,
	472	"name": "user_id",
	473	"entityType": "columns",
	474	"schema": "public",
	475	"table": "relay_environment_links"
	476	},
	477	{
	478	"type": "varchar(191)",
	479	"typeSchema": null,
	480	"notNull": true,
	481	"dimensions": 0,
	482	"default": null,
	483	"generated": null,
	484	"identity": null,
	485	"name": "environment_id",
	486	"entityType": "columns",
	487	"schema": "public",
	488	"table": "relay_environment_links"
	489	},
	490	{
	491	"type": "text",
	492	"typeSchema": null,
	493	"notNull": true,
	494	"dimensions": 0,
	495	"default": "'T3 Environment'",
	496	"generated": null,
	497	"identity": null,
	498	"name": "environment_label",
	499	"entityType": "columns",
	500	"schema": "public",
	501	"table": "relay_environment_links"
	502	},
	503	{
	504	"type": "text",
	505	"typeSchema": null,
	506	"notNull": true,
	507	"dimensions": 0,
	508	"default": null,
	509	"generated": null,
	510	"identity": null,
	511	"name": "environment_public_key",
	512	"entityType": "columns",
	513	"schema": "public",
	514	"table": "relay_environment_links"
	515	},
	516	{
	517	"type": "text",
	518	"typeSchema": null,
	519	"notNull": true,
	520	"dimensions": 0,
	521	"default": null,
	522	"generated": null,
	523	"identity": null,
	524	"name": "endpoint_http_base_url",
	525	"entityType": "columns",
	526	"schema": "public",
	527	"table": "relay_environment_links"
	528	},
	529	{
	530	"type": "text",
	531	"typeSchema": null,
	532	"notNull": true,
	533	"dimensions": 0,
	534	"default": null,
	535	"generated": null,
	536	"identity": null,
	537	"name": "endpoint_ws_base_url",
	538	"entityType": "columns",
	539	"schema": "public",
	540	"table": "relay_environment_links"
	541	},
	542	{
	543	"type": "varchar(32)",
	544	"typeSchema": null,
	545	"notNull": true,
	546	"dimensions": 0,
	547	"default": null,
	548	"generated": null,
	549	"identity": null,
	550	"name": "endpoint_provider_kind",
	551	"entityType": "columns",
	552	"schema": "public",
	553	"table": "relay_environment_links"
	554	},
	555	{
	556	"type": "boolean",
	557	"typeSchema": null,
	558	"notNull": true,
	559	"dimensions": 0,
	560	"default": "true",
	561	"generated": null,
	562	"identity": null,
	563	"name": "notifications_enabled",
	564	"entityType": "columns",
	565	"schema": "public",
	566	"table": "relay_environment_links"
	567	},
	568	{
	569	"type": "boolean",
	570	"typeSchema": null,
	571	"notNull": true,
	572	"dimensions": 0,
	573	"default": "true",
	574	"generated": null,
	575	"identity": null,
	576	"name": "live_activities_enabled",
	577	"entityType": "columns",
	578	"schema": "public",
	579	"table": "relay_environment_links"
	580	},
	581	{
	582	"type": "boolean",
	583	"typeSchema": null,
	584	"notNull": true,
	585	"dimensions": 0,
	586	"default": "false",
	587	"generated": null,
	588	"identity": null,
	589	"name": "managed_tunnels_enabled",
	590	"entityType": "columns",
	591	"schema": "public",
	592	"table": "relay_environment_links"
	593	},
	594	{
	595	"type": "boolean",
	596	"typeSchema": null,
	597	"notNull": true,
	598	"dimensions": 0,
	599	"default": "false",
	600	"generated": null,
	601	"identity": null,
	602	"name": "hold_webhooks_while_offline",
	603	"entityType": "columns",
	604	"schema": "public",
	605	"table": "relay_environment_links"
	606	},
	607	{
	608	"type": "varchar(191)",
	609	"typeSchema": null,
	610	"notNull": false,
	611	"dimensions": 0,
	612	"default": null,
	613	"generated": null,
	614	"identity": null,
	615	"name": "created_by_device_id",
	616	"entityType": "columns",
	617	"schema": "public",
	618	"table": "relay_environment_links"
	619	},
	620	{
	621	"type": "varchar(64)",
	622	"typeSchema": null,
	623	"notNull": false,
	624	"dimensions": 0,
	625	"default": null,
	626	"generated": null,
	627	"identity": null,
	628	"name": "revoked_at",
	629	"entityType": "columns",
	630	"schema": "public",
	631	"table": "relay_environment_links"
	632	},
	633	{
	634	"type": "varchar(64)",
	635	"typeSchema": null,
	636	"notNull": true,
	637	"dimensions": 0,
	638	"default": null,
	639	"generated": null,
	640	"identity": null,
	641	"name": "created_at",
	642	"entityType": "columns",
	643	"schema": "public",
	644	"table": "relay_environment_links"
	645	},
	646	{
	647	"type": "varchar(64)",
	648	"typeSchema": null,
	649	"notNull": true,
	650	"dimensions": 0,
	651	"default": null,
	652	"generated": null,
	653	"identity": null,
	654	"name": "updated_at",
	655	"entityType": "columns",
	656	"schema": "public",
	657	"table": "relay_environment_links"
	658	},
	659	{
	660	"type": "varchar(255)",
	661	"typeSchema": null,
	662	"notNull": true,
	663	"dimensions": 0,
	664	"default": null,
	665	"generated": null,
	666	"identity": null,
	667	"name": "user_id",
	668	"entityType": "columns",
	669	"schema": "public",
	670	"table": "relay_live_activities"
	671	},
	672	{
	673	"type": "varchar(255)",
	674	"typeSchema": null,
	675	"notNull": true,
	676	"dimensions": 0,
	677	"default": null,
	678	"generated": null,
	679	"identity": null,
	680	"name": "device_id",
	681	"entityType": "columns",
	682	"schema": "public",
	683	"table": "relay_live_activities"
	684	},
	685	{
	686	"type": "text",
	687	"typeSchema": null,
	688	"notNull": false,
	689	"dimensions": 0,
	690	"default": null,
	691	"generated": null,
	692	"identity": null,
	693	"name": "activity_push_token",
	694	"entityType": "columns",
	695	"schema": "public",
	696	"table": "relay_live_activities"
	697	},
	698	{
	699	"type": "varchar(64)",
	700	"typeSchema": null,
	701	"notNull": false,
	702	"dimensions": 0,
	703	"default": null,
	704	"generated": null,
	705	"identity": null,
	706	"name": "remote_start_queued_at",
	707	"entityType": "columns",
	708	"schema": "public",
	709	"table": "relay_live_activities"
	710	},
	711	{
	712	"type": "varchar(64)",
	713	"typeSchema": null,
	714	"notNull": false,
	715	"dimensions": 0,
	716	"default": null,
	717	"generated": null,
	718	"identity": null,
	719	"name": "remote_started_at",
	720	"entityType": "columns",
	721	"schema": "public",
	722	"table": "relay_live_activities"
	723	},
	724	{
	725	"type": "varchar(64)",
	726	"typeSchema": null,
	727	"notNull": false,
	728	"dimensions": 0,
	729	"default": null,
	730	"generated": null,
	731	"identity": null,
	732	"name": "ended_at",
	733	"entityType": "columns",
	734	"schema": "public",
	735	"table": "relay_live_activities"
	736	},
	737	{
	738	"type": "jsonb",
	739	"typeSchema": null,
	740	"notNull": false,
	741	"dimensions": 0,
	742	"default": null,
	743	"generated": null,
	744	"identity": null,
	745	"name": "last_aggregate_json",
	746	"entityType": "columns",
	747	"schema": "public",
	748	"table": "relay_live_activities"
	749	},
	750	{
	751	"type": "varchar(64)",
	752	"typeSchema": null,
	753	"notNull": false,
	754	"dimensions": 0,
	755	"default": null,
	756	"generated": null,
	757	"identity": null,
	758	"name": "last_live_activity_delivery_at",
	759	"entityType": "columns",
	760	"schema": "public",
	761	"table": "relay_live_activities"
	762	},
	763	{
	764	"type": "varchar(64)",
	765	"typeSchema": null,
	766	"notNull": true,
	767	"dimensions": 0,
	768	"default": null,
	769	"generated": null,
	770	"identity": null,
	771	"name": "created_at",
	772	"entityType": "columns",
	773	"schema": "public",
	774	"table": "relay_live_activities"
	775	},
	776	{
	777	"type": "varchar(64)",
	778	"typeSchema": null,
	779	"notNull": true,
	780	"dimensions": 0,
	781	"default": null,
	782	"generated": null,
	783	"identity": null,
	784	"name": "updated_at",
	785	"entityType": "columns",
	786	"schema": "public",
	787	"table": "relay_live_activities"
	788	},
	789	{
	790	"type": "varchar(191)",
	791	"typeSchema": null,
	792	"notNull": true,
	793	"dimensions": 0,
	794	"default": null,
	795	"generated": null,
	796	"identity": null,
	797	"name": "user_id",
	798	"entityType": "columns",
	799	"schema": "public",
	800	"table": "relay_managed_endpoint_allocations"
	801	},
	802	{
	803	"type": "varchar(191)",
	804	"typeSchema": null,
	805	"notNull": true,
	806	"dimensions": 0,
	807	"default": null,
	808	"generated": null,
	809	"identity": null,
	810	"name": "environment_id",
	811	"entityType": "columns",
	812	"schema": "public",
	813	"table": "relay_managed_endpoint_allocations"
	814	},
	815	{
	816	"type": "text",
	817	"typeSchema": null,
	818	"notNull": true,
	819	"dimensions": 0,
	820	"default": null,
	821	"generated": null,
	822	"identity": null,
	823	"name": "hostname",
	824	"entityType": "columns",
	825	"schema": "public",
	826	"table": "relay_managed_endpoint_allocations"
	827	},
	828	{
	829	"type": "varchar(191)",
	830	"typeSchema": null,
	831	"notNull": false,
	832	"dimensions": 0,
	833	"default": null,
	834	"generated": null,
	835	"identity": null,
	836	"name": "tunnel_id",
	837	"entityType": "columns",
	838	"schema": "public",
	839	"table": "relay_managed_endpoint_allocations"
	840	},
	841	{
	842	"type": "text",
	843	"typeSchema": null,
	844	"notNull": true,
	845	"dimensions": 0,
	846	"default": null,
	847	"generated": null,
	848	"identity": null,
	849	"name": "tunnel_name",
	850	"entityType": "columns",
	851	"schema": "public",
	852	"table": "relay_managed_endpoint_allocations"
	853	},
	854	{
	855	"type": "varchar(191)",
	856	"typeSchema": null,
	857	"notNull": false,
	858	"dimensions": 0,
	859	"default": null,
	860	"generated": null,
	861	"identity": null,
	862	"name": "dns_record_id",
	863	"entityType": "columns",
	864	"schema": "public",
	865	"table": "relay_managed_endpoint_allocations"
	866	},
	867	{
	868	"type": "varchar(64)",
	869	"typeSchema": null,
	870	"notNull": false,
	871	"dimensions": 0,
	872	"default": null,
	873	"generated": null,
	874	"identity": null,
	875	"name": "ready_at",
	876	"entityType": "columns",
	877	"schema": "public",
	878	"table": "relay_managed_endpoint_allocations"
	879	},
	880	{
	881	"type": "varchar(64)",
	882	"typeSchema": null,
	883	"notNull": false,
	884	"dimensions": 0,
	885	"default": null,
	886	"generated": null,
	887	"identity": null,
	888	"name": "recovery_enabled_at",
	889	"entityType": "columns",
	890	"schema": "public",
	891	"table": "relay_managed_endpoint_allocations"
	892	},
	893	{
	894	"type": "text",
	895	"typeSchema": null,
	896	"notNull": false,
	897	"dimensions": 0,
	898	"default": null,
	899	"generated": null,
	900	"identity": null,
	901	"name": "recovery_environment_public_key",
	902	"entityType": "columns",
	903	"schema": "public",
	904	"table": "relay_managed_endpoint_allocations"
	905	},
	906	{
	907	"type": "jsonb",
	908	"typeSchema": null,
	909	"notNull": false,
	910	"dimensions": 0,
	911	"default": null,
	912	"generated": null,
	913	"identity": null,
	914	"name": "origin",
	915	"entityType": "columns",
	916	"schema": "public",
	917	"table": "relay_managed_endpoint_allocations"
	918	},
	919	{
	920	"type": "integer",
	921	"typeSchema": null,
	922	"notNull": true,
	923	"dimensions": 0,
	924	"default": "0",
	925	"generated": null,
	926	"identity": null,
	927	"name": "generation",
	928	"entityType": "columns",
	929	"schema": "public",
	930	"table": "relay_managed_endpoint_allocations"
	931	},
	932	{
	933	"type": "varchar(64)",
	934	"typeSchema": null,
	935	"notNull": true,
	936	"dimensions": 0,
	937	"default": null,
	938	"generated": null,
	939	"identity": null,
	940	"name": "created_at",
	941	"entityType": "columns",
	942	"schema": "public",
	943	"table": "relay_managed_endpoint_allocations"
	944	},
	945	{
	946	"type": "varchar(64)",
	947	"typeSchema": null,
	948	"notNull": true,
	949	"dimensions": 0,
	950	"default": null,
	951	"generated": null,
	952	"identity": null,
	953	"name": "updated_at",
	954	"entityType": "columns",
	955	"schema": "public",
	956	"table": "relay_managed_endpoint_allocations"
	957	},
	958	{
	959	"type": "varchar(191)",
	960	"typeSchema": null,
	961	"notNull": true,
	962	"dimensions": 0,
	963	"default": null,
	964	"generated": null,
	965	"identity": null,
	966	"name": "user_id",
	967	"entityType": "columns",
	968	"schema": "public",
	969	"table": "relay_managed_tunnel_limits"
	970	},
	971	{
	972	"type": "integer",
	973	"typeSchema": null,
	974	"notNull": true,
	975	"dimensions": 0,
	976	"default": null,
	977	"generated": null,
	978	"identity": null,
	979	"name": "max_tunnels",
	980	"entityType": "columns",
	981	"schema": "public",
	982	"table": "relay_managed_tunnel_limits"
	983	},
	984	{
	985	"type": "varchar(64)",
	986	"typeSchema": null,
	987	"notNull": true,
	988	"dimensions": 0,
	989	"default": null,
	990	"generated": null,
	991	"identity": null,
	992	"name": "created_at",
	993	"entityType": "columns",
	994	"schema": "public",
	995	"table": "relay_managed_tunnel_limits"
	996	},
	997	{
	998	"type": "varchar(64)",
	999	"typeSchema": null,
	1000	"notNull": true,
	1001	"dimensions": 0,
	1002	"default": null,
	1003	"generated": null,
	1004	"identity": null,
	1005	"name": "updated_at",
	1006	"entityType": "columns",
	1007	"schema": "public",
	1008	"table": "relay_managed_tunnel_limits"
	1009	},
	1010	{
	1011	"type": "varchar(255)",
	1012	"typeSchema": null,
	1013	"notNull": true,
	1014	"dimensions": 0,
	1015	"default": null,
	1016	"generated": null,
	1017	"identity": null,
	1018	"name": "user_id",
	1019	"entityType": "columns",
	1020	"schema": "public",
	1021	"table": "relay_mobile_devices"
	1022	},
	1023	{
	1024	"type": "varchar(255)",
	1025	"typeSchema": null,
	1026	"notNull": true,
	1027	"dimensions": 0,
	1028	"default": null,
	1029	"generated": null,
	1030	"identity": null,
	1031	"name": "device_id",
	1032	"entityType": "columns",
	1033	"schema": "public",
	1034	"table": "relay_mobile_devices"
	1035	},
	1036	{
	1037	"type": "text",
	1038	"typeSchema": null,
	1039	"notNull": true,
	1040	"dimensions": 0,
	1041	"default": "'iOS device'",
	1042	"generated": null,
	1043	"identity": null,
	1044	"name": "label",
	1045	"entityType": "columns",
	1046	"schema": "public",
	1047	"table": "relay_mobile_devices"
	1048	},
	1049	{
	1050	"type": "varchar(16)",
	1051	"typeSchema": null,
	1052	"notNull": true,
	1053	"dimensions": 0,
	1054	"default": null,
	1055	"generated": null,
	1056	"identity": null,
	1057	"name": "platform",
	1058	"entityType": "columns",
	1059	"schema": "public",
	1060	"table": "relay_mobile_devices"
	1061	},
	1062	{
	1063	"type": "integer",
	1064	"typeSchema": null,
	1065	"notNull": false,
	1066	"dimensions": 0,
	1067	"default": null,
	1068	"generated": null,
	1069	"identity": null,
	1070	"name": "ios_major_version",
	1071	"entityType": "columns",
	1072	"schema": "public",
	1073	"table": "relay_mobile_devices"
	1074	},
	1075	{
	1076	"type": "integer",
	1077	"typeSchema": null,
	1078	"notNull": false,
	1079	"dimensions": 0,
	1080	"default": null,
	1081	"generated": null,
	1082	"identity": null,
	1083	"name": "android_api_level",
	1084	"entityType": "columns",
	1085	"schema": "public",
	1086	"table": "relay_mobile_devices"
	1087	},
	1088	{
	1089	"type": "varchar(64)",
	1090	"typeSchema": null,
	1091	"notNull": false,
	1092	"dimensions": 0,
	1093	"default": null,
	1094	"generated": null,
	1095	"identity": null,
	1096	"name": "app_version",
	1097	"entityType": "columns",
	1098	"schema": "public",
	1099	"table": "relay_mobile_devices"
	1100	},
	1101	{
	1102	"type": "varchar(255)",
	1103	"typeSchema": null,
	1104	"notNull": false,
	1105	"dimensions": 0,
	1106	"default": null,
	1107	"generated": null,
	1108	"identity": null,
	1109	"name": "bundle_id",
	1110	"entityType": "columns",
	1111	"schema": "public",
	1112	"table": "relay_mobile_devices"
	1113	},
	1114	{
	1115	"type": "varchar(16)",
	1116	"typeSchema": null,
	1117	"notNull": false,
	1118	"dimensions": 0,
	1119	"default": null,
	1120	"generated": null,
	1121	"identity": null,
	1122	"name": "aps_environment",
	1123	"entityType": "columns",
	1124	"schema": "public",
	1125	"table": "relay_mobile_devices"
	1126	},
	1127	{
	1128	"type": "text",
	1129	"typeSchema": null,
	1130	"notNull": false,
	1131	"dimensions": 0,
	1132	"default": null,
	1133	"generated": null,
	1134	"identity": null,
	1135	"name": "push_token",
	1136	"entityType": "columns",
	1137	"schema": "public",
	1138	"table": "relay_mobile_devices"
	1139	},
	1140	{
	1141	"type": "text",
	1142	"typeSchema": null,
	1143	"notNull": false,
	1144	"dimensions": 0,
	1145	"default": null,
	1146	"generated": null,
	1147	"identity": null,
	1148	"name": "push_to_start_token",
	1149	"entityType": "columns",
	1150	"schema": "public",
	1151	"table": "relay_mobile_devices"
	1152	},
	1153	{
	1154	"type": "jsonb",
	1155	"typeSchema": null,
	1156	"notNull": true,
	1157	"dimensions": 0,
	1158	"default": null,
	1159	"generated": null,
	1160	"identity": null,
	1161	"name": "preferences_json",
	1162	"entityType": "columns",
	1163	"schema": "public",
	1164	"table": "relay_mobile_devices"
	1165	},
	1166	{
	1167	"type": "varchar(64)",
	1168	"typeSchema": null,
	1169	"notNull": true,
	1170	"dimensions": 0,
	1171	"default": null,
	1172	"generated": null,
	1173	"identity": null,
	1174	"name": "created_at",
	1175	"entityType": "columns",
	1176	"schema": "public",
	1177	"table": "relay_mobile_devices"
	1178	},
	1179	{
	1180	"type": "varchar(64)",
	1181	"typeSchema": null,
	1182	"notNull": true,
	1183	"dimensions": 0,
	1184	"default": null,
	1185	"generated": null,
	1186	"identity": null,
	1187	"name": "updated_at",
	1188	"entityType": "columns",
	1189	"schema": "public",
	1190	"table": "relay_mobile_devices"
	1191	},
	1192	{
	1193	"nameExplicit": true,
	1194	"columns": [
	1195	{
	1196	"value": "updated_at",
	1197	"isExpression": false,
	1198	"asc": true,
	1199	"nullsFirst": false,
	1200	"opclass": null
	1201	}
	1202	],
	1203	"isUnique": false,
	1204	"where": null,
	1205	"with": "",
	1206	"method": "btree",
	1207	"concurrently": false,
	1208	"name": "idx_relay_agent_activity_rows_updated",
	1209	"entityType": "indexes",
	1210	"schema": "public",
	1211	"table": "relay_agent_activity_rows"
	1212	},
	1213	{
	1214	"nameExplicit": true,
	1215	"columns": [
	1216	{
	1217	"value": "environment_id",
	1218	"isExpression": false,
	1219	"asc": true,
	1220	"nullsFirst": false,
	1221	"opclass": null
	1222	},
	1223	{
	1224	"value": "thread_id",
	1225	"isExpression": false,
	1226	"asc": true,
	1227	"nullsFirst": false,
	1228	"opclass": null
	1229	},
	1230	{
	1231	"value": "created_at",
	1232	"isExpression": false,
	1233	"asc": true,
	1234	"nullsFirst": false,
	1235	"opclass": null
	1236	}
	1237	],
	1238	"isUnique": false,
	1239	"where": null,
	1240	"with": "",
	1241	"method": "btree",
	1242	"concurrently": false,
	1243	"name": "idx_relay_delivery_attempts_environment",
	1244	"entityType": "indexes",
	1245	"schema": "public",
	1246	"table": "relay_delivery_attempts"
	1247	},
	1248	{
	1249	"nameExplicit": true,
	1250	"columns": [
	1251	{
	1252	"value": "source_job_id",
	1253	"isExpression": false,
	1254	"asc": true,
	1255	"nullsFirst": false,
	1256	"opclass": null
	1257	}
	1258	],
	1259	"isUnique": true,
	1260	"where": null,
	1261	"with": "",
	1262	"method": "btree",
	1263	"concurrently": false,
	1264	"name": "idx_relay_delivery_attempts_source_job",
	1265	"entityType": "indexes",
	1266	"schema": "public",
	1267	"table": "relay_delivery_attempts"
	1268	},
	1269	{
	1270	"nameExplicit": true,
	1271	"columns": [
	1272	{
	1273	"value": "expires_at",
	1274	"isExpression": false,
	1275	"asc": true,
	1276	"nullsFirst": false,
	1277	"opclass": null
	1278	}
	1279	],
	1280	"isUnique": false,
	1281	"where": null,
	1282	"with": "",
	1283	"method": "btree",
	1284	"concurrently": false,
	1285	"name": "idx_relay_dpop_proofs_expires_at",
	1286	"entityType": "indexes",
	1287	"schema": "public",
	1288	"table": "relay_dpop_proofs"
	1289	},
	1290	{
	1291	"nameExplicit": true,
	1292	"columns": [
	1293	{
	1294	"value": "credential_hash",
	1295	"isExpression": false,
	1296	"asc": true,
	1297	"nullsFirst": false,
	1298	"opclass": null
	1299	}
	1300	],
	1301	"isUnique": true,
	1302	"where": null,
	1303	"with": "",
	1304	"method": "btree",
	1305	"concurrently": false,
	1306	"name": "idx_relay_environment_credentials_hash",
	1307	"entityType": "indexes",
	1308	"schema": "public",
	1309	"table": "relay_environment_credentials"
	1310	},
	1311	{
	1312	"nameExplicit": true,
	1313	"columns": [
	1314	{
	1315	"value": "environment_id",
	1316	"isExpression": false,
	1317	"asc": true,
	1318	"nullsFirst": false,
	1319	"opclass": null
	1320	},
	1321	{
	1322	"value": "revoked_at",
	1323	"isExpression": false,
	1324	"asc": true,
	1325	"nullsFirst": false,
	1326	"opclass": null
	1327	}
	1328	],
	1329	"isUnique": false,
	1330	"where": null,
	1331	"with": "",
	1332	"method": "btree",
	1333	"concurrently": false,
	1334	"name": "idx_relay_environment_credentials_environment",
	1335	"entityType": "indexes",
	1336	"schema": "public",
	1337	"table": "relay_environment_credentials"
	1338	},
	1339	{
	1340	"nameExplicit": true,
	1341	"columns": [
	1342	{
	1343	"value": "environment_id",
	1344	"isExpression": false,
	1345	"asc": true,
	1346	"nullsFirst": false,
	1347	"opclass": null
	1348	},
	1349	{
	1350	"value": "environment_public_key",
	1351	"isExpression": false,
	1352	"asc": true,
	1353	"nullsFirst": false,
	1354	"opclass": null
	1355	},
	1356	{
	1357	"value": "revoked_at",
	1358	"isExpression": false,
	1359	"asc": true,
	1360	"nullsFirst": false,
	1361	"opclass": null
	1362	}
	1363	],
	1364	"isUnique": false,
	1365	"where": null,
	1366	"with": "",
	1367	"method": "btree",
	1368	"concurrently": false,
	1369	"name": "idx_relay_environment_credentials_environment_key",
	1370	"entityType": "indexes",
	1371	"schema": "public",
	1372	"table": "relay_environment_credentials"
	1373	},
	1374	{
	1375	"nameExplicit": true,
	1376	"columns": [
	1377	{
	1378	"value": "environment_id",
	1379	"isExpression": false,
	1380	"asc": true,
	1381	"nullsFirst": false,
	1382	"opclass": null
	1383	},
	1384	{
	1385	"value": "revoked_at",
	1386	"isExpression": false,
	1387	"asc": true,
	1388	"nullsFirst": false,
	1389	"opclass": null
	1390	}
	1391	],
	1392	"isUnique": false,
	1393	"where": null,
	1394	"with": "",
	1395	"method": "btree",
	1396	"concurrently": false,
	1397	"name": "idx_relay_environment_links_environment",
	1398	"entityType": "indexes",
	1399	"schema": "public",
	1400	"table": "relay_environment_links"
	1401	},
	1402	{
	1403	"nameExplicit": true,
	1404	"columns": [
	1405	{
	1406	"value": "activity_push_token",
	1407	"isExpression": false,
	1408	"asc": true,
	1409	"nullsFirst": false,
	1410	"opclass": null
	1411	}
	1412	],
	1413	"isUnique": true,
	1414	"where": null,
	1415	"with": "",
	1416	"method": "btree",
	1417	"concurrently": false,
	1418	"name": "idx_relay_live_activities_activity_push_token",
	1419	"entityType": "indexes",
	1420	"schema": "public",
	1421	"table": "relay_live_activities"
	1422	},
	1423	{
	1424	"nameExplicit": true,
	1425	"columns": [
	1426	{
	1427	"value": "hostname",
	1428	"isExpression": false,
	1429	"asc": true,
	1430	"nullsFirst": false,
	1431	"opclass": null
	1432	}
	1433	],
	1434	"isUnique": true,
	1435	"where": null,
	1436	"with": "",
	1437	"method": "btree",
	1438	"concurrently": false,
	1439	"name": "idx_relay_managed_endpoint_allocations_hostname",
	1440	"entityType": "indexes",
	1441	"schema": "public",
	1442	"table": "relay_managed_endpoint_allocations"
	1443	},
	1444	{
	1445	"nameExplicit": true,
	1446	"columns": [
	1447	{
	1448	"value": "tunnel_name",
	1449	"isExpression": false,
	1450	"asc": true,
	1451	"nullsFirst": false,
	1452	"opclass": null
	1453	}
	1454	],
	1455	"isUnique": true,
	1456	"where": null,
	1457	"with": "",
	1458	"method": "btree",
	1459	"concurrently": false,
	1460	"name": "idx_relay_managed_endpoint_allocations_tunnel_name",
	1461	"entityType": "indexes",
	1462	"schema": "public",
	1463	"table": "relay_managed_endpoint_allocations"
	1464	},
	1465	{
	1466	"nameExplicit": true,
	1467	"columns": [
	1468	{
	1469	"value": "push_token",
	1470	"isExpression": false,
	1471	"asc": true,
	1472	"nullsFirst": false,
	1473	"opclass": null
	1474	}
	1475	],
	1476	"isUnique": true,
	1477	"where": null,
	1478	"with": "",
	1479	"method": "btree",
	1480	"concurrently": false,
	1481	"name": "idx_relay_mobile_devices_push_token",
	1482	"entityType": "indexes",
	1483	"schema": "public",
	1484	"table": "relay_mobile_devices"
	1485	},
	1486	{
	1487	"nameExplicit": true,
	1488	"columns": [
	1489	{
	1490	"value": "push_to_start_token",
	1491	"isExpression": false,
	1492	"asc": true,
	1493	"nullsFirst": false,
	1494	"opclass": null
	1495	}
	1496	],
	1497	"isUnique": true,
	1498	"where": null,
	1499	"with": "",
	1500	"method": "btree",
	1501	"concurrently": false,
	1502	"name": "idx_relay_mobile_devices_push_to_start_token",
	1503	"entityType": "indexes",
	1504	"schema": "public",
	1505	"table": "relay_mobile_devices"
	1506	},
	1507	{
	1508	"columns": ["environment_id", "environment_public_key", "thread_id"],
	1509	"nameExplicit": false,
	1510	"name": "relay_agent_activity_rows_pkey",
	1511	"entityType": "pks",
	1512	"schema": "public",
	1513	"table": "relay_agent_activity_rows"
	1514	},
	1515	{
	1516	"columns": ["thumbprint", "jti"],
	1517	"nameExplicit": false,
	1518	"name": "relay_dpop_proofs_pkey",
	1519	"entityType": "pks",
	1520	"schema": "public",
	1521	"table": "relay_dpop_proofs"
	1522	},
	1523	{
	1524	"columns": ["user_id", "environment_id"],
	1525	"nameExplicit": false,
	1526	"name": "relay_environment_links_pkey",
	1527	"entityType": "pks",
	1528	"schema": "public",
	1529	"table": "relay_environment_links"
	1530	},
	1531	{
	1532	"columns": ["user_id", "device_id"],
	1533	"nameExplicit": false,
	1534	"name": "relay_live_activities_pkey",
	1535	"entityType": "pks",
	1536	"schema": "public",
	1537	"table": "relay_live_activities"
	1538	},
	1539	{
	1540	"columns": ["user_id", "environment_id"],
	1541	"nameExplicit": false,
	1542	"name": "relay_managed_endpoint_allocations_pkey",
	1543	"entityType": "pks",
	1544	"schema": "public",
	1545	"table": "relay_managed_endpoint_allocations"
	1546	},
	1547	{
	1548	"columns": ["user_id", "device_id"],
	1549	"nameExplicit": false,
	1550	"name": "relay_mobile_devices_pkey",
	1551	"entityType": "pks",
	1552	"schema": "public",
	1553	"table": "relay_mobile_devices"
	1554	},
	1555	{
	1556	"columns": ["id"],
	1557	"nameExplicit": false,
	1558	"name": "relay_delivery_attempts_pkey",
	1559	"schema": "public",
	1560	"table": "relay_delivery_attempts",
	1561	"entityType": "pks"
	1562	},
	1563	{
	1564	"columns": ["credential_id"],
	1565	"nameExplicit": false,
	1566	"name": "relay_environment_credentials_pkey",
	1567	"schema": "public",
	1568	"table": "relay_environment_credentials",
	1569	"entityType": "pks"
	1570	},
	1571	{
	1572	"columns": ["user_id"],
	1573	"nameExplicit": false,
	1574	"name": "relay_managed_tunnel_limits_pkey",
	1575	"schema": "public",
	1576	"table": "relay_managed_tunnel_limits",
	1577	"entityType": "pks"
	1578	}
	1579	],
	1580	"renames": []
	1581	}

infra/relay/package.json+1

...	...	@@ -13,6 +13,7 @@
13	13	"dependencies": {
14	14	"@clerk/backend": "catalog:",
15	15	"@effect/sql-pg": "catalog:",
	16	"@effect/sql-sqlite-do": "catalog:",
16	17	"@noble/curves": "catalog:",
17	18	"@noble/hashes": "catalog:",
18	19	"@t3tools/client-runtime": "workspace:*",

infra/relay/src/agentActivity/AgentActivityPublisher.test.ts+1

...	...	@@ -95,6 +95,7 @@ function makeEnvironmentLinks(
95	95	listForUser: () => Effect.succeed([]),
96	96	getForUser: () => Effect.succeed(null),
97	97	findActiveManagedForEnvironment: () => Effect.succeed([]),
	98	setHoldWebhooksWhileOffline: () => Effect.void,
98	99	revokeForUser: () => Effect.succeed(false),
99	100	...overrides,
100	101	};

infra/relay/src/agentActivity/FcmDeliveries.test.ts+1

...	...	@@ -172,6 +172,7 @@ function harness() {
172	172	),
173	173	listForUser: () => Effect.succeed([]),
174	174	findActiveManagedForEnvironment: () => Effect.succeed([]),
	175	setHoldWebhooksWhileOffline: () => Effect.void,
175	176	revokeForUser: () => Effect.succeed(false),
176	177	getForUser: (input) =>
177	178	Effect.sync(() =>

infra/relay/src/agentActivity/MobileRegistrations.test.ts+1

...	...	@@ -117,6 +117,7 @@ function makeEnvironmentLinks(
117	117	listForUser: () => Effect.succeed([]),
118	118	getForUser: () => Effect.succeed(null),
119	119	findActiveManagedForEnvironment: () => Effect.succeed([]),
	120	setHoldWebhooksWhileOffline: () => Effect.void,
120	121	revokeForUser: () => Effect.succeed(false),
121	122	...overrides,
122	123	};

infra/relay/src/deploymentConfig.ts+13

...	...	@@ -124,3 +124,16 @@ export function managedEndpointTunnelNamePrefix(stage: string): string {
124	124	export function managedEndpointTunnelName(stage: string, hash: string): string {
125	125	return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`;
126	126	}
	127
	128	/**
	129	* A managed endpoint's public key in webhook URLs: the hash suffix its tunnel
	130	* name ends with. The hash covers user and environment, so one key names
	131	* exactly one link, unlike the environment id, which any account can claim.
	132	*/
	133	export const MANAGED_ENDPOINT_KEY_PATTERN = new RegExp(
	134	`^[0-9a-f]{${MANAGED_ENDPOINT_HASH_LENGTH}}$`,
	135	);
	136
	137	export function managedEndpointTunnelNameForKey(stage: string, endpointKey: string): string {
	138	return `${managedEndpointTunnelNamePrefix(stage)}${endpointKey}`;
	139	}

infra/relay/src/environments/EnvironmentConnector.test.ts+2

...	...	@@ -194,6 +194,7 @@ function makeAllocations(
194	194	},
195	195	): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] {
196	196	return {
	197	getByTunnelName: () => Effect.die("unused getByTunnelName"),
197	198	get: () => Effect.succeed(allocation),
198	199	reserve: () => Effect.die("unused"),
199	200	recordTunnel: () => Effect.die("unused"),
...	...	@@ -230,6 +231,7 @@ function makeLinks(
230	231	...overrides,
231	232	}),
232	233	findActiveManagedForEnvironment: () => Effect.succeed([]),
	234	setHoldWebhooksWhileOffline: () => Effect.void,
233	235	revokeForUser: () => Effect.succeed(false),
234	236	};
235	237	}

infra/relay/src/environments/EnvironmentLinker.test.ts+1

...	...	@@ -127,6 +127,7 @@ function testLayer(input?: {
127	127	listForUser: () => Effect.succeed([]),
128	128	getForUser: () => Effect.succeed(null),
129	129	findActiveManagedForEnvironment: () => Effect.succeed([]),
	130	setHoldWebhooksWhileOffline: () => Effect.void,
130	131	revokeForUser: () => Effect.succeed(false),
131	132	}),
132	133	Layer.succeed(EnvironmentCredentials.EnvironmentCredentials, {

infra/relay/src/environments/EnvironmentLinks.test.ts+39

...	...	@@ -122,6 +122,45 @@ describe("EnvironmentLinks", () => {
122	122	);
123	123	});
124	124
	125	it.effect("carries the webhook-hold opt-in over only from links with the same key", () => {
	126	const inserted: Array<Record<string, unknown>> = [];
	127	const fakeDb = {
	128	insert: () => ({
	129	values: (values: Record<string, unknown>) => {
	130	inserted.push(values);
	131	return { onConflictDoUpdate: () => Effect.void };
	132	},
	133	}),
	134	} as unknown as RelayDb.RelayDb["Service"];
	135
	136	return Effect.gen(function* () {
	137	const links = yield* EnvironmentLinks.EnvironmentLinks;
	138	yield* links.upsert({
	139	userId: "user-1",
	140	request: {
	141	notificationsEnabled: false,
	142	liveActivitiesEnabled: false,
	143	managedTunnelsEnabled: true,
	144	} as never,
	145	proof: {
	146	environmentId: "env-1",
	147	environmentPublicKey: "public-key-1",
	148	descriptor: { label: "Laptop" },
	149	} as never,
	150	endpoint: { httpBaseUrl: "https://a.example", wsBaseUrl: "wss://a.example" } as never,
	151	});
	152	const query = new PgDialect().sqlToQuery(inserted[0]?.holdWebhooksWhileOffline as never);
	153	// An environment id is public: another account linking it with its own
	154	// key must not switch the opt-in on for this one.
	155	expect(query.sql).toContain("environment_public_key");
	156	expect(query.params).toEqual(["env-1", "public-key-1"]);
	157	}).pipe(
	158	Effect.provide(
	159	EnvironmentLinks.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb))),
	160	),
	161	);
	162	});
	163
125	164	it.effect("revokes only the active link owned by the requesting user", () => {
126	165	const updateValues: Array<Record<string, unknown>> = [];
127	166	const whereConditions: Array<unknown> = [];

infra/relay/src/environments/EnvironmentLinks.ts+65-5

...	...	@@ -9,7 +9,7 @@ import * as DateTime from "effect/DateTime";
9	9	import * as Effect from "effect/Effect";
10	10	import * as Layer from "effect/Layer";
11	11	import * as Schema from "effect/Schema";
12		import { and, eq, isNull, or } from "drizzle-orm";
	12	import { and, eq, isNull, or, sql } from "drizzle-orm";
13	13
14	14	import * as RelayDb from "../db.ts";
15	15	import { relayEnvironmentLinks } from "../persistence/schema.ts";
...	...	@@ -126,13 +126,30 @@ export class EnvironmentLinks extends Context.Service<
126	126	readonly userId: string;
127	127	readonly environmentId: string;
128	128	}) => Effect.Effect<RelayLinkedEnvironmentRecord \| null, EnvironmentLinkLookupPersistenceError>;
129		/** Active relay-managed links for an environment, across all users (webhook forwarding). */
	129	/**
	130	* Active relay-managed links for an environment, narrowed to one user or to
	131	* links proven by one environment key. The environment id alone is public
	132	* and any account can link it, so callers acting on it must narrow.
	133	*/
130	134	readonly findActiveManagedForEnvironment: (input: {
131	135	readonly environmentId: string;
	136	readonly userId?: string;
	137	readonly environmentPublicKey?: string;
132	138	}) => Effect.Effect<
133		ReadonlyArray<RelayLinkedEnvironmentRecord & { readonly userId: string }>,
	139	ReadonlyArray<
	140	RelayLinkedEnvironmentRecord & {
	141	readonly userId: string;
	142	readonly holdWebhooksWhileOffline: boolean;
	143	}
	144	>,
134	145	EnvironmentLinkEnvironmentLookupPersistenceError
135	146	>;
	147	/** Sets the webhook-hold opt-in on the active links proven by one environment key. */
	148	readonly setHoldWebhooksWhileOffline: (input: {
	149	readonly environmentId: string;
	150	readonly environmentPublicKey: string;
	151	readonly holdWebhooksWhileOffline: boolean;
	152	}) => Effect.Effect<void, EnvironmentLinkEnvironmentLookupPersistenceError>;
136	153	readonly revokeForUser: (input: {
137	154	readonly userId: string;
138	155	readonly environmentId: string;
...	...	@@ -166,12 +183,24 @@ const make = Effect.gen(function* () {
166	183	const now = DateTime.formatIso(yield* DateTime.now);
167	184	const { request, proof } = input;
168	185	const environmentId = proof.environmentId;
	186	// The webhook-hold opt-in belongs to the environment: a new or re-made
	187	// link carries it over from the environment's other active links. Only
	188	// links proven by the same key count; an environment id is public, so
	189	// anyone can link one and switch the opt-in on for their own link.
	190	const inheritedHoldWebhooks = sql<boolean>`EXISTS (
	191	SELECT 1 FROM ${relayEnvironmentLinks} AS other
	192	WHERE other.environment_id = ${environmentId}
	193	AND other.environment_public_key = ${proof.environmentPublicKey}
	194	AND other.revoked_at IS NULL
	195	AND other.hold_webhooks_while_offline
	196	)`;
169	197	const { endpoint } = input;
170	198	yield* db
171	199	.insert(relayEnvironmentLinks)
172	200	.values({
173	201	userId: input.userId,
174	202	environmentId,
	203	holdWebhooksWhileOffline: inheritedHoldWebhooks,
175	204	environmentLabel: proof.descriptor.label,
176	205	environmentPublicKey: proof.environmentPublicKey,
177	206	endpointHttpBaseUrl: endpoint.httpBaseUrl,
...	...	@@ -199,6 +228,7 @@ const make = Effect.gen(function* () {
199	228	createdByDeviceId: request.deviceId ?? null,
200	229	revokedAt: null,
201	230	updatedAt: now,
	231	holdWebhooksWhileOffline: inheritedHoldWebhooks,
202	232	},
203	233	})
204	234	.pipe(
...	...	@@ -355,6 +385,7 @@ const make = Effect.gen(function* () {
355	385	endpointWsBaseUrl: relayEnvironmentLinks.endpointWsBaseUrl,
356	386	endpointProviderKind: relayEnvironmentLinks.endpointProviderKind,
357	387	createdAt: relayEnvironmentLinks.createdAt,
	388	holdWebhooksWhileOffline: relayEnvironmentLinks.holdWebhooksWhileOffline,
358	389	})
359	390	.from(relayEnvironmentLinks)
360	391	.where(
...	...	@@ -363,10 +394,13 @@ const make = Effect.gen(function* () {
363	394	isNull(relayEnvironmentLinks.revokedAt),
364	395	eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"),
365	396	eq(relayEnvironmentLinks.managedTunnelsEnabled, true),
	397	input.userId === undefined ? undefined : eq(relayEnvironmentLinks.userId, input.userId),
	398	input.environmentPublicKey === undefined
	399	? undefined
	400	: eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey),
366	401	),
367	402	)
368		// One row per user who linked this environment; every row is checked
369		// until one has a ready endpoint, so none may be cut off.
	403	// At most one row per user who linked this environment.
370	404	.pipe(
371	405	Effect.map((rows) =>
372	406	rows.map((row) => ({
...	...	@@ -382,6 +416,7 @@ const make = Effect.gen(function* () {
382	416	},
383	417	environmentPublicKey: row.environmentPublicKey,
384	418	linkedAt: row.createdAt,
	419	holdWebhooksWhileOffline: row.holdWebhooksWhileOffline,
385	420	})),
386	421	),
387	422	Effect.mapError(
...	...	@@ -394,6 +429,31 @@ const make = Effect.gen(function* () {
394	429	);
395	430	}),
396	431
	432	setHoldWebhooksWhileOffline: Effect.fn(
	433	"relay.environment_links.set_hold_webhooks_while_offline",
	434	)(function* (input) {
	435	yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId });
	436	yield* db
	437	.update(relayEnvironmentLinks)
	438	.set({ holdWebhooksWhileOffline: input.holdWebhooksWhileOffline })
	439	.where(
	440	and(
	441	eq(relayEnvironmentLinks.environmentId, input.environmentId),
	442	eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey),
	443	isNull(relayEnvironmentLinks.revokedAt),
	444	),
	445	)
	446	.pipe(
	447	Effect.mapError(
	448	(cause) =>
	449	new EnvironmentLinkEnvironmentLookupPersistenceError({
	450	environmentId: input.environmentId,
	451	cause,
	452	}),
	453	),
	454	);
	455	}),
	456
397	457	revokeForUser: Effect.fn("relay.environment_links.revoke_for_user")(function* (input) {
398	458	yield* Effect.annotateCurrentSpan({
399	459	"relay.environment_id": input.environmentId,

infra/relay/src/environments/ManagedEndpointAllocations.ts+28

...	...	@@ -64,6 +64,7 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro
64	64	"claim-deprovision",
65	65	"remove",
66	66	"remove-claimed",
	67	"get-by-tunnel-name",
67	68	]),
68	69	stage: Schema.Literals(["database-request", "resolve-reservation"]),
69	70	userId: Schema.String,
...	...	@@ -132,6 +133,10 @@ export class ManagedEndpointAllocations extends Context.Service<
132	133	readonly get: (
133	134	input: ManagedEndpointAllocationKey,
134	135	) => Effect.Effect<ManagedEndpointAllocation \| null, ManagedEndpointAllocationPersistenceError>;
	136	/** The allocation that owns a tunnel name; tunnel names are unique. */
	137	readonly getByTunnelName: (
	138	tunnelName: string,
	139	) => Effect.Effect<ManagedEndpointAllocation \| null, ManagedEndpointAllocationPersistenceError>;
135	140	readonly reserve: (
136	141	input: ReserveManagedEndpointAllocationInput,
137	142	) => Effect.Effect<ManagedEndpointAllocation, ManagedEndpointAllocationPersistenceError>;
...	...	@@ -230,6 +235,29 @@ export const make = Effect.gen(function* () {
230	235	),
231	236	);
232	237	}),
	238	getByTunnelName: Effect.fn("relay.managed_endpoint_allocations.get_by_tunnel_name")(function* (
	239	tunnelName: string,
	240	) {
	241	return yield* db
	242	.select(allocationSelection)
	243	.from(relayManagedEndpointAllocations)
	244	.where(eq(relayManagedEndpointAllocations.tunnelName, tunnelName))
	245	.limit(1)
	246	.pipe(
	247	Effect.map((rows) => rows[0] ?? null),
	248	Effect.mapError(
	249	(cause) =>
	250	new ManagedEndpointAllocationPersistenceError({
	251	operation: "get-by-tunnel-name",
	252	stage: "database-request",
	253	userId: "",
	254	environmentId: "",
	255	tunnelName,
	256	cause,
	257	}),
	258	),
	259	);
	260	}),
233	261	reserve: Effect.fn("relay.managed_endpoint_allocations.reserve")(function* (
234	262	input: ReserveManagedEndpointAllocationInput,
235	263	) {

infra/relay/src/environments/ManagedEndpointProvider.test.ts+1

...	...	@@ -204,6 +204,7 @@ function makeAllocations(calls: AllocationCall[] = []) {
204	204	}
205	205	};
206	206	return ManagedEndpointAllocations.ManagedEndpointAllocations.of({
	207	getByTunnelName: () => Effect.die("unused getByTunnelName"),
207	208	get: (input) =>
208	209	Effect.sync(() => {
209	210	calls.push({ operation: "get", input });

infra/relay/src/environments/ManagedEndpointReaper.test.ts+1

...	...	@@ -166,6 +166,7 @@ function harness(input?: {
166	166	}),
167	167	});
168	168	const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
	169	getByTunnelName: () => Effect.die("unused"),
169	170	get: () => Effect.die("unused"),
170	171	reserve: () => Effect.die("unused"),
171	172	recordTunnel: () => Effect.die("unused"),

infra/relay/src/hooks/HookForwarder.test.ts+364-19

...	...	@@ -1,9 +1,13 @@
1	1	import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
2	2	import * as NodeServices from "@effect/platform-node/NodeServices";
	3	import * as NodeCrypto from "node:crypto";
	4	import * as EffectNodeCrypto from "@effect/platform-node/NodeCrypto";
3	5	import { describe, expect, it } from "@effect/vitest";
4	6	import { RelayApi } from "@t3tools/contracts/relay";
	7	import * as DateTime from "effect/DateTime";
5	8	import * as Deferred from "effect/Deferred";
6	9	import * as Duration from "effect/Duration";
	10	import * as Clock from "effect/Clock";
7	11	import * as Effect from "effect/Effect";
8	12	import * as Fiber from "effect/Fiber";
9	13	import * as Layer from "effect/Layer";
...	...	@@ -18,6 +22,7 @@ import * as Etag from "effect/http/Etag";
18	22	import * as HttpRouter from "effect/http/HttpRouter";
19	23	import * as HttpApi from "effect/http-api/HttpApi";
20	24	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
	25	import * as HttpMiddleware from "effect/http/HttpMiddleware";
21	26	import * as HttpServerRequest from "effect/http/HttpServerRequest";
22	27	import * as HttpServerResponse from "effect/http/HttpServerResponse";
23	28
...	...	@@ -31,6 +36,15 @@ import {
31	36	traceRelayHttpRequestWith,
32	37	} from "../http/Api.ts";
33	38	import * as HookForwarder from "./HookForwarder.ts";
	39	import { RELAY_HOOK_DELIVERY_TYP, verifyRelayJwt } from "@t3tools/shared/relayJwt";
	40	import * as HookInbox from "./HookInbox.ts";
	41	import type { HeldHook } from "./HookInboxStore.ts";
	42	import { RELAY_HOOK_UPSTREAM_TIMEOUT_MS } from "./upstream.ts";
	43
	44	const mintKeys = NodeCrypto.generateKeyPairSync("ed25519", {
	45	privateKeyEncoding: { format: "pem", type: "pkcs8" },
	46	publicKeyEncoding: { format: "pem", type: "spki" },
	47	});
34	48
35	49	const settings: RelayConfiguration.RelayConfiguration["Service"] = {
36	50	relayIssuer: "https://relay.example.test",
...	...	@@ -39,19 +53,20 @@ const settings: RelayConfiguration.RelayConfiguration["Service"] = {
39	53	clerkPublishableKey: "pk_test_test",
40	54	clerkJwtAudience: "t3-code-relay",
41	55	apnsDeliveryJobSigningSecret: Redacted.make("apns-delivery-secret"),
42		cloudMintPrivateKey: Redacted.make("cloud-mint-private-key"),
43		cloudMintPublicKey: "cloud-mint-public-key",
	56	cloudMintPrivateKey: Redacted.make(mintKeys.privateKey),
	57	cloudMintPublicKey: mintKeys.publicKey,
44	58	managedEndpointBaseDomain: "example.test",
45		managedEndpointNamespace: undefined,
	59	managedEndpointNamespace: "dev",
46	60	};
47	61
48	62	const environmentId = "env-hook";
	63	const endpointKey = "0123456789abcdef";
49	64	const readyAllocation: ManagedEndpointAllocations.ManagedEndpointAllocation = {
50	65	userId: "user_1",
51	66	environmentId,
52	67	hostname: "env.example.test",
53	68	tunnelId: "tunnel-id",
54		tunnelName: "tunnel-name",
	69	tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`,
55	70	dnsRecordId: "dns-record-id",
56	71	readyAt: "2026-05-25T00:00:00.000Z",
57	72	origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
...	...	@@ -70,6 +85,7 @@ const managedLink = {
70	85	},
71	86	environmentPublicKey: "public-key",
72	87	linkedAt: "2026-05-25T00:00:00.000Z",
	88	holdWebhooksWhileOffline: false,
73	89	};
74	90
75	91	interface Harness {
...	...	@@ -79,11 +95,15 @@ interface Harness {
79	95	readonly links?: ReadonlyArray<typeof managedLink>;
80	96	readonly allocation?: ManagedEndpointAllocations.ManagedEndpointAllocation \| null;
81	97	readonly allow?: (key: string) => boolean;
	98	readonly allowEndpoint?: (endpointKey: string) => boolean;
	99	/** Inbox capacity; hold reports full once this many requests are held. */
	100	readonly inboxCapacity?: number;
82	101	}
83	102
84	103	function makeHarness(options: Harness = {}) {
85	104	const sent: Array<HttpClientRequest.HttpClientRequest> = [];
86	105	const rateLimitKeys: Array<string> = [];
	106	const held: Array<HeldHook & { readonly baseUrl: string }> = [];
87	107	const execute =
88	108	options.execute ??
89	109	((request: HttpClientRequest.HttpClientRequest) =>
...	...	@@ -100,12 +120,19 @@ function makeHarness(options: Harness = {}) {
100	120	Layer.mock(EnvironmentLinks.EnvironmentLinks, {
101	121	findActiveManagedForEnvironment: (input) =>
102	122	Effect.succeed(
103		input.environmentId === environmentId ? (options.links ?? [managedLink]) : [],
	123	(options.links ?? [managedLink]).filter(
	124	(link) =>
	125	link.environmentId === input.environmentId &&
	126	(input.userId === undefined \|\| link.userId === input.userId),
	127	),
104	128	),
105	129	}),
106	130	Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {
107		get: () =>
108		Effect.succeed(options.allocation === undefined ? readyAllocation : options.allocation),
	131	getByTunnelName: (tunnelName) => {
	132	const allocation =
	133	options.allocation === undefined ? readyAllocation : options.allocation;
	134	return Effect.succeed(allocation?.tunnelName === tunnelName ? allocation : null);
	135	},
109	136	}),
110	137	Layer.succeed(
111	138	HttpClient.HttpClient,
...	...	@@ -114,12 +141,23 @@ function makeHarness(options: Harness = {}) {
114	141	return execute(request);
115	142	}),
116	143	),
	144	Layer.mock(HookInbox.HookInbox, {
	145	hold: ({ hook, baseUrl }) =>
	146	Effect.sync(() => {
	147	if (held.length >= (options.inboxCapacity ?? Infinity)) return false;
	148	held.push({ ...hook, baseUrl });
	149	return true;
	150	}),
	151	}),
	152	EffectNodeCrypto.layer,
117	153	Layer.succeed(HookForwarder.HookRateLimiter, {
118		allow: (key) =>
	154	allowHook: (key) =>
119	155	Effect.sync(() => {
120	156	rateLimitKeys.push(key);
121	157	return options.allow ? options.allow(key) : true;
122	158	}),
	159	allowEndpoint: (key) =>
	160	Effect.sync(() => (options.allowEndpoint ? options.allowEndpoint(key) : true)),
123	161	}),
124	162	),
125	163	),
...	...	@@ -144,11 +182,11 @@ function makeHarness(options: Harness = {}) {
144	182	),
145	183	);
146	184	});
147		return { sent, rateLimitKeys, send, httpEffect };
	185	return { sent, rateLimitKeys, held, send, httpEffect };
148	186	}
149	187
150	188	const hookUrl = (path = "hook-1/secret-token", query = "") =>
151		`https://relay.test/v1/hooks/${environmentId}/${path}${query}`;
	189	`https://relay.test/v1/hooks/${endpointKey}/${path}${query}`;
152	190
153	191	const readBody = (response: HttpServerResponse.HttpServerResponse) =>
154	192	Effect.promise(() => HttpServerResponse.toWeb(response).arrayBuffer()).pipe(
...	...	@@ -210,6 +248,36 @@ describe("HookForwarder", () => {
210	248	}),
211	249	);
212	250
	251	it.effect("signs each forward for the environment, replacing any copy a sender sent", () =>
	252	Effect.gen(function* () {
	253	const harness = makeHarness();
	254	yield* harness.send(
	255	new Request(hookUrl("hook-1/tok"), {
	256	method: "POST",
	257	headers: { "x-t3-relay-delivery": "forged", "x-t3-relay-delivery-id": "forged" },
	258	body: "{}",
	259	}),
	260	);
	261	const sent = harness.sent[0]!;
	262	const payload = yield* verifyRelayJwt({
	263	publicKey: mintKeys.publicKey,
	264	token: sent.headers["x-t3-relay-delivery"]!,
	265	typ: RELAY_HOOK_DELIVERY_TYP,
	266	issuer: settings.relayIssuer,
	267	audience: `t3-env:${environmentId}`,
	268	nowEpochSeconds: Math.floor((yield* Clock.currentTimeMillis) / 1_000),
	269	});
	270	// The proof names exactly the delivery the environment receives.
	271	expect(payload).toMatchObject({
	272	environmentId,
	273	hookId: "hook-1",
	274	deliveryId: sent.headers["x-t3-relay-delivery-id"],
	275	receivedAt: sent.headers["x-t3-relay-received-at"],
	276	});
	277	expect(sent.headers["x-t3-relay-delivery-id"]).not.toBe("forged");
	278	}),
	279	);
	280
213	281	it.effect("passes upstream status, body and content-type through", () =>
214	282	Effect.gen(function* () {
215	283	const harness = makeHarness({
...	...	@@ -229,11 +297,24 @@ describe("HookForwarder", () => {
229	297	expect(response.headers["content-type"]).toBe("application/json");
230	298	expect(response.headers["set-cookie"]).toBeUndefined();
231	299	expect(response.headers["access-control-allow-origin"]).toBeUndefined();
	300	// Served from the relay's origin, so it may never render or run there.
	301	expect(response.headers["x-content-type-options"]).toBe("nosniff");
	302	expect(response.headers["content-security-policy"]).toBe("sandbox; default-src 'none'");
232	303	expect(new TextDecoder().decode(yield* readBody(response))).toBe('{"error":"bad_signature"}');
233	304	expect(harness.sent[0]?.method).toBe("GET");
234	305	}),
235	306	);
236	307
	308	it.effect("never forwards or holds HEAD, which can carry no body", () =>
	309	Effect.gen(function* () {
	310	const harness = makeHarness({ execute: () => Effect.die("must not be sent") });
	311	const response = yield* harness.send(new Request(hookUrl(), { method: "HEAD" }));
	312	expect(response.status).toBeGreaterThanOrEqual(400);
	313	expect(response.status).toBeLessThan(500);
	314	expect(harness.held).toHaveLength(0);
	315	}),
	316	);
	317
237	318	it.effect("does not follow or relay upstream redirects", () =>
238	319	Effect.gen(function* () {
239	320	const harness = makeHarness({
...	...	@@ -284,14 +365,17 @@ describe("HookForwarder", () => {
284	365	}),
285	366	);
286	367
287		it.effect("returns 404 for unknown environments and unready endpoints", () =>
	368	it.effect("returns 404 for unknown endpoints and unready endpoints", () =>
288	369	Effect.gen(function* () {
289	370	const unknown = makeHarness();
290		const response = yield* unknown.send(
291		new Request("https://relay.test/v1/hooks/other-env/hook-1/token", { method: "POST" }),
292		);
293		expect(response.status).toBe(404);
294		expect(yield* readJson(response)).toEqual({ error: "hook_not_found" });
	371	for (const key of ["fedcba9876543210", environmentId]) {
	372	const response = yield* unknown.send(
	373	new Request(`https://relay.test/v1/hooks/${key}/hook-1/token`, { method: "POST" }),
	374	);
	375	expect(response.status).toBe(404);
	376	expect(yield* readJson(response)).toEqual({ error: "hook_not_found" });
	377	}
	378	expect(unknown.sent).toHaveLength(0);
295	379
296	380	const unready = makeHarness({ allocation: { ...readyAllocation, readyAt: null } });
297	381	const unreadyResponse = yield* unready.send(new Request(hookUrl(), { method: "POST" }));
...	...	@@ -300,6 +384,42 @@ describe("HookForwarder", () => {
300	384	}),
301	385	);
302	386
	387	it.effect("forwards only to the link that owns the endpoint key", () =>
	388	Effect.gen(function* () {
	389	// Another account linked the same environment id under its own key; its
	390	// link must not receive this endpoint's hooks, whatever order rows come in.
	391	const intruder = {
	392	...managedLink,
	393	userId: "user_attacker",
	394	environmentPublicKey: "attacker-key",
	395	endpoint: { ...managedLink.endpoint, httpBaseUrl: "https://attacker.example.test/" },
	396	};
	397	const harness = makeHarness({ links: [intruder, managedLink] });
	398	const response = yield* harness.send(new Request(hookUrl(), { method: "POST", body: "{}" }));
	399	expect(response.status).toBe(200);
	400	expect(harness.sent.map((request) => new URL(request.url).host)).toEqual([
	401	"env.example.test",
	402	]);
	403
	404	// Without the owner's link, the key resolves to nothing at all.
	405	const orphaned = makeHarness({ links: [intruder] });
	406	const orphanedResponse = yield* orphaned.send(
	407	new Request(hookUrl(), { method: "POST", body: "{}" }),
	408	);
	409	expect(orphanedResponse.status).toBe(404);
	410	expect(orphaned.sent).toHaveLength(0);
	411	}),
	412	);
	413
	414	it.effect("returns 429 when the endpoint's overall budget is spent", () =>
	415	Effect.gen(function* () {
	416	const harness = makeHarness({ allowEndpoint: () => false });
	417	const response = yield* harness.send(new Request(hookUrl(), { method: "POST" }));
	418	expect(response.status).toBe(429);
	419	expect(harness.sent).toHaveLength(0);
	420	}),
	421	);
	422
303	423	it.effect("maps tunnel-offline and network failures to 503", () =>
304	424	Effect.gen(function* () {
305	425	const offline = makeHarness({
...	...	@@ -353,13 +473,32 @@ describe("HookForwarder", () => {
353	473	.send(new Request(hookUrl(), { method: "POST", body: "{}" }))
354	474	.pipe(Effect.forkChild);
355	475	yield* Deferred.await(reachedUpstream);
356		yield* TestClock.adjust(Duration.millis(HookForwarder.RELAY_HOOK_UPSTREAM_TIMEOUT_MS));
	476	yield* TestClock.adjust(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS));
357	477	const response = yield* Fiber.join(fiber);
358	478	expect(response.status).toBe(504);
359	479	expect(yield* readJson(response)).toEqual({ error: "environment_timeout" });
360	480	}),
361	481	);
362	482
	483	it.effect("holds a timed-out request with the time it arrived", () =>
	484	Effect.gen(function* () {
	485	const reachedUpstream = yield* Deferred.make<void>();
	486	const harness = makeHarness({
	487	links: [{ ...managedLink, holdWebhooksWhileOffline: true }],
	488	execute: () =>
	489	Deferred.succeed(reachedUpstream, undefined).pipe(Effect.andThen(Effect.never)),
	490	});
	491	const arrivedAt = DateTime.formatIso(yield* DateTime.now);
	492	const fiber = yield* harness
	493	.send(new Request(hookUrl(), { method: "POST", body: "{}" }))
	494	.pipe(Effect.forkChild);
	495	yield* Deferred.await(reachedUpstream);
	496	yield* TestClock.adjust(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS));
	497	expect((yield* Fiber.join(fiber)).status).toBe(202);
	498	expect(harness.held[0]?.receivedAt).toBe(arrivedAt);
	499	}),
	500	);
	501
363	502	it.effect("answers OPTIONS without a CORS preflight or forwarding", () =>
364	503	Effect.gen(function* () {
365	504	const harness = makeHarness();
...	...	@@ -454,9 +593,215 @@ describe("HookForwarder", () => {
454	593	expect(serialized).not.toContain("also-secret");
455	594	expect(serialized).not.toContain("header-secret");
456	595	const server = spans.find((span) => span.kind === "server");
457		expect(server?.attributes.get("url.path")).toBe(
458		`/v1/hooks/${environmentId}/hook-1/<redacted>`,
	596	expect(server?.attributes.get("url.path")).toBe(`/v1/hooks/${endpointKey}/hook-1/<redacted>`);
	597	}),
	598	);
	599
	600	it.effect("records one redacted server span even inside the worker's own HTTP tracer", () =>
	601	Effect.gen(function* () {
	602	const spans: Array<Tracer.NativeSpan> = [];
	603	const tracer = Tracer.make({
	604	span: (options) => {
	605	const span = new Tracer.NativeSpan(options);
	606	spans.push(span);
	607	return span;
	608	},
	609	});
	610	const harness = makeHarness();
	611	const handler = yield* harness.httpEffect;
	612	// As the worker runtime runs it: its own tracer around ours, off for hook
	613	// paths. This checks the predicate; whether alchemy applies it per event
	614	// is only visible on a deployed worker (see worker.ts).
	615	yield* HttpMiddleware.tracer(
	616	traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)),
	617	).pipe(
	618	Effect.provideService(HttpMiddleware.TracerDisabledWhen, (request) =>
	619	HookForwarder.isRelayHookPath(request.url),
	620	),
	621	Effect.withTracer(tracer),
	622	Effect.provideService(
	623	HttpServerRequest.HttpServerRequest,
	624	HttpServerRequest.fromWeb(
	625	new Request(hookUrl("hook-1/super-secret-token"), {
	626	method: "POST",
	627	headers: { traceparent: "00-11111111111111111111111111111111-2222222222222222-01" },
	628	body: "{}",
	629	}),
	630	),
	631	),
	632	);
	633	yield* Effect.yieldNow;
	634	const servers = spans.filter((span) => span.kind === "server");
	635	expect(servers).toHaveLength(1);
	636	expect(servers[0]?.attributes.get("url.path")).toBe(
	637	`/v1/hooks/${endpointKey}/hook-1/<redacted>`,
459	638	);
	639	expect(spans.every((span) => span.traceId !== "11111111111111111111111111111111")).toBe(true);
	640	}),
	641	);
	642
	643	it.effect("joins the environment to its trace and records what the environment did", () =>
	644	Effect.gen(function* () {
	645	const spans: Array<Tracer.NativeSpan> = [];
	646	const tracer = Tracer.make({
	647	span: (options) => {
	648	const span = new Tracer.NativeSpan(options);
	649	spans.push(span);
	650	return span;
	651	},
	652	});
	653	const respondWith = (outcome: string) =>
	654	makeHarness({
	655	execute: (request) =>
	656	Effect.succeed(
	657	HttpClientResponse.fromWeb(
	658	request,
	659	new Response('{"deliveryId":"d"}', {
	660	status: 202,
	661	headers: { "x-t3-hook-outcome": outcome },
	662	}),
	663	),
	664	),
	665	});
	666	const forward = (harness: ReturnType<typeof makeHarness>) =>
	667	Effect.gen(function* () {
	668	const handler = yield* harness.httpEffect;
	669	return yield* traceRelayHttpRequestWith(
	670	handler,
	671	Layer.succeed(Tracer.Tracer, tracer),
	672	).pipe(
	673	Effect.provideService(
	674	HttpServerRequest.HttpServerRequest,
	675	HttpServerRequest.fromWeb(
	676	new Request(hookUrl(), {
	677	method: "POST",
	678	// A sender's own trace context never reaches the environment.
	679	headers: {
	680	traceparent: "00-11111111111111111111111111111111-2222222222222222-01",
	681	"x-b3-traceid": "33333333333333333333333333333333",
	682	},
	683	body: "{}",
	684	}),
	685	),
	686	),
	687	);
	688	});
	689
	690	const duplicate = respondWith("duplicate");
	691	expect((yield* forward(duplicate)).status).toBe(202);
	692	yield* Effect.yieldNow;
	693	const forwardSpan = spans.find((span) => span.name === "relay.hooks.forward");
	694	expect(forwardSpan?.attributes.get("relay.hook.upstream_outcome")).toBe("duplicate");
	695	const sent = duplicate.sent[0]!;
	696	expect(sent.headers.traceparent).toContain(forwardSpan!.traceId);
	697	expect(sent.headers.traceparent).not.toContain("1111111111");
	698	expect(sent.headers["x-b3-traceid"]).toBeUndefined();
	699
	700	// Only a plain outcome name is recorded.
	701	spans.length = 0;
	702	yield* forward(respondWith("<script>alert(1)</script>"));
	703	yield* Effect.yieldNow;
	704	expect(
	705	spans
	706	.find((span) => span.name === "relay.hooks.forward")
	707	?.attributes.has("relay.hook.upstream_outcome"),
	708	).toBe(false);
460	709	}),
461	710	);
	711
	712	describe("holding requests while the environment is offline", () => {
	713	const offline = (request: HttpClientRequest.HttpClientRequest) =>
	714	Effect.fail(
	715	new HttpClientError.HttpClientError({
	716	reason: new HttpClientError.TransportError({ request, cause: new Error("offline") }),
	717	}),
	718	);
	719
	720	it.effect("holds when cloudflared answers that the local server is down", () =>
	721	Effect.gen(function* () {
	722	for (const status of [502, 503, 504, 530]) {
	723	const harness = makeHarness({
	724	links: [{ ...managedLink, holdWebhooksWhileOffline: true }],
	725	execute: (request) =>
	726	Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status }))),
	727	});
	728	const response = yield* harness.send(
	729	new Request(hookUrl(), { method: "POST", body: "{}" }),
	730	);
	731	expect(response.status).toBe(202);
	732	expect(harness.held).toHaveLength(1);
	733	}
	734	}),
	735	);
	736
	737	it.effect("stays a plain proxy when the environment has not opted in", () =>
	738	Effect.gen(function* () {
	739	const harness = makeHarness({ execute: offline });
	740	const response = yield* harness.send(
	741	new Request(hookUrl(), { method: "POST", body: "{}" }),
	742	);
	743	expect(response.status).toBe(503);
	744	expect(harness.held).toHaveLength(0);
	745	}),
	746	);
	747
	748	it.effect("holds the exact request and answers 202 once opted in", () =>
	749	Effect.gen(function* () {
	750	const harness = makeHarness({
	751	execute: offline,
	752	links: [{ ...managedLink, holdWebhooksWhileOffline: true }],
	753	});
	754	const body = new Uint8Array([0, 255, 10]);
	755	const response = yield* harness.send(
	756	new Request(hookUrl("%68ook-1/tok%2Fen", "?a=1"), {
	757	method: "POST",
	758	body,
	759	headers: { "x-t3-relay-delivery-id": "forged", "x-sig": "s" },
	760	}),
	761	);
	762	expect(response.status).toBe(202);
	763	const [hook] = harness.held;
	764	expect(hook?.baseUrl).toBe("https://env.example.test/");
	765	expect(hook?.rawToken).toBe("tok%2Fen");
	766	expect(hook?.query).toBe("a=1");
	767	expect([...(hook?.body ?? [])]).toEqual([0, 255, 10]);
	768	expect(hook?.headers["x-sig"]).toBe("s");
	769	// Every spelling of the hook id shares one per-hook cap.
	770	expect(hook?.hookKey).toBe("hook-1");
	771	// The sender cannot choose the delivery id.
	772	expect(hook?.headers["x-t3-relay-delivery-id"]).toBeUndefined();
	773	expect(hook?.id).not.toBe("forged");
	774	}),
	775	);
	776
	777	it.effect("answers 503 inbox_full when the environment's inbox is full", () =>
	778	Effect.gen(function* () {
	779	const harness = makeHarness({
	780	execute: offline,
	781	links: [{ ...managedLink, holdWebhooksWhileOffline: true }],
	782	inboxCapacity: 0,
	783	});
	784	const response = yield* harness.send(
	785	new Request(hookUrl(), { method: "POST", body: "{}" }),
	786	);
	787	expect(response.status).toBe(503);
	788	expect(yield* readJson(response)).toEqual({ error: "inbox_full" });
	789	}),
	790	);
	791
	792	it.effect("tags every forward with a relay delivery id", () =>
	793	Effect.gen(function* () {
	794	const harness = makeHarness();
	795	yield* harness.send(
	796	new Request(hookUrl(), {
	797	method: "POST",
	798	body: "{}",
	799	headers: { "x-t3-relay-delivery-id": "forged" },
	800	}),
	801	);
	802	const id = harness.sent[0]?.headers["x-t3-relay-delivery-id"];
	803	expect(id).toMatch(/^[0-9a-f-]{36}$/);
	804	}),
	805	);
	806	});
462	807	});

infra/relay/src/hooks/HookForwarder.ts+253-105

...	...	@@ -1,28 +1,57 @@
	1	import * as Clock from "effect/Clock";
1	2	import * as Context from "effect/Context";
2		import * as Duration from "effect/Duration";
	3	import * as Crypto from "effect/Crypto";
	4	import * as DateTime from "effect/DateTime";
3	5	import * as Effect from "effect/Effect";
4	6	import * as Layer from "effect/Layer";
5	7	import * as Option from "effect/Option";
	8	import * as Redacted from "effect/Redacted";
6	9	import * as Result from "effect/Result";
7	10	import * as Schema from "effect/Schema";
8	11	import * as Stream from "effect/Stream";
9	12	import * as HttpClient from "effect/http/HttpClient";
10		import * as HttpClientRequest from "effect/http/HttpClientRequest";
11		import type * as HttpClientResponse from "effect/http/HttpClientResponse";
12	13	import type * as HttpServerRequest from "effect/http/HttpServerRequest";
13	14	import * as HttpServerResponse from "effect/http/HttpServerResponse";
14	15	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
15		import { RelayApi } from "@t3tools/contracts/relay";
	16	import { EnvironmentId } from "@t3tools/contracts";
	17	import { RelayApi, type RelayHookDeliveryProofPayload } from "@t3tools/contracts/relay";
	18	import {
	19	normalizeRelayIssuer,
	20	RELAY_HOOK_DELIVERY_HEADER,
	21	RELAY_HOOK_DELIVERY_TYP,
	22	signRelayJwt,
	23	} from "@t3tools/shared/relayJwt";
16	24
17	25	import * as RelayConfiguration from "../Config.ts";
18		import { validateManagedEndpoint, withoutRedirects } from "../environments/EnvironmentConnector.ts";
	26	import {
	27	MANAGED_ENDPOINT_KEY_PATTERN,
	28	managedEndpointTunnelNameForKey,
	29	} from "../deploymentConfig.ts";
	30	import { validateManagedEndpoint } from "../environments/EnvironmentConnector.ts";
19	31	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";
20	32	import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts";
	33	import * as HookInbox from "./HookInbox.ts";
	34	import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts";
21	35
22	36	export const RELAY_HOOK_PATH_PREFIX = "/v1/hooks/";
23	37	export const RELAY_HOOK_MAX_BODY_BYTES = 1_048_576;
24		export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000;
25	38	export const RELAY_HOOK_RATE_LIMIT = { limit: 60, periodSeconds: 60 } as const;
	39	/**
	40	* Hook budgets are per URL, and a sender who knows an endpoint key can mint
	41	* new URLs for free, so every endpoint also has one overall budget.
	42	*/
	43	export const RELAY_HOOK_ENDPOINT_RATE_LIMIT = { limit: 600, periodSeconds: 60 } as const;
	44	/**
	45	* Upstream statuses that mean the environment did not take the request: the
	46	* tunnel has no origin (530) or cloudflared cannot reach the local server
	47	* (502, 503, 504) while it restarts.
	48	*/
	49	export const ENVIRONMENT_UNREACHABLE_STATUSES: ReadonlySet<number> = new Set([
	50	502,
	51	503,
	52	504,
	53	TUNNEL_OFFLINE_STATUS,
	54	]);
26	55
27	56	const DROPPED_REQUEST_HEADERS = new Set([
28	57	"host",
...	...	@@ -34,12 +63,16 @@ const DROPPED_REQUEST_HEADERS = new Set([
34	63	"content-length",
35	64	"cookie",
36	65	"x-real-ip",
	66	// Only the relay may set this; a sender could otherwise collide delivery ids.
	67	"x-t3-relay-delivery-id",
	68	"x-t3-relay-received-at",
	69	RELAY_HOOK_DELIVERY_HEADER,
	70	// The environment trusts trace context only from the relay, which sets its own.
	71	"traceparent",
	72	"tracestate",
	73	"b3",
37	74	]);
38		const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-"];
39		// Cloudflare answers 530 when the tunnel for a hostname has no connected origin.
40		const TUNNEL_OFFLINE_STATUS = 530;
41		/** The environment answers with a small JSON status; anything past this is cut off. */
42		const MAX_RESPONSE_BYTES = 64 * 1024;
	75	const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-", "x-b3-"];
43	76
44	77	export const isRelayHookPath = (url: string): boolean => url.startsWith(RELAY_HOOK_PATH_PREFIX);
45	78
...	...	@@ -55,13 +88,13 @@ export const redactRelayHookUrl = (url: string): string => {
55	88
56	89	/**
57	90	* Request budget for public hook forwarding, keyed by a hash of the hook URL
58		* (environment, hook and token). Requests with a wrong token get their own
	91	* (endpoint, hook and token). Requests with a wrong token get their own
59	92	* budget, so they cannot use up a real sender's; the environment rejects them.
60	93	* Built from decoded segments, because the environment decodes them too: two
61	94	* spellings of one token (`token`, `%74oken`) must share one budget.
62	95	*/
63	96	const hookBudgetKey = (hook: {
64		readonly environmentId: string;
	97	readonly endpointKey: string;
65	98	readonly hookId: string;
66	99	readonly token: string;
67	100	}) =>
...	...	@@ -70,7 +103,7 @@ const hookBudgetKey = (hook: {
70	103	"SHA-256",
71	104	// Length-prefixed, so no segment contents can make two keys collide.
72	105	new TextEncoder().encode(
73		[hook.environmentId, hook.hookId, hook.token]
	106	[hook.endpointKey, hook.hookId, hook.token]
74	107	.map((part) => `${part.length}:${part}`)
75	108	.join(""),
76	109	),
...	...	@@ -83,7 +116,12 @@ const hookBudgetKey = (hook: {
83	116
84	117	export class HookRateLimiter extends Context.Service<
85	118	HookRateLimiter,
86		{ readonly allow: (key: string) => Effect.Effect<boolean> }
	119	{
	120	/** One hook URL's budget, keyed by `hookBudgetKey`. */
	121	readonly allowHook: (key: string) => Effect.Effect<boolean>;
	122	/** One endpoint's overall budget, keyed by its endpoint key. */
	123	readonly allowEndpoint: (endpointKey: string) => Effect.Effect<boolean>;
	124	}
87	125	>()("t3code-relay/hooks/HookForwarder/HookRateLimiter") {}
88	126
89	127	export class HookForwarder extends Context.Service<
...	...	@@ -96,25 +134,69 @@ export class HookForwarder extends Context.Service<
96	134	>()("t3code-relay/hooks/HookForwarder") {}
97	135
98	136	class HookBodyTooLarge extends Schema.TaggedError<HookBodyTooLarge>()("HookBodyTooLarge", {}) {}
99		class ResponseTooLarge extends Schema.TaggedError<ResponseTooLarge>()("ResponseTooLarge", {}) {}
100	137
101	138	const errorResponse = (status: number, error: string, headers?: Record<string, string>) =>
102	139	HttpServerResponse.jsonUnsafe({ error }, { status, ...(headers ? { headers } : {}) });
103	140
104	141	const hookNotFound = () => errorResponse(404, "hook_not_found");
105	142
	143	/** Longer than the inbox holds a request, so a held delivery's proof still verifies. */
	144	const DELIVERY_PROOF_LIFETIME_SECONDS = 25 * 60 * 60;
	145
	146	const signDeliveryProof = (input: {
	147	readonly settings: RelayConfiguration.RelayConfiguration["Service"];
	148	readonly environmentId: string;
	149	readonly deliveryId: string;
	150	readonly receivedAt: string;
	151	readonly hookId: string;
	152	readonly jti: string;
	153	}) =>
	154	Effect.gen(function* () {
	155	const now = Math.floor((yield* Clock.currentTimeMillis) / 1_000);
	156	return yield* signRelayJwt({
	157	privateKey: Redacted.value(input.settings.cloudMintPrivateKey),
	158	typ: RELAY_HOOK_DELIVERY_TYP,
	159	payload: {
	160	iss: normalizeRelayIssuer(input.settings.relayIssuer),
	161	aud: `t3-env:${input.environmentId}`,
	162	sub: input.environmentId,
	163	jti: input.jti,
	164	iat: now,
	165	exp: now + DELIVERY_PROOF_LIFETIME_SECONDS,
	166	environmentId: EnvironmentId.make(input.environmentId),
	167	deliveryId: input.deliveryId,
	168	receivedAt: input.receivedAt,
	169	hookId: input.hookId,
	170	} satisfies RelayHookDeliveryProofPayload,
	171	});
	172	}).pipe(Effect.orDie);
	173
	174	/** Methods a webhook can arrive with; HEAD reaches the GET route and is refused. */
	175	const FORWARDED_METHODS = new Set(["GET", "POST", "PUT", "PATCH"]);
	176
	177	/**
	178	* Whatever the environment answers is served from the relay's own origin, so
	179	* a body must never render or run there.
	180	*/
	181	const SANDBOXED_RESPONSE_HEADERS = {
	182	"x-content-type-options": "nosniff",
	183	"content-security-policy": "sandbox; default-src 'none'",
	184	} as const;
	185
106	186	function parseHookPath(url: string) {
107	187	const queryIndex = url.indexOf("?");
108	188	const path = queryIndex === -1 ? url : url.slice(0, queryIndex);
109	189	const search = queryIndex === -1 ? "" : url.slice(queryIndex);
110	190	const segments = path.split("/");
111		// ["", "v1", "hooks", environmentId, hookId, token]
	191	// ["", "v1", "hooks", endpointKey, hookId, token]
112	192	if (segments.length !== 6) return null;
113		const [, , , rawEnvironmentId, rawHookId, rawToken] = segments;
114		if (!rawEnvironmentId \|\| !rawHookId \|\| !rawToken) return null;
	193	const [, , , endpointKey, rawHookId, rawToken] = segments;
	194	if (!endpointKey \|\| !MANAGED_ENDPOINT_KEY_PATTERN.test(endpointKey) \|\| !rawHookId \|\| !rawToken) {
	195	return null;
	196	}
115	197	try {
116	198	return {
117		environmentId: decodeURIComponent(rawEnvironmentId),
	199	endpointKey,
118	200	hookId: decodeURIComponent(rawHookId),
119	201	token: decodeURIComponent(rawToken),
120	202	// Forward the encoded segments byte-for-byte; the environment decodes them.
...	...	@@ -156,20 +238,18 @@ function forwardedHeaders(headers: Readonly<Record<string, string>>): Record<str
156	238	const hasNoBody = (request: HttpServerRequest.HttpServerRequest) =>
157	239	request.source instanceof Request && request.source.body === null;
158	240
159		/** Collects a byte stream, failing with `tooLarge` once it passes `maxBytes` rather than buffering it all. */
160		const collectCapped = <E, R, E2>(
161		stream: Stream.Stream<Uint8Array, E, R>,
162		maxBytes: number,
163		tooLarge: () => E2,
164		) =>
	241	const readCappedBody = (request: HttpServerRequest.HttpServerRequest) =>
165	242	Effect.suspend(() => {
	243	if (hasNoBody(request)) {
	244	return Effect.succeed(new Uint8Array(0));
	245	}
166	246	const chunks: Array<Uint8Array> = [];
167	247	let total = 0;
168		return stream.pipe(
	248	return request.stream.pipe(
169	249	Stream.runForEach((chunk) => {
170	250	total += chunk.length;
171		if (total > maxBytes) {
172		return Effect.fail(tooLarge());
	251	if (total > RELAY_HOOK_MAX_BODY_BYTES) {
	252	return Effect.fail(new HookBodyTooLarge());
173	253	}
174	254	chunks.push(chunk);
175	255	return Effect.void;
...	...	@@ -186,19 +266,47 @@ const collectCapped = <E, R, E2>(
186	266	);
187	267	});
188	268
189		const readCappedBody = (request: HttpServerRequest.HttpServerRequest) =>
190		hasNoBody(request)
191		? Effect.succeed(new Uint8Array(0))
192		: collectCapped(request.stream, RELAY_HOOK_MAX_BODY_BYTES, () => new HookBodyTooLarge());
193
194		const readCappedResponse = (response: HttpClientResponse.HttpClientResponse) =>
195		collectCapped(response.stream, MAX_RESPONSE_BYTES, () => new ResponseTooLarge()).pipe(
196		// A response without a body, such as a redirect, has no stream at all.
197		Effect.catchIf(
198		(error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError",
199		() => Effect.succeed(new Uint8Array(0)),
200		),
	269	/**
	270	* The ready managed endpoint a webhook URL's endpoint key names, with whether
	271	* its link opted in to holding webhooks while offline. The key is the tunnel
	272	* name's hash of user and environment, so it names exactly one allocation and
	273	* at most one active link; nobody else can link their way onto it.
	274	*/
	275	export const resolveHookEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* (
	276	endpointKey: string,
	277	) {
	278	const links = yield* EnvironmentLinks.EnvironmentLinks;
	279	const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
	280	const settings = yield* RelayConfiguration.RelayConfiguration;
	281	if (!settings.managedEndpointNamespace) return null;
	282	const allocation = yield* allocations.getByTunnelName(
	283	managedEndpointTunnelNameForKey(settings.managedEndpointNamespace, endpointKey),
201	284	);
	285	if (allocation === null) return null;
	286	const [link] = yield* links.findActiveManagedForEnvironment({
	287	environmentId: allocation.environmentId,
	288	userId: allocation.userId,
	289	});
	290	if (!link) return null;
	291	const result = validateManagedEndpoint({
	292	link,
	293	allocation,
	294	baseDomain: settings.managedEndpointBaseDomain,
	295	});
	296	if (Result.isFailure(result)) return null;
	297	return {
	298	...result.success,
	299	environmentId: allocation.environmentId,
	300	holdWhileOffline: link.holdWebhooksWhileOffline,
	301	};
	302	});
	303
	304	/** The endpoint key of an environment's own managed endpoint, for authenticated callers. */
	305	export const endpointKeyForTunnelName = (namespace: string, tunnelName: string): string \| null => {
	306	const prefix = managedEndpointTunnelNameForKey(namespace, "");
	307	const key = tunnelName.startsWith(prefix) ? tunnelName.slice(prefix.length) : "";
	308	return MANAGED_ENDPOINT_KEY_PATTERN.test(key) ? key : null;
	309	};
202	310
203	311	const make = Effect.gen(function* () {
204	312	const links = yield* EnvironmentLinks.EnvironmentLinks;
...	...	@@ -206,39 +314,38 @@ const make = Effect.gen(function* () {
206	314	const settings = yield* RelayConfiguration.RelayConfiguration;
207	315	const httpClient = yield* HttpClient.HttpClient;
208	316	const rateLimiter = yield* HookRateLimiter;
209
210		const resolveEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* (
211		environmentId: string,
212		) {
213		const candidates = yield* links.findActiveManagedForEnvironment({ environmentId });
214		for (const link of candidates) {
215		const allocation = yield* allocations.get({ userId: link.userId, environmentId });
216		const result = validateManagedEndpoint({
217		link,
218		allocation,
219		baseDomain: settings.managedEndpointBaseDomain,
220		});
221		if (Result.isSuccess(result)) {
222		return result.success;
223		}
224		}
225		return null;
226		});
	317	const inbox = yield* HookInbox.HookInbox;
	318	const crypto = yield* Crypto.Crypto;
227	319
228	320	const handle = Effect.fn("relay.hooks.forward")(function* (
229	321	request: HttpServerRequest.HttpServerRequest,
230	322	) {
231	323	const outcome = (value: string) => Effect.annotateCurrentSpan({ "relay.hook.outcome": value });
	324	if (!FORWARDED_METHODS.has(request.method)) {
	325	yield* outcome("method_not_allowed");
	326	return errorResponse(405, "method_not_allowed", { allow: "GET, POST, PUT, PATCH" });
	327	}
	328	// When the sender called, not when the environment failed to answer.
	329	const receivedAt = DateTime.formatIso(yield* DateTime.now);
232	330	const parsed = parseHookPath(request.url);
233	331	if (!parsed) {
234	332	yield* outcome("invalid_path");
235	333	return hookNotFound();
236	334	}
237	335	yield* Effect.annotateCurrentSpan({
238		"relay.environment_id": parsed.environmentId,
	336	"relay.hook.endpoint_key": parsed.endpointKey,
239	337	"relay.hook_id": parsed.hookId,
240	338	});
241		if (!(yield* rateLimiter.allow(yield* hookBudgetKey(parsed)))) {
	339	// A coarse budget per endpoint first, so minting new hook ids or tokens
	340	// cannot buy unlimited lookups and forwards, or fill the inbox.
	341	const endpointAllowed = yield* rateLimiter.allowEndpoint(parsed.endpointKey);
	342	const hookAllowed =
	343	endpointAllowed && (yield* rateLimiter.allowHook(yield* hookBudgetKey(parsed)));
	344	if (!endpointAllowed \|\| !hookAllowed) {
	345	// Which budget ran out: the whole endpoint's, or this one hook URL's.
	346	yield* Effect.annotateCurrentSpan({
	347	"relay.hook.rate_limit": endpointAllowed ? "hook" : "endpoint",
	348	});
242	349	yield* outcome("rate_limited");
243	350	return errorResponse(429, "rate_limited", {
244	351	"retry-after": String(RELAY_HOOK_RATE_LIMIT.periodSeconds),
...	...	@@ -250,10 +357,13 @@ const make = Effect.gen(function* () {
250	357	return errorResponse(413, "payload_too_large");
251	358	}
252	359
253		const endpoint = yield* resolveEndpoint(parsed.environmentId).pipe(
	360	const endpoint = yield* resolveHookEndpoint(parsed.endpointKey).pipe(
	361	Effect.provideService(EnvironmentLinks.EnvironmentLinks, links),
	362	Effect.provideService(ManagedEndpointAllocations.ManagedEndpointAllocations, allocations),
	363	Effect.provideService(RelayConfiguration.RelayConfiguration, settings),
254	364	Effect.catch((error) =>
255	365	Effect.logWarning("Failed to resolve hook endpoint", {
256		environmentId: parsed.environmentId,
	366	endpointKey: parsed.endpointKey,
257	367	errorTag: error._tag,
258	368	}).pipe(Effect.as(null)),
259	369	),
...	...	@@ -262,6 +372,10 @@ const make = Effect.gen(function* () {
262	372	yield* outcome("not_found");
263	373	return hookNotFound();
264	374	}
	375	yield* Effect.annotateCurrentSpan({
	376	"relay.environment_id": endpoint.environmentId,
	377	"relay.hook.hold_while_offline": endpoint.holdWhileOffline,
	378	});
265	379
266	380	const body =
267	381	request.method === "GET"
...	...	@@ -276,67 +390,101 @@ const make = Effect.gen(function* () {
276	390	return errorResponse(400, "invalid_body");
277	391	}
278	392
279		const baseUrl = endpoint.httpBaseUrl.endsWith("/")
280		? endpoint.httpBaseUrl
281		: `${endpoint.httpBaseUrl}/`;
282		const headers = forwardedHeaders(request.headers);
283		let upstreamRequest = HttpClientRequest.make(
284		request.method as "GET" \| "POST" \| "PUT" \| "PATCH",
285		)(`${baseUrl}api/hooks/${parsed.rawHookId}/${parsed.rawToken}${parsed.search}`, { headers });
286		if (request.method !== "GET") {
287		upstreamRequest = HttpClientRequest.bodyUint8Array(
288		upstreamRequest,
289		body.success,
290		headers["content-type"],
291		);
292		}
	393	// One id per request, so a request that reached the environment before a
	394	// timeout and is later delivered from the inbox runs only once.
	395	yield* Effect.annotateCurrentSpan({ "relay.hook.body_bytes": body.success.byteLength });
	396	const deliveryId = yield* crypto.randomUUIDv4.pipe(Effect.orDie);
	397	// Proves to the environment that this delivery id, receive time and
	398	// trace context came from the relay. Signed once here and stored with a
	399	// held request, so the inbox never needs the signing key.
	400	const proof = yield* signDeliveryProof({
	401	settings,
	402	environmentId: endpoint.environmentId,
	403	deliveryId,
	404	receivedAt,
	405	hookId: parsed.hookId,
	406	jti: yield* crypto.randomUUIDv4.pipe(Effect.orDie),
	407	});
	408	const hook = {
	409	id: deliveryId,
	410	receivedAt,
	411	method: request.method,
	412	rawHookId: parsed.rawHookId,
	413	rawToken: parsed.rawToken,
	414	hookKey: parsed.hookId,
	415	query: parsed.search.replace(/^\?/, ""),
	416	headers: { ...forwardedHeaders(request.headers), [RELAY_HOOK_DELIVERY_HEADER]: proof },
	417	body: body.success,
	418	};
	419	// Held only for environments that opted in; otherwise the relay is a plain proxy.
	420	const holdOrFail = (status: 503 \| 504, error: string) =>
	421	Effect.gen(function* () {
	422	if (!endpoint.holdWhileOffline) {
	423	yield* outcome(error);
	424	return errorResponse(status, error);
	425	}
	426	const stored = yield* inbox
	427	.hold({
	428	endpointKey: parsed.endpointKey,
	429	baseUrl: endpoint.httpBaseUrl,
	430	hook,
	431	})
	432	.pipe(
	433	Effect.catch((cause) =>
	434	Effect.logWarning("Could not hold webhook request", {
	435	environmentId: endpoint.environmentId,
	436	errorTag: cause._tag,
	437	}).pipe(Effect.as(null)),
	438	),
	439	);
	440	if (stored === null) {
	441	yield* outcome(error);
	442	return errorResponse(status, error);
	443	}
	444	if (!stored) {
	445	// The inbox span carries which cap refused it (relay.inbox.refused).
	446	yield* outcome("inbox_full");
	447	return errorResponse(503, "inbox_full");
	448	}
	449	yield* outcome("held");
	450	return HttpServerResponse.jsonUnsafe({ queued: true }, { status: 202 });
	451	});
293	452
294		const upstream = yield* httpClient.execute(upstreamRequest).pipe(
295		Effect.flatMap((response) =>
296		readCappedResponse(response).pipe(
297		Effect.map((body) => ({
298		status: response.status,
299		contentType: response.headers["content-type"],
300		body,
301		})),
302		),
303		),
304		withoutRedirects,
305		// The client span would record url.full, which carries the token.
306		Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
307		Effect.timeoutOption(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)),
	453	const upstream = yield* sendUpstream(endpoint.httpBaseUrl, hook).pipe(
	454	Effect.provideService(HttpClient.HttpClient, httpClient),
308	455	Effect.result,
309	456	);
310	457	if (Result.isFailure(upstream)) {
311		yield* outcome("environment_unavailable");
312		return errorResponse(503, "environment_unavailable");
	458	yield* Effect.annotateCurrentSpan({ "relay.hook.upstream_error": upstream.failure._tag });
	459	return yield* holdOrFail(503, "environment_unavailable");
313	460	}
314	461	if (Option.isNone(upstream.success)) {
315		yield* outcome("environment_timeout");
316		return errorResponse(504, "environment_timeout");
	462	return yield* holdOrFail(504, "environment_timeout");
317	463	}
318	464	const response = upstream.success.value;
319		if (response.status === TUNNEL_OFFLINE_STATUS) {
320		yield* outcome("environment_unavailable");
321		return errorResponse(503, "environment_unavailable");
	465	if (ENVIRONMENT_UNREACHABLE_STATUSES.has(response.status)) {
	466	yield* Effect.annotateCurrentSpan({ "relay.hook.upstream_status": response.status });
	467	return yield* holdOrFail(503, "environment_unavailable");
322	468	}
323	469	yield* Effect.annotateCurrentSpan({
324	470	"relay.hook.outcome": "forwarded",
325	471	"relay.hook.upstream_status": response.status,
	472	...(response.outcome === undefined
	473	? {}
	474	: { "relay.hook.upstream_outcome": response.outcome }),
326	475	});
327	476	// Only content-type is passed through: no location (redirects are never
328	477	// followed or relayed), no cookies, no upstream infrastructure headers.
329		const contentTypeHeaders = response.contentType
330		? { "content-type": response.contentType }
331		: undefined;
	478	const headers = {
	479	...SANDBOXED_RESPONSE_HEADERS,
	480	...(response.contentType ? { "content-type": response.contentType } : {}),
	481	};
332	482	if (response.body.length === 0) {
333		return HttpServerResponse.empty({
334		status: response.status,
335		...(contentTypeHeaders ? { headers: contentTypeHeaders } : {}),
336		});
	483	return HttpServerResponse.empty({ status: response.status, headers });
337	484	}
338	485	return HttpServerResponse.uint8Array(response.body, {
339	486	status: response.status,
	487	headers,
340	488	...(response.contentType ? { contentType: response.contentType } : {}),
341	489	});
342	490	});

infra/relay/src/hooks/HookInbox.ts created+40

...	...	@@ -0,0 +1,40 @@
	1	import * as Context from "effect/Context";
	2	import * as Effect from "effect/Effect";
	3	import * as Schema from "effect/Schema";
	4
	5	import type { HeldHook } from "./HookInboxStore.ts";
	6
	7	export class HookInboxError extends Schema.TaggedError<HookInboxError>()("HookInboxError", {
	8	operation: Schema.Literals(["hold", "wake", "clear"]),
	9	endpointKey: Schema.String,
	10	cause: Schema.Defect(),
	11	}) {
	12	override get message(): string {
	13	return `Hook inbox '${this.operation}' failed for endpoint '${this.endpointKey}'`;
	14	}
	15	}
	16
	17	/**
	18	* Webhook requests held for environments that opted in, while they are
	19	* offline. Each managed endpoint's requests live in its own `HookInboxObject`,
	20	* named by endpoint key, which delivers them itself once the environment is back.
	21	*/
	22	export class HookInbox extends Context.Service<
	23	HookInbox,
	24	{
	25	/** False when the environment's inbox is full and nothing was stored. */
	26	readonly hold: (input: {
	27	readonly endpointKey: string;
	28	readonly baseUrl: string;
	29	readonly hook: HeldHook;
	30	}) => Effect.Effect<boolean, HookInboxError>;
	31	/** Delivers what is waiting now; true when anything was waiting. */
	32	readonly wake: (input: {
	33	readonly endpointKey: string;
	34	readonly baseUrl: string;
	35	}) => Effect.Effect<boolean, HookInboxError>;
	36	readonly clear: (input: {
	37	readonly endpointKey: string;
	38	}) => Effect.Effect<void, HookInboxError>;
	39	}
	40	>()("t3code-relay/hooks/HookInbox") {}

infra/relay/src/hooks/HookInboxObject.ts created+154

...	...	@@ -0,0 +1,154 @@
	1	import * as SqliteClient from "@effect/sql-sqlite-do/SqliteClient";
	2	import type * as Alchemy from "alchemy";
	3	import * as Cloudflare from "alchemy/Cloudflare";
	4	import * as Clock from "effect/Clock";
	5	import * as Effect from "effect/Effect";
	6	import * as Option from "effect/Option";
	7	import * as Result from "effect/Result";
	8	import * as FetchHttpClient from "effect/http/FetchHttpClient";
	9
	10	import * as HookInboxStore from "./HookInboxStore.ts";
	11	import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts";
	12
	13	/** Statuses that mean the environment is not there; the whole inbox waits and backs off. */
	14	const UNREACHABLE_STATUSES = new Set([502, 503, 504, TUNNEL_OFFLINE_STATUS]);
	15	/**
	16	* Statuses that mean the environment is there but did not take this request
	17	* yet: its task's queue is full (429) or it failed while handling it (500).
	18	* The environment drops a delivery id it has already run, so retrying is safe.
	19	*/
	20	const BUSY_STATUSES = new Set([429, 500]);
	21	/** When a run itself fails, the next one is tried after this long. */
	22	const RUN_FAILURE_RETRY_MS = 60_000;
	23
	24	type Call<A> = Effect.Effect<A, never, Alchemy.RuntimeContext>;
	25
	26	export interface HookInboxObjectShape {
	27	/** Holds a request for `baseUrl`; false when the inbox is full and nothing was stored. */
	28	readonly hold: (hook: HookInboxStore.HeldHook, baseUrl: string) => Call<boolean>;
	29	/** The environment is back at `baseUrl`: deliver what is waiting now. */
	30	readonly wake: (baseUrl: string) => Call<boolean>;
	31	readonly clear: () => Call<void>;
	32	}
	33
	34	/**
	35	* One per managed endpoint, addressed by endpoint key. Holds webhook requests
	36	* the environment could not take, in SQLite, and pushes them back through
	37	* its tunnel from the alarm, oldest first, backing off while it stays away.
	38	*/
	39	export class HookInboxObject extends Cloudflare.DurableObject<
	40	HookInboxObject,
	41	HookInboxObjectShape
	42	>()("HookInboxObject") {}
	43
	44	/**
	45	* Each call into an inbox is its own trace: the alarm has no parent, and a
	46	* hold arrives over Durable Object RPC without the forwarding span. The
	47	* object's name is the endpoint key, which the forward spans carry too.
	48	*/
	49	const withInboxSpan =
	50	(name: string, inboxId: string) =>
	51	<A, E, R>(effect: Effect.Effect<A, E, R>) =>
	52	effect.pipe(
	53	Effect.withSpan(name, { root: true, attributes: { "relay.hook.endpoint_key": inboxId } }),
	54	);
	55
	56	const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) =>
	57	sendUpstream(baseUrl, hook).pipe(
	58	Effect.result,
	59	Effect.map((result) => {
	60	// Unreachable or timed out: a timeout may still have run it, and the
	61	// environment drops a delivery id it has already seen.
	62	if (Result.isFailure(result)) {
	63	return { outcome: "unreachable" as const, reason: result.failure._tag };
	64	}
	65	if (Option.isNone(result.success))
	66	return { outcome: "unreachable" as const, reason: "timeout" };
	67	const status = result.success.value.status;
	68	const upstreamOutcome = result.success.value.outcome;
	69	const outcome: HookInboxStore.DeliveryOutcome = UNREACHABLE_STATUSES.has(status)
	70	? "unreachable"
	71	: BUSY_STATUSES.has(status)
	72	? "busy"
	73	: "delivered";
	74	return { outcome, reason: `status ${status}`, upstreamOutcome };
	75	}),
	76	Effect.tap(({ upstreamOutcome }) =>
	77	upstreamOutcome === undefined
	78	? Effect.void
	79	: Effect.annotateCurrentSpan({ "relay.hook.upstream_outcome": upstreamOutcome }),
	80	),
	81	Effect.tap(({ outcome, reason }) =>
	82	outcome === "delivered"
	83	? Effect.void
	84	: Effect.logInfo("Held webhook not delivered yet", {
	85	outcome,
	86	reason,
	87	deliveryId: hook.id,
	88	}),
	89	),
	90	Effect.map(({ outcome }) => outcome),
	91	Effect.provide(FetchHttpClient.layer),
	92	);
	93
	94	export const HookInboxObjectLive = HookInboxObject.make(
	95	Effect.gen(function* () {
	96	const state = yield* Cloudflare.DurableObjectState;
	97	// The init phase returns the per-instance Effect, which alchemy runs once
	98	// per object; only that inner Effect may touch storage.
	99	// @effect-diagnostics-next-line returnEffectInGen:off
	100	return Effect.gen(function* () {
	101	const sql = SqliteClient.layer({ storage: state.raw.storage });
	102	// Inboxes are opened by endpoint key, so spans line up with the
	103	// forward spans that held their requests.
	104	const inboxId = state.raw.id.name ?? state.raw.id.toString();
	105	const run = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
	106	effect.pipe(Effect.provide(sql), Effect.orDie);
	107	yield* run(HookInboxStore.migrate);
	108
	109	/** Moves the alarm to `at`, unless one is already due sooner. */
	110	const scheduleBy = (at: number) =>
	111	Effect.gen(function* () {
	112	const current = yield* state.storage.getAlarm();
	113	if (current === null \|\| current > at) yield* state.storage.setAlarm(at);
	114	});
	115
	116	return {
	117	hold: (hook: HookInboxStore.HeldHook, baseUrl: string) =>
	118	Effect.gen(function* () {
	119	const dueAt = yield* run(HookInboxStore.hold(hook, baseUrl));
	120	yield* Effect.annotateCurrentSpan({ "relay.inbox.stored": dueAt !== null });
	121	if (dueAt === null) return false;
	122	yield* scheduleBy(dueAt);
	123	return true;
	124	}).pipe(withInboxSpan("relay.inbox.hold", inboxId)),
	125	wake: (baseUrl: string) =>
	126	Effect.gen(function* () {
	127	const pending = yield* run(HookInboxStore.wake(baseUrl));
	128	yield* Effect.annotateCurrentSpan({ "relay.inbox.pending": pending });
	129	if (pending) yield* state.storage.setAlarm(yield* Clock.currentTimeMillis);
	130	return pending;
	131	}).pipe(withInboxSpan("relay.inbox.wake", inboxId)),
	132	clear: () =>
	133	Effect.gen(function* () {
	134	yield* run(HookInboxStore.clear);
	135	yield* state.storage.deleteAlarm();
	136	}).pipe(withInboxSpan("relay.inbox.clear", inboxId)),
	137	alarm: () =>
	138	run(HookInboxStore.deliverDue(deliver)).pipe(
	139	// A wake during this run may already have asked for an earlier
	140	// run; a backoff must not push it out.
	141	Effect.flatMap((nextAt) => (nextAt === null ? Effect.void : scheduleBy(nextAt))),
	142	Effect.catchCause((cause) =>
	143	Effect.logWarning("Held webhook delivery run failed", { cause }).pipe(
	144	Effect.andThen(Effect.annotateCurrentSpan({ "relay.inbox.run_result": "failed" })),
	145	Effect.andThen(Clock.currentTimeMillis),
	146	Effect.flatMap((now) => scheduleBy(now + RUN_FAILURE_RETRY_MS)),
	147	),
	148	),
	149	withInboxSpan("relay.inbox.deliver", inboxId),
	150	),
	151	};
	152	});
	153	}),
	154	);

infra/relay/src/hooks/HookInboxStore.test.ts created+260

...	...	@@ -0,0 +1,260 @@
	1	import { describe, expect, it } from "@effect/vitest";
	2	import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
	3	import * as Clock from "effect/Clock";
	4	import * as DateTime from "effect/DateTime";
	5	import * as Duration from "effect/Duration";
	6	import * as Effect from "effect/Effect";
	7	import * as SqlClient from "effect/sql/SqlClient";
	8	import * as TestClock from "effect/testing/TestClock";
	9
	10	import * as HookInboxStore from "./HookInboxStore.ts";
	11
	12	const BASE_URL = "https://env.example.test/";
	13
	14	const hook = (id: string, overrides: Partial<HookInboxStore.HeldHook> = {}) =>
	15	Effect.map(Clock.currentTimeMillis, (now): HookInboxStore.HeldHook => ({
	16	id,
	17	receivedAt: DateTime.formatIso(DateTime.makeUnsafe(now)),
	18	method: "POST",
	19	rawHookId: "hook-1",
	20	hookKey: "hook-1",
	21	rawToken: "tok%2Fen",
	22	query: "a=1",
	23	headers: { "content-type": "application/json", "x-sig": "s" },
	24	body: new Uint8Array([0, 255, 10]),
	25	...overrides,
	26	}));
	27
	28	/** Delivers with `outcome` per request and records what the environment was sent. */
	29	const deliverer = (outcome: (hook: HookInboxStore.HeldHook) => HookInboxStore.DeliveryOutcome) => {
	30	const sent: Array<{ readonly baseUrl: string; readonly hook: HookInboxStore.HeldHook }> = [];
	31	const send = (baseUrl: string, held: HookInboxStore.HeldHook) =>
	32	Effect.sync(() => {
	33	sent.push({ baseUrl, hook: held });
	34	return outcome(held);
	35	});
	36	return { sent, send };
	37	};
	38
	39	const withInbox = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
	40	HookInboxStore.migrate.pipe(
	41	Effect.andThen(effect),
	42	Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" })),
	43	);
	44
	45	describe("HookInboxStore", () => {
	46	it.effect("delivers held requests oldest first, byte for byte", () =>
	47	withInbox(
	48	Effect.gen(function* () {
	49	const first = yield* hook("first");
	50	expect(yield* HookInboxStore.hold(first, BASE_URL)).not.toBeNull();
	51	yield* HookInboxStore.hold(yield* hook("second"), BASE_URL);
	52	const { sent, send } = deliverer(() => "delivered");
	53	expect(yield* HookInboxStore.deliverDue(send)).toBeNull();
	54	expect(sent.map((entry) => entry.hook.id)).toEqual(["first", "second"]);
	55	expect(sent[0]).toEqual({ baseUrl: BASE_URL, hook: first });
	56	// Delivered requests are gone.
	57	expect(yield* HookInboxStore.deliverDue(send)).toBeNull();
	58	expect(sent).toHaveLength(2);
	59	}),
	60	),
	61	);
	62
	63	it.effect("keeps a request the environment did not take and backs off", () =>
	64	withInbox(
	65	Effect.gen(function* () {
	66	yield* HookInboxStore.hold(yield* hook("first"), BASE_URL);
	67	yield* HookInboxStore.hold(yield* hook("second"), BASE_URL);
	68	const offline = deliverer(() => "unreachable");
	69	const now = yield* Clock.currentTimeMillis;
	70	// Stops at the first failure, so order is kept.
	71	expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
	72	expect(offline.sent.map((entry) => entry.hook.id)).toEqual(["first"]);
	73	expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
	74
	75	const online = deliverer(() => "delivered");
	76	expect(yield* HookInboxStore.deliverDue(online.send)).toBeNull();
	77	expect(online.sent.map((entry) => entry.hook.id)).toEqual(["first", "second"]);
	78	}),
	79	),
	80	);
	81
	82	it.effect("lets other hooks through while one hook's environment is busy", () =>
	83	withInbox(
	84	Effect.gen(function* () {
	85	const stuck = { rawHookId: "stuck", hookKey: "stuck" };
	86	yield* HookInboxStore.hold(yield* hook("stuck-1", stuck), BASE_URL);
	87	yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL);
	88	yield* HookInboxStore.hold(yield* hook("stuck-2", stuck), BASE_URL);
	89	const busy = deliverer((held) => (held.hookKey === "stuck" ? "busy" : "delivered"));
	90	const now = yield* Clock.currentTimeMillis;
	91	// The other hook is delivered; the busy hook keeps its order and runs again soon.
	92	expect(yield* HookInboxStore.deliverDue(busy.send)).toBe(now);
	93	expect(busy.sent.map((entry) => entry.hook.id)).toEqual(["stuck-1", "other-1"]);
	94	// Only the busy hook is left, so the next run waits rather than spinning.
	95	expect(yield* HookInboxStore.deliverDue(busy.send)).toBe(now + 30_000);
	96
	97	const drained = deliverer(() => "delivered");
	98	expect(yield* HookInboxStore.deliverDue(drained.send)).toBeNull();
	99	expect(drained.sent.map((entry) => entry.hook.id)).toEqual(["stuck-1", "stuck-2"]);
	100	}),
	101	),
	102	);
	103
	104	it.effect("reaches a hook queued behind busy hooks' full backlogs in the same run", () =>
	105	withInbox(
	106	Effect.gen(function* () {
	107	for (const name of ["stuck-a", "stuck-b"]) {
	108	const stuck = { rawHookId: name, hookKey: name };
	109	for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) {
	110	yield* HookInboxStore.hold(yield* hook(`${name}-${index}`, stuck), BASE_URL);
	111	}
	112	}
	113	yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL);
	114	const busy = deliverer((held) => (held.hookKey.startsWith("stuck") ? "busy" : "delivered"));
	115	yield* HookInboxStore.deliverDue(busy.send);
	116	expect(busy.sent.map((entry) => entry.hook.id)).toEqual([
	117	"stuck-a-0",
	118	"stuck-b-0",
	119	"other-1",
	120	]);
	121	}),
	122	),
	123	);
	124
	125	it.effect("drops a held request it cannot read instead of stalling on it", () =>
	126	withInbox(
	127	Effect.gen(function* () {
	128	yield* HookInboxStore.hold(yield* hook("broken"), BASE_URL);
	129	yield* HookInboxStore.hold(yield* hook("fine"), BASE_URL);
	130	const sql = yield* SqlClient.SqlClient;
	131	yield* sql`UPDATE held_hooks SET headers = 'not json' WHERE id = 'broken'`;
	132	const { sent, send } = deliverer(() => "delivered");
	133	expect(yield* HookInboxStore.deliverDue(send)).toBeNull();
	134	expect(sent.map((entry) => entry.hook.id)).toEqual(["fine"]);
	135	}),
	136	),
	137	);
	138
	139	it.effect("retries every 10 s for 3 minutes, then backs off to 10 minutes", () =>
	140	Effect.gen(function* () {
	141	const delays = yield* Effect.forEach(
	142	[1, 18, 19, 20, 21, 23, 24, 40],
	143	HookInboxStore.retryDelayMs,
	144	);
	145	expect(delays).toEqual([10_000, 10_000, 30_000, 60_000, 120_000, 480_000, 600_000, 600_000]);
	146	}),
	147	);
	148
	149	it.effect("waking resets the backoff and reports whether anything waits", () =>
	150	withInbox(
	151	Effect.gen(function* () {
	152	expect(yield* HookInboxStore.wake(BASE_URL)).toBe(false);
	153	yield* HookInboxStore.hold(yield* hook("first"), "https://old.example.test/");
	154	const offline = deliverer(() => "unreachable");
	155	yield* HookInboxStore.deliverDue(offline.send);
	156	yield* HookInboxStore.deliverDue(offline.send);
	157
	158	// Past the fast phase: the environment was away a while.
	159	for (let failure = 0; failure < 20; failure++) {
	160	yield* HookInboxStore.deliverDue(offline.send);
	161	}
	162	const now = yield* Clock.currentTimeMillis;
	163	expect(yield* HookInboxStore.deliverDue(offline.send)).toBeGreaterThan(now + 60_000);
	164
	165	// A wake means its tunnel just connected, which Cloudflare may not
	166	// route to for a few minutes: back to retrying every 10 s, sent to
	167	// where the environment is now.
	168	expect(yield* HookInboxStore.wake(BASE_URL)).toBe(true);
	169	expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
	170	expect(offline.sent.at(-1)?.baseUrl).toBe(BASE_URL);
	171	}),
	172	),
	173	);
	174
	175	it.effect("starts the schedule over once the inbox drains", () =>
	176	withInbox(
	177	Effect.gen(function* () {
	178	yield* HookInboxStore.hold(yield* hook("old"), BASE_URL);
	179	const offline = deliverer(() => "unreachable");
	180	for (let failure = 0; failure < 30; failure++) {
	181	yield* HookInboxStore.deliverDue(offline.send);
	182	}
	183	// Nothing got through before the request expired.
	184	yield* TestClock.adjust(Duration.hours(25));
	185	expect(yield* HookInboxStore.deliverDue(offline.send)).toBeNull();
	186
	187	const now = yield* Clock.currentTimeMillis;
	188	expect(yield* HookInboxStore.hold(yield* hook("new"), BASE_URL)).toBe(now + 10_000);
	189	}),
	190	),
	191	);
	192
	193	it.effect("drops requests older than 24 hours", () =>
	194	withInbox(
	195	Effect.gen(function* () {
	196	yield* HookInboxStore.hold(yield* hook("old"), BASE_URL);
	197	yield* TestClock.adjust(Duration.hours(1));
	198	yield* HookInboxStore.hold(yield* hook("new"), BASE_URL);
	199	yield* TestClock.adjust(Duration.minutes(23 * 60 + 1));
	200	const { sent, send } = deliverer(() => "delivered");
	201	yield* HookInboxStore.deliverDue(send);
	202	expect(sent.map((entry) => entry.hook.id)).toEqual(["new"]);
	203	}),
	204	),
	205	);
	206
	207	it.effect("refuses requests past the per-hook cap", () =>
	208	withInbox(
	209	Effect.gen(function* () {
	210	for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) {
	211	expect(yield* HookInboxStore.hold(yield* hook(`a-${index}`), BASE_URL)).not.toBeNull();
	212	}
	213	expect(yield* HookInboxStore.hold(yield* hook("one-too-many"), BASE_URL)).toBeNull();
	214	// Another hook still has room.
	215	const other = yield* hook("other", { rawHookId: "hook-2", hookKey: "hook-2" });
	216	expect(yield* HookInboxStore.hold(other, BASE_URL)).not.toBeNull();
	217	}),
	218	),
	219	);
	220
	221	it.effect("refuses a request that would pass the byte cap", () =>
	222	withInbox(
	223	Effect.gen(function* () {
	224	const big = new Uint8Array(HookInboxStore.HOOK_INBOX_MAX_BYTES - 10);
	225	expect(
	226	yield* HookInboxStore.hold(yield* hook("big", { body: big }), BASE_URL),
	227	).not.toBeNull();
	228	const small = new Uint8Array(11);
	229	expect(
	230	yield* HookInboxStore.hold(
	231	yield* hook("small", { rawHookId: "x", hookKey: "x", body: small }),
	232	BASE_URL,
	233	),
	234	).toBeNull();
	235	}),
	236	),
	237	);
	238
	239	it.effect("stores a request id once", () =>
	240	withInbox(
	241	Effect.gen(function* () {
	242	const first = yield* hook("same");
	243	expect(yield* HookInboxStore.hold(first, BASE_URL)).not.toBeNull();
	244	expect(yield* HookInboxStore.hold(first, BASE_URL)).toBeNull();
	245	}),
	246	),
	247	);
	248
	249	it.effect("clear drops everything held", () =>
	250	withInbox(
	251	Effect.gen(function* () {
	252	yield* HookInboxStore.hold(yield* hook("first"), BASE_URL);
	253	yield* HookInboxStore.clear;
	254	const { sent, send } = deliverer(() => "delivered");
	255	expect(yield* HookInboxStore.deliverDue(send)).toBeNull();
	256	expect(sent).toHaveLength(0);
	257	}),
	258	),
	259	);
	260	});

infra/relay/src/hooks/HookInboxStore.ts created+380

...	...	@@ -0,0 +1,380 @@
	1	import * as Clock from "effect/Clock";
	2	import * as DateTime from "effect/DateTime";
	3	import * as Duration from "effect/Duration";
	4	import * as Effect from "effect/Effect";
	5	import * as Option from "effect/Option";
	6	import * as Pull from "effect/Pull";
	7	import * as Schedule from "effect/Schedule";
	8	import * as Schema from "effect/Schema";
	9	import * as SqlClient from "effect/sql/SqlClient";
	10
	11	/**
	12	* The storage and delivery rules of one environment's held webhook requests.
	13	* Runs inside the environment's `HookInboxObject` Durable Object against its SQLite
	14	* storage; written against `SqlClient` so tests can run it on any SQLite.
	15	*/
	16
	17	/** How long a held request waits for its environment. */
	18	const HOOK_INBOX_TTL_MS = 24 * 60 * 60 * 1000;
	19	/** Requests one environment may have waiting at once. */
	20	const HOOK_INBOX_MAX_REQUESTS = 1_000;
	21	/** Body bytes one environment may have waiting at once. */
	22	export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576;
	23	/**
	24	* Requests one hook may have waiting at once. The relay cannot check tokens,
	25	* so this keeps junk sent to one hook id from crowding out the others.
	26	*/
	27	export const HOOK_INBOX_MAX_PER_HOOK = 100;
	28	/** Requests pushed per alarm run; the next run starts right away while more wait. */
	29	const DELIVERIES_PER_RUN = 20;
	30	/** Requests read at a time; a run keeps reading past busy hooks' backlogs. */
	31	const ROWS_READ_PER_PAGE = 50;
	32	/** Wait before trying a hook whose environment answered busy again. */
	33	const BUSY_RETRY_MS = 30_000;
	34	const MAX_RETRY_DELAY = Duration.minutes(10);
	35	/** Past this many failures the schedule is at its cap, so stepping further changes nothing. */
	36	const MAX_COUNTED_FAILURES = 25;
	37
	38	/**
	39	* Delays between delivery attempts while the environment is unreachable,
	40	* indexed by consecutive failures since it was last reached or woke us.
	41	* A wake means its tunnel just connected, but Cloudflare can take a few
	42	* minutes to route the hostname to it, so the first 3 minutes retry every
	43	* 10 s. After that the environment is likely gone again: 30 s, 1 min, 2 min,
	44	* ... up to 10 min.
	45	*/
	46	export const retrySchedule = Schedule.spaced("10 seconds").pipe(
	47	Schedule.upTo({ times: 18 }),
	48	Schedule.concat(
	49	Schedule.exponential("30 seconds").pipe(
	50	Schedule.modifyDelay(({ duration }) =>
	51	Effect.succeed(Duration.min(Duration.fromInputUnsafe(duration), MAX_RETRY_DELAY)),
	52	),
	53	),
	54	),
	55	);
	56
	57	export interface HeldHook {
	58	readonly id: string;
	59	readonly receivedAt: string;
	60	readonly method: string;
	61	/** Path segments exactly as the sender sent them; the environment decodes them. */
	62	readonly rawHookId: string;
	63	readonly rawToken: string;
	64	/** The decoded hook id, so every spelling of one hook shares its cap. */
	65	readonly hookKey: string;
	66	/** Without the leading `?`. */
	67	readonly query: string;
	68	readonly headers: Readonly<Record<string, string>>;
	69	readonly body: Uint8Array;
	70	}
	71
	72	/**
	73	* `delivered` deletes the request. `unreachable` keeps it and backs off the
	74	* whole inbox, since nothing else will get through either. `busy` keeps it
	75	* and its hook's later requests for the next run, but lets other hooks'
	76	* requests go ahead, so one stuck task cannot hold up the rest.
	77	*/
	78	export type DeliveryOutcome = "delivered" \| "busy" \| "unreachable";
	79
	80	/** The `retrySchedule` delay after `failures` consecutive unreachable attempts. */
	81	export const retryDelayMs = Effect.fn("HookInboxStore.retryDelayMs")(function* (failures: number) {
	82	const step = yield* Schedule.toStep(retrySchedule);
	83	let delay = MAX_RETRY_DELAY;
	84	for (
	85	let attempt = 0;
	86	attempt < Math.min(Math.max(1, failures), MAX_COUNTED_FAILURES);
	87	attempt++
	88	) {
	89	const next = yield* step(0, undefined).pipe(
	90	Effect.map(([, duration]) => Option.some(duration)),
	91	// The schedule never ends; stay at the cap if it ever does.
	92	Pull.catchDone(() => Effect.succeedNone),
	93	);
	94	if (Option.isNone(next)) break;
	95	delay = next.value;
	96	}
	97	return Duration.toMillis(delay);
	98	});
	99
	100	const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String));
	101	const encodeHeaders = Schema.encodeSync(HeadersJson);
	102	const decodeHeaders = Schema.decodeUnknownOption(HeadersJson);
	103
	104	interface HeldHookRow {
	105	readonly seq: number;
	106	readonly id: string;
	107	readonly received_at: string;
	108	readonly method: string;
	109	readonly raw_hook_id: string;
	110	readonly raw_token: string;
	111	readonly hook_key: string;
	112	readonly query: string;
	113	readonly headers: string;
	114	readonly body: Uint8Array;
	115	}
	116
	117	interface TargetRow {
	118	readonly base_url: string;
	119	readonly failures: number;
	120	}
	121
	122	const iso = (epochMillis: number) => DateTime.formatIso(DateTime.makeUnsafe(epochMillis));
	123
	124	export const migrate = Effect.gen(function* () {
	125	const sql = yield* SqlClient.SqlClient;
	126	yield* sql`
	127	CREATE TABLE IF NOT EXISTS held_hooks (
	128	seq INTEGER PRIMARY KEY AUTOINCREMENT,
	129	id TEXT NOT NULL UNIQUE,
	130	received_at TEXT NOT NULL,
	131	method TEXT NOT NULL,
	132	raw_hook_id TEXT NOT NULL,
	133	raw_token TEXT NOT NULL,
	134	hook_key TEXT NOT NULL,
	135	query TEXT NOT NULL,
	136	headers TEXT NOT NULL,
	137	body BLOB NOT NULL
	138	)
	139	`;
	140	yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (hook_key)`;
	141	// Where to push, and how many attempts in a row have failed. One row.
	142	yield* sql`
	143	CREATE TABLE IF NOT EXISTS held_hooks_target (
	144	id INTEGER PRIMARY KEY CHECK (id = 1),
	145	base_url TEXT NOT NULL,
	146	failures INTEGER NOT NULL DEFAULT 0
	147	)
	148	`;
	149	});
	150
	151	const setTarget = (baseUrl: string, options: { readonly resetFailures: boolean }) =>
	152	Effect.gen(function* () {
	153	const sql = yield* SqlClient.SqlClient;
	154	yield* options.resetFailures
	155	? sql`
	156	INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl})
	157	ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url, failures = 0
	158	`
	159	: sql`
	160	INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl})
	161	ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url
	162	`;
	163	});
	164
	165	const readTarget = Effect.gen(function* () {
	166	const sql = yield* SqlClient.SqlClient;
	167	const rows = yield* sql<TargetRow>`SELECT base_url, failures FROM held_hooks_target WHERE id = 1`;
	168	return rows[0] ?? null;
	169	});
	170
	171	const resetFailures = Effect.gen(function* () {
	172	const sql = yield* SqlClient.SqlClient;
	173	yield* sql`UPDATE held_hooks_target SET failures = 0 WHERE id = 1`;
	174	});
	175
	176	/**
	177	* Counts one more unreachable attempt. Incremented in place, so a wake that
	178	* reset the count while this run's request was in flight is not overwritten.
	179	*/
	180	const countFailure = Effect.gen(function* () {
	181	const sql = yield* SqlClient.SqlClient;
	182	const rows = yield* sql<{ readonly failures: number }>`
	183	UPDATE held_hooks_target SET failures = min(failures + 1, ${MAX_COUNTED_FAILURES})
	184	WHERE id = 1 RETURNING failures
	185	`;
	186	return rows[0]?.failures ?? 1;
	187	});
	188
	189	const hasPending = Effect.gen(function* () {
	190	const sql = yield* SqlClient.SqlClient;
	191	const rows = yield* sql<{ readonly id: string }>`SELECT id FROM held_hooks LIMIT 1`;
	192	return rows.length > 0;
	193	});
	194
	195	/**
	196	* Stores a request for later delivery to `baseUrl`. Returns the time the
	197	* first attempt is due, or null when the inbox is full and nothing was stored.
	198	*/
	199	export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook, baseUrl: string) {
	200	const sql = yield* SqlClient.SqlClient;
	201	// One statement, so the caps hold however many requests arrive at once.
	202	const inserted = yield* sql<{ readonly id: string }>`
	203	INSERT INTO held_hooks
	204	(id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body)
	205	SELECT ${hook.id}, ${hook.receivedAt}, ${hook.method}, ${hook.rawHookId}, ${hook.rawToken},
	206	${hook.hookKey}, ${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body}
	207	WHERE (SELECT count(*) FROM held_hooks) < ${HOOK_INBOX_MAX_REQUESTS}
	208	AND (SELECT count(*) FROM held_hooks WHERE hook_key = ${hook.hookKey})
	209	< ${HOOK_INBOX_MAX_PER_HOOK}
	210	AND (SELECT coalesce(sum(length(body)), 0) FROM held_hooks) + ${hook.body.byteLength}
	211	<= ${HOOK_INBOX_MAX_BYTES}
	212	ON CONFLICT (id) DO NOTHING
	213	RETURNING id
	214	`;
	215	if (inserted.length === 0) {
	216	yield* Effect.annotateCurrentSpan({ "relay.inbox.refused": yield* refusalReason(hook) });
	217	return null;
	218	}
	219	yield* setTarget(baseUrl, { resetFailures: false });
	220	const target = yield* readTarget;
	221	const backlog = yield* backlogSize;
	222	yield* Effect.annotateCurrentSpan({
	223	"relay.inbox.held_count": backlog.count,
	224	"relay.inbox.held_bytes": backlog.bytes,
	225	});
	226	return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(target?.failures ?? 0));
	227	});
	228
	229	const backlogSize = Effect.gen(function* () {
	230	const sql = yield* SqlClient.SqlClient;
	231	const rows = yield* sql<{ readonly count: number; readonly bytes: number }>`
	232	SELECT count(*) AS count, coalesce(sum(length(body)), 0) AS bytes FROM held_hooks
	233	`;
	234	return rows[0] ?? { count: 0, bytes: 0 };
	235	});
	236
	237	/** Which cap refused a request, or that it was already held, for traces. */
	238	const refusalReason = (hook: HeldHook) =>
	239	Effect.gen(function* () {
	240	const sql = yield* SqlClient.SqlClient;
	241	const existing = yield* sql<{ readonly id: string }>`
	242	SELECT id FROM held_hooks WHERE id = ${hook.id}
	243	`;
	244	if (existing.length > 0) return "already_held";
	245	const backlog = yield* backlogSize;
	246	if (backlog.count >= HOOK_INBOX_MAX_REQUESTS) return "max_requests";
	247	if (backlog.bytes + hook.body.byteLength > HOOK_INBOX_MAX_BYTES) return "max_bytes";
	248	return "max_per_hook";
	249	});
	250
	251	/**
	252	* The environment is reachable again at `baseUrl`. Returns whether anything
	253	* is waiting, so the caller can deliver right away.
	254	*/
	255	export const wake = Effect.fn("HookInboxStore.wake")(function* (baseUrl: string) {
	256	if (!(yield* hasPending)) return false;
	257	yield* setTarget(baseUrl, { resetFailures: true });
	258	return true;
	259	});
	260
	261	export const clear = Effect.gen(function* () {
	262	const sql = yield* SqlClient.SqlClient;
	263	yield* sql`DELETE FROM held_hooks`;
	264	yield* sql`DELETE FROM held_hooks_target`;
	265	});
	266
	267	/**
	268	* Pushes the oldest held requests to the environment, one at a time, in
	269	* order per hook. Stops at the first sign the environment is unreachable;
	270	* skips past a hook whose environment answered busy. Drops requests older
	271	* than the TTL. Returns when the next run is due, or null when nothing is left.
	272	*/
	273	export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
	274	send: (baseUrl: string, hook: HeldHook) => Effect.Effect<DeliveryOutcome, never, R>,
	275	) {
	276	const sql = yield* SqlClient.SqlClient;
	277	const startedAt = yield* Clock.currentTimeMillis;
	278	const expired = yield* sql<{ readonly id: string }>`
	279	DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)}
	280	RETURNING id
	281	`;
	282	const target = yield* readTarget;
	283	/** Oldest held requests after `afterSeq`, skipping hooks already found busy. */
	284	const readPage = (afterSeq: number, skip: ReadonlySet<string>) =>
	285	sql<HeldHookRow>`
	286	SELECT seq, id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body
	287	FROM held_hooks
	288	WHERE seq > ${afterSeq}
	289	${skip.size === 0 ? sql`` : sql`AND hook_key NOT IN ${sql.in([...skip])}`}
	290	ORDER BY seq LIMIT ${ROWS_READ_PER_PAGE}
	291	`;
	292	let batch = yield* readPage(0, new Set());
	293	if (batch.length === 0 \|\| target === null) {
	294	if (target === null) yield* sql`DELETE FROM held_hooks`;
	295	// Empty, so the next request held starts the schedule from the top.
	296	else yield* resetFailures;
	297	yield* Effect.annotateCurrentSpan({ "relay.inbox.expired": expired.length });
	298	return null;
	299	}
	300
	301	let failures = target.failures;
	302	let sent = 0;
	303	let delivered = 0;
	304	let unreadable = 0;
	305	let longestWaitMs = 0;
	306	const busyHooks = new Set<string>();
	307	/** What this run did, for the alarm's span; never request contents. */
	308	const annotateRun = (result: string) =>
	309	Effect.gen(function* () {
	310	const backlog = yield* backlogSize;
	311	yield* Effect.annotateCurrentSpan({
	312	"relay.inbox.run_result": result,
	313	"relay.inbox.sent": sent,
	314	"relay.inbox.delivered": delivered,
	315	"relay.inbox.busy_hooks": busyHooks.size,
	316	"relay.inbox.expired": expired.length,
	317	"relay.inbox.unreadable": unreadable,
	318	"relay.inbox.consecutive_failures": failures,
	319	"relay.inbox.longest_wait_ms": longestWaitMs,
	320	"relay.inbox.held_count": backlog.count,
	321	"relay.inbox.held_bytes": backlog.bytes,
	322	});
	323	});
	324	// Pages run out only once every non-busy request has been tried, so a
	325	// backlog behind busy hooks never hides another hook's requests.
	326	pages: while (batch.length > 0) {
	327	for (const row of batch) {
	328	if (sent === DELIVERIES_PER_RUN) break pages;
	329	// Later requests to a busy hook wait their turn, so its order is kept.
	330	if (busyHooks.has(row.hook_key)) continue;
	331	const headers = decodeHeaders(row.headers);
	332	if (Option.isNone(headers)) {
	333	// Unreadable: it can never be delivered, and must not block the rest.
	334	yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
	335	unreadable += 1;
	336	continue;
	337	}
	338	sent += 1;
	339	const outcome = yield* send(target.base_url, {
	340	id: row.id,
	341	receivedAt: row.received_at,
	342	method: row.method,
	343	rawHookId: row.raw_hook_id,
	344	rawToken: row.raw_token,
	345	hookKey: row.hook_key,
	346	query: row.query,
	347	headers: headers.value,
	348	body: row.body,
	349	});
	350	if (outcome === "unreachable") {
	351	failures = yield* countFailure;
	352	yield* annotateRun("unreachable");
	353	return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures));
	354	}
	355	if (failures !== 0) {
	356	failures = 0;
	357	yield* resetFailures;
	358	}
	359	if (outcome === "busy") {
	360	busyHooks.add(row.hook_key);
	361	continue;
	362	}
	363	yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
	364	delivered += 1;
	365	const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at));
	366	longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs);
	367	}
	368	batch = yield* readPage(batch.at(-1)!.seq, busyHooks);
	369	}
	370	if (!(yield* hasPending)) {
	371	yield* resetFailures;
	372	yield* annotateRun("drained");
	373	return null;
	374	}
	375	// Everything left this run was busy: give those tasks a moment to drain.
	376	const now = yield* Clock.currentTimeMillis;
	377	const idle = delivered === 0 && busyHooks.size > 0;
	378	yield* annotateRun(idle ? "busy" : "more_pending");
	379	return idle ? now + BUSY_RETRY_MS : now;
	380	});

infra/relay/src/hooks/upstream.ts created+125

...	...	@@ -0,0 +1,125 @@
	1	import * as Duration from "effect/Duration";
	2	import * as Effect from "effect/Effect";
	3	import * as Option from "effect/Option";
	4	import * as Schema from "effect/Schema";
	5	import * as Stream from "effect/Stream";
	6	import * as HttpClient from "effect/http/HttpClient";
	7	import * as HttpClientRequest from "effect/http/HttpClientRequest";
	8	import * as HttpMethod from "effect/http/HttpMethod";
	9	import * as HttpTraceContext from "effect/http/HttpTraceContext";
	10	import type * as HttpClientResponse from "effect/http/HttpClientResponse";
	11
	12	import { withoutRedirects } from "../environments/EnvironmentConnector.ts";
	13
	14	/** Set by the relay on every forward; the environment uses it as the delivery id. */
	15	const RELAY_DELIVERY_ID_HEADER = "x-t3-relay-delivery-id";
	16	/** When the relay received the request; the environment trusts it only with the delivery id. */
	17	const RELAY_RECEIVED_AT_HEADER = "x-t3-relay-received-at";
	18	/** The environment answers with a small JSON status; anything past this is cut off. */
	19	const MAX_RESPONSE_BYTES = 64 * 1024;
	20	export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000;
	21	// Cloudflare answers 530 when the tunnel for a hostname has no connected origin.
	22	export const TUNNEL_OFFLINE_STATUS = 530;
	23
	24	/** A webhook request as the relay sends it on to the environment. */
	25	export interface UpstreamHook {
	26	readonly id: string;
	27	readonly receivedAt: string;
	28	readonly method: string;
	29	/** Path segments exactly as the sender sent them; the environment decodes them. */
	30	readonly rawHookId: string;
	31	readonly rawToken: string;
	32	/** Without the leading `?`. */
	33	readonly query: string;
	34	readonly headers: Readonly<Record<string, string>>;
	35	readonly body: Uint8Array;
	36	}
	37
	38	export interface UpstreamResponse {
	39	readonly status: number;
	40	readonly contentType: string \| undefined;
	41	/** What the environment did with the request (`x-t3-hook-outcome`). */
	42	readonly outcome: string \| undefined;
	43	readonly body: Uint8Array;
	44	}
	45
	46	/** Outcome names are short identifiers; anything else is not recorded. */
	47	const OUTCOME_PATTERN = /^[a-z_]{1,32}$/;
	48
	49	class ResponseTooLarge extends Schema.TaggedError<ResponseTooLarge>()("ResponseTooLarge", {}) {}
	50
	51	/** Reads a response body, failing once it passes the cap rather than buffering it all. */
	52	const readCapped = (response: HttpClientResponse.HttpClientResponse) =>
	53	Effect.suspend(() => {
	54	const chunks: Array<Uint8Array> = [];
	55	let total = 0;
	56	return response.stream.pipe(
	57	Stream.runForEach((chunk) => {
	58	total += chunk.length;
	59	if (total > MAX_RESPONSE_BYTES) return Effect.fail(new ResponseTooLarge());
	60	chunks.push(chunk);
	61	return Effect.void;
	62	}),
	63	Effect.map(() => {
	64	const body = new Uint8Array(total);
	65	let offset = 0;
	66	for (const chunk of chunks) {
	67	body.set(chunk, offset);
	68	offset += chunk.length;
	69	}
	70	return body;
	71	}),
	72	// A response without a body, such as a redirect, has no stream at all.
	73	Effect.catchIf(
	74	(error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError",
	75	() => Effect.succeed(new Uint8Array(0)),
	76	),
	77	);
	78	});
	79
	80	/**
	81	* Sends a webhook request through the environment's tunnel. Fails when the
	82	* environment cannot be reached, and succeeds with None on timeout.
	83	*/
	84	export const sendUpstream = (baseUrl: string, hook: UpstreamHook) =>
	85	Effect.gen(function* () {
	86	const httpClient = yield* HttpClient.HttpClient;
	87	const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
	88	// The environment's span joins this trace. Set by hand: the client span
	89	// that would propagate it is off, because it records the token in url.full.
	90	const parent = yield* Effect.currentSpan.pipe(Effect.option);
	91	// `hook.headers` carries the signed delivery proof, set once when the
	92	// relay received the request, so a held request sends the same proof.
	93	const headers: Record<string, string> = {
	94	...hook.headers,
	95	...(Option.isSome(parent) ? HttpTraceContext.toHeaders(parent.value) : {}),
	96	[RELAY_DELIVERY_ID_HEADER]: hook.id,
	97	[RELAY_RECEIVED_AT_HEADER]: hook.receivedAt,
	98	};
	99	let request = HttpClientRequest.make(hook.method as "GET" \| "POST" \| "PUT" \| "PATCH")(
	100	`${base}api/hooks/${hook.rawHookId}/${hook.rawToken}${hook.query ? `?${hook.query}` : ""}`,
	101	{ headers },
	102	);
	103	if (HttpMethod.hasBody(request.method)) {
	104	request = HttpClientRequest.bodyUint8Array(request, hook.body, headers["content-type"]);
	105	}
	106	return yield* httpClient.execute(request).pipe(
	107	Effect.flatMap((response) =>
	108	readCapped(response).pipe(
	109	Effect.map((body): UpstreamResponse => {
	110	const outcome = response.headers["x-t3-hook-outcome"];
	111	return {
	112	status: response.status,
	113	contentType: response.headers["content-type"],
	114	outcome: outcome !== undefined && OUTCOME_PATTERN.test(outcome) ? outcome : undefined,
	115	body,
	116	};
	117	}),
	118	),
	119	),
	120	withoutRedirects,
	121	// The client span would record url.full, which carries the token.
	122	Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
	123	Effect.timeoutOption(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)),
	124	);
	125	});

infra/relay/src/http/Api.test.ts+63-2

...	...	@@ -61,6 +61,7 @@ import {
61	61	import * as RelayConfiguration from "../Config.ts";
62	62	import * as RelayDb from "../db.ts";
63	63	import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";
	64	import * as HookInbox from "../hooks/HookInbox.ts";
64	65	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";
65	66	import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts";
66	67	import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts";
...	...	@@ -124,6 +125,7 @@ describe("device listing compatibility", () => {
124	125	Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
125	126	Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),
126	127	Layer.mock(RelayDb.RelayTransactions, {}),
	128	Layer.mock(HookInbox.HookInbox, {}),
127	129	),
128	130	),
129	131	Layer.provide(
...	...	@@ -317,8 +319,12 @@ function relayUnlinkTestLayer(input?: {
317	319	readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"];
318	320	readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"];
319	321	readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"];
	322	readonly clearInbox?: HookInbox.HookInbox["Service"]["clear"];
320	323	}) {
321	324	return Layer.mergeAll(
	325	Layer.mock(HookInbox.HookInbox, {
	326	clear: input?.clearInbox ?? (() => Effect.void),
	327	}),
322	328	Layer.succeed(
323	329	RelayDb.RelayTransactions,
324	330	RelayDb.RelayTransactions.of({
...	...	@@ -332,7 +338,8 @@ function relayUnlinkTestLayer(input?: {
332	338	listDeliveryUsersForEnvironment: () => Effect.die("unused listDeliveryUsersForEnvironment"),
333	339	listForUser: () => Effect.die("unused listForUser"),
334	340	getForUser: input?.getForUser ?? (() => Effect.succeed(null)),
335		findActiveManagedForEnvironment: () => Effect.succeed([]),
	341	findActiveManagedForEnvironment: () => Effect.die("unused findActiveManagedForEnvironment"),
	342	setHoldWebhooksWhileOffline: () => Effect.die("unused setHoldWebhooksWhileOffline"),
336	343	revokeForUser: input?.revokeForUser ?? (() => Effect.succeed(false)),
337	344	}),
338	345	),
...	...	@@ -899,6 +906,53 @@ describe("relay environment unlink", () => {
899	906	);
900	907	});
901	908
	909	it.effect("drops the unlinked endpoint's held webhooks, even if clearing fails", () => {
	910	const cleared: Array<string> = [];
	911	const endpointKey = "0123456789abcdef";
	912	const unlink = (clearFails: boolean) =>
	913	unlinkEnvironmentRecord({
	914	userId: "user-1",
	915	environmentId: "environment-1",
	916	managedEndpointNamespace: "dev",
	917	}).pipe(
	918	Effect.provide(
	919	relayUnlinkTestLayer({
	920	getForUser: () => Effect.succeed(linkedEnvironmentRecord),
	921	revokeForUser: () => Effect.succeed(true),
	922	prepareDeprovision: () =>
	923	Effect.succeed({
	924	userId: "user-1",
	925	environmentId: "environment-1",
	926	hostname: "dev-0123456789abcdef.example.test",
	927	tunnelId: "tunnel-1",
	928	tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`,
	929	dnsRecordId: "dns-1",
	930	readyAt: "2026-07-28T00:00:00.000Z",
	931	origin: null,
	932	updatedAt: "2026-07-28T00:00:00.000Z",
	933	generation: 1,
	934	}),
	935	clearInbox: (input) =>
	936	clearFails
	937	? Effect.fail(
	938	new HookInbox.HookInboxError({
	939	operation: "clear",
	940	endpointKey: input.endpointKey,
	941	cause: new Error("unavailable"),
	942	}),
	943	)
	944	: Effect.sync(() => void cleared.push(input.endpointKey)),
	945	}),
	946	),
	947	);
	948	return Effect.gen(function* () {
	949	expect(yield* unlink(false)).toBe(true);
	950	expect(cleared).toEqual([endpointKey]);
	951	// The link is already revoked; a failed clear must not fail the unlink.
	952	expect(yield* unlink(true)).toBe(true);
	953	});
	954	});
	955
902	956	it.effect("commits database revocation before deprovisioning the managed endpoint", () => {
903	957	const calls: Array<string> = [];
904	958	const deprovisionTarget = {
...	...	@@ -1215,7 +1269,14 @@ describe("relay routing fallback", () => {
1215	1269	Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),
1216	1270	),
1217	1271	),
1218		Layer.provide([publisher, signatures]),
	1272	Layer.provide([
	1273	publisher,
	1274	signatures,
	1275	Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
	1276	Layer.mock(HookInbox.HookInbox, {}),
	1277	Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {}),
	1278	Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings),
	1279	]),
1219	1280	),
1220	1281	),
1221	1282	Layer.provide(auth),

infra/relay/src/http/Api.ts+136-9

...	...	@@ -66,6 +66,8 @@ import * as DpopProofs from "../auth/DpopProofs.ts";
66	66	import * as RelayTokens from "../auth/RelayTokens.ts";
67	67	import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";
68	68	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";
	69	import * as HookForwarder from "../hooks/HookForwarder.ts";
	70	import * as HookInbox from "../hooks/HookInbox.ts";
69	71	import * as LiveActivities from "../agentActivity/LiveActivities.ts";
70	72	import * as RelayConfiguration from "../Config.ts";
71	73	import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts";
...	...	@@ -188,6 +190,12 @@ export const relayDocsRedirectRoute = HttpRouter.add(
188	190	// contains the exact child span that stalled, and the response still carries
189	191	// the traceparent back to the client.
190	192	export const RELAY_REQUEST_DEADLINE_MS = 9_000;
	193	/**
	194	* Webhook forwarding reads a body of up to 1 MiB, waits up to the upstream
	195	* timeout, and may then hold the request, so it needs more room than an API
	196	* call; cutting it off before the hold would drop a request it should keep.
	197	*/
	198	export const RELAY_HOOK_REQUEST_DEADLINE_MS = 25_000;
191	199
192	200	const relayRequestDeadline = <E, R>(
193	201	httpEffect: Effect.Effect<
...	...	@@ -195,9 +203,10 @@ const relayRequestDeadline = <E, R>(
195	203	E,
196	204	HttpServerRequest.HttpServerRequest \| R
197	205	>,
	206	deadlineMs = RELAY_REQUEST_DEADLINE_MS,
198	207	) =>
199	208	httpEffect.pipe(
200		Effect.timeoutOption(Duration.millis(RELAY_REQUEST_DEADLINE_MS)),
	209	Effect.timeoutOption(Duration.millis(deadlineMs)),
201	210	Effect.flatMap(
202	211	Option.match({
203	212	onNone: () =>
...	...	@@ -206,7 +215,7 @@ const relayRequestDeadline = <E, R>(
206	215	yield* Effect.logError("relay request exceeded deadline", {
207	216	"http.method": request.method,
208	217	"http.url": request.url,
209		"relay.request.deadline_ms": RELAY_REQUEST_DEADLINE_MS,
	218	"relay.request.deadline_ms": deadlineMs,
210	219	});
211	220	yield* Effect.annotateCurrentSpan({
212	221	"relay.request.deadline_exceeded": true,
...	...	@@ -221,6 +230,9 @@ const relayRequestDeadline = <E, R>(
221	230	),
222	231	);
223	232
	233	/** Trace context headers in every format the tracer reads (W3C and B3). */
	234	const SENDER_TRACE_HEADER = /^(traceparent\|tracestate\|b3\|x-b3-.*)$/i;
	235
224	236	export const traceRelayHttpRequest = <E, R>(
225	237	httpEffect: Effect.Effect<
226	238	HttpServerResponse.HttpServerResponse,
...	...	@@ -238,18 +250,29 @@ export const traceRelayHttpRequest = <E, R>(
238	250	return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow));
239	251	}
240	252	// Hook URLs carry a secret token: the tracer and deadline log see a redacted
241		// request, while the route itself still receives the original.
242		const redacted = request.modify({ url: redactRelayHookUrl(request.url) });
	253	// request, while the route itself still receives the original. A webhook
	254	// sender's trace context is dropped, so it cannot pick the trace our relay
	255	// and environment spans land in.
	256	const redacted = request.modify({
	257	url: redactRelayHookUrl(request.url),
	258	headers: Headers.removeMany(
	259	request.headers,
	260	Object.keys(request.headers).filter((name) => SENDER_TRACE_HEADER.test(name)),
	261	),
	262	});
243	263	return yield* HttpMiddleware.tracer(
244	264	appendRelayTraceContextResponseHeader.pipe(
245	265	Effect.andThen(
246	266	relayRequestDeadline(
247	267	httpEffect.pipe(Effect.provideService(HttpServerRequest.HttpServerRequest, request)),
	268	RELAY_HOOK_REQUEST_DEADLINE_MS,
248	269	),
249	270	),
250	271	),
251	272	).pipe(
252	273	Effect.provideService(HttpServerRequest.HttpServerRequest, redacted),
	274	// The worker disables its own span for hook paths; this one is ours.
	275	Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
253	276	Effect.ensuring(Effect.yieldNow),
254	277	);
255	278	});
...	...	@@ -494,7 +517,12 @@ export const revokeEnvironmentLinkRecord = Effect.fn(
494	517	});
495	518
496	519	export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnvironmentRecord")(
497		function* (input: { readonly userId: string; readonly environmentId: string }) {
	520	function* (input: {
	521	readonly userId: string;
	522	readonly environmentId: string;
	523	/** The stage's tunnel-name namespace, to find this link's held webhook requests. */
	524	readonly managedEndpointNamespace?: string \| undefined;
	525	}) {
498	526	const links = yield* EnvironmentLinks.EnvironmentLinks;
499	527	const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
500	528	const deprovisionTarget = yield* managedEndpointProvider.prepareDeprovision({
...	...	@@ -513,7 +541,6 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron
513	541	environmentId: link.environmentId,
514	542	environmentPublicKey: link.environmentPublicKey,
515	543	});
516
517	544	// External teardown cannot share the SQL transaction. Run it only after
518	545	// revocation commits so a database failure leaves a fully usable active
519	546	// link. Still run teardown when the link is already revoked, allowing a
...	...	@@ -523,10 +550,31 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron
523	550	environmentId: input.environmentId,
524	551	target: deprovisionTarget,
525	552	});
	553	// Requests held for this link's endpoint go with it. Best effort: the link
	554	// is already gone, and its inbox drops anything left after its TTL.
	555	const endpointKey =
	556	deprovisionTarget && input.managedEndpointNamespace
	557	? HookForwarder.endpointKeyForTunnelName(
	558	input.managedEndpointNamespace,
	559	deprovisionTarget.tunnelName,
	560	)
	561	: null;
	562	if (endpointKey !== null) {
	563	const inbox = yield* HookInbox.HookInbox;
	564	yield* inbox.clear({ endpointKey }).pipe(
	565	Effect.catch((error) =>
	566	Effect.logWarning("Could not clear held webhook requests", {
	567	environmentId: input.environmentId,
	568	errorTag: error._tag,
	569	}),
	570	),
	571	);
	572	}
526	573	if (!deprovisioned) {
527		const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input);
528		if (retryTarget !== null && (yield* links.getForUser(input)) === null) {
529		yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget });
	574	const key = { userId: input.userId, environmentId: input.environmentId };
	575	const retryTarget = yield* managedEndpointProvider.prepareDeprovision(key);
	576	if (retryTarget !== null && (yield* links.getForUser(key)) === null) {
	577	yield* managedEndpointProvider.deprovision({ ...key, target: retryTarget });
530	578	}
531	579	}
532	580	return unlinked;
...	...	@@ -926,6 +974,7 @@ export const clientApi = HttpApiBuilder.group(
926	974	const unlinked = yield* unlinkEnvironmentRecord({
927	975	userId,
928	976	environmentId: params.environmentId,
	977	managedEndpointNamespace: config.managedEndpointNamespace,
929	978	}).pipe(
930	979	Effect.catchTags({
931	980	SqlError: () => relayInternalErrorResponse("internal_error"),
...	...	@@ -1127,6 +1176,40 @@ export const serverApi = HttpApiBuilder.group(
1127	1176	Effect.fnUntraced(function* (handlers) {
1128	1177	const publisher = yield* AgentActivityPublisher.AgentActivityPublisher;
1129	1178	const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures;
	1179	const links = yield* EnvironmentLinks.EnvironmentLinks;
	1180	const inbox = yield* HookInbox.HookInbox;
	1181	const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
	1182	const settings = yield* RelayConfiguration.RelayConfiguration;
	1183	const requireOwnEnvironment = (environmentId: string) =>
	1184	Effect.gen(function* () {
	1185	const principal = yield* RelayEnvironmentPrincipal;
	1186	if (principal.environmentId !== environmentId) {
	1187	return yield* new HttpApiError.Unauthorized({});
	1188	}
	1189	});
	1190	/**
	1191	* Endpoint keys of the managed links the calling environment key proved.
	1192	* The environment id alone would also match other accounts' links of it.
	1193	*/
	1194	const ownEndpointKeys = (environmentId: string) =>
	1195	Effect.gen(function* () {
	1196	const principal = yield* RelayEnvironmentPrincipal;
	1197	const namespace = settings.managedEndpointNamespace;
	1198	if (!namespace) return [];
	1199	const ownLinks = yield* links.findActiveManagedForEnvironment({
	1200	environmentId,
	1201	environmentPublicKey: principal.environmentPublicKey,
	1202	});
	1203	const keys: Array<string> = [];
	1204	for (const link of ownLinks) {
	1205	const allocation = yield* allocations.get({ userId: link.userId, environmentId });
	1206	const key = allocation
	1207	? HookForwarder.endpointKeyForTunnelName(namespace, allocation.tunnelName)
	1208	: null;
	1209	if (key !== null) keys.push(key);
	1210	}
	1211	return keys;
	1212	});
1130	1213	const activityHandlers = handlers.handle(
1131	1214	"publishAgentActivity",
1132	1215	Effect.fn("relay.api.server.publishAgentActivity")(
...	...	@@ -1336,6 +1419,48 @@ export const serverApi = HttpApiBuilder.group(
1336	1419	}),
1337	1420	mapRelayCommonApiErrors("not_authorized"),
1338	1421	),
	1422	)
	1423	.handle(
	1424	"updateLinkPreferences",
	1425	Effect.fn("relay.api.server.updateLinkPreferences")(function* ({ params, payload }) {
	1426	yield* requireOwnEnvironment(params.environmentId);
	1427	const principal = yield* RelayEnvironmentPrincipal;
	1428	yield* links.setHoldWebhooksWhileOffline({
	1429	environmentId: params.environmentId,
	1430	environmentPublicKey: principal.environmentPublicKey,
	1431	holdWebhooksWhileOffline: payload.holdWebhooksWhileOffline,
	1432	});
	1433	// Opting out also drops what is already held, rather than delivering
	1434	// it later to an environment that said it does not want it.
	1435	if (!payload.holdWebhooksWhileOffline) {
	1436	for (const endpointKey of yield* ownEndpointKeys(params.environmentId)) {
	1437	yield* inbox.clear({ endpointKey });
	1438	}
	1439	}
	1440	return payload;
	1441	}, mapRelayCommonApiErrors("not_authorized")),
	1442	)
	1443	.handle(
	1444	"wakeHeldHooks",
	1445	Effect.fn("relay.api.server.wakeHeldHooks")(function* ({ params }) {
	1446	yield* requireOwnEnvironment(params.environmentId);
	1447	let pending = false;
	1448	for (const endpointKey of yield* ownEndpointKeys(params.environmentId)) {
	1449	const endpoint = yield* HookForwarder.resolveHookEndpoint(endpointKey).pipe(
	1450	Effect.provideService(EnvironmentLinks.EnvironmentLinks, links),
	1451	Effect.provideService(
	1452	ManagedEndpointAllocations.ManagedEndpointAllocations,
	1453	allocations,
	1454	),
	1455	Effect.provideService(RelayConfiguration.RelayConfiguration, settings),
	1456	);
	1457	if (endpoint === null) continue;
	1458	if (yield* inbox.wake({ endpointKey, baseUrl: endpoint.httpBaseUrl })) {
	1459	pending = true;
	1460	}
	1461	}
	1462	return { pending };
	1463	}, mapRelayCommonApiErrors("not_authorized")),
1339	1464	);
1340	1465	}),
1341	1466	);
...	...	@@ -1378,6 +1503,8 @@ const RelayCommonPersistenceError = Schema.Union([
1378	1503	AgentActivityRows.AgentActivityRowListPersistenceError,
1379	1504	LiveActivities.LiveActivityDeliveryMarkPersistenceError,
1380	1505	DeliveryAttempts.DeliveryAttemptRecordPersistenceError,
	1506	EnvironmentLinks.EnvironmentLinkEnvironmentLookupPersistenceError,
	1507	HookInbox.HookInboxError,
1381	1508	]);
1382	1509	type RelayCommonPersistenceError = typeof RelayCommonPersistenceError.Type;
1383	1510	const isRelayCommonPersistenceError = Schema.is(RelayCommonPersistenceError);

infra/relay/src/persistence/schema.ts+2

...	...	@@ -74,6 +74,8 @@ export const relayEnvironmentLinks = pgTable(
74	74	notificationsEnabled: boolean("notifications_enabled").notNull().default(true),
75	75	liveActivitiesEnabled: boolean("live_activities_enabled").notNull().default(true),
76	76	managedTunnelsEnabled: boolean("managed_tunnels_enabled").notNull().default(false),
	77	// Opt-in: hold webhook requests while the environment is offline.
	78	holdWebhooksWhileOffline: boolean("hold_webhooks_while_offline").notNull().default(false),
77	79	createdByDeviceId: varchar("created_by_device_id", { length: 191 }),
78	80	revokedAt: varchar("revoked_at", { length: 64 }),
79	81	createdAt: varchar("created_at", { length: 64 }).notNull(),

infra/relay/src/worker.ts+78-5

...	...	@@ -1,4 +1,5 @@
1	1	import * as Alchemy from "alchemy";
	2	import * as Axiom from "alchemy/Axiom";
2	3	import * as Cloudflare from "alchemy/Cloudflare";
3	4	import * as Drizzle from "alchemy/Drizzle/Postgres";
4	5	import * as Config from "effect/Config";
...	...	@@ -11,6 +12,7 @@ import * as Option from "effect/Option";
11	12	import * as Redacted from "effect/Redacted";
12	13	import * as Stream from "effect/Stream";
13	14	import * as Etag from "effect/http/Etag";
	15	import * as HttpMiddleware from "effect/http/HttpMiddleware";
14	16	import * as HttpPlatform from "effect/http/HttpPlatform";
15	17	import * as HttpRouter from "effect/http/HttpRouter";
16	18	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
...	...	@@ -74,6 +76,8 @@ import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts"
74	76	import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts";
75	77	import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts";
76	78	import * as HookForwarder from "./hooks/HookForwarder.ts";
	79	import * as HookInbox from "./hooks/HookInbox.ts";
	80	import { HookInboxObject, HookInboxObjectLive } from "./hooks/HookInboxObject.ts";
77	81
78	82	const webcryptoLayer = Layer.succeed(
79	83	Crypto.Crypto,
...	...	@@ -190,6 +194,8 @@ export const ApiLive = Api.make(
190	194	const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);
191	195	const managedEndpointZoneName = yield* managedEndpointZone.name;
192	196	const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig;
	197	// Keys are endpoint keys or hashes over them, which already differ per
	198	// stage, so stages sharing an account cannot collide in these namespaces.
193	199	const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", {
194	200	namespaceId: 1001,
195	201	simple: {
...	...	@@ -197,6 +203,14 @@ export const ApiLive = Api.make(
197	203	period: HookForwarder.RELAY_HOOK_RATE_LIMIT.periodSeconds,
198	204	},
199	205	});
	206	const hookEndpointRateLimit = yield* Cloudflare.RateLimit("HOOK_ENDPOINT_RATE_LIMIT", {
	207	namespaceId: 1002,
	208	simple: {
	209	limit: HookForwarder.RELAY_HOOK_ENDPOINT_RATE_LIMIT.limit,
	210	period: HookForwarder.RELAY_HOOK_ENDPOINT_RATE_LIMIT.periodSeconds,
	211	},
	212	});
	213	const hookInboxes = yield* HookInboxObject;
200	214
201	215	//
202	216	// 3. Runtime layers and app construction
...	...	@@ -228,6 +242,31 @@ export const ApiLive = Api.make(
228	242	}).pipe(Effect.map(makeRelayTraceLayer)),
229	243	);
230	244
	245	// Each managed endpoint's held webhook requests live in its own Durable Object.
	246	const inboxCall =
	247	<A>(operation: HookInbox.HookInboxError["operation"], endpointKey: string) =>
	248	(effect: Effect.Effect<A, never, Alchemy.RuntimeContext>) =>
	249	effect.pipe(
	250	Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
	251	Effect.catchCause((cause) =>
	252	Effect.fail(
	253	new HookInbox.HookInboxError({
	254	operation,
	255	endpointKey,
	256	cause: Cause.squash(cause),
	257	}),
	258	),
	259	),
	260	);
	261	const hookInboxLayer = Layer.succeed(HookInbox.HookInbox, {
	262	hold: ({ endpointKey, baseUrl, hook }) =>
	263	hookInboxes.getByName(endpointKey).hold(hook, baseUrl).pipe(inboxCall("hold", endpointKey)),
	264	wake: ({ endpointKey, baseUrl }) =>
	265	hookInboxes.getByName(endpointKey).wake(baseUrl).pipe(inboxCall("wake", endpointKey)),
	266	clear: ({ endpointKey }) =>
	267	hookInboxes.getByName(endpointKey).clear().pipe(inboxCall("clear", endpointKey)),
	268	});
	269
231	270	const runtimeLayer = Layer.empty.pipe(
232	271	Layer.provideMerge(MobileRegistrations.layer),
233	272	Layer.provideMerge(AgentActivityPublisher.layer),
...	...	@@ -269,7 +308,7 @@ export const ApiLive = Api.make(
269	308	Layer.provideMerge(
270	309	ApnsDeliveryQueue.layerCloudflareQueues(apnsDeliveryQueueSender, alchemyRuntimeContext),
271	310	),
272		Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer)),
	311	Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer, hookInboxLayer)),
273	312	Layer.provideMerge(EnvironmentCredentials.layer),
274	313	Layer.provideMerge(
275	314	Layer.mergeAll(
...	...	@@ -291,17 +330,25 @@ export const ApiLive = Api.make(
291	330	);
292	331
293	332	// Fails open: a limiter outage must not drop webhooks the environment would accept.
294		const hookRateLimiterLayer = Layer.succeed(HookForwarder.HookRateLimiter, {
295		allow: (key) =>
296		hookRateLimit.limit({ key }).pipe(
	333	const allowWith =
	334	(limiter: typeof hookRateLimit) =>
	335	(key: string): Effect.Effect<boolean> =>
	336	limiter.limit({ key }).pipe(
297	337	Effect.map((result) => result.success),
298	338	Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
299	339	Effect.catch((error) =>
300	340	Effect.logWarning("Hook rate limiter unavailable", { error: error.message }).pipe(
	341	// Visible on the forward span, so an outage that disables limits shows up.
	342	Effect.andThen(
	343	Effect.annotateCurrentSpan({ "relay.hook.rate_limiter_failed_open": true }),
	344	),
301	345	Effect.as(true),
302	346	),
303	347	),
304		),
	348	);
	349	const hookRateLimiterLayer = Layer.succeed(HookForwarder.HookRateLimiter, {
	350	allowHook: allowWith(hookRateLimit),
	351	allowEndpoint: allowWith(hookEndpointRateLimit),
305	352	});
306	353
307	354	const appLayer = Layer.merge(
...	...	@@ -426,6 +473,32 @@ export const ApiLive = Api.make(
426	473	Layer.provideMerge(Cloudflare.Tunnel.ReadWriteTunnelBinding),
427	474	Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),
428	475	Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),
	476	Layer.provideMerge(HookInboxObjectLive),
	477	// The worker runtime opens its own HTTP span around ours. For webhook
	478	// paths it would record the raw URL, token included, and adopt the
	479	// sender's traceparent, so only our redacted span covers those.
	480	// Registered as telemetry: request-time context is assembled per
	481	// event, and only these layers are built into it.
	482	Layer.provideMerge(
	483	Alchemy.Telemetry.layer(
	484	Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) =>
	485	HookForwarder.isRelayHookPath(request.url),
	486	),
	487	),
	488	),
	489	// Exports spans from events the HTTP tracer does not wrap, notably
	490	// HookInboxObject calls and alarms, to the same Axiom dataset.
	491	Layer.provideMerge(
	492	Layer.unwrap(
	493	Effect.map(RelayObservability, (observability) =>
	494	Axiom.Telemetry({
	495	serviceName: "t3code-relay",
	496	token: observability.workerIngestToken,
	497	traces: observability.traces,
	498	}),
	499	),
	500	),
	501	),
429	502	),
430	503	),
431	504	),

packages/client-runtime/package.json+4

...	...	@@ -35,6 +35,10 @@
35	35	"types": "./src/userMessage.ts",
36	36	"default": "./src/userMessage.ts"
37	37	},
	38	"./scheduled-task-webhook": {
	39	"types": "./src/scheduledTaskWebhook.ts",
	40	"default": "./src/scheduledTaskWebhook.ts"
	41	},
38	42	"./connection": {
39	43	"types": "./src/connection/index.ts",
40	44	"default": "./src/connection/index.ts"

packages/client-runtime/src/scheduledTaskWebhook.test.ts created+13

...	...	@@ -0,0 +1,13 @@
	1	import { describe, expect, it } from "vite-plus/test";
	2
	3	import { parseMaxDeliveryAge } from "./scheduledTaskWebhook.ts";
	4
	5	describe("parseMaxDeliveryAge", () => {
	6	it("treats blank as no limit and rejects values the server would not accept", () => {
	7	expect(parseMaxDeliveryAge("")).toBeNull();
	8	expect(parseMaxDeliveryAge(" 45 ")).toBe(45);
	9	for (const invalid of ["0", "1.5", "-3", "abc", "1441"]) {
	10	expect(parseMaxDeliveryAge(invalid)).toBeUndefined();
	11	}
	12	});
	13	});

packages/client-runtime/src/scheduledTaskWebhook.ts created+13

...	...	@@ -0,0 +1,13 @@
	1	import { MAX_WEBHOOK_DELIVERY_AGE_MINUTES } from "@t3tools/contracts";
	2
	3	/** Prompt a new webhook task starts with: just the body, so request headers stay out unless the user adds them. */
	4	export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{body}}";
	5
	6	/** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */
	7	export function parseMaxDeliveryAge(value: string): number \| null \| undefined {
	8	if (value.trim() === "") return null;
	9	const minutes = Number(value.trim());
	10	return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES
	11	? minutes
	12	: undefined;
	13	}

packages/contracts/src/environmentHttp.ts+5

...	...	@@ -404,11 +404,16 @@ export const EnvironmentCloudLinkStateResult = Schema.Struct({
404	404	// Optional so newer clients tolerate older environment servers.
405	405	managedTunnelActive: Schema.optional(Schema.Boolean),
406	406	publishAgentActivity: Schema.Boolean,
	407	// Opt-in: T3 Connect holds webhook requests while this environment is
	408	// offline. Optional so newer clients tolerate older environment servers.
	409	holdWebhooksWhileOffline: Schema.optional(Schema.Boolean),
407	410	});
408	411	export type EnvironmentCloudLinkStateResult = typeof EnvironmentCloudLinkStateResult.Type;
409	412
410	413	export const EnvironmentCloudPreferencesRequest = Schema.Struct({
411	414	publishAgentActivity: Schema.Boolean,
	415	// Omit to leave the current value unchanged.
	416	holdWebhooksWhileOffline: Schema.optional(Schema.Boolean),
412	417	});
413	418	export type EnvironmentCloudPreferencesRequest = typeof EnvironmentCloudPreferencesRequest.Type;
414	419

packages/contracts/src/relay.ts+50-3

...	...	@@ -864,6 +864,21 @@ export const RelayCloudEnvironmentHealthProofPayload = Schema.Struct({
864	864	export type RelayCloudEnvironmentHealthProofPayload =
865	865	typeof RelayCloudEnvironmentHealthProofPayload.Type;
866	866
	867	/**
	868	* Sent with every webhook the relay forwards, signed with the relay's mint key
	869	* (`x-t3-relay-delivery`). The environment trusts the relay's delivery id,
	870	* receive time and trace context only when this verifies, since its webhook
	871	* URL can also be called directly.
	872	*/
	873	export const RelayHookDeliveryProofPayload = Schema.Struct({
	874	...RelaySignedJwtRegisteredClaims,
	875	environmentId: EnvironmentId,
	876	deliveryId: TrimmedNonEmptyString,
	877	receivedAt: TrimmedNonEmptyString,
	878	hookId: TrimmedNonEmptyString,
	879	});
	880	export type RelayHookDeliveryProofPayload = typeof RelayHookDeliveryProofPayload.Type;
	881
867	882	export const RelayCloudEnvironmentHealthProof = TrimmedNonEmptyString;
868	883	export type RelayCloudEnvironmentHealthProof = typeof RelayCloudEnvironmentHealthProof.Type;
869	884
...	...	@@ -1136,6 +1151,18 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient")
1136	1151	.annotate(OpenApi.Description, "DPoP-authenticated client access to linked environments.")
1137	1152	.middleware(RelayDpopClientAuth);
1138	1153
	1154	export const RelayEnvironmentLinkPreferencesRequest = Schema.Struct({
	1155	holdWebhooksWhileOffline: Schema.Boolean,
	1156	});
	1157	export type RelayEnvironmentLinkPreferencesRequest =
	1158	typeof RelayEnvironmentLinkPreferencesRequest.Type;
	1159
	1160	export const RelayWakeHeldHooksResponse = Schema.Struct({
	1161	/** True when the relay was holding requests and has started delivering them. */
	1162	pending: Schema.Boolean,
	1163	});
	1164	export type RelayWakeHeldHooksResponse = typeof RelayWakeHeldHooksResponse.Type;
	1165
1139	1166	const RelayServerGroup = HttpApiGroup.make("server")
1140	1167	.add(
1141	1168	HttpApiEndpoint.post(
...	...	@@ -1171,6 +1198,24 @@ const RelayServerGroup = HttpApiGroup.make("server")
1171	1198	error: RelayAgentActivityPublishErrors,
1172	1199	},
1173	1200	).annotate(OpenApi.Summary, "Publish agent activity"),
	1201	HttpApiEndpoint.post(
	1202	"updateLinkPreferences",
	1203	"/v1/environments/:environmentId/link-preferences",
	1204	{
	1205	params: Schema.Struct({ environmentId: EnvironmentId }),
	1206	payload: RelayEnvironmentLinkPreferencesRequest,
	1207	success: RelayEnvironmentLinkPreferencesRequest,
	1208	error: RelayAuthAndInternalErrors,
	1209	},
	1210	).annotate(OpenApi.Summary, "Update an environment's link preferences"),
	1211	HttpApiEndpoint.post("wakeHeldHooks", "/v1/environments/:environmentId/hooks/wake", {
	1212	params: Schema.Struct({ environmentId: EnvironmentId }),
	1213	success: RelayWakeHeldHooksResponse,
	1214	error: RelayAuthAndInternalErrors,
	1215	}).annotate(
	1216	OpenApi.Summary,
	1217	"Deliver webhook requests held while the environment was offline now",
	1218	),
1174	1219	)
1175	1220	.annotate(OpenApi.Description, "Environment-authenticated activity publication.")
1176	1221	.middleware(RelayEnvironmentAuth);
...	...	@@ -1179,14 +1224,16 @@ const RelayServerGroup = HttpApiGroup.make("server")
1179	1224	* Public, stateless webhook forwarding to an environment's managed tunnel.
1180	1225	* Unauthenticated: the token in the path is the environment's credential, and
1181	1226	* the relay only routes and forwards. Raw, so the body reaches the
1182		* environment byte for byte and signatures still verify there.
	1227	* environment byte for byte and signatures still verify there. `endpointKey`
	1228	* names one managed endpoint (its tunnel's hash of user and environment), not
	1229	* the environment id, which any account can link.
1183	1230	*/
1184	1231	const RelayHookParams = Schema.Struct({
1185		environmentId: Schema.String,
	1232	endpointKey: Schema.String,
1186	1233	hookId: Schema.String,
1187	1234	token: Schema.String,
1188	1235	});
1189		const RELAY_HOOK_PATH = "/v1/hooks/:environmentId/:hookId/:token";
	1236	const RELAY_HOOK_PATH = "/v1/hooks/:endpointKey/:hookId/:token";
1190	1237	const relayHookEndpoint = { params: RelayHookParams } as const;
1191	1238	const RelayHooksGroup = HttpApiGroup.make("hooks")
1192	1239	.add(

packages/contracts/src/scheduledTask.ts+14

...	...	@@ -73,11 +73,23 @@ export const ScheduledTaskWebhookSignature = Schema.Struct(
73	73	).annotate({ description: "Optional HMAC-SHA256 signature check over the raw request body." });
74	74	export type ScheduledTaskWebhookSignature = typeof ScheduledTaskWebhookSignature.Type;
75	75
	76	/** Matches how long the relay holds a request for an offline environment. */
	77	export const MAX_WEBHOOK_DELIVERY_AGE_MINUTES = 24 * 60;
	78
	79	const WebhookMaxDeliveryAgeMinutes = Schema.Int.check(
	80	Schema.isBetween({ minimum: 1, maximum: MAX_WEBHOOK_DELIVERY_AGE_MINUTES }),
	81	).annotate({
	82	description:
	83	"Skip requests the relay held longer than this many minutes while the environment was offline. Null runs every request.",
	84	});
	85
76	86	const ScheduledTaskWebhookSchedule = Schema.Struct({
77	87	type: Schema.Literal("webhook").annotate({
78	88	description: "Run when the task's webhook URL receives a request.",
79	89	}),
80	90	signature: Schema.NullOr(ScheduledTaskWebhookSignature),
	91	// Optional so rows saved before this setting existed still decode.
	92	maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)),
81	93	}).annotate({
82	94	description:
83	95	"Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.",
...	...	@@ -99,6 +111,7 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({
99	111	).annotate({
100	112	description: "Signature check; omit or null to accept requests by URL token only.",
101	113	}),
	114	maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)),
102	115	}).annotate({
103	116	description:
104	117	"Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.",
...	...	@@ -241,6 +254,7 @@ export const ScheduledTaskWebhookDeliveryOutcome = Schema.Literals([
241	254	"rejected_signature",
242	255	"disabled",
243	256	"rate_limited",
	257	"expired",
244	258	]);
245	259	export type ScheduledTaskWebhookDeliveryOutcome = typeof ScheduledTaskWebhookDeliveryOutcome.Type;
246	260

packages/shared/src/relayJwt.ts+3

...	...	@@ -11,6 +11,9 @@ export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt";
11	11	export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt";
12	12	export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt";
13	13	export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt";
	14	export const RELAY_HOOK_DELIVERY_TYP = "t3-relay-hook-delivery+jwt";
	15	/** Header carrying the signed proof that a webhook request came from the relay. */
	16	export const RELAY_HOOK_DELIVERY_HEADER = "x-t3-relay-delivery";
14	17
15	18	export class RelayJwtError extends Schema.TaggedError<RelayJwtError>()("RelayJwtError", {
16	19	operation: Schema.Literals(["sign", "verify"]),

pnpm-lock.yaml+16-10

...	...	@@ -9,6 +9,9 @@ catalogs:
9	9	'@effect/openapi-generator':
10	10	specifier: 4.0.1
11	11	version: 4.0.1
	12	'@effect/sql-sqlite-do':
	13	specifier: 4.0.1
	14	version: 4.0.1
12	15	'@effect/tsgo':
13	16	specifier: 0.41.0
14	17	version: 0.41.0
...	...	@@ -837,6 +840,9 @@ importers:
837	840	'@effect/sql-pg':
838	841	specifier: 4.0.1
839	842	version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
	843	'@effect/sql-sqlite-do':
	844	specifier: 'catalog:'
	845	version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
840	846	'@noble/curves':
841	847	specifier: 'catalog:'
842	848	version: 1.9.1
...	...	@@ -854,10 +860,10 @@ importers:
854	860	version: link:../../packages/shared
855	861	alchemy:
856	862	specifier: 2.0.0-beta.80
857		version: 2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(824b7c95c4257cf54626a40f00f69c9b)
	863	version: 2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(d02b4e60a6a3c6b542dfa700209cf5ce)
858	864	drizzle-orm:
859	865	specifier: 1.0.0-rc.5-ab785fc
860		version: 1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d)
	866	version: 1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c)
861	867	effect:
862	868	specifier: 4.0.1
863	869	version: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)
...	...	@@ -2359,8 +2365,8 @@ packages:
2359	2365	peerDependencies:
2360	2366	effect: 4.0.1
2361	2367
2362		'@effect/sql-sqlite-do@4.0.0':
2363		resolution: {integrity: sha512-XkRZmQOPblzwjmAJ0RWSOiLlTsKkf1EigZsxUNuIuk4uZ/pnufSqvDpXubCwv8iyRTbOLFMmM+EyOEH5V5XqAw==}
	2368	'@effect/sql-sqlite-do@4.0.1':
	2369	resolution: {integrity: sha512-tS+qQok3xKsycnnhvIDX+6nvlM4Y2ouvru3voOzxtDQqX96NSOMX2xReLf4Feb6GN3jnzgEEtVGDDWk3YLrpsg==}
2364	2370	peerDependencies:
2365	2371	effect: 4.0.1
2366	2372
...	...	@@ -13036,7 +13042,7 @@ snapshots:
13036	13042	effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)
13037	13043	optional: true
13038	13044
13039		'@effect/sql-sqlite-do@4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))':
	13045	'@effect/sql-sqlite-do@4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))':
13040	13046	dependencies:
13041	13047	effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)
13042	13048
...	...	@@ -16619,7 +16625,7 @@ snapshots:
16619	16625	json-schema-traverse: 1.0.0
16620	16626	require-from-string: 2.0.2
16621	16627
16622		alchemy@2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(824b7c95c4257cf54626a40f00f69c9b):
	16628	alchemy@2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(d02b4e60a6a3c6b542dfa700209cf5ce):
16623	16629	dependencies:
16624	16630	'@alchemy.run/cloudflare-runtime': 2.0.0-beta.80(@distilled.cloud/cloudflare@1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)))(@effect/platform-node@4.0.1(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(redis@6.2.1)(utf-8-validate@6.0.6))(@types/node@24.12.4)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(rolldown@1.2.5)(typescript@7.0.2)
16625	16631	'@alchemy.run/floci': 2.0.0-beta.80(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
...	...	@@ -16642,7 +16648,7 @@ snapshots:
16642	16648	'@distilled.cloud/stripe': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16643	16649	'@distilled.cloud/zerossl': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16644	16650	'@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16645		'@effect/sql-sqlite-do': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
	16651	'@effect/sql-sqlite-do': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16646	16652	'@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6)
16647	16653	'@neon/functions': 0.11.0(hono@4.13.7)
16648	16654	'@octokit/rest': 22.0.1
...	...	@@ -16662,7 +16668,7 @@ snapshots:
16662	16668	'@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16663	16669	'@effect/vitest': 4.0.1(patch_hash=359f6fb2f7b3ec145bb72208edb9034f02489791aa2491a55cdbd69bd56ee0d2)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0)
16664	16670	drizzle-kit: 1.0.0-rc.5-ab785fc
16665		drizzle-orm: 1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d)
	16671	drizzle-orm: 1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c)
16666	16672	mongodb: 6.21.0(@aws-sdk/credential-providers@3.1062.0)(socks@2.8.9)
16667	16673	pg: 8.23.0
16668	16674	vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0)'
...	...	@@ -17817,13 +17823,13 @@ snapshots:
17817	17823	get-tsconfig: 4.14.3
17818	17824	jiti: 2.7.0
17819	17825
17820		drizzle-orm@1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d):
	17826	drizzle-orm@1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c):
17821	17827	optionalDependencies:
17822	17828	'@cloudflare/workers-types': 4.20260604.1
17823	17829	'@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
17824	17830	'@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
17825	17831	'@effect/sql-sqlite-bun': 4.0.0-rc.112(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
17826		'@effect/sql-sqlite-do': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
	17832	'@effect/sql-sqlite-do': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
17827	17833	'@electric-sql/pglite': 0.3.15
17828	17834	'@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6)
17829	17835	bun-types: 1.3.14

pnpm-workspace.yaml+2

...	...	@@ -35,6 +35,7 @@ catalog:
35	35	"@effect/platform-node": 4.0.1
36	36	"@effect/platform-node-shared": 4.0.1
37	37	"@effect/sql-pg": 4.0.1
	38	"@effect/sql-sqlite-do": 4.0.1
38	39	"@effect/tsgo": 0.41.0
39	40	"@effect/vitest": 4.0.1
40	41	"@legendapp/list": 3.3.5
...	...	@@ -75,6 +76,7 @@ minimumReleaseAgeExclude:
75	76	- "@effect/platform-node-shared@4.0.1"
76	77	- "@effect/platform-node@4.0.1"
77	78	- "@effect/sql-pg@4.0.1"
	79	- "@effect/sql-sqlite-do@4.0.1"
78	80	- "@effect/vitest@4.0.1"
79	81	- alchemy@2.0.0-beta.80
80	82	- effect@4.0.1

author	Julius Marminge <julius0216@outlook.com> 2026-10-05 15:14:53-07:00
committer	GitHub <noreply@github.com> 2026-10-05 15:14:53-07:00
log	f33b060cafc96d7c3229744315c230a3b8d1f631
tree	0d84f75282264d93612e1103ebc56c2cfd5ad8cf
parent	ea4b44343d3c8f71cf49a72a266d00eac7389435
signature	Signed by PGP key B5690EEEBB952194

feat(relay,server,web,mobile): opt-in to hold webhooks while offline (#15487)