Commit f33b060cafc96d7c3229744315c230a3b8d1f631 | the-knife

apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx+59-15

 @@ -4,7 +4,10 @@ import type {
   ScheduledTask,
   ScheduledTaskUpsertInput,
 } from "@t3tools/contracts";
-import { resolveEnvironmentMachineKind } from "@t3tools/contracts";
+import {
+  MAX_WEBHOOK_DELIVERY_AGE_MINUTES,
+  resolveEnvironmentMachineKind,
+} from "@t3tools/contracts";
 import type { MenuAction } from "@react-native-menu/menu";
 import { DateTimePicker } from "@expo/ui/community/datetime-picker";
 import {
 @@ -12,6 +15,10 @@ import {
   squashAtomCommandFailure,
   type AtomCommandResult,
 } from "@t3tools/client-runtime/state/runtime";
+import {
+  DEFAULT_WEBHOOK_PROMPT,
+  parseMaxDeliveryAge,
+} from "@t3tools/client-runtime/scheduled-task-webhook";
 import {
   useCallback,
   useEffect,
 @@ -59,7 +66,6 @@ import { SettingsSection } from "./components/SettingsSection";
 import { useSettingsEnvironmentFilter, type SettingsTarget } from "./settings-environment-filter";
 import {
   editDraft,
-  DEFAULT_WEBHOOK_PROMPT,
   scheduledTaskDefaultModel,
   scheduleFromDraft,
   type ScheduledTaskDraft as Draft,
 @@ -132,7 +138,7 @@ function FormField(props: {
   readonly label: string;
   readonly value: string;
   readonly onChange: (value: string) => void;
-  readonly keyboardType?: "decimal-pad";
+  readonly keyboardType?: "decimal-pad" | "number-pad";
   readonly disabled?: boolean;
   readonly placeholder?: string;
   readonly borderTop?: boolean;
 @@ -611,6 +617,16 @@ function TaskForm({
         ? { ...draft.schedule, signature: liveTask.schedule.signature }
         : draft.schedule,
     );
+    if (
+      draft.schedule.mode === "webhook" &&
+      parseMaxDeliveryAge(draft.schedule.maxDeliveryAgeMinutes) === undefined
+    ) {
+      Alert.alert(
+        "Invalid age limit",
+        `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`,
+      );
+      return;
+    }
     if (
       !draft.title.trim() ||
       !draft.prompt.trim() ||
 @@ -823,7 +839,7 @@ function TaskForm({
             options={[
               { value: "fixed_time", label: "At a time" },
               { value: "interval", label: "Interval" },
-              { value: "webhook", label: "Webhook" },
+              { value: "webhook", label: "On webhook" },
             ]}
             selected={draft.schedule.mode}
             onSelect={(mode) => {
 @@ -910,13 +926,31 @@ function TaskForm({
             />
           </>
         ) : draft.schedule.mode === "webhook" ? (
-          <WebhookScheduleDetails
+          <>
-            environmentId={environmentId}
+            <WebhookScheduleDetails
-            task={
+              environmentId={environmentId}
-              tasks.data?.tasks.find((task) => task.id === draft.task?.id) ?? draft.task ?? null
+              // The live row, so a rotated URL shows up without reopening the form.
-            }
+              // Once the list has loaded, a missing task is gone; don't keep showing its URL.
-            signatureConfigured={draft.schedule.signature !== null}
+              task={
-          />
+                tasks.data
+                  ? (tasks.data.tasks.find((task) => task.id === draft.task?.id) ?? null)
+                  : draft.task
+              }
+              signatureConfigured={draft.schedule.signature !== null}
+              disabled={saving || environmentUnavailable}
+            />
+            <FormField
+              label="Skip requests older than (minutes)"
+              value={draft.schedule.maxDeliveryAgeMinutes}
+              placeholder="Run every request"
+              keyboardType="number-pad"
+              disabled={saving}
+              borderTop
+              onChange={(maxDeliveryAgeMinutes) =>
+                setDraft({ ...draft, schedule: { ...draft.schedule, maxDeliveryAgeMinutes } })
+              }
+            />
+          </>
         ) : (
           <>
             <FormField
 @@ -976,11 +1010,14 @@ function WebhookScheduleDetails({
   environmentId,
   task,
   signatureConfigured,
+  disabled,
 }: {
   readonly environmentId: EnvironmentId;
   readonly task: ScheduledTask | null;
   readonly signatureConfigured: boolean;
+  readonly disabled: boolean;
 }) {
+  const [rotating, setRotating] = useState(false);
   const rotate = useAtomCommand(serverEnvironment.rotateScheduledTaskWebhookToken, {
     label: "scheduled task rotate webhook token",
     reportFailure: false,
 @@ -1023,27 +1060,34 @@ function WebhookScheduleDetails({
           ) : null}
           <Pressable
             accessibilityRole="button"
+            accessibilityState={{ disabled: disabled || rotating }}
+            disabled={disabled || rotating}
             onPress={() =>
               Alert.alert("Rotate URL?", "The current URL stops working immediately.", [
                 { text: "Cancel", style: "cancel" },
                 {
                   text: "Rotate",
                   style: "destructive",
-                  onPress: () =>
+                  onPress: () => {
+                    setRotating(true);
                     void rotate({ environmentId, input: { id: task.id } }).then((result) => {
+                      setRotating(false);
                       if (result._tag === "Failure" && !isAtomCommandInterrupted(result)) {
                         Alert.alert(
                           "Could not rotate URL",
                           String(squashAtomCommandFailure(result)),
                         );
                       }
-                    }),
+                    });
+                  },
                 },
               ])
             }
-            className="min-h-11 justify-center active:opacity-70"
+            className="min-h-11 justify-center active:opacity-70 disabled:opacity-50"
           >
-            <Text className="text-base text-danger-foreground">Rotate URL</Text>
+            <Text className="text-base text-danger-foreground">
+              {rotating ? "Rotating…" : "Rotate URL"}
+            </Text>
           </Pressable>
         </>
       )}

apps/mobile/src/features/settings/scheduledTaskDraft.test.ts+20-2

...	@@ -32,7 +32,25 @@ describe("scheduleDraftForTask", () => {	...	@@ -32,7 +32,25 @@ describe("scheduleDraftForTask", () => {
32	it("round-trips a webhook schedule without a signature", () => {	32	it("round-trips a webhook schedule without a signature", () => {
33	const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } });	33	const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } });
34	expect(draft.mode).toBe("webhook");	34	expect(draft.mode).toBe("webhook");
35	expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null });	35	expect(scheduleFromDraft(draft)).toEqual({
		36	type: "webhook",
		37	signature: null,
		38	maxDeliveryAgeMinutes: null,
		39	});
		40	});
		41
		42	it("round-trips a webhook max age and treats blank input as no limit", () => {
		43	const draft = scheduleDraftForTask({
		44	schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 45 },
		45	});
		46	expect(draft.maxDeliveryAgeMinutes).toBe("45");
		47	expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 45 });
		48	// An invalid limit is an invalid schedule, never a silently removed one.
		49	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "1.5" })).toBeNull();
		50	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "0" })).toBeNull();
		51	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({
		52	maxDeliveryAgeMinutes: null,
		53	});
36	});	54	});
37		55
38	it("keeps a webhook signature on save without sending a secret", () => {	56	it("keeps a webhook signature on save without sending a secret", () => {
...	@@ -44,7 +62,7 @@ describe("scheduleDraftForTask", () => {	...	@@ -44,7 +62,7 @@ describe("scheduleDraftForTask", () => {
44	const saved = scheduleFromDraft(	62	const saved = scheduleFromDraft(
45	scheduleDraftForTask({ schedule: { type: "webhook", signature } }),	63	scheduleDraftForTask({ schedule: { type: "webhook", signature } }),
46	);	64	);
47	expect(saved).toEqual({ type: "webhook", signature });	65	expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null });
48	expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false);	66	expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false);
49	});	67	});
50	});	68	});

apps/mobile/src/features/settings/scheduledTaskDraft.ts+17-4

 @@ -9,6 +9,7 @@ import type {
 } from "@t3tools/contracts";
 import { DEFAULT_SERVER_SETTINGS } from "@t3tools/contracts";
+import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook";
 import {
   resolveProjectSettings,
   type LegacyProjectSettingsFields,
 @@ -44,6 +45,8 @@ export type ScheduleDraft = {
   readonly intervalMinutes: string;
   /** A webhook signature check configured elsewhere; mobile keeps it but does not edit it. */
   readonly signature: ScheduledTaskWebhookSignature | null;
+  /** Minutes as typed; empty runs every held request regardless of age. */
+  readonly maxDeliveryAgeMinutes: string;
 };
 export const DEFAULT_SCHEDULE: ScheduleDraft = {
 @@ -52,11 +55,9 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = {
   weekdays: [1, 2, 3, 4, 5],
   intervalMinutes: "15",
   signature: null,
+  maxDeliveryAgeMinutes: "",
 };
-/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */
-export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}";
 export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): ScheduleDraft {
   switch (task.schedule.type) {
     case "fixed_time":
 @@ -74,12 +75,22 @@ export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): Sch
         intervalMinutes: String(Math.max(1, task.schedule.everyMs / 60_000)),
       };
     case "webhook":
-      return { ...DEFAULT_SCHEDULE, mode: "webhook", signature: task.schedule.signature };
+      return {
+        ...DEFAULT_SCHEDULE,
+        mode: "webhook",
+        signature: task.schedule.signature,
+        maxDeliveryAgeMinutes:
+          task.schedule.maxDeliveryAgeMinutes == null
+            ? ""
+            : String(task.schedule.maxDeliveryAgeMinutes),
+      };
   }
 }
 export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSchedule | null {
   if (draft.mode === "webhook") {
+    const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes);
+    if (maxDeliveryAgeMinutes === undefined) return null;
     // No secret is sent, so the server keeps the stored one.
     return {
       type: "webhook",
 @@ -91,6 +102,7 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
               encoding: draft.signature.encoding,
               prefix: draft.signature.prefix,
             },
+      maxDeliveryAgeMinutes,
     };
   }
   if (draft.mode === "interval") {
 @@ -145,6 +157,7 @@ function draftSignature(draft: ScheduledTaskDraft): string {
     draft.schedule.timeOfDay,
     [...draft.schedule.weekdays].sort((a, b) => a - b),
     draft.schedule.intervalMinutes,
+    draft.schedule.maxDeliveryAgeMinutes,
     draft.workspace,
     draft.baseRef,
     draft.checkoutPath,

apps/server/src/cloud/ManagedEndpointRuntime.test.ts+30

...	@@ -209,6 +209,36 @@ describe("CloudManagedEndpointRuntime", () => {	...	@@ -209,6 +209,36 @@ describe("CloudManagedEndpointRuntime", () => {
209	}),	209	}),
210	);	210	);
211		211
		212	it.effect("signals each registered tunnel connection", () =>
		213	Effect.gen(function* () {
		214	const output = yield* Queue.unbounded<Uint8Array>();
		215	const spawner = ChildProcessSpawner.make(() =>
		216	Effect.gen(function* () {
		217	const handle = makeHandle({
		218	pid: 700,
		219	onKill: () => {},
		220	output: Stream.fromQueue(output),
		221	});
		222	yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore));
		223	return handle;
		224	}),
		225	);
		226	const runtime = yield* buildCloudManagedEndpointRuntime(spawner);
		227	yield* runtime.applyConfig({
		228	providerKind: "cloudflare_tunnel",
		229	connectorToken: "token",
		230	tunnelId: "tunnel-1",
		231	});
		232	yield* Queue.offer(
		233	output,
		234	new TextEncoder().encode(
		235	"2026-10-04T06:30:43Z INF Registered tunnel connection connIndex=0 event=0\n",
		236	),
		237	);
		238	expect(Option.isSome(yield* Stream.runHead(runtime.tunnelConnected))).toBe(true);
		239	}),
		240	);
		241
212	it.effect("recovers a rejected tunnel without waiting for the connector to exit", () =>	242	it.effect("recovers a rejected tunnel without waiting for the connector to exit", () =>
213	Effect.gen(function* () {	243	Effect.gen(function* () {
214	const output = yield* Queue.unbounded<Uint8Array>();	244	const output = yield* Queue.unbounded<Uint8Array>();

apps/server/src/cloud/ManagedEndpointRuntime.ts+8-1

apps/server/src/cloud/config.ts+31

...	@@ -16,6 +16,7 @@ export const RELAY_URL_SECRET = "cloud-relay-url";	...	@@ -16,6 +16,7 @@ export const RELAY_URL_SECRET = "cloud-relay-url";
16	export const RELAY_ISSUER_SECRET = "cloud-relay-issuer";	16	export const RELAY_ISSUER_SECRET = "cloud-relay-issuer";
17	export const RELAY_ENVIRONMENT_CREDENTIAL_SECRET = "cloud-relay-environment-credential";	17	export const RELAY_ENVIRONMENT_CREDENTIAL_SECRET = "cloud-relay-environment-credential";
18	export const PUBLISH_AGENT_ACTIVITY_SECRET = "cloud-publish-agent-activity";	18	export const PUBLISH_AGENT_ACTIVITY_SECRET = "cloud-publish-agent-activity";
		19	export const HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET = "cloud-hold-webhooks-while-offline";
19		20
20	export const encodeEndpointRuntimeConfigJson = Schema.encodeEffect(	21	export const encodeEndpointRuntimeConfigJson = Schema.encodeEffect(
21	Schema.fromJsonString(RelayManagedEndpointRuntimeConfig),	22	Schema.fromJsonString(RelayManagedEndpointRuntimeConfig),
...	@@ -73,3 +74,33 @@ export const readAgentActivityPublishingActive = (	...	@@ -73,3 +74,33 @@ export const readAgentActivityPublishingActive = (
73	environmentCredential !== ""	74	environmentCredential !== ""
74	);	75	);
75	}).pipe(Effect.orElseSucceed(() => false));	76	}).pipe(Effect.orElseSucceed(() => false));
		77
		78	const readSecretString = (
		79	secrets: ServerSecretStore.ServerSecretStore["Service"],
		80	name: string,
		81	): Effect.Effect<string \| null> =>
		82	secrets.get(name).pipe(
		83	Effect.map((bytes) =>
		84	Option.isSome(bytes) && bytes.value.length > 0 ? new TextDecoder().decode(bytes.value) : null,
		85	),
		86	Effect.orElseSucceed(() => null),
		87	);
		88
		89	/** The relay URL and environment credential, or null when not linked to T3 Connect. */
		90	export const readRelayConnection = (secrets: ServerSecretStore.ServerSecretStore["Service"]) =>
		91	Effect.all([
		92	readSecretString(secrets, RELAY_URL_SECRET),
		93	readSecretString(secrets, RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
		94	]).pipe(
		95	Effect.map(([url, environmentCredential]) =>
		96	url && environmentCredential ? { url, environmentCredential } : null,
		97	),
		98	);
		99
		100	/** Whether this environment opted in to T3 Connect holding webhooks while it is offline. */
		101	export const readHoldWebhooksWhileOffline = (
		102	secrets: ServerSecretStore.ServerSecretStore["Service"],
		103	) =>
		104	readSecretString(secrets, HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET).pipe(
		105	Effect.map((value) => value === "true"),
		106	);

apps/server/src/cloud/http.test.ts+2

...	@@ -249,6 +249,7 @@ describe("reconcileDesiredCloudLink", () => {	...	@@ -249,6 +249,7 @@ describe("reconcileDesiredCloudLink", () => {
249	applyConfig: unusedSecretStoreOperation,	249	applyConfig: unusedSecretStoreOperation,
250	recoveryRequests: Stream.empty,	250	recoveryRequests: Stream.empty,
251	requestRecovery: () => Effect.void,	251	requestRecovery: () => Effect.void,
		252	tunnelConnected: Stream.empty,
252	withLinkStateLock: (effect) => effect,	253	withLinkStateLock: (effect) => effect,
253	} satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]),	254	} satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]),
254	),	255	),
...	@@ -410,6 +411,7 @@ describe("releaseManagedTunnelOnShutdown", () => {	...	@@ -410,6 +411,7 @@ describe("releaseManagedTunnelOnShutdown", () => {
410	}),	411	}),
411	recoveryRequests: Stream.empty,	412	recoveryRequests: Stream.empty,
412	requestRecovery: () => Effect.void,	413	requestRecovery: () => Effect.void,
		414	tunnelConnected: Stream.empty,
413	withLinkStateLock: (effect) => effect,	415	withLinkStateLock: (effect) => effect,
414	}),	416	}),
415	),	417	),

apps/server/src/cloud/http.ts+71-16

...	@@ -8,6 +8,7 @@ import {	...	@@ -8,6 +8,7 @@ import {
8	EnvironmentCloudRelayConfigResult,	8	EnvironmentCloudRelayConfigResult,
9	EnvironmentHttpApi,	9	EnvironmentHttpApi,
10	EnvironmentHttpBadRequestError,	10	EnvironmentHttpBadRequestError,
		11	type EnvironmentCloudPreferencesRequest,
11	EnvironmentHttpConflictError,	12	EnvironmentHttpConflictError,
12	EnvironmentHttpInternalServerError,	13	EnvironmentHttpInternalServerError,
13	EnvironmentHttpUnauthorizedError,	14	EnvironmentHttpUnauthorizedError,
...	@@ -69,6 +70,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";	...	@@ -69,6 +70,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
69	import { requireEnvironmentScope } from "../auth/http.ts";	70	import { requireEnvironmentScope } from "../auth/http.ts";
70	import * as ServerConfig from "../config.ts";	71	import * as ServerConfig from "../config.ts";
71	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";	72	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
		73	import { makeRelayEnvironmentClient } from "../relay/relayEnvironmentClient.ts";
72	import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";	74	import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
73	import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";	75	import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
74	import {	76	import {
...	@@ -86,6 +88,9 @@ import {	...	@@ -86,6 +88,9 @@ import {
86	encodeEndpointRuntimeConfigJson,	88	encodeEndpointRuntimeConfigJson,
87	encodeConfirmedOriginJson,	89	encodeConfirmedOriginJson,
88	PUBLISH_AGENT_ACTIVITY_SECRET,	90	PUBLISH_AGENT_ACTIVITY_SECRET,
		91	HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
		92	readHoldWebhooksWhileOffline,
		93	readRelayConnection,
89	RELAY_ENVIRONMENT_CREDENTIAL_SECRET,	94	RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
90	RELAY_ISSUER_SECRET,	95	RELAY_ISSUER_SECRET,
91	RELAY_URL_SECRET,	96	RELAY_URL_SECRET,
...	@@ -1277,17 +1282,24 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(	...	@@ -1277,17 +1282,24 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
1277	const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* (	1282	const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* (
1278	dependencies: CloudHttpDependencies,	1283	dependencies: CloudHttpDependencies,
1279	) {	1284	) {
1280	const [cloudUserId, relayUrl, relayIssuer, endpointRuntimeConfig, publishAgentActivity] =	1285	const [
1281	yield* Effect.all(	1286	cloudUserId,
1282	[	1287	relayUrl,
1283	dependencies.secrets.get(CLOUD_LINKED_USER_ID),	1288	relayIssuer,
1284	dependencies.secrets.get(RELAY_URL_SECRET),	1289	endpointRuntimeConfig,
1285	dependencies.secrets.get(RELAY_ISSUER_SECRET),	1290	publishAgentActivity,
1286	dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),	1291	holdWebhooks,
1287	dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),	1292	] = yield* Effect.all(
1288	],	1293	[
1289	{ concurrency: 5 },	1294	dependencies.secrets.get(CLOUD_LINKED_USER_ID),
1290	);	1295	dependencies.secrets.get(RELAY_URL_SECRET),
		1296	dependencies.secrets.get(RELAY_ISSUER_SECRET),
		1297	dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
		1298	dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
		1299	dependencies.secrets.get(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
		1300	],
		1301	{ concurrency: 6 },
		1302	);
1291	return {	1303	return {
1292	linked: Option.isSome(cloudUserId),	1304	linked: Option.isSome(cloudUserId),
1293	cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null,	1305	cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null,
...	@@ -1299,6 +1311,8 @@ const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function	...	@@ -1299,6 +1311,8 @@ const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function
1299	publishAgentActivity: Option.isSome(publishAgentActivity)	1311	publishAgentActivity: Option.isSome(publishAgentActivity)
1300	? bytesToString(publishAgentActivity.value) === "true"	1312	? bytesToString(publishAgentActivity.value) === "true"
1301	: false,	1313	: false,
		1314	holdWebhooksWhileOffline:
		1315	Option.isSome(holdWebhooks) && bytesToString(holdWebhooks.value) === "true",
1302	} satisfies EnvironmentCloudLinkStateResult;	1316	} satisfies EnvironmentCloudLinkStateResult;
1303	});	1317	});
1304		1318
...	@@ -1329,8 +1343,9 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(	...	@@ -1329,8 +1343,9 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
1329	dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),	1343	dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
1330	dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),	1344	dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN),
1331	dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),	1345	dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
		1346	dependencies.secrets.remove(HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET),
1332	],	1347	],
1333	{ concurrency: 8 },	1348	{ concurrency: 9 },
1334	);	1349	);
1335	yield* setCliDesiredCloudLink(false);	1350	yield* setCliDesiredCloudLink(false);
1336	return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;	1351	return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
...	@@ -1343,12 +1358,52 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(	...	@@ -1343,12 +1358,52 @@ const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
1343	),	1358	),
1344	);	1359	);
1345		1360
		1361	const pushHoldWebhooksWhileOffline = Effect.fn("environment.cloud.pushHoldWebhooksWhileOffline")(
		1362	function* (dependencies: CloudHttpDependencies, holdWebhooksWhileOffline: boolean) {
		1363	const connection = yield* readRelayConnection(dependencies.secrets);
		1364	if (connection === null) {
		1365	return yield* new EnvironmentHttpBadRequestError({
		1366	message: "Link this environment to T3 Connect first.",
		1367	});
		1368	}
		1369	const environmentId = yield* dependencies.environment.getEnvironmentId;
		1370	const client = yield* makeRelayEnvironmentClient(connection);
		1371	yield* client.server
		1372	.updateLinkPreferences({
		1373	params: { environmentId },
		1374	payload: { holdWebhooksWhileOffline },
		1375	})
		1376	.pipe(
		1377	Effect.timeout("10 seconds"),
		1378	Effect.catch(
		1379	failEnvironmentCloudInternalError("Could not update T3 Connect webhook settings."),
		1380	),
		1381	);
		1382	},
		1383	);
		1384
1346	const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")(	1385	const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")(
1347	function* (	1386	function* (dependencies: CloudHttpDependencies, payload: EnvironmentCloudPreferencesRequest) {
1348	dependencies: CloudHttpDependencies,
1349	payload: { readonly publishAgentActivity: boolean },
1350	) {
1351	yield* requireEnvironmentScope(AuthRelayWriteScope);	1387	yield* requireEnvironmentScope(AuthRelayWriteScope);
		1388	if (payload.holdWebhooksWhileOffline !== undefined) {
		1389	// The relay decides whether to hold a request, so it is told first; the
		1390	// local copy is only saved once the relay has the same value, and the
		1391	// relay is put back if that save fails.
		1392	const previous = yield* readHoldWebhooksWhileOffline(dependencies.secrets);
		1393	yield* pushHoldWebhooksWhileOffline(dependencies, payload.holdWebhooksWhileOffline);
		1394	yield* dependencies.secrets
		1395	.set(
		1396	HOLD_WEBHOOKS_WHILE_OFFLINE_SECRET,
		1397	stringToBytes(String(payload.holdWebhooksWhileOffline)),
		1398	)
		1399	.pipe(
		1400	Effect.tapError(() =>
		1401	previous === payload.holdWebhooksWhileOffline
		1402	? Effect.void
		1403	: pushHoldWebhooksWhileOffline(dependencies, previous).pipe(Effect.ignore),
		1404	),
		1405	);
		1406	}
1352	yield* dependencies.secrets.set(	1407	yield* dependencies.secrets.set(
1353	PUBLISH_AGENT_ACTIVITY_SECRET,	1408	PUBLISH_AGENT_ACTIVITY_SECRET,
1354	stringToBytes(String(payload.publishAgentActivity)),	1409	stringToBytes(String(payload.publishAgentActivity)),

apps/server/src/observability/Metrics.ts+24

...	@@ -70,6 +70,30 @@ export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total"	...	@@ -70,6 +70,30 @@ export const terminalRestartsTotal = Metric.counter("t3_terminal_restarts_total"
70	description: "Total terminal restart requests handled.",	70	description: "Total terminal restart requests handled.",
71	});	71	});
72		72
		73	/**
		74	* One per webhook request that reached a task, by `outcome` (accepted,
		75	* not_found, rejected_signature, disabled, rate_limited, queue_full, expired,
		76	* prompt_too_long, error) and `source` (relay or direct).
		77	*/
		78	export const webhookDeliveriesTotal = Metric.counter("t3_webhook_deliveries_total", {
		79	description: "Webhook requests handled, by outcome and source.",
		80	});
		81
		82	export const webhookDeliveryDuration = Metric.timer("t3_webhook_delivery_duration", {
		83	description: "Time to verify, log, and enqueue one webhook request.",
		84	});
		85
		86	/** How long a relay-held request waited before this environment got it. */
		87	export const webhookHeldDelay = Metric.timer("t3_webhook_held_delay", {
		88	description:
		89	"Time between the relay receiving a webhook request and the environment handling it.",
		90	});
		91
		92	/** Runs started by webhook deliveries, by `outcome` (started, skipped, failed). */
		93	export const webhookRunsTotal = Metric.counter("t3_webhook_runs_total", {
		94	description: "Runs started from webhook deliveries, by outcome.",
		95	});
		96
73	export const metricAttributes = (	97	export const metricAttributes = (
74	attributes: Readonly<Record<string, unknown>>,	98	attributes: Readonly<Record<string, unknown>>,
75	): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes));	99	): ReadonlyArray<[string, string]> => Object.entries(compactMetricAttributes(attributes));

apps/server/src/persistence/Migrations.ts+2

...	@@ -71,6 +71,7 @@ import Migration0054 from "./Migrations/054_ProjectionThreadsAutoSettleDisabledA	...	@@ -71,6 +71,7 @@ import Migration0054 from "./Migrations/054_ProjectionThreadsAutoSettleDisabledA
71	import Migration0055 from "./Migrations/055_OrchestrationV2.ts";	71	import Migration0055 from "./Migrations/055_OrchestrationV2.ts";
72	import Migration0056 from "./Migrations/056_RemoveRedundantProjectionIndexes.ts";	72	import Migration0056 from "./Migrations/056_RemoveRedundantProjectionIndexes.ts";
73	import Migration0057 from "./Migrations/057_ScheduledTaskWebhooks.ts";	73	import Migration0057 from "./Migrations/057_ScheduledTaskWebhooks.ts";
		74	import Migration0058 from "./Migrations/058_WebhookRelayDeliveries.ts";
74		75
75	/**	76	/**
76	* Migration loader with all migrations defined inline.	77	* Migration loader with all migrations defined inline.
...	@@ -142,6 +143,7 @@ export const migrationEntries = [	...	@@ -142,6 +143,7 @@ export const migrationEntries = [
142	[55, "OrchestrationV2", Migration0055],	143	[55, "OrchestrationV2", Migration0055],
143	[56, "RemoveRedundantProjectionIndexes", Migration0056],	144	[56, "RemoveRedundantProjectionIndexes", Migration0056],
144	[57, "ScheduledTaskWebhooks", Migration0057],	145	[57, "ScheduledTaskWebhooks", Migration0057],
		146	[58, "WebhookRelayDeliveries", Migration0058],
145	] as const;	147	] as const;
146		148
147	export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);	149	export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);

apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts+3-1

apps/server/src/persistence/Migrations/058_WebhookRelayDeliveries.ts created+22

...	@@ -0,0 +1,22 @@
1	import * as Effect from "effect/Effect";
2	import * as SqlClient from "effect/sql/SqlClient";
3
4	export default Effect.gen(function* () {
5	const sql = yield* SqlClient.SqlClient;
6
7	// Relay delivery ids already handled, kept longer than the relay holds a
8	// request so a replay can never run twice. The delivery log keeps only the
9	// newest 50 rows per task, which is too short for that.
10	yield* sql`
11	CREATE TABLE IF NOT EXISTS scheduled_task_webhook_relay_deliveries (
12	relay_delivery_id TEXT PRIMARY KEY,
13	task_id TEXT NOT NULL,
14	seen_at TEXT NOT NULL
15	)
16	`;
17
18	yield* sql`
19	CREATE INDEX IF NOT EXISTS idx_scheduled_task_webhook_relay_deliveries_seen
20	ON scheduled_task_webhook_relay_deliveries(seen_at)
21	`;
22	});

apps/server/src/persistence/reconcileV2PreviewMigration.test.ts+2

...	@@ -38,6 +38,7 @@ describe("V2 preview upgrade", () => {	...	@@ -38,6 +38,7 @@ describe("V2 preview upgrade", () => {
38	[54, "ProjectionThreadsAutoSettleDisabledAt"],	38	[54, "ProjectionThreadsAutoSettleDisabledAt"],
39	[56, "RemoveRedundantProjectionIndexes"],	39	[56, "RemoveRedundantProjectionIndexes"],
40	[57, "ScheduledTaskWebhooks"],	40	[57, "ScheduledTaskWebhooks"],
		41	[58, "WebhookRelayDeliveries"],
41	]);	42	]);
42	assert.deepStrictEqual(yield* runMigrations(), []);	43	assert.deepStrictEqual(yield* runMigrations(), []);
43	assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);	44	assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
...	@@ -118,6 +119,7 @@ describe("V2 preview upgrade", () => {	...	@@ -118,6 +119,7 @@ describe("V2 preview upgrade", () => {
118	[54, "ProjectionThreadsAutoSettleDisabledAt"],	119	[54, "ProjectionThreadsAutoSettleDisabledAt"],
119	[56, "RemoveRedundantProjectionIndexes"],	120	[56, "RemoveRedundantProjectionIndexes"],
120	[57, "ScheduledTaskWebhooks"],	121	[57, "ScheduledTaskWebhooks"],
		122	[58, "WebhookRelayDeliveries"],
121	]);	123	]);
122	}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),	124	}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
123	);	125	);

apps/server/src/relay/HeldHooksWaker.ts created+52

...	@@ -0,0 +1,52 @@
1	import * as Effect from "effect/Effect";
2	import * as Layer from "effect/Layer";
3	import * as Schedule from "effect/Schedule";
4	import * as Stream from "effect/Stream";
5
6	import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
7	import * as CloudManagedEndpointRuntime from "../cloud/ManagedEndpointRuntime.ts";
8	import { readHoldWebhooksWhileOffline, readRelayConnection } from "../cloud/config.ts";
9	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
10	import { makeRelayEnvironmentClient } from "./relayEnvironmentClient.ts";
11
12	/**
13	* Tells T3 Connect this environment is reachable again, so the relay delivers
14	* the webhook requests it held while we were offline now rather than at its
15	* next backoff step. Nothing happens unless the environment opted in.
16	*/
17	const wakeHeldHooks = Effect.fn("HeldHooksWaker.wake")(function* () {
18	const secrets = yield* ServerSecretStore.ServerSecretStore;
19	if (!(yield* readHoldWebhooksWhileOffline(secrets))) return false;
20	const connection = yield* readRelayConnection(secrets);
21	if (connection === null) return false;
22	const environmentId = yield* (yield* ServerEnvironment.ServerEnvironment).getEnvironmentId;
23	const client = yield* makeRelayEnvironmentClient(connection);
24	const { pending } = yield* client.server.wakeHeldHooks({ params: { environmentId } });
25	yield* Effect.annotateCurrentSpan({ "relay.inbox.pending": pending });
26	return pending;
27	});
28
29	/** Wakes held webhooks each time the managed tunnel connects. */
30	export const layer = Layer.effectDiscard(
31	Effect.gen(function* () {
32	const runtime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime;
33	const wake = wakeHeldHooks().pipe(
34	Effect.timeout("10 seconds"),
35	// The relay retries on its own schedule too, so a few tries are enough.
36	Effect.retry({ schedule: Schedule.exponential("2 seconds"), times: 3 }),
37	Effect.tap((pending) =>
38	pending ? Effect.logInfo("T3 Connect is delivering held webhook requests") : Effect.void,
39	),
40	Effect.catchCause((cause) =>
41	Effect.logWarning("Could not ask T3 Connect to deliver held webhook requests", { cause }),
42	),
43	);
44	yield* runtime.tunnelConnected.pipe(
45	// cloudflared registers several connections per (re)connect within a
46	// few seconds; they are one wake.
47	Stream.debounce("3 seconds"),
48	Stream.runForEach(() => wake),
49	Effect.forkScoped,
50	);
51	}),
52	);

apps/server/src/relay/relayEnvironmentClient.ts created+21

...	@@ -0,0 +1,21 @@
1	import { RelayApi } from "@t3tools/contracts/relay";
2	import * as Effect from "effect/Effect";
3	import * as FetchHttpClient from "effect/http/FetchHttpClient";
4	import * as HttpClient from "effect/http/HttpClient";
5	import * as HttpClientRequest from "effect/http/HttpClientRequest";
6	import * as HttpApiClient from "effect/http-api/HttpApiClient";
7
8	/**
9	* A typed RelayApi client that authenticates as this environment, for the
10	* environment-credential endpoints (link preferences, held webhooks).
11	*/
12	export const makeRelayEnvironmentClient = (connection: {
13	readonly url: string;
14	readonly environmentCredential: string;
15	}) =>
16	HttpApiClient.make(RelayApi, {
17	baseUrl: connection.url,
18	transformClient: HttpClient.mapRequest(
19	HttpClientRequest.setHeader("authorization", `Bearer ${connection.environmentCredential}`),
20	),
21	}).pipe(Effect.provide(FetchHttpClient.layer));

apps/server/src/scheduledTasks/ScheduledTaskService.ts+345-125

...	@@ -1,6 +1,7 @@	...	@@ -1,6 +1,7 @@
1	import {	1	import {
2	CommandId,	2	CommandId,
3	MessageId,	3	MessageId,
		4	PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
4	ScheduledTask,	5	ScheduledTask,
5	ScheduledTaskError,	6	ScheduledTaskError,
6	ScheduledTaskId,	7	ScheduledTaskId,
...	@@ -27,6 +28,8 @@ import * as Crypto from "effect/Crypto";	...	@@ -27,6 +28,8 @@ import * as Crypto from "effect/Crypto";
27	import * as Data from "effect/Data";	28	import * as Data from "effect/Data";
28	import * as DateTime from "effect/DateTime";	29	import * as DateTime from "effect/DateTime";
29	import * as Effect from "effect/Effect";	30	import * as Effect from "effect/Effect";
		31	import * as Duration from "effect/Duration";
		32	import * as Metric from "effect/Metric";
30	import * as Layer from "effect/Layer";	33	import * as Layer from "effect/Layer";
31	import * as Option from "effect/Option";	34	import * as Option from "effect/Option";
32	import * as PubSub from "effect/PubSub";	35	import * as PubSub from "effect/PubSub";
...	@@ -38,10 +41,16 @@ import * as Stream from "effect/Stream";	...	@@ -38,10 +41,16 @@ import * as Stream from "effect/Stream";
38	import * as SqlClient from "effect/sql/SqlClient";	41	import * as SqlClient from "effect/sql/SqlClient";
39		42
40	import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";	43	import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
		44	import * as Metrics from "../observability/Metrics.ts";
41	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";	45	import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
42	import * as Scheduler from "../scheduling/Scheduler.ts";	46	import * as Scheduler from "../scheduling/Scheduler.ts";
43	import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";	47	import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";
44	import { renderWebhookPrompt, type WebhookRequest } from "./webhookTemplate.ts";	48	import {
		49	redactHeaders,
		50	redactQuery,
		51	renderWebhookPrompt,
		52	type WebhookRequest,
		53	} from "./webhookTemplate.ts";
45	import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts";	54	import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts";
46		55
47	/** Path prefix of the environment route that receives webhook requests. */	56	/** Path prefix of the environment route that receives webhook requests. */
...	@@ -50,21 +59,44 @@ export const WEBHOOK_ROUTE_PREFIX = "/api/hooks";	...	@@ -50,21 +59,44 @@ export const WEBHOOK_ROUTE_PREFIX = "/api/hooks";
50	const WEBHOOK_DELIVERY_RETENTION = 50;	59	const WEBHOOK_DELIVERY_RETENTION = 50;
51	/** Body text kept in the delivery log. Larger bodies are cut and flagged. */	60	/** Body text kept in the delivery log. Larger bodies are cut and flagged. */
52	const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;	61	const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
		62	/** Rendered prompt text kept in the delivery log. */
		63	const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024;
53	/** Deliveries one task may hold at once, running or waiting their turn. */	64	/** Deliveries one task may hold at once, running or waiting their turn. */
54	const WEBHOOK_MAX_QUEUED_PER_TASK = 20;	65	const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
55	/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */	66	/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */
56	const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;	67	const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;
57		68
58	/** Where a webhook task's public URL points. `relayUrl` is null when the environment is not linked to T3 Connect. */	69	/**
		70	* Where a webhook task's public URL points: `${relayHookBaseUrl}/${taskId}/${token}`.
		71	* Null when the environment has no managed tunnel on T3 Connect; clients then show the path.
		72	*/
59	interface WebhookOrigin {	73	interface WebhookOrigin {
60	readonly environmentId: string;	74	readonly relayHookBaseUrl: string \| null;
61	readonly relayUrl: string \| null;	75	}
		76
		77	const ENDPOINT_KEY = /^[0-9a-f]{16}$/;
		78
		79	/**
		80	* The relay's hook URL prefix for this environment. The relay finds the
		81	* environment by its managed tunnel's key (the tunnel name's last segment),
		82	* so the URL never reveals the environment id.
		83	*/
		84	export function relayHookBaseUrl(input: {
		85	readonly relayUrl: string;
		86	readonly tunnelName: string \| undefined;
		87	}): string \| null {
		88	const endpointKey = input.tunnelName?.split("-").at(-1);
		89	const relayUrl = input.relayUrl.replace(/\/+$/, "");
		90	if (relayUrl === "" \|\| endpointKey === undefined \|\| !ENDPOINT_KEY.test(endpointKey)) {
		91	return null;
		92	}
		93	return `${relayUrl}/v1/hooks/${endpointKey}`;
62	}	94	}
63		95
64	export class ScheduledTaskWebhookOrigin extends Context.Reference<Effect.Effect<WebhookOrigin>>(	96	export class ScheduledTaskWebhookOrigin extends Context.Reference<Effect.Effect<WebhookOrigin>>(
65	"t3/scheduledTasks/ScheduledTaskWebhookOrigin",	97	"t3/scheduledTasks/ScheduledTaskWebhookOrigin",
66	{	98	{
67	defaultValue: () => Effect.succeed({ environmentId: "local", relayUrl: null }),	99	defaultValue: () => Effect.succeed({ relayHookBaseUrl: null }),
68	},	100	},
69	) {}	101	) {}
70		102
...	@@ -83,15 +115,45 @@ export interface WebhookTriggerRequest extends WebhookRequest {	...	@@ -83,15 +115,45 @@ export interface WebhookTriggerRequest extends WebhookRequest {
83	readonly hookId: string;	115	readonly hookId: string;
84	readonly token: string;	116	readonly token: string;
85	readonly body: Uint8Array;	117	readonly body: Uint8Array;
		118	/** Set by T3 Connect; the same id is never dispatched twice. */
		119	readonly relayDeliveryId?: string;
		120	/** When the relay received a held request; defaults to now. */
		121	readonly receivedAt?: string;
86	}	122	}
87		123
88	/** What the HTTP route should answer. `not_found` covers unknown hooks and wrong tokens alike. */	124	/** What the HTTP route should answer. `not_found` covers unknown hooks and wrong tokens alike. */
		125	/**
		126	* What happened to one webhook request, as recorded in metrics and spans.
		127	* Sent back to the relay as `x-t3-hook-outcome`; never request contents.
		128	*/
		129	export type WebhookDeliveryOutcome =
		130	\| "accepted"
		131	\| "duplicate"
		132	\| "prompt_too_long"
		133	\| "queue_full"
		134	\| "rate_limited"
		135	\| "disabled"
		136	\| "rejected_signature"
		137	\| "expired"
		138	\| "not_found"
		139	\| "error";
		140
89	export type WebhookTriggerResult =	141	export type WebhookTriggerResult =
90	\| { readonly _tag: "accepted"; readonly deliveryId: ScheduledTaskWebhookDeliveryId }	142	\| {
		143	readonly _tag: "accepted";
		144	readonly deliveryId: ScheduledTaskWebhookDeliveryId;
		145	/** A 202 covers more than a started run; this says which. */
		146	readonly outcome: "accepted" \| "duplicate" \| "prompt_too_long";
		147	}
91	\| { readonly _tag: "not_found" }	148	\| { readonly _tag: "not_found" }
92	\| { readonly _tag: "rejected_signature" }	149	\| { readonly _tag: "rejected_signature" }
93	\| { readonly _tag: "disabled" }	150	\| { readonly _tag: "disabled" }
94	\| { readonly _tag: "rate_limited" };	151	\| {
		152	readonly _tag: "rate_limited";
		153	/** Too many requests to this hook, or too many runs already waiting. */
		154	readonly outcome: "rate_limited" \| "queue_full";
		155	}
		156	\| { readonly _tag: "expired" };
95		157
96	const decodeTask = Schema.decodeUnknownEffect(ScheduledTask);	158	const decodeTask = Schema.decodeUnknownEffect(ScheduledTask);
97	const decodeTaskId = Schema.decodeUnknownOption(ScheduledTaskId);	159	const decodeTaskId = Schema.decodeUnknownOption(ScheduledTaskId);
...	@@ -228,19 +290,6 @@ function errorMessage(error: unknown): string {	...	@@ -228,19 +290,6 @@ function errorMessage(error: unknown): string {
228	return String(error);	290	return String(error);
229	}	291	}
230		292
231	/** Headers kept out of the delivery log because they commonly carry credentials. */
232	const REDACTED_HEADER =
233	/^(authorization\|proxy-authorization\|cookie\|set-cookie)$\|token\|secret\|signature\|key\|password\|auth/i;
234
235	function redactHeaders(headers: Readonly<Record<string, string>>): Record<string, string> {
236	return Object.fromEntries(
237	Object.entries(headers).map(([name, value]) => [
238	name,
239	REDACTED_HEADER.test(name) ? "[redacted]" : value,
240	]),
241	);
242	}
243
244	function webhookPath(taskId: string, token: string): string {	293	function webhookPath(taskId: string, token: string): string {
245	return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`;	294	return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`;
246	}	295	}
...	@@ -250,13 +299,10 @@ function webhookEndpoint(	...	@@ -250,13 +299,10 @@ function webhookEndpoint(
250	origin: WebhookOrigin \| null,	299	origin: WebhookOrigin \| null,
251	): ScheduledTask["webhook"] {	300	): ScheduledTask["webhook"] {
252	if (row.webhook_token === null) return undefined;	301	if (row.webhook_token === null) return undefined;
253	const relayUrl = origin?.relayUrl?.replace(/\/+$/, "") ?? null;	302	const base = origin?.relayHookBaseUrl ?? null;
254	return {	303	return {
255	path: webhookPath(row.task_id, row.webhook_token),	304	path: webhookPath(row.task_id, row.webhook_token),
256	url:	305	url: base === null ? null : `${base}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`,
257	relayUrl === null \|\| origin === null
258	? null
259	: `${relayUrl}/v1/hooks/${encodeURIComponent(origin.environmentId)}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`,
260	hasSecret: row.webhook_secret !== null,	306	hasSecret: row.webhook_secret !== null,
261	};	307	};
262	}	308	}
...	@@ -985,6 +1031,7 @@ export const layer = Layer.effect(	...	@@ -985,6 +1031,7 @@ export const layer = Layer.effect(
985	encoding: input.schedule.signature.encoding,	1031	encoding: input.schedule.signature.encoding,
986	prefix: input.schedule.signature.prefix,	1032	prefix: input.schedule.signature.prefix,
987	},	1033	},
		1034	maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null,
988	}	1035	}
989	: input.schedule;	1036	: input.schedule;
990	const webhook =	1037	const webhook =
...	@@ -1197,6 +1244,7 @@ export const layer = Layer.effect(	...	@@ -1197,6 +1244,7 @@ export const layer = Layer.effect(
1197	readonly signatureVerified: boolean;	1244	readonly signatureVerified: boolean;
1198	readonly missing: ReadonlyArray<string>;	1245	readonly missing: ReadonlyArray<string>;
1199	readonly renderedPrompt: string \| null;	1246	readonly renderedPrompt: string \| null;
		1247	readonly error?: string;
1200	}) => {	1248	}) => {
1201	const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;	1249	const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1202	const loggedBody = truncated	1250	const loggedBody = truncated
...	@@ -1215,12 +1263,22 @@ export const layer = Layer.effect(	...	@@ -1215,12 +1263,22 @@ export const layer = Layer.effect(
1215	)	1263	)
1216	SELECT	1264	SELECT
1217	${input.id}, ${input.taskId}, ${input.receivedAt}, ${input.request.method},	1265	${input.id}, ${input.taskId}, ${input.receivedAt}, ${input.request.method},
1218	${input.request.query}, ${encodeHeadersJson(redactHeaders(input.request.headers))},	1266	${redactQuery(input.request.query)},
		1267	${encodeHeadersJson(redactHeaders(input.request.headers))},
1219	${loggedBody},	1268	${loggedBody},
1220	${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome},	1269	${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome},
1221	${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)},	1270	${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)},
1222	${input.renderedPrompt}, NULL	1271	${input.renderedPrompt?.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT) ?? null},
		1272	${input.error ?? null}
1223	WHERE EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.taskId})	1273	WHERE EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.taskId})
		1274	-- A held request retried after a rate limit reuses its relay
		1275	-- delivery id; the newer attempt replaces the logged one.
		1276	ON CONFLICT (delivery_id) DO UPDATE SET
		1277	outcome = excluded.outcome,
		1278	signature_verified = excluded.signature_verified,
		1279	missing_fields_json = excluded.missing_fields_json,
		1280	rendered_prompt = excluded.rendered_prompt,
		1281	error = excluded.error
1224	`;	1282	`;
1225	yield* sql`	1283	yield* sql`
1226	DELETE FROM scheduled_task_webhook_deliveries	1284	DELETE FROM scheduled_task_webhook_deliveries
...	@@ -1295,10 +1353,55 @@ export const layer = Layer.effect(	...	@@ -1295,10 +1353,55 @@ export const layer = Layer.effect(
1295	// run; scoped to the service so shutdown interrupts it.	1353	// run; scoped to the service so shutdown interrupts it.
1296	const serviceScope = yield* Effect.scope;	1354	const serviceScope = yield* Effect.scope;
1297		1355
		1356	/**
		1357	* Records one handled request on the span and in metrics. `outcome` is
		1358	* finer than the result tag: it separates a full queue and an oversized
		1359	* prompt from the rest, which is what an operator needs to act on.
		1360	*/
		1361	const observeDelivery = (
		1362	request: WebhookTriggerRequest,
		1363	outcome: WebhookDeliveryOutcome,
		1364	receivedAt: DateTime.DateTime \| undefined,
		1365	now: DateTime.DateTime \| undefined,
		1366	) =>
		1367	Effect.gen(function* () {
		1368	const source = request.relayDeliveryId === undefined ? "direct" : "relay";
		1369	yield* Effect.annotateCurrentSpan({
		1370	"scheduled_task.webhook.outcome": outcome,
		1371	"scheduled_task.webhook.source": source,
		1372	});
		1373	yield* Metrics.increment(Metrics.webhookDeliveriesTotal, { outcome, source });
		1374	if (request.receivedAt !== undefined && receivedAt !== undefined && now !== undefined) {
		1375	const heldMs = Math.max(
		1376	0,
		1377	DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt),
		1378	);
		1379	yield* Effect.annotateCurrentSpan({ "scheduled_task.webhook.held_ms": heldMs });
		1380	yield* Metric.update(Metrics.webhookHeldDelay, Duration.millis(heldMs));
		1381	}
		1382	});
		1383
1298	const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) =>	1384	const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) =>
		1385	triggerWebhookUnobserved(request).pipe(
		1386	Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)),
		1387	Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }),
		1388	Effect.withSpan("ScheduledTaskService.triggerWebhook", {
		1389	attributes: {
		1390	"scheduled_task.webhook.method": request.method,
		1391	"scheduled_task.webhook.body_bytes": request.body.byteLength,
		1392	"scheduled_task.webhook.source":
		1393	request.relayDeliveryId === undefined ? "direct" : "relay",
		1394	},
		1395	}),
		1396	);
		1397
		1398	const triggerWebhookUnobserved = (request: WebhookTriggerRequest) =>
1299	Effect.gen(function* () {	1399	Effect.gen(function* () {
1300	const taskId = decodeTaskId(request.hookId);	1400	const taskId = decodeTaskId(request.hookId);
1301	if (Option.isNone(taskId)) return { _tag: "not_found" as const };	1401	const notFound = observeDelivery(request, "not_found", undefined, undefined).pipe(
		1402	Effect.as({ _tag: "not_found" as const }),
		1403	);
		1404	if (Option.isNone(taskId)) return yield* notFound;
1302	const rows = yield* getRows(taskId.value).pipe(	1405	const rows = yield* getRows(taskId.value).pipe(
1303	Effect.mapError((cause) =>	1406	Effect.mapError((cause) =>
1304	taskError("Could not load schedule task.", { taskId: taskId.value, cause }),	1407	taskError("Could not load schedule task.", { taskId: taskId.value, cause }),
...	@@ -1312,110 +1415,227 @@ export const layer = Layer.effect(	...	@@ -1312,110 +1415,227 @@ export const layer = Layer.effect(
1312	row.webhook_token === null \|\|	1415	row.webhook_token === null \|\|
1313	!constantTimeEquals(request.token, row.webhook_token)	1416	!constantTimeEquals(request.token, row.webhook_token)
1314	) {	1417	) {
1315	return { _tag: "not_found" as const };	1418	return yield* notFound;
1316	}	1419	}
1317	const task = yield* decodeRow(row);	1420	const task = yield* decodeRow(row);
1318	if (task.schedule.type !== "webhook") return { _tag: "not_found" as const };	1421	yield* Effect.annotateCurrentSpan({ "scheduled_task.id": task.id });
		1422	const schedule = task.schedule;
		1423	if (schedule.type !== "webhook") return yield* notFound;
1319		1424
1320	const receivedAt = yield* localNow;	1425	const now = yield* localNow;
1321	const deliveryUuid = yield* crypto.randomUUIDv4.pipe(	1426	// Anyone can reach the tunnel directly and set the relay's header, so
1322	Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })),	1427	// a receive time is never later than now: a future one would pin the
		1428	// delivery in the log and slip past the task's max age.
		1429	const receivedAt =
		1430	request.receivedAt === undefined
		1431	? now
		1432	: DateTime.min(
		1433	Option.getOrElse(DateTime.make(request.receivedAt), () => now),
		1434	now,
		1435	);
		1436	const observe = (outcome: WebhookDeliveryOutcome) =>
		1437	observeDelivery(request, outcome, receivedAt, now);
		1438	const deliveryId = ScheduledTaskWebhookDeliveryId.make(
		1439	request.relayDeliveryId === undefined
		1440	? `delivery:${yield* crypto.randomUUIDv4.pipe(
		1441	Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })),
		1442	)}`
		1443	: `delivery:relay:${request.relayDeliveryId}`,
1323	);	1444	);
1324	const deliveryId = ScheduledTaskWebhookDeliveryId.make(`delivery:${deliveryUuid}`);	1445	// A held request may already have reached this environment directly
1325	const log = (	1446	// before a timeout; it runs once. Claimed in its own table, kept longer
1326	outcome: ScheduledTaskWebhookDeliveryOutcome,	1447	// than the relay holds a request, because the delivery log is trimmed.
1327	details: {	1448	// A rate-limited delivery stays held on the relay, so it must give up
1328	readonly signatureVerified?: boolean;	1449	// its claim or the next pass would treat it as already delivered.
1329	readonly missing?: ReadonlyArray<string>;	1450	const releaseClaim = <A>(result: A) =>
1330	readonly renderedPrompt?: string;	1451	request.relayDeliveryId === undefined
1331	} = {},	1452	? Effect.succeed(result)
1332	) =>	1453	: sql`
1333	recordDelivery({	1454	DELETE FROM scheduled_task_webhook_relay_deliveries
1334	id: deliveryId,	1455	WHERE relay_delivery_id = ${request.relayDeliveryId}
1335	taskId: task.id,	1456	`.pipe(Effect.ignore, Effect.as(result));
1336	receivedAt: iso(receivedAt),	1457	// From the claim until the run is forked nothing may interrupt: a
1337	request,	1458	// request dropped in between (the relay hangs up after its timeout)
1338	outcome,	1459	// would leave a claimed delivery that never runs, or a queue slot
1339	signatureVerified: details.signatureVerified ?? false,	1460	// that is never released. Everything in here is local and quick.
1340	missing: details.missing ?? [],	1461	return yield* Effect.uninterruptible(
1341	renderedPrompt: details.renderedPrompt ?? null,	1462	Effect.gen(function* () {
1342	});	1463	if (request.relayDeliveryId !== undefined) {
		1464	const claimed = yield* sql<{ relay_delivery_id: string }>`
		1465	INSERT INTO scheduled_task_webhook_relay_deliveries
		1466	(relay_delivery_id, task_id, seen_at)
		1467	VALUES (${request.relayDeliveryId}, ${task.id}, ${iso(now)})
		1468	ON CONFLICT (relay_delivery_id) DO NOTHING
		1469	RETURNING relay_delivery_id
		1470	`.pipe(
		1471	Effect.mapError((cause) =>
		1472	taskError("Could not record webhook delivery.", { taskId: task.id, cause }),
		1473	),
		1474	);
		1475	if (claimed.length === 0) {
		1476	// A held request this environment already ran: accepted, not run twice.
		1477	yield* observe("duplicate");
		1478	return { _tag: "accepted" as const, deliveryId, outcome: "duplicate" as const };
		1479	}
		1480	// Older claims can no longer be replayed by the relay.
		1481	yield* sql`
		1482	DELETE FROM scheduled_task_webhook_relay_deliveries
		1483	WHERE seen_at < ${iso(DateTime.subtract(now, { hours: 48 }))}
		1484	`.pipe(Effect.ignore);
		1485	}
		1486	const log = (
		1487	outcome: ScheduledTaskWebhookDeliveryOutcome,
		1488	details: {
		1489	readonly signatureVerified?: boolean;
		1490	readonly missing?: ReadonlyArray<string>;
		1491	readonly renderedPrompt?: string;
		1492	readonly error?: string;
		1493	} = {},
		1494	) =>
		1495	recordDelivery({
		1496	id: deliveryId,
		1497	taskId: task.id,
		1498	receivedAt: iso(receivedAt),
		1499	request,
		1500	outcome,
		1501	signatureVerified: details.signatureVerified ?? false,
		1502	missing: details.missing ?? [],
		1503	renderedPrompt: details.renderedPrompt ?? null,
		1504	...(details.error === undefined ? {} : { error: details.error }),
		1505	});
		1506
		1507	// Only the first rejected request in a window is logged, so a flood
		1508	// cannot write rows or push the real deliveries out of the log.
		1509	const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now));
		1510	if (slot !== "allowed") {
		1511	if (slot === "first_rejected") yield* log("rate_limited");
		1512	yield* observe("rate_limited");
		1513	return yield* releaseClaim({
		1514	_tag: "rate_limited" as const,
		1515	outcome: "rate_limited" as const,
		1516	});
		1517	}
		1518	if (!task.enabled) {
		1519	yield* log("disabled");
		1520	yield* observe("disabled");
		1521	return { _tag: "disabled" as const };
		1522	}
		1523	const signature = schedule.signature;
		1524	if (signature !== null) {
		1525	const verified =
		1526	row.webhook_secret !== null &&
		1527	verifyWebhookSignature({
		1528	signature,
		1529	secret: row.webhook_secret,
		1530	headers: request.headers,
		1531	body: request.body,
		1532	});
		1533	if (!verified) {
		1534	yield* log("rejected_signature");
		1535	yield* observe("rejected_signature");
		1536	return { _tag: "rejected_signature" as const };
		1537	}
		1538	}
		1539	const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null;
		1540	if (
		1541	maxAgeMinutes !== null &&
		1542	DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) >
		1543	maxAgeMinutes * 60_000
		1544	) {
		1545	yield* log("expired", { signatureVerified: signature !== null });
		1546	yield* observe("expired");
		1547	return { _tag: "expired" as const };
		1548	}
1343		1549
1344	// Only the first rejected request in a window is logged, so a flood	1550	const rendered = renderWebhookPrompt(task.prompt, request);
1345	// cannot write rows or push the real deliveries out of the log.	1551	// A provider refuses a turn this long, so it is not started. The
1346	const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(receivedAt));	1552	// delivery is not retryable, so the claim is kept. Providers trim
1347	if (slot !== "allowed") {	1553	// the prompt before checking, so whitespace around it is free.
1348	if (slot === "first_rejected") yield* log("rate_limited");	1554	if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
1349	return { _tag: "rate_limited" as const };	1555	yield* log("dispatch_failed", {
1350	}	1556	signatureVerified: signature !== null,
1351	if (!task.enabled) {	1557	missing: rendered.missing,
1352	yield* log("disabled");	1558	renderedPrompt: rendered.prompt,
1353	return { _tag: "disabled" as const };	1559	error: "The filled-in prompt is too long.",
1354	}	1560	});
1355	const signature = task.schedule.signature;	1561	yield* observe("prompt_too_long");
1356	if (signature !== null) {	1562	return { _tag: "accepted" as const, deliveryId, outcome: "prompt_too_long" as const };
1357	const verified =	1563	}
1358	row.webhook_secret !== null &&	1564	// Bound the deliveries one task holds, so steady traffic to a stuck
1359	verifyWebhookSignature({	1565	// task cannot pile up parked fibers. A refused request is not logged,
1360	signature,	1566	// so it cannot push real deliveries out of the log.
1361	secret: row.webhook_secret,	1567	const queueKey = `${task.id}\u0000${task.createdAt}`;
1362	headers: request.headers,	1568	const queued = yield* Ref.modify(webhookQueued, (counts) => {
1363	body: request.body,	1569	const count = counts.get(queueKey) ?? 0;
		1570	return count >= WEBHOOK_MAX_QUEUED_PER_TASK
		1571	? ([false, counts] as const)
		1572	: ([true, new Map(counts).set(queueKey, count + 1)] as const);
1364	});	1573	});
1365	if (!verified) {	1574	if (!queued) {
1366	yield* log("rejected_signature");	1575	// The task is busy with WEBHOOK_MAX_QUEUED_PER_TASK deliveries already.
1367	return { _tag: "rejected_signature" as const };	1576	yield* observe("queue_full");
1368	}	1577	return yield* releaseClaim({
1369	}	1578	_tag: "rate_limited" as const,
1370		1579	outcome: "queue_full" as const,
1371	const rendered = renderWebhookPrompt(task.prompt, request);	1580	});
1372	// Bound the deliveries one task holds, so steady traffic to a stuck	1581	}
1373	// task cannot pile up parked fibers. A refused request is not logged,	1582	// Entries leave the map when their count reaches zero, so a deleted
1374	// so it cannot push real deliveries out of the log.	1583	// task's key does not linger once its last delivery finishes.
1375	const queueKey = `${task.id}\u0000${task.createdAt}`;	1584	const release = Ref.update(webhookQueued, (counts) => {
1376	const queued = yield* Ref.modify(webhookQueued, (counts) => {	1585	const next = new Map(counts);
1377	const count = counts.get(queueKey) ?? 0;	1586	const count = (next.get(queueKey) ?? 1) - 1;
1378	return count >= WEBHOOK_MAX_QUEUED_PER_TASK	1587	if (count <= 0) next.delete(queueKey);
1379	? ([false, counts] as const)	1588	else next.set(queueKey, count);
1380	: ([true, new Map(counts).set(queueKey, count + 1)] as const);	1589	return next;
1381	});	1590	});
1382	if (!queued) return { _tag: "rate_limited" as const };	1591	yield* log("accepted", {
1383	// Entries leave the map when their count reaches zero, so a deleted	1592	signatureVerified: signature !== null,
1384	// task's key does not linger once its last delivery finishes.	1593	missing: rendered.missing,
1385	const release = Ref.update(webhookQueued, (counts) => {	1594	renderedPrompt: rendered.prompt,
1386	const next = new Map(counts);	1595	}).pipe(Effect.onError(() => release));
1387	const count = (next.get(queueKey) ?? 1) - 1;	1596	yield* observe("accepted");
1388	if (count <= 0) next.delete(queueKey);	1597	const permit = yield* webhookPermit(task.id);
1389	else next.set(queueKey, count);	1598	const runOutcome = (outcome: "started" \| "skipped" \| "failed") =>
1390	return next;	1599	Effect.all([
1391	});	1600	Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }),
1392	yield* log("accepted", {	1601	Metrics.increment(Metrics.webhookRunsTotal, { outcome }),
1393	signatureVerified: signature !== null,	1602	]);
1394	missing: rendered.missing,	1603	yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe(
1395	renderedPrompt: rendered.prompt,	1604	Effect.flatMap((completed) =>
1396	}).pipe(Effect.onError(() => release));	1605	completed.lastRunStatus === "failed"
1397	const permit = yield* webhookPermit(task.id);	1606	? runOutcome("failed").pipe(
1398	yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe(	1607	Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1399	Effect.flatMap((completed) =>	1608	)
1400	completed.lastRunStatus === "failed"	1609	: runOutcome("started"),
1401	? markDeliveryFailed(deliveryId, "The run failed to start.")	1610	),
1402	: Effect.void,	1611	Effect.catchTag("WebhookDeliverySkipped", (skipped) =>
1403	),	1612	runOutcome("skipped").pipe(
1404	Effect.catchTag("WebhookDeliverySkipped", (skipped) =>	1613	Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)),
1405	markDeliveryFailed(deliveryId, skipped.reason),	1614	),
1406	),	1615	),
1407	// The log is readable over RPC, so it gets a fixed reason; the	1616	// The log is readable over RPC, so it gets a fixed reason; the
1408	// cause, which can carry request data, stays in the server log.	1617	// cause, which can carry request data, stays in the server log.
1409	Effect.catchCause((cause) =>	1618	Effect.catchCause((cause) =>
1410	Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe(	1619	Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe(
1411	Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),	1620	Effect.andThen(runOutcome("failed")),
1412	),	1621	Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1413	),	1622	),
1414	permit.withPermits(1),	1623	),
1415	Effect.ensuring(release),	1624	permit.withPermits(1),
1416	Effect.forkIn(serviceScope),	1625	// Its own trace: the request that triggered it has already been answered.
		1626	Effect.withSpan("ScheduledTaskService.runWebhookDelivery", {
		1627	root: true,
		1628	attributes: {
		1629	"scheduled_task.id": task.id,
		1630	"scheduled_task.webhook.delivery_id": deliveryId,
		1631	},
		1632	}),
		1633	Effect.ensuring(release),
		1634	Effect.forkIn(serviceScope),
		1635	);
		1636	return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const };
		1637	}),
1417	);	1638	);
1418	return { _tag: "accepted" as const, deliveryId };
1419	});	1639	});
1420		1640
1421	return ScheduledTaskService.of({	1641	return ScheduledTaskService.of({

apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts+308-2

...	@@ -3,10 +3,14 @@ import * as NodeCrypto from "node:crypto";	...	@@ -3,10 +3,14 @@ import * as NodeCrypto from "node:crypto";
3	import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";	3	import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
4	import { assert, it } from "@effect/vitest";	4	import { assert, it } from "@effect/vitest";
5	import { ScheduledTaskUpsertInput } from "@t3tools/contracts";	5	import { ScheduledTaskUpsertInput } from "@t3tools/contracts";
		6	import * as DateTime from "effect/DateTime";
6	import * as Deferred from "effect/Deferred";	7	import * as Deferred from "effect/Deferred";
7	import * as Effect from "effect/Effect";	8	import * as Effect from "effect/Effect";
		9	import * as Fiber from "effect/Fiber";
8	import * as Layer from "effect/Layer";	10	import * as Layer from "effect/Layer";
		11	import * as Metric from "effect/Metric";
9	import * as Queue from "effect/Queue";	12	import * as Queue from "effect/Queue";
		13	import * as EffectScheduler from "effect/Scheduler";
10	import * as Schema from "effect/Schema";	14	import * as Schema from "effect/Schema";
11	import * as TestClock from "effect/testing/TestClock";	15	import * as TestClock from "effect/testing/TestClock";
12		16
...	@@ -63,7 +67,7 @@ const withService = <A, E>(	...	@@ -63,7 +67,7 @@ const withService = <A, E>(
63	readonly service: ScheduledTaskService.ScheduledTaskService["Service"];	67	readonly service: ScheduledTaskService.ScheduledTaskService["Service"];
64	readonly launches: Queue.Queue<LaunchInput>;	68	readonly launches: Queue.Queue<LaunchInput>;
65	}) => Effect.Effect<A, E, never>,	69	}) => Effect.Effect<A, E, never>,
66	options: { readonly gate?: Deferred.Deferred<void> } = {},	70	options: { readonly gate?: Deferred.Deferred<void>; readonly relayHookBaseUrl?: string } = {},
67	) =>	71	) =>
68	Effect.gen(function* () {	72	Effect.gen(function* () {
69	const launches = yield* Queue.unbounded<LaunchInput>();	73	const launches = yield* Queue.unbounded<LaunchInput>();
...	@@ -78,6 +82,10 @@ const withService = <A, E>(	...	@@ -78,6 +82,10 @@ const withService = <A, E>(
78	),	82	),
79	}),	83	}),
80	Layer.mock(ThreadManagementService.ThreadManagementService)({}),	84	Layer.mock(ThreadManagementService.ThreadManagementService)({}),
		85	Layer.succeed(
		86	ScheduledTaskService.ScheduledTaskWebhookOrigin,
		87	Effect.succeed({ relayHookBaseUrl: options.relayHookBaseUrl ?? null }),
		88	),
81	);	89	);
82	return yield* Effect.gen(function* () {	90	return yield* Effect.gen(function* () {
83	const service = yield* ScheduledTaskService.ScheduledTaskService;	91	const service = yield* ScheduledTaskService.ScheduledTaskService;
...	@@ -120,6 +128,35 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>	...	@@ -120,6 +128,35 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
120	),	128	),
121	);	129	);
122		130
		131	it("builds the relay hook URL from the managed tunnel's key, never the environment id", () => {
		132	const relayUrl = "https://relay.example.com/";
		133	assert.equal(
		134	ScheduledTaskService.relayHookBaseUrl({
		135	relayUrl,
		136	tunnelName: "t3coderelay-managedendpoint-dev-julius-0123456789abcdef",
		137	}),
		138	"https://relay.example.com/v1/hooks/0123456789abcdef",
		139	);
		140	for (const tunnelName of [undefined, "t3coderelay-managedendpoint", "x-0123456789ABCDEF"]) {
		141	assert.isNull(ScheduledTaskService.relayHookBaseUrl({ relayUrl, tunnelName }));
		142	}
		143	});
		144
		145	it.effect("gives webhook tasks a relay URL when the environment has a managed tunnel", () =>
		146	withService(
		147	({ service }) =>
		148	Effect.gen(function* () {
		149	const { task } = yield* service.upsert(yield* webhookTaskInput());
		150	const token = task.webhook!.path.split("/").at(-1);
		151	assert.equal(
		152	task.webhook?.url,
		153	`https://relay.example.com/v1/hooks/0123456789abcdef/scheduled-task%3Ahook/${token}`,
		154	);
		155	}),
		156	{ relayHookBaseUrl: "https://relay.example.com/v1/hooks/0123456789abcdef" },
		157	),
		158	);
		159
123	it.effect("answers not found for a wrong token or unknown hook without logging", () =>	160	it.effect("answers not found for a wrong token or unknown hook without logging", () =>
124	withService(({ service }) =>	161	withService(({ service }) =>
125	Effect.gen(function* () {	162	Effect.gen(function* () {
...	@@ -168,6 +205,7 @@ it.effect("checks the configured signature and keeps the secret write-only", ()	...	@@ -168,6 +205,7 @@ it.effect("checks the configured signature and keeps the secret write-only", ()
168	assert.deepEqual(task.schedule, {	205	assert.deepEqual(task.schedule, {
169	type: "webhook",	206	type: "webhook",
170	signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },	207	signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
		208	maxDeliveryAgeMinutes: null,
171	});	209	});
172	assert.isTrue(task.webhook?.hasSecret);	210	assert.isTrue(task.webhook?.hasSecret);
173		211
...	@@ -298,12 +336,13 @@ it.effect("keeps the newest 50 deliveries when they share a timestamp", () =>	...	@@ -298,12 +336,13 @@ it.effect("keeps the newest 50 deliveries when they share a timestamp", () =>
298	),	336	),
299	);	337	);
300		338
301	it.effect("keeps credential headers out of the delivery log", () =>	339	it.effect("keeps credential headers and query values out of the delivery log", () =>
302	withService(({ service }) =>	340	withService(({ service }) =>
303	Effect.gen(function* () {	341	Effect.gen(function* () {
304	const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));	342	const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
305	yield* service.triggerWebhook(	343	yield* service.triggerWebhook(
306	requestFor(task, {	344	requestFor(task, {
		345	query: "page=2&api_key=k",
307	headers: {	346	headers: {
308	"content-type": "application/json",	347	"content-type": "application/json",
309	authorization: "Bearer sender-token",	348	authorization: "Bearer sender-token",
...	@@ -317,6 +356,7 @@ it.effect("keeps credential headers out of the delivery log", () =>	...	@@ -317,6 +356,7 @@ it.effect("keeps credential headers out of the delivery log", () =>
317	id: task.id,	356	id: task.id,
318	deliveryId: summary!.id,	357	deliveryId: summary!.id,
319	});	358	});
		359	assert.equal(delivery.query, "page=2&api_key=[redacted]");
320	assert.equal(delivery.headers.authorization, "[redacted]");	360	assert.equal(delivery.headers.authorization, "[redacted]");
321	assert.equal(delivery.headers["x-webhook-key"], "[redacted]");	361	assert.equal(delivery.headers["x-webhook-key"], "[redacted]");
322	assert.equal(delivery.headers["x-github-event"], "push");	362	assert.equal(delivery.headers["x-github-event"], "push");
...	@@ -454,6 +494,209 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () =>	...	@@ -454,6 +494,209 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () =>
454	),	494	),
455	);	495	);
456		496
		497	it.effect("counts the prompt as the provider does, without surrounding whitespace", () =>
		498	withService(({ service, launches }) =>
		499	Effect.gen(function* () {
		500	const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body.text}}" }));
		501	// Over the limit as sent, within it once the padding is trimmed.
		502	const text = `{"text":"${" ".repeat(1_000)}${"x".repeat(119_990)}${"\\n".repeat(1_000)}"}`;
		503	const result = yield* service.triggerWebhook(
		504	requestFor(task, { body: new TextEncoder().encode(text), bodyText: text }),
		505	);
		506	assert.equal(result._tag, "accepted");
		507	yield* Queue.take(launches);
		508	}),
		509	),
		510	);
		511
		512	it.effect("does not start a run when the filled-in prompt is too long", () =>
		513	withService(({ service, launches }) =>
		514	Effect.gen(function* () {
		515	const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body}}" }));
		516	const text = "x".repeat(200_000);
		517	const body = new TextEncoder().encode(text);
		518	const result = yield* service.triggerWebhook(requestFor(task, { body, bodyText: text }));
		519	assert.equal(result._tag, "accepted");
		520	assert.equal(yield* Queue.size(launches), 0);
		521	const { delivery } = yield* service.getWebhookDelivery({
		522	id: task.id,
		523	deliveryId: result._tag === "accepted" ? result.deliveryId : ("" as never),
		524	});
		525	assert.equal(delivery.outcome, "dispatch_failed");
		526	assert.equal(delivery.error, "The filled-in prompt is too long.");
		527	assert.equal(delivery.renderedPrompt?.length, 64 * 1024);
		528	// The queue slot was never taken: a normal delivery still runs.
		529	const ok = yield* service.triggerWebhook(requestFor(task));
		530	assert.equal(ok._tag, "accepted");
		531	yield* Queue.take(launches);
		532	}),
		533	),
		534	);
		535
		536	it.effect("a held request already delivered directly runs only once", () =>
		537	withService(({ service, launches }) =>
		538	Effect.gen(function* () {
		539	const { task } = yield* service.upsert(yield* webhookTaskInput());
		540	const direct = yield* service.triggerWebhook(
		541	requestFor(task, { relayDeliveryId: "relay-1" }),
		542	);
		543	const replayed = yield* service.triggerWebhook(
		544	requestFor(task, { relayDeliveryId: "relay-1", receivedAt: "2026-10-04T10:00:00.000Z" }),
		545	);
		546	assert.equal(direct._tag, "accepted");
		547	// Same delivery, answered the same way, but recorded as a duplicate.
		548	assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed);
		549	yield* Queue.take(launches);
		550	const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
		551	assert.equal(logged.length, 1);
		552	}),
		553	),
		554	);
		555
		556	it.effect("a held request whose sender hung up mid-request still runs", () =>
		557	withService(({ service, launches }) =>
		558	Effect.gen(function* () {
		559	const { task } = yield* service.upsert(yield* webhookTaskInput());
		560	// The relay times out and the request fiber is interrupted. A small
		561	// operation budget makes the request yield often, so stepping the
		562	// interrupt one yield later each time lands it at every point in the
		563	// request (one takes about 40 yields), then the relay retries.
		564	for (let step = 0; step < 60; step++) {
		565	const request = requestFor(task, { relayDeliveryId: `hung-up-${step}` });
		566	const fiber = yield* service
		567	.triggerWebhook(request)
		568	.pipe(Effect.provideService(EffectScheduler.MaxOpsBeforeYield, 8), Effect.forkChild);
		569	for (let yields = 0; yields < step; yields++) yield* Effect.yieldNow;
		570	yield* Fiber.interrupt(fiber);
		571	const retried = yield* service.triggerWebhook(request);
		572	assert.equal(retried._tag, "accepted");
		573	const deliveryId = retried._tag === "accepted" ? retried.deliveryId : undefined;
		574	// Logged and run exactly once, whether or not the first attempt got through.
		575	yield* service.getWebhookDelivery({ id: task.id, deliveryId: deliveryId! });
		576	const launch = yield* Queue.take(launches);
		577	assert.include(launch.commandId, `hung-up-${step}`);
		578	// Keep each step in a fresh rate-limit window.
		579	yield* TestClock.adjust("61 seconds");
		580	}
		581	assert.equal(yield* Queue.size(launches), 0);
		582	}),
		583	),
		584	);
		585
		586	it.effect("a held request runs once even after the log has trimmed it", () =>
		587	withService(({ service, launches }) =>
		588	Effect.gen(function* () {
		589	const { task } = yield* service.upsert(yield* webhookTaskInput());
		590	const first = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
		591	assert.equal(first._tag, "accepted");
		592	yield* Queue.take(launches);
		593	// Push the original row out of the 50-row delivery log.
		594	const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
		595	yield* Effect.forEach(Array.from({ length: 55 }), () =>
		596	service.triggerWebhook(requestFor(paused.task)),
		597	);
		598	yield* service.upsert(yield* webhookTaskInput());
		599	const replay = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
		600	assert.equal(replay._tag, "accepted");
		601	assert.equal(yield* Queue.size(launches), 0);
		602	}),
		603	),
		604	);
		605
		606	it.effect("a rate-limited held request can run on a later pass", () =>
		607	withService(({ service, launches }) =>
		608	Effect.gen(function* () {
		609	const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
		610	// Spend the task's 60-a-minute budget.
		611	yield* Effect.forEach(Array.from({ length: 60 }), () =>
		612	service.triggerWebhook(requestFor(task)),
		613	);
		614	const limited = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" }));
		615	assert.equal(limited._tag, "rate_limited");
		616	yield* service.upsert(yield* webhookTaskInput());
		617	yield* TestClock.adjust("61 seconds");
		618	const retried = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" }));
		619	assert.equal(retried._tag, "accepted");
		620	yield* Queue.take(launches);
		621	}),
		622	),
		623	);
		624
		625	it.effect("logs a held request at the time the relay received it", () =>
		626	withService(({ service }) =>
		627	Effect.gen(function* () {
		628	const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
		629	const receivedAt = DateTime.formatIso(DateTime.subtract(yield* DateTime.now, { minutes: 5 }));
		630	yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "relay-2", receivedAt }));
		631	const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
		632	assert.equal(delivery?.receivedAt, receivedAt);
		633	}),
		634	),
		635	);
		636
		637	it.effect("a receive time in the future counts as now", () =>
		638	withService(({ service, launches }) =>
		639	Effect.gen(function* () {
		640	const { task } = yield* service.upsert(
		641	yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }),
		642	);
		643	const now = yield* DateTime.now;
		644	const result = yield* service.triggerWebhook(
		645	requestFor(task, {
		646	relayDeliveryId: "future",
		647	receivedAt: DateTime.formatIso(DateTime.add(now, { days: 365 })),
		648	}),
		649	);
		650	assert.equal(result._tag, "accepted");
		651	yield* Queue.take(launches);
		652	const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
		653	assert.equal(delivery?.receivedAt, DateTime.formatIso(now));
		654	}),
		655	),
		656	);
		657
		658	it.effect("skips a held request older than the task's max age", () =>
		659	withService(({ service, launches }) =>
		660	Effect.gen(function* () {
		661	const { task } = yield* service.upsert(
		662	yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }),
		663	);
		664	const now = yield* DateTime.now;
		665	const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
		666	const fresh = DateTime.formatIso(DateTime.subtract(now, { minutes: 5 }));
		667	const tooOld = yield* service.triggerWebhook(
		668	requestFor(task, { relayDeliveryId: "old", receivedAt: old }),
		669	);
		670	assert.equal(tooOld._tag, "expired");
		671	assert.equal(yield* Queue.size(launches), 0);
		672	const ok = yield* service.triggerWebhook(
		673	requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }),
		674	);
		675	assert.equal(ok._tag, "accepted");
		676	const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
		677	(delivery) => delivery.outcome,
		678	);
		679	assert.includeMembers(outcomes, ["expired", "accepted"]);
		680	}),
		681	),
		682	);
		683
		684	it.effect("runs a held request of any age when no max age is set", () =>
		685	withService(({ service }) =>
		686	Effect.gen(function* () {
		687	const { task } = yield* service.upsert(yield* webhookTaskInput());
		688	const now = yield* DateTime.now;
		689	const result = yield* service.triggerWebhook(
		690	requestFor(task, {
		691	relayDeliveryId: "ancient",
		692	receivedAt: DateTime.formatIso(DateTime.subtract(now, { hours: 23 })),
		693	}),
		694	);
		695	assert.equal(result._tag, "accepted");
		696	}),
		697	),
		698	);
		699
457	it.effect("deleting a task removes its delivery log", () =>	700	it.effect("deleting a task removes its delivery log", () =>
458	withService(({ service }) =>	701	withService(({ service }) =>
459	Effect.gen(function* () {	702	Effect.gen(function* () {
...	@@ -464,3 +707,66 @@ it.effect("deleting a task removes its delivery log", () =>	...	@@ -464,3 +707,66 @@ it.effect("deleting a task removes its delivery log", () =>
464	}),	707	}),
465	),	708	),
466	);	709	);
		710
		711	const signatureFor = (secret: string) =>
		712	`sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;
		713
		714	/** Count recorded by `t3_webhook_deliveries_total` for one outcome and source. */
		715	const deliveriesCounted = (outcome: string, source: "relay" \| "direct") =>
		716	Metric.snapshot.pipe(
		717	Effect.map((snapshots) => {
		718	const found = snapshots.find(
		719	(snapshot) =>
		720	snapshot.id === "t3_webhook_deliveries_total" &&
		721	snapshot.attributes?.outcome === outcome &&
		722	snapshot.attributes?.source === source,
		723	);
		724	return found?.type === "Counter" ? Number(found.state.count) : 0;
		725	}),
		726	);
		727
		728	it.effect("counts each handled request by what happened to it", () =>
		729	withService(({ service, launches }) =>
		730	Effect.gen(function* () {
		731	const { task } = yield* service.upsert(
		732	yield* webhookTaskInput({
		733	schedule: {
		734	type: "webhook",
		735	signature: {
		736	header: "x-hub-signature-256",
		737	encoding: "hex",
		738	prefix: "sha256=",
		739	secret: "github-secret",
		740	},
		741	},
		742	}),
		743	);
		744	const before = {
		745	accepted: yield* deliveriesCounted("accepted", "direct"),
		746	rejected: yield* deliveriesCounted("rejected_signature", "direct"),
		747	notFound: yield* deliveriesCounted("not_found", "direct"),
		748	relayAccepted: yield* deliveriesCounted("accepted", "relay"),
		749	duplicate: yield* deliveriesCounted("duplicate", "relay"),
		750	};
		751	const signed = {
		752	"content-type": "application/json",
		753	"x-hub-signature-256": signatureFor("github-secret"),
		754	};
		755	yield* service.triggerWebhook(requestFor(task, { headers: signed }));
		756	yield* Queue.take(launches);
		757	yield* service.triggerWebhook(requestFor(task));
		758	yield* service.triggerWebhook(requestFor(task, { token: "wrong" }));
		759	// A request the relay held, then the same request again.
		760	const relayed = requestFor(task, { headers: signed, relayDeliveryId: "relay-1" });
		761	yield* service.triggerWebhook(relayed);
		762	yield* Queue.take(launches);
		763	yield* service.triggerWebhook(relayed);
		764
		765	assert.equal((yield* deliveriesCounted("accepted", "direct")) - before.accepted, 1);
		766	assert.equal((yield* deliveriesCounted("rejected_signature", "direct")) - before.rejected, 1);
		767	assert.equal((yield* deliveriesCounted("not_found", "direct")) - before.notFound, 1);
		768	assert.equal((yield* deliveriesCounted("accepted", "relay")) - before.relayAccepted, 1);
		769	assert.equal((yield* deliveriesCounted("duplicate", "relay")) - before.duplicate, 1);
		770	}),
		771	),
		772	);

apps/server/src/scheduledTasks/relayDeliveryProof.ts created+74

...	@@ -0,0 +1,74 @@
1	/**
2	* Whether a webhook request really came through this environment's relay.
3	*
4	* The relay's delivery id, receive time and trace context arrive as plain
5	* headers, and the webhook URL can also be called directly, so they are only
6	* trusted alongside a proof the relay signed with its mint key. A request
7	* without a valid proof is handled as a direct request.
8	*/
9	import { RelayHookDeliveryProofPayload } from "@t3tools/contracts/relay";
10	import {
11	normalizeRelayIssuer,
12	RELAY_HOOK_DELIVERY_HEADER,
13	RELAY_HOOK_DELIVERY_TYP,
14	verifyRelayJwt,
15	} from "@t3tools/shared/relayJwt";
16	import * as Clock from "effect/Clock";
17	import * as Effect from "effect/Effect";
18	import * as Option from "effect/Option";
19	import * as Schema from "effect/Schema";
20
21	import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
22	import { CLOUD_MINT_PUBLIC_KEY, RELAY_ISSUER_SECRET, RELAY_URL_SECRET } from "../cloud/config.ts";
23	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
24
25	/** Covers the relay's 24-hour hold plus a margin. */
26	const MAX_PROOF_AGE_SECONDS = 25 * 60 * 60;
27
28	const decodePayload = Schema.decodeUnknownOption(RelayHookDeliveryProofPayload);
29	const text = (bytes: Option.Option<Uint8Array>) =>
30	Option.map(bytes, (value) => new TextDecoder().decode(value));
31
32	/** The relay's own delivery id and receive time, when the request proves it came from the relay. */
33	export const makeRelayDeliveryVerifier = Effect.gen(function* () {
34	const secrets = yield* ServerSecretStore.ServerSecretStore;
35	const environment = yield* ServerEnvironment.ServerEnvironment;
36	return (input: { readonly headers: Readonly<Record<string, string>>; readonly hookId: string }) =>
37	Effect.gen(function* () {
38	const proof = input.headers[RELAY_HOOK_DELIVERY_HEADER];
39	const deliveryId = input.headers["x-t3-relay-delivery-id"];
40	const receivedAt = input.headers["x-t3-relay-received-at"];
41	if (proof === undefined \|\| deliveryId === undefined \|\| receivedAt === undefined) {
42	return Option.none();
43	}
44	const publicKey = text(yield* secrets.get(CLOUD_MINT_PUBLIC_KEY));
45	const issuer = Option.orElse(text(yield* secrets.get(RELAY_ISSUER_SECRET)), () =>
46	Option.none<string>(),
47	);
48	const relayUrl = text(yield* secrets.get(RELAY_URL_SECRET));
49	const relayIssuer = Option.isSome(issuer) ? issuer : relayUrl;
50	// Not linked to T3 Connect: no relay can be delivering to us.
51	if (Option.isNone(publicKey) \|\| Option.isNone(relayIssuer)) return Option.none();
52	const environmentId = yield* environment.getEnvironmentId;
53	const payload = yield* verifyRelayJwt({
54	publicKey: publicKey.value,
55	token: proof,
56	typ: RELAY_HOOK_DELIVERY_TYP,
57	issuer: normalizeRelayIssuer(relayIssuer.value),
58	audience: `t3-env:${environmentId}`,
59	nowEpochSeconds: Math.floor((yield* Clock.currentTimeMillis) / 1_000),
60	maxTokenAge: MAX_PROOF_AGE_SECONDS,
61	}).pipe(Effect.map(decodePayload), Effect.orElseSucceed(Option.none));
62	// The proof must name exactly this delivery, so it cannot be lifted onto another one.
63	if (
64	Option.isNone(payload) \|\|
65	payload.value.environmentId !== environmentId \|\|
66	payload.value.deliveryId !== deliveryId \|\|
67	payload.value.receivedAt !== receivedAt \|\|
68	payload.value.hookId !== input.hookId
69	) {
70	return Option.none();
71	}
72	return Option.some({ deliveryId, receivedAt });
73	}).pipe(Effect.orElseSucceed(Option.none), Effect.withSpan("webhook.verifyRelayDelivery"));
74	});

apps/server/src/scheduledTasks/webhookRoute.test.ts+204-20

...	@@ -1,6 +1,9 @@	...	@@ -1,6 +1,9 @@
1	import { describe, expect, it } from "@effect/vitest";	1	import { describe, expect, it } from "@effect/vitest";
		2	import * as NodeCrypto from "node:crypto";
		3	import * as Clock from "effect/Clock";
2	import * as Effect from "effect/Effect";	4	import * as Effect from "effect/Effect";
3	import * as Layer from "effect/Layer";	5	import * as Layer from "effect/Layer";
		6	import * as Option from "effect/Option";
4	import * as NodeServices from "@effect/platform-node/NodeServices";	7	import * as NodeServices from "@effect/platform-node/NodeServices";
5	import * as Etag from "effect/http/Etag";	8	import * as Etag from "effect/http/Etag";
6	import * as HttpPlatform from "effect/http/HttpPlatform";	9	import * as HttpPlatform from "effect/http/HttpPlatform";
...	@@ -8,8 +11,10 @@ import * as HttpRouter from "effect/http/HttpRouter";	...	@@ -8,8 +11,10 @@ import * as HttpRouter from "effect/http/HttpRouter";
8	import * as HttpApi from "effect/http-api/HttpApi";	11	import * as HttpApi from "effect/http-api/HttpApi";
9	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";	12	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
10		13
		14	import { RELAY_HOOK_DELIVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt";
11	import {	15	import {
12	EnvironmentHttpApi,	16	EnvironmentHttpApi,
		17	EnvironmentId,
13	ScheduledTaskWebhookDeliveryId,	18	ScheduledTaskWebhookDeliveryId,
14	ScheduledTaskError,	19	ScheduledTaskError,
15	} from "@t3tools/contracts";	20	} from "@t3tools/contracts";
...	@@ -18,19 +23,76 @@ import {	...	@@ -18,19 +23,76 @@ import {
18	type WebhookTriggerRequest,	23	type WebhookTriggerRequest,
19	type WebhookTriggerResult,	24	type WebhookTriggerResult,
20	} from "./ScheduledTaskService.ts";	25	} from "./ScheduledTaskService.ts";
		26	import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
		27	import { CLOUD_MINT_PUBLIC_KEY, RELAY_ISSUER_SECRET } from "../cloud/config.ts";
		28	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
21	import { WEBHOOK_MAX_BODY_BYTES, webhookHttpApiLayer } from "./webhookRoute.ts";	29	import { WEBHOOK_MAX_BODY_BYTES, webhookHttpApiLayer } from "./webhookRoute.ts";
22		30
23	class WebhookTestApi extends HttpApi.make("environment").add(EnvironmentHttpApi.groups.webhooks) {}	31	class WebhookTestApi extends HttpApi.make("environment").add(EnvironmentHttpApi.groups.webhooks) {}
24		32
		33	const environmentId = EnvironmentId.make("env-1");
		34	const relayIssuer = "https://relay.example.test";
		35	const mintKeys = NodeCrypto.generateKeyPairSync("ed25519", {
		36	privateKeyEncoding: { format: "pem", type: "pkcs8" },
		37	publicKeyEncoding: { format: "pem", type: "spki" },
		38	});
		39	/** The secrets a T3 Connect-linked environment holds, keyed by name. */
		40	const linkedSecrets: ReadonlyMap<string, string> = new Map([
		41	[CLOUD_MINT_PUBLIC_KEY, mintKeys.publicKey],
		42	[RELAY_ISSUER_SECRET, relayIssuer],
		43	]);
		44
		45	/** A proof as the relay signs it for one delivery. */
		46	const relayProof = Effect.fn("relayProof")(function* (claims: {
		47	readonly deliveryId: string;
		48	readonly receivedAt: string;
		49	readonly hookId: string;
		50	readonly privateKey?: string;
		51	}) {
		52	const now = Math.floor((yield* Clock.currentTimeMillis) / 1_000);
		53	return yield* signRelayJwt({
		54	privateKey: claims.privateKey ?? mintKeys.privateKey,
		55	typ: RELAY_HOOK_DELIVERY_TYP,
		56	payload: {
		57	iss: relayIssuer,
		58	aud: `t3-env:${environmentId}`,
		59	sub: environmentId,
		60	jti: `proof:${claims.deliveryId}`,
		61	iat: now,
		62	exp: now + 3_600,
		63	environmentId,
		64	deliveryId: claims.deliveryId,
		65	receivedAt: claims.receivedAt,
		66	hookId: claims.hookId,
		67	},
		68	});
		69	});
		70
25	const handlerFor = (	71	const handlerFor = (
26	trigger: (	72	trigger: (
27	request: WebhookTriggerRequest,	73	request: WebhookTriggerRequest,
28	) => Effect.Effect<WebhookTriggerResult, ScheduledTaskError>,	74	) => Effect.Effect<WebhookTriggerResult, ScheduledTaskError>,
		75	secrets: ReadonlyMap<string, string> = linkedSecrets,
29	) =>	76	) =>
30	HttpRouter.toWebHandler(	77	HttpRouter.toWebHandler(
31	HttpApiBuilder.layer(WebhookTestApi).pipe(	78	HttpApiBuilder.layer(WebhookTestApi).pipe(
32	Layer.provide(webhookHttpApiLayer),	79	Layer.provide(webhookHttpApiLayer),
33	Layer.provide(Layer.mock(ScheduledTaskService)({ triggerWebhook: trigger })),	80	Layer.provide(Layer.mock(ScheduledTaskService)({ triggerWebhook: trigger })),
		81	Layer.provide(
		82	Layer.mock(ServerSecretStore.ServerSecretStore)({
		83	get: (name) =>
		84	Effect.succeed(
		85	Option.map(Option.fromUndefinedOr(secrets.get(name)), (value) =>
		86	new TextEncoder().encode(value),
		87	),
		88	),
		89	}),
		90	),
		91	Layer.provide(
		92	Layer.mock(ServerEnvironment.ServerEnvironment)({
		93	getEnvironmentId: Effect.succeed(environmentId),
		94	}),
		95	),
34	Layer.provide(	96	Layer.provide(
35	HttpPlatform.layer.pipe(	97	HttpPlatform.layer.pipe(
36	Layer.provideMerge(NodeServices.layer),	98	Layer.provideMerge(NodeServices.layer),
...	@@ -56,6 +118,7 @@ describe("webhook route", () => {	...	@@ -56,6 +118,7 @@ describe("webhook route", () => {
56	return Effect.succeed({	118	return Effect.succeed({
57	_tag: "accepted",	119	_tag: "accepted",
58	deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"),	120	deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"),
		121	outcome: "accepted",
59	});	122	});
60	});	123	});
61	try {	124	try {
...	@@ -77,25 +140,142 @@ describe("webhook route", () => {	...	@@ -77,25 +140,142 @@ describe("webhook route", () => {
77	}	140	}
78	});	141	});
79		142
80	it("maps service outcomes to status codes", async () => {	143	// Live clock: the server checks proofs against real time.
81	const cases: ReadonlyArray<[WebhookTriggerResult["_tag"], number]> = [	144	it.live("trusts the relay's delivery id and receive time only with its signed proof", () =>
82	["not_found", 404],	145	Effect.gen(function* () {
83	["rejected_signature", 401],	146	const received: Array<WebhookTriggerRequest> = [];
84	["disabled", 409],	147	const { handler, dispose } = handlerFor((request) => {
85	["rate_limited", 429],	148	received.push(request);
86	];	149	return Effect.succeed({
87	for (const [tag, status] of cases) {	150	_tag: "accepted",
88	const { handler, dispose } = handlerFor(() =>	151	deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:1"),
89	Effect.succeed({ _tag: tag } as WebhookTriggerResult),	152	outcome: "accepted",
		153	});
		154	});
		155	const send = (path: string, headers: Record<string, string>) =>
		156	Effect.promise(() => handler(post(path, "{}", headers)));
		157	const receivedAt = "2026-10-04T10:00:00.000Z";
		158	const relayHeaders = {
		159	"x-t3-relay-delivery-id": "relay-1",
		160	"x-t3-relay-received-at": receivedAt,
		161	};
		162	const forged = NodeCrypto.generateKeyPairSync("ed25519", {
		163	privateKeyEncoding: { format: "pem", type: "pkcs8" },
		164	publicKeyEncoding: { format: "pem", type: "spki" },
		165	});
		166	const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" });
		167	const forgedProof = yield* relayProof({
		168	deliveryId: "relay-1",
		169	receivedAt,
		170	hookId: "id",
		171	privateKey: forged.privateKey,
		172	});
		173	// 0: signed by the relay for exactly this delivery.
		174	yield* send("/api/hooks/id/tok", { ...relayHeaders, "x-t3-relay-delivery": proof });
		175	// 1: a direct caller claiming to be the relay, without a proof.
		176	yield* send("/api/hooks/id/tok", relayHeaders);
		177	// 2: a proof signed with any other key.
		178	yield* send("/api/hooks/id/tok", { ...relayHeaders, "x-t3-relay-delivery": forgedProof });
		179	// 3: a real proof lifted onto another delivery id.
		180	yield* send("/api/hooks/id/tok", {
		181	...relayHeaders,
		182	"x-t3-relay-delivery-id": "relay-2",
		183	"x-t3-relay-delivery": proof,
		184	});
		185	// 4: a real proof for another hook.
		186	yield* send("/api/hooks/other/tok", { ...relayHeaders, "x-t3-relay-delivery": proof });
		187	yield* Effect.promise(() => dispose());
		188	expect(received.map((request) => request.relayDeliveryId)).toEqual([
		189	"relay-1",
		190	undefined,
		191	undefined,
		192	undefined,
		193	undefined,
		194	]);
		195	expect(received[0]?.receivedAt).toBe(receivedAt);
		196	expect(received.slice(1).every((request) => request.receivedAt === undefined)).toBe(true);
		197	}),
		198	);
		199
		200	// Live clock: the server checks proofs against real time.
		201	it.live("trusts no relay headers on an environment not linked to T3 Connect", () =>
		202	Effect.gen(function* () {
		203	const received: Array<WebhookTriggerRequest> = [];
		204	const { handler, dispose } = handlerFor((request) => {
		205	received.push(request);
		206	return Effect.succeed({ _tag: "not_found" });
		207	}, new Map());
		208	const receivedAt = "2026-10-04T10:00:00.000Z";
		209	const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" });
		210	yield* Effect.promise(() =>
		211	handler(
		212	post("/api/hooks/id/tok", "{}", {
		213	"x-t3-relay-delivery-id": "relay-1",
		214	"x-t3-relay-received-at": receivedAt,
		215	"x-t3-relay-delivery": proof,
		216	}),
		217	),
90	);	218	);
		219	yield* Effect.promise(() => dispose());
		220	expect(received[0]?.relayDeliveryId).toBeUndefined();
		221	}),
		222	);
		223
		224	it("maps service outcomes to status codes and names each outcome for the relay", async () => {
		225	const deliveryId = ScheduledTaskWebhookDeliveryId.make("delivery:1");
		226	const cases: ReadonlyArray<[WebhookTriggerResult, number, string]> = [
		227	[{ _tag: "accepted", deliveryId, outcome: "accepted" }, 202, "accepted"],
		228	// Same status as a started run; only the header tells them apart.
		229	[{ _tag: "accepted", deliveryId, outcome: "duplicate" }, 202, "duplicate"],
		230	[{ _tag: "accepted", deliveryId, outcome: "prompt_too_long" }, 202, "prompt_too_long"],
		231	[{ _tag: "not_found" }, 404, "not_found"],
		232	[{ _tag: "rejected_signature" }, 401, "rejected_signature"],
		233	[{ _tag: "disabled" }, 409, "disabled"],
		234	[{ _tag: "rate_limited", outcome: "rate_limited" }, 429, "rate_limited"],
		235	[{ _tag: "rate_limited", outcome: "queue_full" }, 429, "queue_full"],
		236	[{ _tag: "expired" }, 410, "expired"],
		237	];
		238	for (const [result, status, outcome] of cases) {
		239	const { handler, dispose } = handlerFor(() => Effect.succeed(result));
91	try {	240	try {
92	expect((await handler(post("/api/hooks/id/tok", "{}"))).status).toBe(status);	241	const response = await handler(post("/api/hooks/id/tok", "{}"));
		242	expect(response.status).toBe(status);
		243	expect(response.headers.get("x-t3-hook-outcome")).toBe(outcome);
93	} finally {	244	} finally {
94	await dispose();	245	await dispose();
95	}	246	}
96	}	247	}
97	});	248	});
98		249
		250	// Live clock: the server checks proofs against real time.
		251	it.live("joins the relay's trace only for requests the relay forwarded", () =>
		252	Effect.gen(function* () {
		253	const parents: Array<string \| undefined> = [];
		254	const { handler, dispose } = handlerFor(() =>
		255	Effect.gen(function* () {
		256	const span = yield* Effect.currentParentSpan.pipe(Effect.option);
		257	parents.push(span._tag === "Some" ? span.value.traceId : undefined);
		258	return { _tag: "not_found" } as const;
		259	}),
		260	);
		261	const send = (headers: Record<string, string>) =>
		262	Effect.promise(() => handler(post("/api/hooks/id/tok", "{}", headers)));
		263	const traceparent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01";
		264	const receivedAt = "2026-10-04T10:00:00.000Z";
		265	const relayHeaders = {
		266	"x-t3-relay-delivery-id": "relay-1",
		267	"x-t3-relay-received-at": receivedAt,
		268	};
		269	const proof = yield* relayProof({ deliveryId: "relay-1", receivedAt, hookId: "id" });
		270	yield* send({ ...relayHeaders, traceparent, "x-t3-relay-delivery": proof });
		271	// Claiming to be the relay without its proof cannot attach to our traces.
		272	yield* send({ ...relayHeaders, traceparent });
		273	yield* send({ traceparent });
		274	yield* Effect.promise(() => dispose());
		275	expect(parents).toEqual(["0af7651916cd43dd8448eb211c80319c", undefined, undefined]);
		276	}),
		277	);
		278
99	it("rejects oversized bodies and malformed paths before reaching the service", async () => {	279	it("rejects oversized bodies and malformed paths before reaching the service", async () => {
100	let calls = 0;	280	let calls = 0;
101	const { handler, dispose } = handlerFor(() => {	281	const { handler, dispose } = handlerFor(() => {
...	@@ -132,16 +312,20 @@ describe("webhook route", () => {	...	@@ -132,16 +312,20 @@ describe("webhook route", () => {
132	}	312	}
133	});	313	});
134		314
135	it("hides service failures behind a 500", async () => {	315	it("hides service failures and defects behind a fixed 500", async () => {
136	const { handler, dispose } = handlerFor(() =>	316	const failures = [
137	Effect.fail(new ScheduledTaskError({ message: "database locked" })),	317	Effect.fail(new ScheduledTaskError({ message: "database locked" })),
138	);	318	Effect.die(new Error("database exploded")),
139	try {	319	];
140	const response = await handler(post("/api/hooks/id/tok", "{}"));	320	for (const failure of failures) {
141	expect(response.status).toBe(500);	321	const { handler, dispose } = handlerFor(() => failure);
142	expect(await response.text()).not.toContain("database");	322	try {
143	} finally {	323	const response = await handler(post("/api/hooks/id/tok", "{}"));
144	await dispose();	324	expect(response.status).toBe(500);
		325	expect(await response.json()).toEqual({ error: "internal_error" });
		326	} finally {
		327	await dispose();
		328	}
145	}	329	}
146	});	330	});
147	});	331	});

apps/server/src/scheduledTasks/webhookRoute.ts+56-15

 @@ -2,27 +2,39 @@ import { EnvironmentHttpApi } from "@t3tools/contracts";
 import * as ByteSize from "effect/ByteSize";
 import * as Effect from "effect/Effect";
 import * as Option from "effect/Option";
+import * as Tracer from "effect/Tracer";
 import * as HttpIncomingMessage from "effect/http/HttpIncomingMessage";
 import type * as HttpServerRequest from "effect/http/HttpServerRequest";
 import * as HttpServerResponse from "effect/http/HttpServerResponse";
+import * as HttpTraceContext from "effect/http/HttpTraceContext";
+import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
 import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
+import * as Metrics from "../observability/Metrics.ts";
+import { makeRelayDeliveryVerifier } from "./relayDeliveryProof.ts";
 import * as ScheduledTaskService from "./ScheduledTaskService.ts";
 /** Largest request body a webhook accepts. The relay enforces the same cap. */
 export const WEBHOOK_MAX_BODY_BYTES = 1024 * 1024;
-const json = (status: number, body: Record<string, string>) =>
+/** Response header naming what happened to the request; the relay records it. */
-  HttpServerResponse.jsonUnsafe(body, { status });
+const WEBHOOK_OUTCOME_HEADER = "x-t3-hook-outcome";
+const json = (status: number, body: Record<string, string>, outcome: string) =>
+  HttpServerResponse.jsonUnsafe(body, { status, headers: { [WEBHOOK_OUTCOME_HEADER]: outcome } });
 /**
  * Handles `/api/hooks/:hookId/:token` for every accepted method. The endpoint
  * is raw so the signature is checked over the exact body bytes; the service
  * checks the token and signature. It is reachable directly, over the managed
- * tunnel, or through the relay's stable `/v1/hooks/...` URL.
+ * tunnel, or through the relay's stable `/v1/hooks/:endpointKey/:hookId/:token`
+ * URL, where the endpoint key is this environment's managed tunnel key.
  */
 const handleWebhook =
-  (scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"]) =>
+  (
+    scheduledTasks: ScheduledTaskService.ScheduledTaskService["Service"],
+    verifyRelayDelivery: Effect.Success<typeof makeRelayDeliveryVerifier>,
+  ) =>
   ({
     params,
     request,
 @@ -33,7 +45,7 @@ const handleWebhook =
     Effect.gen(function* () {
       const contentLength = Number(request.headers["content-length"] ?? "0");
       if (!Number.isFinite(contentLength) || contentLength > WEBHOOK_MAX_BODY_BYTES) {
-        return json(413, { error: "body_too_large" });
+        return json(413, { error: "body_too_large" }, "body_too_large");
       }
       // Chunked requests carry no content-length, so the reader itself is capped.
       const body = yield* request.arrayBuffer.pipe(
 @@ -44,9 +56,15 @@ const handleWebhook =
         ),
         Effect.option,
       );
-      if (Option.isNone(body)) return json(413, { error: "body_too_large_or_unreadable" });
+      // Refused before a task is looked up, so the service never sees them.
+      const tooLarge = (error: string) =>
+        Metrics.increment(Metrics.webhookDeliveriesTotal, {
+          outcome: "body_too_large",
+          source: request.headers["x-t3-relay-delivery-id"] ? "relay" : "direct",
+        }).pipe(Effect.as(json(413, { error }, "body_too_large")));
+      if (Option.isNone(body)) return yield* tooLarge("body_too_large_or_unreadable");
       if (body.value.byteLength > WEBHOOK_MAX_BODY_BYTES) {
-        return json(413, { error: "body_too_large" });
+        return yield* tooLarge("body_too_large");
       }
       const headers: Record<string, string> = {};
 @@ -54,7 +72,18 @@ const handleWebhook =
         if (typeof value === "string") headers[name.toLowerCase()] = value;
       }
       const queryIndex = request.url.indexOf("?");
+      // The relay's delivery id and receive time count only with its signed
+      // proof; the URL can also be called directly. The receive time matters
+      // for requests the relay held while we were offline.
+      const relay = yield* verifyRelayDelivery({ headers, hookId: params.hookId });
+      const relayDeliveryId = Option.isSome(relay) ? relay.value.deliveryId : undefined;
+      const relayReceivedAt = Option.isSome(relay) ? relay.value.receivedAt : undefined;
+      // A relay delivery joins the relay's trace, and goes to the T3 Connect
+      // tracer with it. Anyone else's traceparent is never trusted.
+      const relayParent = Option.isSome(relay)
+        ? HttpTraceContext.fromHeaders(request.headers)
+        : Option.none<Tracer.ExternalSpan>();
       const result = yield* scheduledTasks
         .triggerWebhook({
           hookId: params.hookId,
 @@ -65,9 +94,16 @@ const handleWebhook =
           headers,
           body: body.value,
           bodyText: new TextDecoder().decode(body.value),
+          ...(relayDeliveryId ? { relayDeliveryId } : {}),
+          ...(relayReceivedAt ? { receivedAt: relayReceivedAt } : {}),
         })
         .pipe(
-          Effect.catch((cause) =>
+          Option.isSome(relayParent)
+            ? (effect) =>
+                effect.pipe(Effect.withParentSpan(relayParent.value), withRelayClientTracing)
+            : (effect) => effect,
+          // Defects too, so the sender only ever sees the fixed error body.
+          Effect.catchCause((cause) =>
             Effect.logWarning("Webhook delivery failed").pipe(
               Effect.annotateLogs({ hookId: params.hookId }),
               Effect.andThen(Effect.logDebug("Webhook delivery failure cause", { cause })),
 @@ -78,17 +114,19 @@ const handleWebhook =
       switch (result._tag) {
         case "accepted":
-          return json(202, { deliveryId: result.deliveryId });
+          return json(202, { deliveryId: result.deliveryId }, result.outcome);
         case "not_found":
-          return json(404, { error: "hook_not_found" });
+          return json(404, { error: "hook_not_found" }, "not_found");
         case "rejected_signature":
-          return json(401, { error: "invalid_signature" });
+          return json(401, { error: "invalid_signature" }, "rejected_signature");
         case "disabled":
-          return json(409, { error: "hook_disabled" });
+          return json(409, { error: "hook_disabled" }, "disabled");
         case "rate_limited":
-          return json(429, { error: "rate_limited" });
+          return json(429, { error: "rate_limited" }, result.outcome);
+        case "expired":
+          return json(410, { error: "delivery_too_old" }, "expired");
         case "error":
-          return json(500, { error: "internal_error" });
+          return json(500, { error: "internal_error" }, "error");
       }
     });
 @@ -96,7 +134,10 @@ export const webhookHttpApiLayer = HttpApiBuilder.group(
   EnvironmentHttpApi,
   "webhooks",
   Effect.fnUntraced(function* (handlers) {
-    const handler = handleWebhook(yield* ScheduledTaskService.ScheduledTaskService);
+    const handler = handleWebhook(
+      yield* ScheduledTaskService.ScheduledTaskService,
+      yield* makeRelayDeliveryVerifier,
+    );
     return handlers
       .handleRaw("webhookPost", handler)
       .handleRaw("webhookPut", handler)

apps/server/src/scheduledTasks/webhookTemplate.test.ts+25

...	@@ -69,6 +69,31 @@ describe("renderWebhookPrompt", () => {	...	@@ -69,6 +69,31 @@ describe("renderWebhookPrompt", () => {
69	assert.deepEqual(plain.missing, ["body.field"]);	69	assert.deepEqual(plain.missing, ["body.field"]);
70	});	70	});
71		71
		72	it("redacts credentials in whole-request placeholders but not named ones", () => {
		73	const request: WebhookRequest = {
		74	...githubPullRequest,
		75	query: "source=github&access_token=q-secret",
		76	headers: {
		77	...githubPullRequest.headers,
		78	authorization: "Bearer h-secret",
		79	"x-hub-signature-256": "sha256=abc",
		80	},
		81	};
		82	const whole = renderWebhookPrompt("{{request}}\|{{headers}}\|{{query}}", request).prompt;
		83	for (const secret of ["q-secret", "h-secret", "sha256=abc"]) {
		84	assert.notInclude(whole, secret);
		85	}
		86	assert.include(whole, "?source=github&access_token=[redacted]\n");
		87	assert.include(whole, "authorization: [redacted]");
		88	assert.include(whole, "x-github-event: pull_request");
		89
		90	const named = renderWebhookPrompt(
		91	"{{headers.authorization}} {{query.access_token}}",
		92	request,
		93	).prompt;
		94	assert.equal(named, "Bearer h-secret q-secret");
		95	});
		96
72	it("does not resolve inherited object properties", () => {	97	it("does not resolve inherited object properties", () => {
73	const rendered = renderWebhookPrompt(	98	const rendered = renderWebhookPrompt(
74	"{{body.constructor}}{{body.__proto__}}",	99	"{{body.constructor}}{{body.__proto__}}",

apps/server/src/scheduledTasks/webhookTemplate.ts+48-6

...	@@ -14,6 +14,10 @@	...	@@ -14,6 +14,10 @@
14	*	14	*
15	* Strings and numbers render as text, objects and arrays as JSON. A path with	15	* Strings and numbers render as text, objects and arrays as JSON. A path with
16	* no value renders empty and is reported in `missing`.	16	* no value renders empty and is reported in `missing`.
		17	*
		18	* Credential-looking headers and query parameters are redacted wherever the
		19	* whole set renders (`{{request}}`, `{{headers}}`, `{{query}}`), as in the
		20	* delivery log. Naming one (`{{headers.authorization}}`) gives its raw value.
17	*/	21	*/
18		22
19	export interface WebhookRequest {	23	export interface WebhookRequest {
...	@@ -31,6 +35,40 @@ export interface RenderedWebhookPrompt {	...	@@ -31,6 +35,40 @@ export interface RenderedWebhookPrompt {
31	readonly missing: ReadonlyArray<string>;	35	readonly missing: ReadonlyArray<string>;
32	}	36	}
33		37
		38	/** Header and query parameter names that commonly carry credentials. */
		39	const CREDENTIAL_NAME =
		40	/^(authorization\|proxy-authorization\|cookie\|set-cookie)$\|token\|secret\|signature\|key\|password\|auth/i;
		41	const REDACTED = "[redacted]";
		42
		43	export function redactHeaders(headers: Readonly<Record<string, string>>): Record<string, string> {
		44	return Object.fromEntries(
		45	Object.entries(headers).map(([name, value]) => [
		46	name,
		47	CREDENTIAL_NAME.test(name) ? REDACTED : value,
		48	]),
		49	);
		50	}
		51
		52	/** Redacts credential-named values in a raw query string, keeping the rest as sent. */
		53	export function redactQuery(query: string): string {
		54	if (query === "") return query;
		55	return query
		56	.split("&")
		57	.map((part) => {
		58	const separator = part.indexOf("=");
		59	if (separator === -1) return part;
		60	const name = part.slice(0, separator);
		61	let decoded = name;
		62	try {
		63	decoded = decodeURIComponent(name.replaceAll("+", " "));
		64	} catch {
		65	// A malformed escape is matched as sent.
		66	}
		67	return CREDENTIAL_NAME.test(decoded) ? `${name}=${REDACTED}` : part;
		68	})
		69	.join("&");
		70	}
		71
34	const PLACEHOLDER = /\{\{\s([^{}]?)\s*\}\}/g;	72	const PLACEHOLDER = /\{\{\s([^{}]?)\s*\}\}/g;
35		73
36	function parseBody(request: WebhookRequest): unknown {	74	function parseBody(request: WebhookRequest): unknown {
...	@@ -64,9 +102,12 @@ function stringify(value: unknown): string {	...	@@ -64,9 +102,12 @@ function stringify(value: unknown): string {
64	}	102	}
65		103
66	function formatWebhookRequest(request: WebhookRequest): string {	104	function formatWebhookRequest(request: WebhookRequest): string {
67	const headerLines = Object.entries(request.headers).map(([name, value]) => `${name}: ${value}`);	105	const headerLines = Object.entries(redactHeaders(request.headers)).map(
		106	([name, value]) => `${name}: ${value}`,
		107	);
		108	const query = redactQuery(request.query);
68	return [	109	return [
69	`${request.method} ${request.path}${request.query ? `?${request.query}` : ""}`,	110	`${request.method} ${request.path}${query ? `?${query}` : ""}`,
70	...headerLines,	111	...headerLines,
71	"",	112	"",
72	request.bodyText,	113	request.bodyText,
...	@@ -90,12 +131,13 @@ export function renderWebhookPrompt(	...	@@ -90,12 +131,13 @@ export function renderWebhookPrompt(
90	return segments.length === 0 ? request.bodyText : lookup(parsedBody(), segments);	131	return segments.length === 0 ? request.bodyText : lookup(parsedBody(), segments);
91	case "headers":	132	case "headers":
92	return segments.length === 0	133	return segments.length === 0
93	? request.headers	134	? redactHeaders(request.headers)
94	: request.headers[segments.join(".").toLowerCase()];	135	: request.headers[segments.join(".").toLowerCase()];
95	case "query": {	136	case "query": {
96	const params = new URLSearchParams(request.query);	137	if (segments.length === 0) {
97	if (segments.length === 0) return Object.fromEntries(params);	138	return Object.fromEntries(new URLSearchParams(redactQuery(request.query)));
98	return params.get(segments.join(".")) ?? undefined;	139	}
		140	return new URLSearchParams(request.query).get(segments.join(".")) ?? undefined;
99	}	141	}
100	default:	142	default:
101	return undefined;	143	return undefined;

apps/server/src/server.ts+32-20

 @@ -119,8 +119,16 @@ import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http
 import * as ReplayMarkers from "./auth/replayMarkers.ts";
 import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
 import { webhookHttpApiLayer } from "./scheduledTasks/webhookRoute.ts";
-import { ScheduledTaskWebhookOrigin } from "./scheduledTasks/ScheduledTaskService.ts";
+import * as HeldHooksWaker from "./relay/HeldHooksWaker.ts";
-import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./cloud/config.ts";
+import {
+  relayHookBaseUrl,
+  ScheduledTaskWebhookOrigin,
+} from "./scheduledTasks/ScheduledTaskService.ts";
+import {
+  CLOUD_ENDPOINT_RUNTIME_CONFIG,
+  decodeRuntimeConfig,
+  RELAY_URL_SECRET,
+} from "./cloud/config.ts";
 import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
 import {
   connectHttpApiLayer,
 @@ -452,24 +460,26 @@ const CloudManagedEndpointRuntimeLive = Layer.mergeAll(
 // to is configured; otherwise clients show the environment-relative path.
 const ScheduledTaskWebhookOriginLive = Layer.effect(
   ScheduledTaskWebhookOrigin,
-  Effect.map(
+  Effect.gen(function* () {
-    Effect.all([ServerEnvironment.ServerEnvironment, ServerSecretStore.ServerSecretStore]),
+    const secrets = yield* ServerSecretStore.ServerSecretStore;
-    ([environment, secrets]) =>
+    // The reference holds an effect so each read sees the current link state.
-      // The reference holds an effect so each read sees the current link state.
+    return Effect.gen(function* () {
-      Effect.gen(function* () {
+      const [relayUrl, tunnelConfig] = yield* Effect.all([
-        const [relayUrl, tunnelConfig] = yield* Effect.all([
+        secrets.get(RELAY_URL_SECRET),
-          secrets.get(RELAY_URL_SECRET),
+        secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
-          secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
+      ]).pipe(Effect.orElseSucceed(() => [Option.none(), Option.none()] as const));
-        ]).pipe(Effect.orElseSucceed(() => [Option.none(), Option.none()] as const));
+      if (Option.isNone(relayUrl) || Option.isNone(tunnelConfig)) {
-        return {
+        return { relayHookBaseUrl: null };
-          environmentId: yield* environment.getEnvironmentId,
+      }
-          relayUrl:
+      const config = decodeRuntimeConfig(new TextDecoder().decode(tunnelConfig.value));
-            Option.isSome(relayUrl) && Option.isSome(tunnelConfig)
+      return {
-              ? new TextDecoder().decode(relayUrl.value) || null
+        relayHookBaseUrl: relayHookBaseUrl({
-              : null,
+          relayUrl: new TextDecoder().decode(relayUrl.value),
-        };
+          tunnelName: Option.isSome(config) ? config.value.tunnelName : undefined,
-      }),
+        }),
-  ),
+      };
+    });
+  }),
 );
 const OrchestrationV2RuntimeLayerLive = OrchestrationV2ProductionLayerLive.pipe(
 @@ -537,6 +547,8 @@ const ProviderInstallationRefreshLive = Layer.effectDiscard(
 const RuntimeCoreDependenciesBaseLive = Layer.mergeAll(
   AgentAwarenessRelay.layer,
+  // Asks T3 Connect to deliver webhooks it held while this environment was offline.
+  HeldHooksWaker.layer,
   ThreadSettlementWorkerLive,
   Layer.effectDiscard(StorageCleanup.make.pipe(Effect.flatMap((service) => service.start()))).pipe(
     Layer.provide(ProjectionStoreV2.layer),

apps/web/src/cloud/linkEnvironment.ts+3-1

 @@ -199,13 +199,15 @@ export function readPrimaryCloudLinkState(input: {
 export function updatePrimaryCloudPreferences(input: {
   readonly target: CloudLinkTarget;
   readonly publishAgentActivity: boolean;
+  readonly holdWebhooksWhileOffline?: boolean;
 }): Effect.Effect<CloudLinkState, CloudEnvironmentLinkError, HttpClient.HttpClient> {
   return Effect.gen(function* () {
     const client = yield* makeEnvironmentHttpApiClient(input.target.httpBaseUrl);
+    const { target: _target, ...payload } = input;
     return yield* client.connect
       .preferences({
         headers: {},
-        payload: input,
+        payload,
       })
       .pipe(
         Effect.mapError(environmentApiError("Could not update environment cloud preferences.")),

apps/web/src/cloud/linkEnvironmentAtoms.ts+5-2

 @@ -41,6 +41,9 @@ export const updatePrimaryEnvironmentPreferences = createRuntimeCommand(connecti
   label: "web:cloud:update-primary-environment-preferences",
   scheduler: cloudLinkScheduler,
   concurrency: cloudLinkConcurrency,
-  execute: (input: { readonly target: CloudLinkTarget; readonly publishAgentActivity: boolean }) =>
+  execute: (input: {
-    updatePrimaryCloudPreferences(input),
+    readonly target: CloudLinkTarget;
+    readonly publishAgentActivity: boolean;
+    readonly holdWebhooksWhileOffline?: boolean;
+  }) => updatePrimaryCloudPreferences(input),
 });

apps/web/src/cloud/primaryCloudLinkState.ts+6-3

 @@ -13,6 +13,7 @@ import { usePrimaryEnvironment } from "../state/environments";
 import { runtime } from "../lib/runtime";
 import { appAtomRegistry } from "../rpc/atomRegistry";
 import { readPrimaryCloudLinkState, type CloudLinkTarget } from "./linkEnvironment";
+import { hasCloudPublicConfig } from "./publicConfig";
 const primaryCloudLinkAtomRuntime = Atom.runtime(
   Layer.effect(
 @@ -62,9 +63,11 @@ export function usePrimaryCloudLinkState() {
         : null,
     [primary],
   );
-  const atom = target
+  // Builds without T3 Connect have no link to read; skip the request.
-    ? primaryCloudLinkStateAtom(targetKey(target))
+  const atom =
-    : EMPTY_PRIMARY_CLOUD_LINK_STATE_ATOM;
+    target && hasCloudPublicConfig()
+      ? primaryCloudLinkStateAtom(targetKey(target))
+      : EMPTY_PRIMARY_CLOUD_LINK_STATE_ATOM;
   const result = useAtomValue(atom);
   const refresh = useCallback(() => {
     refreshPrimaryCloudLinkState(target);

apps/web/src/cloud/useCloudLinkController.ts+7

...	@@ -21,6 +21,8 @@ import { resolveRelayClerkTokenOptions } from "./publicConfig";	...	@@ -21,6 +21,8 @@ import { resolveRelayClerkTokenOptions } from "./publicConfig";
21	export interface CloudLinkDesiredState {	21	export interface CloudLinkDesiredState {
22	readonly managedTunnel: boolean;	22	readonly managedTunnel: boolean;
23	readonly publish: boolean;	23	readonly publish: boolean;
		24	/** Omit to leave the webhook-hold setting as it is. */
		25	readonly holdWebhooksWhileOffline?: boolean;
24	}	26	}
25		27
26	/**	28	/**
...	@@ -75,6 +77,7 @@ export function useCloudLinkController() {	...	@@ -75,6 +77,7 @@ export function useCloudLinkController() {
75	const managedTunnelActive =	77	const managedTunnelActive =
76	primaryCloudLinkState.data?.managedTunnelActive ?? primaryCloudLinkState.data?.linked ?? false;	78	primaryCloudLinkState.data?.managedTunnelActive ?? primaryCloudLinkState.data?.linked ?? false;
77	const publishAgentActivity = primaryCloudLinkState.data?.publishAgentActivity ?? false;	79	const publishAgentActivity = primaryCloudLinkState.data?.publishAgentActivity ?? false;
		80	const holdWebhooksWhileOffline = primaryCloudLinkState.data?.holdWebhooksWhileOffline ?? false;
78	const linked = primaryCloudLinkState.data?.linked ?? false;	81	const linked = primaryCloudLinkState.data?.linked ?? false;
79		82
80	const reconcileCloudState = async (desired: CloudLinkDesiredState): Promise<boolean> => {	83	const reconcileCloudState = async (desired: CloudLinkDesiredState): Promise<boolean> => {
...	@@ -132,6 +135,9 @@ export function useCloudLinkController() {	...	@@ -132,6 +135,9 @@ export function useCloudLinkController() {
132	const prefResult = await updatePrimaryEnvironmentPreferences({	135	const prefResult = await updatePrimaryEnvironmentPreferences({
133	target,	136	target,
134	publishAgentActivity: desired.publish,	137	publishAgentActivity: desired.publish,
		138	...(desired.holdWebhooksWhileOffline === undefined
		139	? {}
		140	: { holdWebhooksWhileOffline: desired.holdWebhooksWhileOffline }),
135	});	141	});
136	if (prefResult._tag === "Failure") {	142	if (prefResult._tag === "Failure") {
137	if (!isAtomCommandInterrupted(prefResult)) {	143	if (!isAtomCommandInterrupted(prefResult)) {
...	@@ -157,6 +163,7 @@ export function useCloudLinkController() {	...	@@ -157,6 +163,7 @@ export function useCloudLinkController() {
157	linked,	163	linked,
158	managedTunnelActive,	164	managedTunnelActive,
159	publishAgentActivity,	165	publishAgentActivity,
		166	holdWebhooksWhileOffline,
160	operationError,	167	operationError,
161	reconcileCloudState,	168	reconcileCloudState,
162	};	169	};

apps/web/src/components/chat/ThreadAutomationsPanel.tsx+20-22

...	@@ -186,28 +186,26 @@ export function ThreadAutomationsPanel(props: {	...	@@ -186,28 +186,26 @@ export function ThreadAutomationsPanel(props: {
186	/>	186	/>
187	<TooltipPopup>Edit automation</TooltipPopup>	187	<TooltipPopup>Edit automation</TooltipPopup>
188	</Tooltip>	188	</Tooltip>
189	<Tooltip>	189	{/* A webhook task runs from its URL; there is no request to run it with. */}
190	<TooltipTrigger	190	{task.schedule.type === "webhook" ? null : (
191	render={	191	<Tooltip>
192	<ThreadDetailsControl	192	<TooltipTrigger
193	size="icon-xs"	193	render={
194	variant="ghost"	194	<ThreadDetailsControl
195	part="icon"	195	size="icon-xs"
196	aria-label={`Run ${task.title} now`}	196	variant="ghost"
197	// A webhook task runs from its URL; there is no request to run it with.	197	part="icon"
198	disabled={	198	aria-label={`Run ${task.title} now`}
199	busyTaskId !== null \|\|	199	disabled={busyTaskId !== null \|\| task.lastRunStatus === "running"}
200	task.lastRunStatus === "running" \|\|	200	onClick={() => void runNow(task)}
201	task.schedule.type === "webhook"	201	>
202	}	202	<PlayIcon className="size-3.5" />
203	onClick={() => void runNow(task)}	203	</ThreadDetailsControl>
204	>	204	}
205	<PlayIcon className="size-3.5" />	205	/>
206	</ThreadDetailsControl>	206	<TooltipPopup>Run now</TooltipPopup>
207	}	207	</Tooltip>
208	/>	208	)}
209	<TooltipPopup>Run now</TooltipPopup>
210	</Tooltip>
211	<Switch	209	<Switch
212	checked={task.enabled}	210	checked={task.enabled}
213	disabled={busyTaskId !== null}	211	disabled={busyTaskId !== null}

apps/web/src/components/settings/ConnectionsSettings.tsx+35

...	@@ -1761,6 +1761,7 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b	...	@@ -1761,6 +1761,7 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b
1761	linkState: primaryCloudLinkState,	1761	linkState: primaryCloudLinkState,
1762	managedTunnelActive,	1762	managedTunnelActive,
1763	publishAgentActivity,	1763	publishAgentActivity,
		1764	holdWebhooksWhileOffline,
1764	operationError,	1765	operationError,
1765	reconcileCloudState,	1766	reconcileCloudState,
1766	} = useCloudLinkController();	1767	} = useCloudLinkController();
...	@@ -1812,6 +1813,25 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b	...	@@ -1812,6 +1813,25 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b
1812	setIsUpdatingPreference(false);	1813	setIsUpdatingPreference(false);
1813	};	1814	};
1814		1815
		1816	const updateHoldWebhooks = async (enabled: boolean) => {
		1817	setIsUpdatingPreference(true);
		1818	const ok = await reconcileCloudState({
		1819	managedTunnel: managedTunnelActive,
		1820	publish: publishAgentActivity,
		1821	holdWebhooksWhileOffline: enabled,
		1822	});
		1823	if (ok) {
		1824	toastManager.add({
		1825	type: "success",
		1826	title: enabled ? "Webhooks held while offline" : "Webhooks no longer held",
		1827	description: enabled
		1828	? "T3 Connect keeps webhook requests for up to 24 hours while this environment is offline."
		1829	: "Requests to an offline environment now fail. Anything already held is still delivered.",
		1830	});
		1831	}
		1832	setIsUpdatingPreference(false);
		1833	};
		1834
1815	return (	1835	return (
1816	<>	1836	<>
1817	{window.desktopBridge ? (	1837	{window.desktopBridge ? (
...	@@ -1846,6 +1866,21 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b	...	@@ -1846,6 +1866,21 @@ function ConfiguredCloudLinkRow({ canManageRelay }: { readonly canManageRelay: b
1846	/>	1866	/>
1847	}	1867	}
1848	/>	1868	/>
		1869	{managedTunnelActive ? (
		1870	<SettingsRow
		1871	title={searchableSetting("hold-webhooks-while-offline").title}
		1872	description="Keep webhook requests for up to 24 hours while this environment is offline, then deliver them. Off: T3 Connect only forwards requests and stores nothing."
		1873	control={
		1874	<CloudLinkSwitch
		1875	ariaLabel="Hold webhook requests while this environment is offline"
		1876	checked={holdWebhooksWhileOffline}
		1877	disabled={!canManageRelay \|\| !isSignedIn \|\| primaryCloudLinkState.isPending \|\| isBusy}
		1878	disabledReason={disabledReason}
		1879	onCheckedChange={(enabled) => void updateHoldWebhooks(enabled)}
		1880	/>
		1881	}
		1882	/>
		1883	) : null}
1849	</>	1884	</>
1850	);	1885	);
1851	}	1886	}

apps/web/src/components/settings/ScheduledTasksSettings.tsx+58-2

...	@@ -23,7 +23,9 @@ import type {	...	@@ -23,7 +23,9 @@ import type {
23	ScheduledTaskWebhookDeliverySummary,	23	ScheduledTaskWebhookDeliverySummary,
24	ThreadId,	24	ThreadId,
25	} from "@t3tools/contracts";	25	} from "@t3tools/contracts";
		26	import { DEFAULT_WEBHOOK_PROMPT } from "@t3tools/client-runtime/scheduled-task-webhook";
26	import {	27	import {
		28	MAX_WEBHOOK_DELIVERY_AGE_MINUTES,
27	MIN_SCHEDULED_TASK_INTERVAL_MS,	29	MIN_SCHEDULED_TASK_INTERVAL_MS,
28	ProviderInstanceId,	30	ProviderInstanceId,
29	resolveEnvironmentMachineKind,	31	resolveEnvironmentMachineKind,
...	@@ -41,8 +43,10 @@ import {	...	@@ -41,8 +43,10 @@ import {
41	deriveProviderInstanceEntries,	43	deriveProviderInstanceEntries,
42	sortProviderInstanceEntries,	44	sortProviderInstanceEntries,
43	} from "../../providerInstances";	45	} from "../../providerInstances";
		46	import { usePrimaryCloudLinkState } from "../../cloud/primaryCloudLinkState";
44	import { requestConfirmDialog } from "../../confirmDialog";	47	import { requestConfirmDialog } from "../../confirmDialog";
45	import { webhookAddress } from "@t3tools/client-runtime/webhook-address";	48	import { webhookAddress } from "@t3tools/client-runtime/webhook-address";
		49	import { Link } from "@tanstack/react-router";
46	import { useCopyToClipboard } from "../../hooks/useCopyToClipboard";	50	import { useCopyToClipboard } from "../../hooks/useCopyToClipboard";
47	import {	51	import {
48	useEnvironment,	52	useEnvironment,
...	@@ -57,7 +61,6 @@ import { WorktreeBaseBranchPicker } from "../WorktreeBaseBranchPicker";	...	@@ -57,7 +61,6 @@ import { WorktreeBaseBranchPicker } from "../WorktreeBaseBranchPicker";
57	import { EnvironmentMachineIcon } from "../EnvironmentMachineIcon";	61	import { EnvironmentMachineIcon } from "../EnvironmentMachineIcon";
58	import { useSettingsScope } from "./SettingsScopeContext";	62	import { useSettingsScope } from "./SettingsScopeContext";
59	import {	63	import {
60	DEFAULT_WEBHOOK_PROMPT,
61	WEBHOOK_SIGNATURE_DEFAULTS,	64	WEBHOOK_SIGNATURE_DEFAULTS,
62	matchesScheduledTaskScope,	65	matchesScheduledTaskScope,
63	scheduleFromDraft,	66	scheduleFromDraft,
...	@@ -130,6 +133,7 @@ const EMPTY_DRAFT: DraftState = {	...	@@ -130,6 +133,7 @@ const EMPTY_DRAFT: DraftState = {
130	signatureEnabled: false,	133	signatureEnabled: false,
131	...WEBHOOK_SIGNATURE_DEFAULTS,	134	...WEBHOOK_SIGNATURE_DEFAULTS,
132	signatureSecret: "",	135	signatureSecret: "",
		136	maxDeliveryAgeMinutes: "",
133	};	137	};
134		138
135	/** Labelled field: a caption sitting above its control. */	139	/** Labelled field: a caption sitting above its control. */
...	@@ -211,11 +215,14 @@ const DELIVERY_OUTCOME_LABELS: Record<ScheduledTaskWebhookDeliveryOutcome, strin	...	@@ -211,11 +215,14 @@ const DELIVERY_OUTCOME_LABELS: Record<ScheduledTaskWebhookDeliveryOutcome, strin
211	rejected_signature: "Bad signature",	215	rejected_signature: "Bad signature",
212	disabled: "Task paused",	216	disabled: "Task paused",
213	rate_limited: "Rate limited",	217	rate_limited: "Rate limited",
		218	expired: "Too old",
214	};	219	};
215		220
216	function deliveryOutcomeVariant(outcome: ScheduledTaskWebhookDeliveryOutcome) {	221	function deliveryOutcomeVariant(outcome: ScheduledTaskWebhookDeliveryOutcome) {
217	if (outcome === "accepted") return "success";	222	if (outcome === "accepted") return "success";
218	if (outcome === "disabled" \|\| outcome === "rate_limited") return "warning";	223	if (outcome === "disabled" \|\| outcome === "rate_limited" \|\| outcome === "expired") {
		224	return "warning";
		225	}
219	return "error";	226	return "error";
220	}	227	}
221		228
...	@@ -725,10 +732,31 @@ function WebhookEndpointField({	...	@@ -725,10 +732,31 @@ function WebhookEndpointField({
725	</Button>	732	</Button>
726	</div>	733	</div>
727	{note !== null ? <p className="text-xs text-muted-foreground">{note}</p> : null}	734	{note !== null ? <p className="text-xs text-muted-foreground">{note}</p> : null}
		735	{endpoint.url !== null ? <WebhookDeliveryMode environmentId={environmentId} /> : null}
728	</div>	736	</div>
729	);	737	);
730	}	738	}
731		739
		740	/**
		741	* Whether T3 Connect forwards requests live or holds them while the
		742	* environment is offline. The setting is per environment and only readable
		743	* for this machine's own environment, so other environments show nothing.
		744	*/
		745	function WebhookDeliveryMode({ environmentId }: { readonly environmentId: EnvironmentId }) {
		746	const cloudLink = usePrimaryCloudLinkState();
		747	if (cloudLink.target?.environmentId !== environmentId \|\| cloudLink.data === null) return null;
		748	return (
		749	<p className="text-xs text-muted-foreground">
		750	{cloudLink.data.holdWebhooksWhileOffline
		751	? "Held for up to 24 hours while this environment is offline. "
		752	: "Forwarded live. Requests fail while this environment is offline. "}
		753	<Link to="/settings/connections" className="underline underline-offset-2">
		754	Change in Connections
		755	</Link>
		756	</p>
		757	);
		758	}
		759
732	function ScheduledTaskEditorDialog({	760	function ScheduledTaskEditorDialog({
733	initialEnvironmentId,	761	initialEnvironmentId,
734	task,	762	task,
...	@@ -830,6 +858,13 @@ function ScheduledTaskEditorDialog({	...	@@ -830,6 +858,13 @@ function ScheduledTaskEditorDialog({
830	return;	858	return;
831	}	859	}
832	const schedule = scheduleFromDraft(draft);	860	const schedule = scheduleFromDraft(draft);
		861	if (schedule === null) {
		862	reportFailure(
		863	"Invalid age limit",
		864	`Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`,
		865	);
		866	return;
		867	}
833	if (	868	if (
834	schedule.type === "webhook" &&	869	schedule.type === "webhook" &&
835	schedule.signature &&	870	schedule.signature &&
...	@@ -1126,6 +1161,27 @@ function ScheduledTaskEditorDialog({	...	@@ -1126,6 +1161,27 @@ function ScheduledTaskEditorDialog({
1126	"Each request runs the prompt. Use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} or {{request}} in the prompt; only what it names reaches the agent."	1161	"Each request runs the prompt. Use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} or {{request}} in the prompt; only what it names reaches the agent."
1127	}	1162	}
1128	</p>	1163	</p>
		1164	<Field
		1165	label="Skip requests older than"
		1166	hint="minutes, optional"
		1167	htmlFor="scheduled-task-max-age"
		1168	>
		1169	<Input
		1170	id="scheduled-task-max-age"
		1171	type="number"
		1172	nativeInput
		1173	min={1}
		1174	max={MAX_WEBHOOK_DELIVERY_AGE_MINUTES}
		1175	placeholder="Run every request"
		1176	value={draft.maxDeliveryAgeMinutes}
		1177	onChange={(event) =>
		1178	setDraft((current) => ({
		1179	...current,
		1180	maxDeliveryAgeMinutes: event.target.value,
		1181	}))
		1182	}
		1183	/>
		1184	</Field>
1129	<div className="flex items-center justify-between gap-4">	1185	<div className="flex items-center justify-between gap-4">
1130	<div className="min-w-0 space-y-1">	1186	<div className="min-w-0 space-y-1">
1131	<Label htmlFor="scheduled-task-signature">Require signature</Label>	1187	<Label htmlFor="scheduled-task-signature">Require signature</Label>

apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts+28-2

...	@@ -181,10 +181,15 @@ describe("webhook scheduled tasks", () => {	...	@@ -181,10 +181,15 @@ describe("webhook scheduled tasks", () => {
181	const draft = taskToDraft(webhookTask);	181	const draft = taskToDraft(webhookTask);
182	expect(draft.scheduleMode).toBe("webhook");	182	expect(draft.scheduleMode).toBe("webhook");
183	expect(draft.signatureSecret).toBe("");	183	expect(draft.signatureSecret).toBe("");
184	expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature });	184	expect(scheduleFromDraft(draft)).toEqual({
		185	type: "webhook",
		186	signature,
		187	maxDeliveryAgeMinutes: null,
		188	});
185	expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({	189	expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({
186	type: "webhook",	190	type: "webhook",
187	signature: { ...signature, secret: "new" },	191	signature: { ...signature, secret: "new" },
		192	maxDeliveryAgeMinutes: null,
188	});	193	});
189	});	194	});
190		195
...	@@ -192,7 +197,28 @@ describe("webhook scheduled tasks", () => {	...	@@ -192,7 +197,28 @@ describe("webhook scheduled tasks", () => {
192	const draft = taskToDraft({ ...webhookTask, schedule: { type: "webhook", signature: null } });	197	const draft = taskToDraft({ ...webhookTask, schedule: { type: "webhook", signature: null } });
193	expect(draft.signatureEnabled).toBe(false);	198	expect(draft.signatureEnabled).toBe(false);
194	expect(draft.signatureHeader).toBe("x-hub-signature-256");	199	expect(draft.signatureHeader).toBe("x-hub-signature-256");
195	expect(scheduleFromDraft(draft)).toEqual({ type: "webhook", signature: null });	200	expect(scheduleFromDraft(draft)).toEqual({
		201	type: "webhook",
		202	signature: null,
		203	maxDeliveryAgeMinutes: null,
		204	});
		205	});
		206
		207	it("round-trips the max age, treats blank as no limit, and rejects an invalid limit", () => {
		208	const draft = taskToDraft({
		209	...webhookTask,
		210	schedule: { type: "webhook", signature: null, maxDeliveryAgeMinutes: 90 },
		211	});
		212	expect(draft.maxDeliveryAgeMinutes).toBe("90");
		213	expect(scheduleFromDraft(draft)).toMatchObject({ maxDeliveryAgeMinutes: 90 });
		214	for (const blank of ["", " "]) {
		215	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: blank })).toMatchObject({
		216	maxDeliveryAgeMinutes: null,
		217	});
		218	}
		219	for (const invalid of ["0", "-5", "1.5", "abc", "1441"]) {
		220	expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: invalid })).toBeNull();
		221	}
196	});	222	});
197	});	223	});
198		224

apps/web/src/components/settings/scheduledTasksSettings.logic.ts+12-4

 @@ -9,6 +9,7 @@ import {
   type ProviderInteractionMode,
   type ServerSettings,
 } from "@t3tools/contracts";
+import { parseMaxDeliveryAge } from "@t3tools/client-runtime/scheduled-task-webhook";
 import {
   resolveProjectSettings,
 @@ -78,6 +79,8 @@ export interface DraftState {
   readonly signaturePrefix: string;
   /** Write-only: empty keeps the secret already stored on the server. */
   readonly signatureSecret: string;
+  /** Minutes as typed; empty runs every held request regardless of age. */
+  readonly maxDeliveryAgeMinutes: string;
 }
 /** GitHub's signature settings, the most common sender. */
 @@ -87,11 +90,11 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = {
   signaturePrefix: "sha256=",
 } as const;
-/** Prompt a new webhook task starts with: the whole request, which the user can narrow down. */
+/** Null when the draft's webhook age limit is invalid; the caller reports it and does not save. */
-export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{request}}";
+export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule | null {
-export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule {
   if (draft.scheduleMode === "webhook") {
+    const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes);
+    if (maxDeliveryAgeMinutes === undefined) return null;
     const secret = draft.signatureSecret.trim();
     return {
       type: "webhook",
 @@ -103,6 +106,7 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul
             ...(secret ? { secret } : {}),
           }
         : null,
+      maxDeliveryAgeMinutes,
     };
   }
   if (draft.scheduleMode === "interval") {
 @@ -159,6 +163,10 @@ export function taskToDraft(task: ScheduledTask): DraftState {
         }
       : { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS }),
     signatureSecret: "",
+    maxDeliveryAgeMinutes:
+      schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null
+        ? String(schedule.maxDeliveryAgeMinutes)
+        : "",
   };
 }

apps/web/src/components/settings/settingsSearch.test.ts+19

...	@@ -170,6 +170,7 @@ describe("searchSettings", () => {	...	@@ -170,6 +170,7 @@ describe("searchSettings", () => {
170	"source-control-writer-model",	170	"source-control-writer-model",
171	"source-control-writing-style",	171	"source-control-writing-style",
172	"t3-connect",	172	"t3-connect",
		173	"hold-webhooks-while-offline",
173	"tailscale-https",	174	"tailscale-https",
174	"wsl-backend",	175	"wsl-backend",
175	"auto-settle-inactive-threads",	176	"auto-settle-inactive-threads",
...	@@ -221,6 +222,24 @@ describe("searchSettings", () => {	...	@@ -221,6 +222,24 @@ describe("searchSettings", () => {
221	expect(browser).toContain("publish-agent-activity");	222	expect(browser).toContain("publish-agent-activity");
222	});	223	});
223		224
		225	it("offers webhook holding only while the managed tunnel is on, like its row", () => {
		226	const availability = {
		227	hasCloudPublicConfig: true,
		228	hasEnvironment: true,
		229	hasProviderSettingsEnvironment: true,
		230	hasMacProviderSettingsEnvironment: false,
		231	canManageLocalBackend: true,
		232	isWslSettingsRowVisible: false,
		233	hasThreadAutoSettlement: false,
		234	};
		235	const itemIds = (managedTunnelActive: boolean) =>
		236	filterAvailableSettingsSearchItems({ ...availability, managedTunnelActive }).map(
		237	(item) => item.id,
		238	);
		239	expect(itemIds(false)).not.toContain("hold-webhooks-while-offline");
		240	expect(itemIds(true)).toContain("hold-webhooks-while-offline");
		241	});
		242
224	it("shows automatic settlement settings when the server supports them", () => {	243	it("shows automatic settlement settings when the server supports them", () => {
225	const available = filterAvailableSettingsSearchItems({	244	const available = filterAvailableSettingsSearchItems({
226	hasCloudPublicConfig: false,	245	hasCloudPublicConfig: false,

apps/web/src/components/settings/settingsSearch.ts+15-1

 @@ -60,6 +60,8 @@ export interface SettingsSearchItem {
   readonly localBackendManagementOnly?: boolean;
   readonly localEnvironmentOnly?: boolean;
   readonly wslAvailableOnly?: boolean;
+  // Its row only renders while this environment's T3 Connect managed tunnel is on.
+  readonly managedTunnelOnly?: boolean;
   /**
    * Sorts after every other match. Keybinding commands mirror rows on other
    * surfaces, so "model" must still lead with Default model, not Model Picker.
 @@ -77,6 +79,7 @@ export interface SettingsSearchAvailability {
   readonly canManageLocalBackend: boolean;
   readonly isWslSettingsRowVisible: boolean;
   readonly hasThreadAutoSettlement: boolean;
+  readonly managedTunnelActive?: boolean;
 }
 /**
 @@ -836,6 +839,16 @@ export const SETTINGS_SEARCH_ITEMS = [
     desktopOnly: true,
     cloudOnly: true,
   },
+  {
+    id: "hold-webhooks-while-offline",
+    localEnvironmentOnly: true,
+    title: "Hold webhooks while offline",
+    to: "/settings/connections",
+    targetId: "connections-environment",
+    searchTerms: ["webhook automations offline queue mailbox t3 connect"],
+    cloudOnly: true,
+    managedTunnelOnly: true,
+  },
   {
     id: "publish-agent-activity",
     localEnvironmentOnly: true,
 @@ -1018,7 +1031,8 @@ export function filterAvailableSettingsSearchItems(
       (!item.localBackendManagementOnly || availability.canManageLocalBackend) &&
       (!item.localEnvironmentOnly || !availability.localEnvironmentDisabled) &&
       (!item.wslAvailableOnly || availability.isWslSettingsRowVisible) &&
-      (!item.requiresThreadAutoSettlement || availability.hasThreadAutoSettlement),
+      (!item.requiresThreadAutoSettlement || availability.hasThreadAutoSettlement) &&
+      (!item.managedTunnelOnly || availability.managedTunnelActive === true),
   );
 }

apps/web/src/components/settings/useAvailableSettingsSearchItems.ts+7

 @@ -1,6 +1,7 @@
 import { useMemo } from "react";
 import { AuthAccessWriteScope } from "@t3tools/contracts";
+import { usePrimaryCloudLinkState } from "~/cloud/primaryCloudLinkState";
 import { hasCloudPublicConfig } from "~/cloud/publicConfig";
 import { isElectron } from "~/env";
 import { isLocalEnvironmentDisabled } from "~/localEnvironment";
 @@ -23,6 +24,10 @@ export function useAvailableSettingsSearchItems(scopeSearch: SettingsScopeSearch
   const desktopWsl = useEnvironmentQuery(
     isElectron && !localEnvironmentDisabled ? desktopWslStateAtom : null,
   );
+  const cloudLinkState = usePrimaryCloudLinkState().data;
+  // Same fallback as the Connections row: older servers imply a tunnel from `linked`.
+  const managedTunnelActive =
+    cloudLinkState?.managedTunnelActive ?? cloudLinkState?.linked ?? false;
   const canManageLocalBackend =
     !localEnvironmentDisabled &&
     (isElectron ||
 @@ -59,8 +64,10 @@ export function useAvailableSettingsSearchItems(scopeSearch: SettingsScopeSearch
         }),
         hasThreadAutoSettlement:
           getThreadAutoSettlementSearchAvailability(environments).eligibleEnvironmentIds.length > 0,
+        managedTunnelActive,
       }),
     [
+      managedTunnelActive,
       canManageLocalBackend,
       desktopWsl.data,
       desktopWsl.error,

docs/internals/t3-connect.md+22-4

...	@@ -5,12 +5,30 @@ credentials for reaching environments, and managed tunnel allocations. After	...	@@ -5,12 +5,30 @@ credentials for reaching environments, and managed tunnel allocations. After
5	bootstrap, clients send application traffic through the environment's tunnel	5	bootstrap, clients send application traffic through the environment's tunnel
6	hostname; the relay Worker does not proxy their HTTP or WebSocket sessions.	6	hostname; the relay Worker does not proxy their HTTP or WebSocket sessions.
7	The one exception is automation webhooks: the relay forwards	7	The one exception is automation webhooks: the relay forwards
8	`/v1/hooks/:environmentId/:hookId/:token` statelessly to the environment's	8	`/v1/hooks/:environmentId/:hookId/:token` to the environment's tunnel so
9	tunnel so senders get a stable URL. It stores nothing, keeps bodies and tokens	9	senders get a stable URL. It keeps bodies and tokens out of its traces and
10	out of its traces, and leaves token and signature verification to the	10	leaves token and signature verification to the environment
11	environment
12	([forwarder](../../infra/relay/src/hooks/HookForwarder.ts)).	11	([forwarder](../../infra/relay/src/hooks/HookForwarder.ts)).
13		12
		13	By default the forwarder stores nothing. An environment can opt in to having
		14	the relay hold requests while it is offline
		15	(`hold_webhooks_while_offline` on its link). Only then does the relay store the
		16	raw request, including the hook token in the path, in a Durable Object for
		17	that environment, with SQLite storage. The object pushes held requests back
		18	through the tunnel from its alarm, oldest first, and backs off while the
		19	environment stays away. When the tunnel reconnects, the environment asks the
		20	relay to deliver right away. Requests are deleted once the environment
		21	answers, after 24 hours, or when no user has the environment linked. The relay
		22	still never checks the token; delivery goes through the same environment route.
		23	Every forward carries `x-t3-relay-delivery-id`, so a request that reached the
		24	environment before a timeout and is delivered again later runs once
		25	([inbox object](../../infra/relay/src/hooks/HookInboxObject.ts)).
		26
		27	A Durable Object, not Postgres or Queues, because held requests are write-once,
		28	read-once bodies of up to 1 MiB that need per-environment order, caps, and
		29	retry timing. Queues cap messages at 128 KB and cannot hold one environment's
		30	requests back while it is away.
		31
14	Clerk, deployment, and native authentication setup live in the	32	Clerk, deployment, and native authentication setup live in the
15	[Connect setup runbook](../operations/connect-setup.md).	33	[Connect setup runbook](../operations/connect-setup.md).
16		34

docs/operations/observability.md+15

...	@@ -386,6 +386,21 @@ Counters tell you volume and failure rate:	...	@@ -386,6 +386,21 @@ Counters tell you volume and failure rate:
386	- `t3_provider_turns_total`	386	- `t3_provider_turns_total`
387	- `t3_git_commands_total`	387	- `t3_git_commands_total`
388		388
		389	Webhooks have their own families:
		390
		391	- `t3_webhook_deliveries_total` by `outcome` and `source` (`relay` or `direct`). Beyond what the
		392	sender sees, `queue_full` means a task already had its limit of deliveries waiting,
		393	`prompt_too_long` means the filled-in prompt passed the provider limit, and `duplicate` means
		394	the relay delivered a request this environment had already run.
		395	- `t3_webhook_runs_total` by `outcome` (`started`, `skipped`, `failed`) for the runs those
		396	deliveries start, which happen after the sender has its answer.
		397	- `t3_webhook_held_delay` for how long requests the relay held waited before arriving.
		398
		399	`ScheduledTaskService.triggerWebhook` spans carry the same outcome per request, and each run
		400	started from a delivery is its own `ScheduledTaskService.runWebhookDelivery` trace. For a request
		401	the relay forwarded, the span also goes to the T3 Connect trace export as a child of the relay's
		402	span; requests that reach the environment directly never join a sender's trace.
		403
389	Use metrics when the question is:	404	Use metrics when the question is:
390		405
391	- "is this always slow?"	406	- "is this always slow?"

docs/operations/relay-observability.md+51

...	@@ -57,3 +57,54 @@ failure means that a signed proof was too old or too far in the future for the r	...	@@ -57,3 +57,54 @@ failure means that a signed proof was too old or too far in the future for the r
57	window. It can point to a date or time problem on either device, but it can also result from a	57	window. It can point to a date or time problem on either device, but it can also result from a
58	delayed request. The client uses this category, and the absence of a category from an older relay,	58	delayed request. The client uses this category, and the absence of a category from an older relay,
59	to decide whether clock skew is confirmed or only one possible cause.	59	to decide whether clock skew is confirmed or only one possible cause.
		60
		61	## Webhooks
		62
		63	A public webhook request is one `relay.hooks.forward` span. Its `relay.hook.outcome` says what
		64	happened: `forwarded`, `held`, `rate_limited`, `inbox_full`, `not_found`, `payload_too_large`,
		65	`environment_unavailable`, or `environment_timeout`. `relay.hook.endpoint_key` identifies the
		66	managed endpoint, and with it the environment. On a forward, `relay.hook.upstream_status` or
		67	`relay.hook.upstream_error` records the environment's answer, and
		68	`relay.hook.upstream_outcome` what it did with the request (`accepted`, `duplicate`,
		69	`prompt_too_long`, `queue_full`, `rejected_signature`, `expired`, `disabled`, ...), from its
		70	`x-t3-hook-outcome` response header. A held request's delivery records the same on its
		71	`relay.inbox.deliver` span. The relay sends its own `traceparent` with each forward and drops any a
		72	sender supplied, and signs each forward (`x-t3-relay-delivery`) so the environment trusts the
		73	relay's delivery id, receive time and trace context only from the relay, so on environments that export to T3 Connect, the environment's
		74	`ScheduledTaskService.triggerWebhook` span lands in the same trace. `relay.hook.rate_limit` says which
		75	budget ran out: `endpoint` or `hook`. `relay.hook.rate_limiter_failed_open` is set when the
		76	Cloudflare rate limiter was unavailable and the request went through unlimited.
		77
		78	Held requests are handled in the endpoint's `HookInboxObject`, and each call into it is its own
		79	trace, not a child of the forward span: `relay.inbox.hold`, `relay.inbox.wake`, and
		80	`relay.inbox.deliver` for each alarm run. Each carries `relay.hook.endpoint_key`, so they join to
		81	forwards. The details are on the child spans: `HookInboxStore.hold` carries
		82	`relay.inbox.refused` when the inbox refused a request (`max_per_hook`, `max_requests`,
		83	`max_bytes`, or `already_held`). `HookInboxStore.deliverDue` carries `relay.inbox.run_result`
		84	(`drained`, `more_pending`, `busy`, or `unreachable`), how many requests it sent and delivered,
		85	the consecutive failures behind the retry delay, the longest time a delivered request waited, and
		86	the backlog left. A run that errored outright has `relay.inbox.run_result = failed` on the root
		87	`relay.inbox.deliver` span instead.
		88
		89	Questions these answer:
		90
		91	```apl
		92	// Webhook outcomes per hour
		93	['t3-code-relay-traces-prod']
		94	\| where name == 'relay.hooks.forward'
		95	\| summarize count() by bin(_time, 1h), outcome = tostring(['attributes.custom']['relay.hook.outcome'])
		96
		97	// Inboxes stuck behind an unreachable environment, by endpoint
		98	['t3-code-relay-traces-prod']
		99	\| where name == 'HookInboxStore.deliverDue'
		100	\| extend c = ['attributes.custom']
		101	\| where tostring(c['relay.inbox.run_result']) == 'unreachable'
		102	\| project trace_id, failures = toint(c['relay.inbox.consecutive_failures']),
		103	backlog = toint(c['relay.inbox.held_count'])
		104	\| join kind=inner (
		105	['t3-code-relay-traces-prod']
		106	\| where name == 'relay.inbox.deliver'
		107	\| project trace_id, endpoint = tostring(['attributes.custom']['relay.hook.endpoint_key'])
		108	) on trace_id
		109	\| summarize runs = count(), failures = max(failures), backlog = max(backlog) by endpoint
		110	```

docs/user/project-settings.md+21-10

...	@@ -61,16 +61,18 @@ using its project, model, and workspace settings. Fixed-time schedules use that	...	@@ -61,16 +61,18 @@ using its project, model, and workspace settings. Fixed-time schedules use that
61	environment's time zone, which may differ from your phone's.	61	environment's time zone, which may differ from your phone's.
62		62
63	You can edit, pause, resume, run immediately, or delete a task from the list.	63	You can edit, pause, resume, run immediately, or delete a task from the list.
		64	Webhook tasks only run when their URL is called, so they can't be run
		65	immediately.
64	Leaving an edited form asks before discarding unsaved changes.	66	Leaving an edited form asks before discarding unsaved changes.
65		67
66	## Webhook automations	68	## Webhook automations
67		69
68	In Settings → Scheduled tasks, choose On webhook as a task's schedule to run it whenever another service	70	In Settings → Scheduled tasks, choose On webhook
69	calls its URL, such as GitHub on a new pull request or a CI job that failed.	71	as a task's schedule to run it whenever another service calls its URL, such as
70	After you save the task, copy its URL from the editor. If the environment uses a	72	GitHub on a new pull request or a CI job that failed. A public URL needs a
71	[T3 Connect](remote-access.md) managed tunnel, the URL is public; otherwise it	73	[T3 Connect](remote-access.md) managed tunnel; after you save the task, copy
72	works anywhere the environment itself is reachable. Rotate replaces the URL and	74	its URL from the editor. Without one, the editor shows only the URL's path.
73	the old one stops working.	75	Rotate replaces the URL and the old one stops working.
74		76
75	The prompt decides what the agent sees. Placeholders pull values out of the	77	The prompt decides what the agent sees. Placeholders pull values out of the
76	request: `{{body.path}}` for a JSON or form field, `{{headers.name}}`,	78	request: `{{body.path}}` for a JSON or form field, `{{headers.name}}`,
...	@@ -81,12 +83,21 @@ pull request link. A placeholder with no value is left empty.	...	@@ -81,12 +83,21 @@ pull request link. A placeholder with no value is left empty.
81	For GitHub, turn on Require signature, keep the header	83	For GitHub, turn on Require signature, keep the header
82	`x-hub-signature-256`, hex encoding and the `sha256=` prefix, and enter the	84	`x-hub-signature-256`, hex encoding and the `sha256=` prefix, and enter the
83	same secret in the repository's webhook settings with content type	85	same secret in the repository's webhook settings with content type
84	`application/json`. Requests without a valid signature are rejected.	86	`application/json`. Requests without a valid signature are rejected. Set this
		87	up on desktop or web; mobile keeps an existing signature check but can't turn
		88	one on.
85		89
86	On desktop and web, pick Deliveries from a task's menu to see recent	90	On desktop and web, pick Deliveries from a task's menu to see recent
87	requests and the prompt each one produced. If the environment is offline, the sender gets an error and	91	requests and the prompt each one produced.
88	nothing runs; redeliver from the sender, such as GitHub's Recent Deliveries,	92
89	once it is back.	93	If the environment is offline, the sender gets an error and nothing runs;
		94	redeliver from the sender, such as GitHub's Recent Deliveries, once it is
		95	back. To have T3 Connect keep requests instead, turn on **Hold webhooks while
		96	offline in Settings → Connections**. T3 Connect then stores requests to a
		97	T3 Connect URL for up to 24 hours and delivers them when the environment
		98	returns. Leave it off if you don't want request bodies stored outside your
		99	machine. To skip requests that waited too long, set **Skip requests older
		100	than** on the task.
90		101
91	## Defaults and inheritance	102	## Defaults and inheritance
92		103

infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/migration.sql created+1

...	@@ -0,0 +1 @@
1	ALTER TABLE "relay_environment_links" ADD COLUMN "hold_webhooks_while_offline" boolean DEFAULT false NOT NULL;
	\ No newline at end of file

infra/relay/migrations/postgres/20261004061459_hold_webhooks_while_offline/snapshot.json created+1581

...	@@ -0,0 +1,1581 @@
1	{
2	"version": "8",
3	"dialect": "postgres",
4	"id": "13972659-45db-4ce9-9818-523b450bceca",
5	"prevIds": ["3809c51e-3821-4a08-818d-e5d28dd3e9b4"],
6	"ddl": [
7	{
8	"isRlsEnabled": false,
9	"name": "relay_agent_activity_rows",
10	"entityType": "tables",
11	"schema": "public"
12	},
13	{
14	"isRlsEnabled": false,
15	"name": "relay_delivery_attempts",
16	"entityType": "tables",
17	"schema": "public"
18	},
19	{
20	"isRlsEnabled": false,
21	"name": "relay_dpop_proofs",
22	"entityType": "tables",
23	"schema": "public"
24	},
25	{
26	"isRlsEnabled": false,
27	"name": "relay_environment_credentials",
28	"entityType": "tables",
29	"schema": "public"
30	},
31	{
32	"isRlsEnabled": false,
33	"name": "relay_environment_links",
34	"entityType": "tables",
35	"schema": "public"
36	},
37	{
38	"isRlsEnabled": false,
39	"name": "relay_live_activities",
40	"entityType": "tables",
41	"schema": "public"
42	},
43	{
44	"isRlsEnabled": false,
45	"name": "relay_managed_endpoint_allocations",
46	"entityType": "tables",
47	"schema": "public"
48	},
49	{
50	"isRlsEnabled": false,
51	"name": "relay_managed_tunnel_limits",
52	"entityType": "tables",
53	"schema": "public"
54	},
55	{
56	"isRlsEnabled": false,
57	"name": "relay_mobile_devices",
58	"entityType": "tables",
59	"schema": "public"
60	},
61	{
62	"type": "varchar(191)",
63	"typeSchema": null,
64	"notNull": true,
65	"dimensions": 0,
66	"default": null,
67	"generated": null,
68	"identity": null,
69	"name": "environment_id",
70	"entityType": "columns",
71	"schema": "public",
72	"table": "relay_agent_activity_rows"
73	},
74	{
75	"type": "text",
76	"typeSchema": null,
77	"notNull": true,
78	"dimensions": 0,
79	"default": null,
80	"generated": null,
81	"identity": null,
82	"name": "environment_public_key",
83	"entityType": "columns",
84	"schema": "public",
85	"table": "relay_agent_activity_rows"
86	},
87	{
88	"type": "varchar(512)",
89	"typeSchema": null,
90	"notNull": true,
91	"dimensions": 0,
92	"default": null,
93	"generated": null,
94	"identity": null,
95	"name": "thread_id",
96	"entityType": "columns",
97	"schema": "public",
98	"table": "relay_agent_activity_rows"
99	},
100	{
101	"type": "jsonb",
102	"typeSchema": null,
103	"notNull": true,
104	"dimensions": 0,
105	"default": null,
106	"generated": null,
107	"identity": null,
108	"name": "state_json",
109	"entityType": "columns",
110	"schema": "public",
111	"table": "relay_agent_activity_rows"
112	},
113	{
114	"type": "varchar(64)",
115	"typeSchema": null,
116	"notNull": true,
117	"dimensions": 0,
118	"default": null,
119	"generated": null,
120	"identity": null,
121	"name": "updated_at",
122	"entityType": "columns",
123	"schema": "public",
124	"table": "relay_agent_activity_rows"
125	},
126	{
127	"type": "varchar(64)",
128	"typeSchema": null,
129	"notNull": true,
130	"dimensions": 0,
131	"default": null,
132	"generated": null,
133	"identity": null,
134	"name": "created_at",
135	"entityType": "columns",
136	"schema": "public",
137	"table": "relay_agent_activity_rows"
138	},
139	{
140	"type": "varchar(36)",
141	"typeSchema": null,
142	"notNull": true,
143	"dimensions": 0,
144	"default": null,
145	"generated": null,
146	"identity": null,
147	"name": "id",
148	"entityType": "columns",
149	"schema": "public",
150	"table": "relay_delivery_attempts"
151	},
152	{
153	"type": "varchar(64)",
154	"typeSchema": null,
155	"notNull": true,
156	"dimensions": 0,
157	"default": null,
158	"generated": null,
159	"identity": null,
160	"name": "created_at",
161	"entityType": "columns",
162	"schema": "public",
163	"table": "relay_delivery_attempts"
164	},
165	{
166	"type": "varchar(255)",
167	"typeSchema": null,
168	"notNull": false,
169	"dimensions": 0,
170	"default": null,
171	"generated": null,
172	"identity": null,
173	"name": "user_id",
174	"entityType": "columns",
175	"schema": "public",
176	"table": "relay_delivery_attempts"
177	},
178	{
179	"type": "varchar(191)",
180	"typeSchema": null,
181	"notNull": false,
182	"dimensions": 0,
183	"default": null,
184	"generated": null,
185	"identity": null,
186	"name": "environment_id",
187	"entityType": "columns",
188	"schema": "public",
189	"table": "relay_delivery_attempts"
190	},
191	{
192	"type": "varchar(512)",
193	"typeSchema": null,
194	"notNull": false,
195	"dimensions": 0,
196	"default": null,
197	"generated": null,
198	"identity": null,
199	"name": "thread_id",
200	"entityType": "columns",
201	"schema": "public",
202	"table": "relay_delivery_attempts"
203	},
204	{
205	"type": "varchar(255)",
206	"typeSchema": null,
207	"notNull": false,
208	"dimensions": 0,
209	"default": null,
210	"generated": null,
211	"identity": null,
212	"name": "device_id",
213	"entityType": "columns",
214	"schema": "public",
215	"table": "relay_delivery_attempts"
216	},
217	{
218	"type": "varchar(64)",
219	"typeSchema": null,
220	"notNull": true,
221	"dimensions": 0,
222	"default": null,
223	"generated": null,
224	"identity": null,
225	"name": "kind",
226	"entityType": "columns",
227	"schema": "public",
228	"table": "relay_delivery_attempts"
229	},
230	{
231	"type": "varchar(64)",
232	"typeSchema": null,
233	"notNull": false,
234	"dimensions": 0,
235	"default": null,
236	"generated": null,
237	"identity": null,
238	"name": "source_job_id",
239	"entityType": "columns",
240	"schema": "public",
241	"table": "relay_delivery_attempts"
242	},
243	{
244	"type": "varchar(16)",
245	"typeSchema": null,
246	"notNull": false,
247	"dimensions": 0,
248	"default": null,
249	"generated": null,
250	"identity": null,
251	"name": "token_suffix",
252	"entityType": "columns",
253	"schema": "public",
254	"table": "relay_delivery_attempts"
255	},
256	{
257	"type": "integer",
258	"typeSchema": null,
259	"notNull": false,
260	"dimensions": 0,
261	"default": null,
262	"generated": null,
263	"identity": null,
264	"name": "apns_status",
265	"entityType": "columns",
266	"schema": "public",
267	"table": "relay_delivery_attempts"
268	},
269	{
270	"type": "text",
271	"typeSchema": null,
272	"notNull": false,
273	"dimensions": 0,
274	"default": null,
275	"generated": null,
276	"identity": null,
277	"name": "apns_reason",
278	"entityType": "columns",
279	"schema": "public",
280	"table": "relay_delivery_attempts"
281	},
282	{
283	"type": "varchar(128)",
284	"typeSchema": null,
285	"notNull": false,
286	"dimensions": 0,
287	"default": null,
288	"generated": null,
289	"identity": null,
290	"name": "apns_id",
291	"entityType": "columns",
292	"schema": "public",
293	"table": "relay_delivery_attempts"
294	},
295	{
296	"type": "text",
297	"typeSchema": null,
298	"notNull": false,
299	"dimensions": 0,
300	"default": null,
301	"generated": null,
302	"identity": null,
303	"name": "transport_error",
304	"entityType": "columns",
305	"schema": "public",
306	"table": "relay_delivery_attempts"
307	},
308	{
309	"type": "varchar(128)",
310	"typeSchema": null,
311	"notNull": true,
312	"dimensions": 0,
313	"default": null,
314	"generated": null,
315	"identity": null,
316	"name": "thumbprint",
317	"entityType": "columns",
318	"schema": "public",
319	"table": "relay_dpop_proofs"
320	},
321	{
322	"type": "varchar(255)",
323	"typeSchema": null,
324	"notNull": true,
325	"dimensions": 0,
326	"default": null,
327	"generated": null,
328	"identity": null,
329	"name": "jti",
330	"entityType": "columns",
331	"schema": "public",
332	"table": "relay_dpop_proofs"
333	},
334	{
335	"type": "integer",
336	"typeSchema": null,
337	"notNull": true,
338	"dimensions": 0,
339	"default": null,
340	"generated": null,
341	"identity": null,
342	"name": "iat",
343	"entityType": "columns",
344	"schema": "public",
345	"table": "relay_dpop_proofs"
346	},
347	{
348	"type": "varchar(64)",
349	"typeSchema": null,
350	"notNull": true,
351	"dimensions": 0,
352	"default": null,
353	"generated": null,
354	"identity": null,
355	"name": "expires_at",
356	"entityType": "columns",
357	"schema": "public",
358	"table": "relay_dpop_proofs"
359	},
360	{
361	"type": "varchar(64)",
362	"typeSchema": null,
363	"notNull": true,
364	"dimensions": 0,
365	"default": null,
366	"generated": null,
367	"identity": null,
368	"name": "created_at",
369	"entityType": "columns",
370	"schema": "public",
371	"table": "relay_dpop_proofs"
372	},
373	{
374	"type": "varchar(64)",
375	"typeSchema": null,
376	"notNull": true,
377	"dimensions": 0,
378	"default": null,
379	"generated": null,
380	"identity": null,
381	"name": "credential_id",
382	"entityType": "columns",
383	"schema": "public",
384	"table": "relay_environment_credentials"
385	},
386	{
387	"type": "varchar(191)",
388	"typeSchema": null,
389	"notNull": true,
390	"dimensions": 0,
391	"default": null,
392	"generated": null,
393	"identity": null,
394	"name": "environment_id",
395	"entityType": "columns",
396	"schema": "public",
397	"table": "relay_environment_credentials"
398	},
399	{
400	"type": "text",
401	"typeSchema": null,
402	"notNull": true,
403	"dimensions": 0,
404	"default": null,
405	"generated": null,
406	"identity": null,
407	"name": "environment_public_key",
408	"entityType": "columns",
409	"schema": "public",
410	"table": "relay_environment_credentials"
411	},
412	{
413	"type": "varchar(191)",
414	"typeSchema": null,
415	"notNull": true,
416	"dimensions": 0,
417	"default": null,
418	"generated": null,
419	"identity": null,
420	"name": "credential_hash",
421	"entityType": "columns",
422	"schema": "public",
423	"table": "relay_environment_credentials"
424	},
425	{
426	"type": "varchar(64)",
427	"typeSchema": null,
428	"notNull": false,
429	"dimensions": 0,
430	"default": null,
431	"generated": null,
432	"identity": null,
433	"name": "revoked_at",
434	"entityType": "columns",
435	"schema": "public",
436	"table": "relay_environment_credentials"
437	},
438	{
439	"type": "varchar(64)",
440	"typeSchema": null,
441	"notNull": true,
442	"dimensions": 0,
443	"default": null,
444	"generated": null,
445	"identity": null,
446	"name": "created_at",
447	"entityType": "columns",
448	"schema": "public",
449	"table": "relay_environment_credentials"
450	},
451	{
452	"type": "varchar(64)",
453	"typeSchema": null,
454	"notNull": true,
455	"dimensions": 0,
456	"default": null,
457	"generated": null,
458	"identity": null,
459	"name": "updated_at",
460	"entityType": "columns",
461	"schema": "public",
462	"table": "relay_environment_credentials"
463	},
464	{
465	"type": "varchar(191)",
466	"typeSchema": null,
467	"notNull": true,
468	"dimensions": 0,
469	"default": null,
470	"generated": null,
471	"identity": null,
472	"name": "user_id",
473	"entityType": "columns",
474	"schema": "public",
475	"table": "relay_environment_links"
476	},
477	{
478	"type": "varchar(191)",
479	"typeSchema": null,
480	"notNull": true,
481	"dimensions": 0,
482	"default": null,
483	"generated": null,
484	"identity": null,
485	"name": "environment_id",
486	"entityType": "columns",
487	"schema": "public",
488	"table": "relay_environment_links"
489	},
490	{
491	"type": "text",
492	"typeSchema": null,
493	"notNull": true,
494	"dimensions": 0,
495	"default": "'T3 Environment'",
496	"generated": null,
497	"identity": null,
498	"name": "environment_label",
499	"entityType": "columns",
500	"schema": "public",
501	"table": "relay_environment_links"
502	},
503	{
504	"type": "text",
505	"typeSchema": null,
506	"notNull": true,
507	"dimensions": 0,
508	"default": null,
509	"generated": null,
510	"identity": null,
511	"name": "environment_public_key",
512	"entityType": "columns",
513	"schema": "public",
514	"table": "relay_environment_links"
515	},
516	{
517	"type": "text",
518	"typeSchema": null,
519	"notNull": true,
520	"dimensions": 0,
521	"default": null,
522	"generated": null,
523	"identity": null,
524	"name": "endpoint_http_base_url",
525	"entityType": "columns",
526	"schema": "public",
527	"table": "relay_environment_links"
528	},
529	{
530	"type": "text",
531	"typeSchema": null,
532	"notNull": true,
533	"dimensions": 0,
534	"default": null,
535	"generated": null,
536	"identity": null,
537	"name": "endpoint_ws_base_url",
538	"entityType": "columns",
539	"schema": "public",
540	"table": "relay_environment_links"
541	},
542	{
543	"type": "varchar(32)",
544	"typeSchema": null,
545	"notNull": true,
546	"dimensions": 0,
547	"default": null,
548	"generated": null,
549	"identity": null,
550	"name": "endpoint_provider_kind",
551	"entityType": "columns",
552	"schema": "public",
553	"table": "relay_environment_links"
554	},
555	{
556	"type": "boolean",
557	"typeSchema": null,
558	"notNull": true,
559	"dimensions": 0,
560	"default": "true",
561	"generated": null,
562	"identity": null,
563	"name": "notifications_enabled",
564	"entityType": "columns",
565	"schema": "public",
566	"table": "relay_environment_links"
567	},
568	{
569	"type": "boolean",
570	"typeSchema": null,
571	"notNull": true,
572	"dimensions": 0,
573	"default": "true",
574	"generated": null,
575	"identity": null,
576	"name": "live_activities_enabled",
577	"entityType": "columns",
578	"schema": "public",
579	"table": "relay_environment_links"
580	},
581	{
582	"type": "boolean",
583	"typeSchema": null,
584	"notNull": true,
585	"dimensions": 0,
586	"default": "false",
587	"generated": null,
588	"identity": null,
589	"name": "managed_tunnels_enabled",
590	"entityType": "columns",
591	"schema": "public",
592	"table": "relay_environment_links"
593	},
594	{
595	"type": "boolean",
596	"typeSchema": null,
597	"notNull": true,
598	"dimensions": 0,
599	"default": "false",
600	"generated": null,
601	"identity": null,
602	"name": "hold_webhooks_while_offline",
603	"entityType": "columns",
604	"schema": "public",
605	"table": "relay_environment_links"
606	},
607	{
608	"type": "varchar(191)",
609	"typeSchema": null,
610	"notNull": false,
611	"dimensions": 0,
612	"default": null,
613	"generated": null,
614	"identity": null,
615	"name": "created_by_device_id",
616	"entityType": "columns",
617	"schema": "public",
618	"table": "relay_environment_links"
619	},
620	{
621	"type": "varchar(64)",
622	"typeSchema": null,
623	"notNull": false,
624	"dimensions": 0,
625	"default": null,
626	"generated": null,
627	"identity": null,
628	"name": "revoked_at",
629	"entityType": "columns",
630	"schema": "public",
631	"table": "relay_environment_links"
632	},
633	{
634	"type": "varchar(64)",
635	"typeSchema": null,
636	"notNull": true,
637	"dimensions": 0,
638	"default": null,
639	"generated": null,
640	"identity": null,
641	"name": "created_at",
642	"entityType": "columns",
643	"schema": "public",
644	"table": "relay_environment_links"
645	},
646	{
647	"type": "varchar(64)",
648	"typeSchema": null,
649	"notNull": true,
650	"dimensions": 0,
651	"default": null,
652	"generated": null,
653	"identity": null,
654	"name": "updated_at",
655	"entityType": "columns",
656	"schema": "public",
657	"table": "relay_environment_links"
658	},
659	{
660	"type": "varchar(255)",
661	"typeSchema": null,
662	"notNull": true,
663	"dimensions": 0,
664	"default": null,
665	"generated": null,
666	"identity": null,
667	"name": "user_id",
668	"entityType": "columns",
669	"schema": "public",
670	"table": "relay_live_activities"
671	},
672	{
673	"type": "varchar(255)",
674	"typeSchema": null,
675	"notNull": true,
676	"dimensions": 0,
677	"default": null,
678	"generated": null,
679	"identity": null,
680	"name": "device_id",
681	"entityType": "columns",
682	"schema": "public",
683	"table": "relay_live_activities"
684	},
685	{
686	"type": "text",
687	"typeSchema": null,
688	"notNull": false,
689	"dimensions": 0,
690	"default": null,
691	"generated": null,
692	"identity": null,
693	"name": "activity_push_token",
694	"entityType": "columns",
695	"schema": "public",
696	"table": "relay_live_activities"
697	},
698	{
699	"type": "varchar(64)",
700	"typeSchema": null,
701	"notNull": false,
702	"dimensions": 0,
703	"default": null,
704	"generated": null,
705	"identity": null,
706	"name": "remote_start_queued_at",
707	"entityType": "columns",
708	"schema": "public",
709	"table": "relay_live_activities"
710	},
711	{
712	"type": "varchar(64)",
713	"typeSchema": null,
714	"notNull": false,
715	"dimensions": 0,
716	"default": null,
717	"generated": null,
718	"identity": null,
719	"name": "remote_started_at",
720	"entityType": "columns",
721	"schema": "public",
722	"table": "relay_live_activities"
723	},
724	{
725	"type": "varchar(64)",
726	"typeSchema": null,
727	"notNull": false,
728	"dimensions": 0,
729	"default": null,
730	"generated": null,
731	"identity": null,
732	"name": "ended_at",
733	"entityType": "columns",
734	"schema": "public",
735	"table": "relay_live_activities"
736	},
737	{
738	"type": "jsonb",
739	"typeSchema": null,
740	"notNull": false,
741	"dimensions": 0,
742	"default": null,
743	"generated": null,
744	"identity": null,
745	"name": "last_aggregate_json",
746	"entityType": "columns",
747	"schema": "public",
748	"table": "relay_live_activities"
749	},
750	{
751	"type": "varchar(64)",
752	"typeSchema": null,
753	"notNull": false,
754	"dimensions": 0,
755	"default": null,
756	"generated": null,
757	"identity": null,
758	"name": "last_live_activity_delivery_at",
759	"entityType": "columns",
760	"schema": "public",
761	"table": "relay_live_activities"
762	},
763	{
764	"type": "varchar(64)",
765	"typeSchema": null,
766	"notNull": true,
767	"dimensions": 0,
768	"default": null,
769	"generated": null,
770	"identity": null,
771	"name": "created_at",
772	"entityType": "columns",
773	"schema": "public",
774	"table": "relay_live_activities"
775	},
776	{
777	"type": "varchar(64)",
778	"typeSchema": null,
779	"notNull": true,
780	"dimensions": 0,
781	"default": null,
782	"generated": null,
783	"identity": null,
784	"name": "updated_at",
785	"entityType": "columns",
786	"schema": "public",
787	"table": "relay_live_activities"
788	},
789	{
790	"type": "varchar(191)",
791	"typeSchema": null,
792	"notNull": true,
793	"dimensions": 0,
794	"default": null,
795	"generated": null,
796	"identity": null,
797	"name": "user_id",
798	"entityType": "columns",
799	"schema": "public",
800	"table": "relay_managed_endpoint_allocations"
801	},
802	{
803	"type": "varchar(191)",
804	"typeSchema": null,
805	"notNull": true,
806	"dimensions": 0,
807	"default": null,
808	"generated": null,
809	"identity": null,
810	"name": "environment_id",
811	"entityType": "columns",
812	"schema": "public",
813	"table": "relay_managed_endpoint_allocations"
814	},
815	{
816	"type": "text",
817	"typeSchema": null,
818	"notNull": true,
819	"dimensions": 0,
820	"default": null,
821	"generated": null,
822	"identity": null,
823	"name": "hostname",
824	"entityType": "columns",
825	"schema": "public",
826	"table": "relay_managed_endpoint_allocations"
827	},
828	{
829	"type": "varchar(191)",
830	"typeSchema": null,
831	"notNull": false,
832	"dimensions": 0,
833	"default": null,
834	"generated": null,
835	"identity": null,
836	"name": "tunnel_id",
837	"entityType": "columns",
838	"schema": "public",
839	"table": "relay_managed_endpoint_allocations"
840	},
841	{
842	"type": "text",
843	"typeSchema": null,
844	"notNull": true,
845	"dimensions": 0,
846	"default": null,
847	"generated": null,
848	"identity": null,
849	"name": "tunnel_name",
850	"entityType": "columns",
851	"schema": "public",
852	"table": "relay_managed_endpoint_allocations"
853	},
854	{
855	"type": "varchar(191)",
856	"typeSchema": null,
857	"notNull": false,
858	"dimensions": 0,
859	"default": null,
860	"generated": null,
861	"identity": null,
862	"name": "dns_record_id",
863	"entityType": "columns",
864	"schema": "public",
865	"table": "relay_managed_endpoint_allocations"
866	},
867	{
868	"type": "varchar(64)",
869	"typeSchema": null,
870	"notNull": false,
871	"dimensions": 0,
872	"default": null,
873	"generated": null,
874	"identity": null,
875	"name": "ready_at",
876	"entityType": "columns",
877	"schema": "public",
878	"table": "relay_managed_endpoint_allocations"
879	},
880	{
881	"type": "varchar(64)",
882	"typeSchema": null,
883	"notNull": false,
884	"dimensions": 0,
885	"default": null,
886	"generated": null,
887	"identity": null,
888	"name": "recovery_enabled_at",
889	"entityType": "columns",
890	"schema": "public",
891	"table": "relay_managed_endpoint_allocations"
892	},
893	{
894	"type": "text",
895	"typeSchema": null,
896	"notNull": false,
897	"dimensions": 0,
898	"default": null,
899	"generated": null,
900	"identity": null,
901	"name": "recovery_environment_public_key",
902	"entityType": "columns",
903	"schema": "public",
904	"table": "relay_managed_endpoint_allocations"
905	},
906	{
907	"type": "jsonb",
908	"typeSchema": null,
909	"notNull": false,
910	"dimensions": 0,
911	"default": null,
912	"generated": null,
913	"identity": null,
914	"name": "origin",
915	"entityType": "columns",
916	"schema": "public",
917	"table": "relay_managed_endpoint_allocations"
918	},
919	{
920	"type": "integer",
921	"typeSchema": null,
922	"notNull": true,
923	"dimensions": 0,
924	"default": "0",
925	"generated": null,
926	"identity": null,
927	"name": "generation",
928	"entityType": "columns",
929	"schema": "public",
930	"table": "relay_managed_endpoint_allocations"
931	},
932	{
933	"type": "varchar(64)",
934	"typeSchema": null,
935	"notNull": true,
936	"dimensions": 0,
937	"default": null,
938	"generated": null,
939	"identity": null,
940	"name": "created_at",
941	"entityType": "columns",
942	"schema": "public",
943	"table": "relay_managed_endpoint_allocations"
944	},
945	{
946	"type": "varchar(64)",
947	"typeSchema": null,
948	"notNull": true,
949	"dimensions": 0,
950	"default": null,
951	"generated": null,
952	"identity": null,
953	"name": "updated_at",
954	"entityType": "columns",
955	"schema": "public",
956	"table": "relay_managed_endpoint_allocations"
957	},
958	{
959	"type": "varchar(191)",
960	"typeSchema": null,
961	"notNull": true,
962	"dimensions": 0,
963	"default": null,
964	"generated": null,
965	"identity": null,
966	"name": "user_id",
967	"entityType": "columns",
968	"schema": "public",
969	"table": "relay_managed_tunnel_limits"
970	},
971	{
972	"type": "integer",
973	"typeSchema": null,
974	"notNull": true,
975	"dimensions": 0,
976	"default": null,
977	"generated": null,
978	"identity": null,
979	"name": "max_tunnels",
980	"entityType": "columns",
981	"schema": "public",
982	"table": "relay_managed_tunnel_limits"
983	},
984	{
985	"type": "varchar(64)",
986	"typeSchema": null,
987	"notNull": true,
988	"dimensions": 0,
989	"default": null,
990	"generated": null,
991	"identity": null,
992	"name": "created_at",
993	"entityType": "columns",
994	"schema": "public",
995	"table": "relay_managed_tunnel_limits"
996	},
997	{
998	"type": "varchar(64)",
999	"typeSchema": null,
1000	"notNull": true,
1001	"dimensions": 0,
1002	"default": null,
1003	"generated": null,
1004	"identity": null,
1005	"name": "updated_at",
1006	"entityType": "columns",
1007	"schema": "public",
1008	"table": "relay_managed_tunnel_limits"
1009	},
1010	{
1011	"type": "varchar(255)",
1012	"typeSchema": null,
1013	"notNull": true,
1014	"dimensions": 0,
1015	"default": null,
1016	"generated": null,
1017	"identity": null,
1018	"name": "user_id",
1019	"entityType": "columns",
1020	"schema": "public",
1021	"table": "relay_mobile_devices"
1022	},
1023	{
1024	"type": "varchar(255)",
1025	"typeSchema": null,
1026	"notNull": true,
1027	"dimensions": 0,
1028	"default": null,
1029	"generated": null,
1030	"identity": null,
1031	"name": "device_id",
1032	"entityType": "columns",
1033	"schema": "public",
1034	"table": "relay_mobile_devices"
1035	},
1036	{
1037	"type": "text",
1038	"typeSchema": null,
1039	"notNull": true,
1040	"dimensions": 0,
1041	"default": "'iOS device'",
1042	"generated": null,
1043	"identity": null,
1044	"name": "label",
1045	"entityType": "columns",
1046	"schema": "public",
1047	"table": "relay_mobile_devices"
1048	},
1049	{
1050	"type": "varchar(16)",
1051	"typeSchema": null,
1052	"notNull": true,
1053	"dimensions": 0,
1054	"default": null,
1055	"generated": null,
1056	"identity": null,
1057	"name": "platform",
1058	"entityType": "columns",
1059	"schema": "public",
1060	"table": "relay_mobile_devices"
1061	},
1062	{
1063	"type": "integer",
1064	"typeSchema": null,
1065	"notNull": false,
1066	"dimensions": 0,
1067	"default": null,
1068	"generated": null,
1069	"identity": null,
1070	"name": "ios_major_version",
1071	"entityType": "columns",
1072	"schema": "public",
1073	"table": "relay_mobile_devices"
1074	},
1075	{
1076	"type": "integer",
1077	"typeSchema": null,
1078	"notNull": false,
1079	"dimensions": 0,
1080	"default": null,
1081	"generated": null,
1082	"identity": null,
1083	"name": "android_api_level",
1084	"entityType": "columns",
1085	"schema": "public",
1086	"table": "relay_mobile_devices"
1087	},
1088	{
1089	"type": "varchar(64)",
1090	"typeSchema": null,
1091	"notNull": false,
1092	"dimensions": 0,
1093	"default": null,
1094	"generated": null,
1095	"identity": null,
1096	"name": "app_version",
1097	"entityType": "columns",
1098	"schema": "public",
1099	"table": "relay_mobile_devices"
1100	},
1101	{
1102	"type": "varchar(255)",
1103	"typeSchema": null,
1104	"notNull": false,
1105	"dimensions": 0,
1106	"default": null,
1107	"generated": null,
1108	"identity": null,
1109	"name": "bundle_id",
1110	"entityType": "columns",
1111	"schema": "public",
1112	"table": "relay_mobile_devices"
1113	},
1114	{
1115	"type": "varchar(16)",
1116	"typeSchema": null,
1117	"notNull": false,
1118	"dimensions": 0,
1119	"default": null,
1120	"generated": null,
1121	"identity": null,
1122	"name": "aps_environment",
1123	"entityType": "columns",
1124	"schema": "public",
1125	"table": "relay_mobile_devices"
1126	},
1127	{
1128	"type": "text",
1129	"typeSchema": null,
1130	"notNull": false,
1131	"dimensions": 0,
1132	"default": null,
1133	"generated": null,
1134	"identity": null,
1135	"name": "push_token",
1136	"entityType": "columns",
1137	"schema": "public",
1138	"table": "relay_mobile_devices"
1139	},
1140	{
1141	"type": "text",
1142	"typeSchema": null,
1143	"notNull": false,
1144	"dimensions": 0,
1145	"default": null,
1146	"generated": null,
1147	"identity": null,
1148	"name": "push_to_start_token",
1149	"entityType": "columns",
1150	"schema": "public",
1151	"table": "relay_mobile_devices"
1152	},
1153	{
1154	"type": "jsonb",
1155	"typeSchema": null,
1156	"notNull": true,
1157	"dimensions": 0,
1158	"default": null,
1159	"generated": null,
1160	"identity": null,
1161	"name": "preferences_json",
1162	"entityType": "columns",
1163	"schema": "public",
1164	"table": "relay_mobile_devices"
1165	},
1166	{
1167	"type": "varchar(64)",
1168	"typeSchema": null,
1169	"notNull": true,
1170	"dimensions": 0,
1171	"default": null,
1172	"generated": null,
1173	"identity": null,
1174	"name": "created_at",
1175	"entityType": "columns",
1176	"schema": "public",
1177	"table": "relay_mobile_devices"
1178	},
1179	{
1180	"type": "varchar(64)",
1181	"typeSchema": null,
1182	"notNull": true,
1183	"dimensions": 0,
1184	"default": null,
1185	"generated": null,
1186	"identity": null,
1187	"name": "updated_at",
1188	"entityType": "columns",
1189	"schema": "public",
1190	"table": "relay_mobile_devices"
1191	},
1192	{
1193	"nameExplicit": true,
1194	"columns": [
1195	{
1196	"value": "updated_at",
1197	"isExpression": false,
1198	"asc": true,
1199	"nullsFirst": false,
1200	"opclass": null
1201	}
1202	],
1203	"isUnique": false,
1204	"where": null,
1205	"with": "",
1206	"method": "btree",
1207	"concurrently": false,
1208	"name": "idx_relay_agent_activity_rows_updated",
1209	"entityType": "indexes",
1210	"schema": "public",
1211	"table": "relay_agent_activity_rows"
1212	},
1213	{
1214	"nameExplicit": true,
1215	"columns": [
1216	{
1217	"value": "environment_id",
1218	"isExpression": false,
1219	"asc": true,
1220	"nullsFirst": false,
1221	"opclass": null
1222	},
1223	{
1224	"value": "thread_id",
1225	"isExpression": false,
1226	"asc": true,
1227	"nullsFirst": false,
1228	"opclass": null
1229	},
1230	{
1231	"value": "created_at",
1232	"isExpression": false,
1233	"asc": true,
1234	"nullsFirst": false,
1235	"opclass": null
1236	}
1237	],
1238	"isUnique": false,
1239	"where": null,
1240	"with": "",
1241	"method": "btree",
1242	"concurrently": false,
1243	"name": "idx_relay_delivery_attempts_environment",
1244	"entityType": "indexes",
1245	"schema": "public",
1246	"table": "relay_delivery_attempts"
1247	},
1248	{
1249	"nameExplicit": true,
1250	"columns": [
1251	{
1252	"value": "source_job_id",
1253	"isExpression": false,
1254	"asc": true,
1255	"nullsFirst": false,
1256	"opclass": null
1257	}
1258	],
1259	"isUnique": true,
1260	"where": null,
1261	"with": "",
1262	"method": "btree",
1263	"concurrently": false,
1264	"name": "idx_relay_delivery_attempts_source_job",
1265	"entityType": "indexes",
1266	"schema": "public",
1267	"table": "relay_delivery_attempts"
1268	},
1269	{
1270	"nameExplicit": true,
1271	"columns": [
1272	{
1273	"value": "expires_at",
1274	"isExpression": false,
1275	"asc": true,
1276	"nullsFirst": false,
1277	"opclass": null
1278	}
1279	],
1280	"isUnique": false,
1281	"where": null,
1282	"with": "",
1283	"method": "btree",
1284	"concurrently": false,
1285	"name": "idx_relay_dpop_proofs_expires_at",
1286	"entityType": "indexes",
1287	"schema": "public",
1288	"table": "relay_dpop_proofs"
1289	},
1290	{
1291	"nameExplicit": true,
1292	"columns": [
1293	{
1294	"value": "credential_hash",
1295	"isExpression": false,
1296	"asc": true,
1297	"nullsFirst": false,
1298	"opclass": null
1299	}
1300	],
1301	"isUnique": true,
1302	"where": null,
1303	"with": "",
1304	"method": "btree",
1305	"concurrently": false,
1306	"name": "idx_relay_environment_credentials_hash",
1307	"entityType": "indexes",
1308	"schema": "public",
1309	"table": "relay_environment_credentials"
1310	},
1311	{
1312	"nameExplicit": true,
1313	"columns": [
1314	{
1315	"value": "environment_id",
1316	"isExpression": false,
1317	"asc": true,
1318	"nullsFirst": false,
1319	"opclass": null
1320	},
1321	{
1322	"value": "revoked_at",
1323	"isExpression": false,
1324	"asc": true,
1325	"nullsFirst": false,
1326	"opclass": null
1327	}
1328	],
1329	"isUnique": false,
1330	"where": null,
1331	"with": "",
1332	"method": "btree",
1333	"concurrently": false,
1334	"name": "idx_relay_environment_credentials_environment",
1335	"entityType": "indexes",
1336	"schema": "public",
1337	"table": "relay_environment_credentials"
1338	},
1339	{
1340	"nameExplicit": true,
1341	"columns": [
1342	{
1343	"value": "environment_id",
1344	"isExpression": false,
1345	"asc": true,
1346	"nullsFirst": false,
1347	"opclass": null
1348	},
1349	{
1350	"value": "environment_public_key",
1351	"isExpression": false,
1352	"asc": true,
1353	"nullsFirst": false,
1354	"opclass": null
1355	},
1356	{
1357	"value": "revoked_at",
1358	"isExpression": false,
1359	"asc": true,
1360	"nullsFirst": false,
1361	"opclass": null
1362	}
1363	],
1364	"isUnique": false,
1365	"where": null,
1366	"with": "",
1367	"method": "btree",
1368	"concurrently": false,
1369	"name": "idx_relay_environment_credentials_environment_key",
1370	"entityType": "indexes",
1371	"schema": "public",
1372	"table": "relay_environment_credentials"
1373	},
1374	{
1375	"nameExplicit": true,
1376	"columns": [
1377	{
1378	"value": "environment_id",
1379	"isExpression": false,
1380	"asc": true,
1381	"nullsFirst": false,
1382	"opclass": null
1383	},
1384	{
1385	"value": "revoked_at",
1386	"isExpression": false,
1387	"asc": true,
1388	"nullsFirst": false,
1389	"opclass": null
1390	}
1391	],
1392	"isUnique": false,
1393	"where": null,
1394	"with": "",
1395	"method": "btree",
1396	"concurrently": false,
1397	"name": "idx_relay_environment_links_environment",
1398	"entityType": "indexes",
1399	"schema": "public",
1400	"table": "relay_environment_links"
1401	},
1402	{
1403	"nameExplicit": true,
1404	"columns": [
1405	{
1406	"value": "activity_push_token",
1407	"isExpression": false,
1408	"asc": true,
1409	"nullsFirst": false,
1410	"opclass": null
1411	}
1412	],
1413	"isUnique": true,
1414	"where": null,
1415	"with": "",
1416	"method": "btree",
1417	"concurrently": false,
1418	"name": "idx_relay_live_activities_activity_push_token",
1419	"entityType": "indexes",
1420	"schema": "public",
1421	"table": "relay_live_activities"
1422	},
1423	{
1424	"nameExplicit": true,
1425	"columns": [
1426	{
1427	"value": "hostname",
1428	"isExpression": false,
1429	"asc": true,
1430	"nullsFirst": false,
1431	"opclass": null
1432	}
1433	],
1434	"isUnique": true,
1435	"where": null,
1436	"with": "",
1437	"method": "btree",
1438	"concurrently": false,
1439	"name": "idx_relay_managed_endpoint_allocations_hostname",
1440	"entityType": "indexes",
1441	"schema": "public",
1442	"table": "relay_managed_endpoint_allocations"
1443	},
1444	{
1445	"nameExplicit": true,
1446	"columns": [
1447	{
1448	"value": "tunnel_name",
1449	"isExpression": false,
1450	"asc": true,
1451	"nullsFirst": false,
1452	"opclass": null
1453	}
1454	],
1455	"isUnique": true,
1456	"where": null,
1457	"with": "",
1458	"method": "btree",
1459	"concurrently": false,
1460	"name": "idx_relay_managed_endpoint_allocations_tunnel_name",
1461	"entityType": "indexes",
1462	"schema": "public",
1463	"table": "relay_managed_endpoint_allocations"
1464	},
1465	{
1466	"nameExplicit": true,
1467	"columns": [
1468	{
1469	"value": "push_token",
1470	"isExpression": false,
1471	"asc": true,
1472	"nullsFirst": false,
1473	"opclass": null
1474	}
1475	],
1476	"isUnique": true,
1477	"where": null,
1478	"with": "",
1479	"method": "btree",
1480	"concurrently": false,
1481	"name": "idx_relay_mobile_devices_push_token",
1482	"entityType": "indexes",
1483	"schema": "public",
1484	"table": "relay_mobile_devices"
1485	},
1486	{
1487	"nameExplicit": true,
1488	"columns": [
1489	{
1490	"value": "push_to_start_token",
1491	"isExpression": false,
1492	"asc": true,
1493	"nullsFirst": false,
1494	"opclass": null
1495	}
1496	],
1497	"isUnique": true,
1498	"where": null,
1499	"with": "",
1500	"method": "btree",
1501	"concurrently": false,
1502	"name": "idx_relay_mobile_devices_push_to_start_token",
1503	"entityType": "indexes",
1504	"schema": "public",
1505	"table": "relay_mobile_devices"
1506	},
1507	{
1508	"columns": ["environment_id", "environment_public_key", "thread_id"],
1509	"nameExplicit": false,
1510	"name": "relay_agent_activity_rows_pkey",
1511	"entityType": "pks",
1512	"schema": "public",
1513	"table": "relay_agent_activity_rows"
1514	},
1515	{
1516	"columns": ["thumbprint", "jti"],
1517	"nameExplicit": false,
1518	"name": "relay_dpop_proofs_pkey",
1519	"entityType": "pks",
1520	"schema": "public",
1521	"table": "relay_dpop_proofs"
1522	},
1523	{
1524	"columns": ["user_id", "environment_id"],
1525	"nameExplicit": false,
1526	"name": "relay_environment_links_pkey",
1527	"entityType": "pks",
1528	"schema": "public",
1529	"table": "relay_environment_links"
1530	},
1531	{
1532	"columns": ["user_id", "device_id"],
1533	"nameExplicit": false,
1534	"name": "relay_live_activities_pkey",
1535	"entityType": "pks",
1536	"schema": "public",
1537	"table": "relay_live_activities"
1538	},
1539	{
1540	"columns": ["user_id", "environment_id"],
1541	"nameExplicit": false,
1542	"name": "relay_managed_endpoint_allocations_pkey",
1543	"entityType": "pks",
1544	"schema": "public",
1545	"table": "relay_managed_endpoint_allocations"
1546	},
1547	{
1548	"columns": ["user_id", "device_id"],
1549	"nameExplicit": false,
1550	"name": "relay_mobile_devices_pkey",
1551	"entityType": "pks",
1552	"schema": "public",
1553	"table": "relay_mobile_devices"
1554	},
1555	{
1556	"columns": ["id"],
1557	"nameExplicit": false,
1558	"name": "relay_delivery_attempts_pkey",
1559	"schema": "public",
1560	"table": "relay_delivery_attempts",
1561	"entityType": "pks"
1562	},
1563	{
1564	"columns": ["credential_id"],
1565	"nameExplicit": false,
1566	"name": "relay_environment_credentials_pkey",
1567	"schema": "public",
1568	"table": "relay_environment_credentials",
1569	"entityType": "pks"
1570	},
1571	{
1572	"columns": ["user_id"],
1573	"nameExplicit": false,
1574	"name": "relay_managed_tunnel_limits_pkey",
1575	"schema": "public",
1576	"table": "relay_managed_tunnel_limits",
1577	"entityType": "pks"
1578	}
1579	],
1580	"renames": []
1581	}

infra/relay/package.json+1

...	@@ -13,6 +13,7 @@	...	@@ -13,6 +13,7 @@
13	"dependencies": {	13	"dependencies": {
14	"@clerk/backend": "catalog:",	14	"@clerk/backend": "catalog:",
15	"@effect/sql-pg": "catalog:",	15	"@effect/sql-pg": "catalog:",
		16	"@effect/sql-sqlite-do": "catalog:",
16	"@noble/curves": "catalog:",	17	"@noble/curves": "catalog:",
17	"@noble/hashes": "catalog:",	18	"@noble/hashes": "catalog:",
18	"@t3tools/client-runtime": "workspace:*",	19	"@t3tools/client-runtime": "workspace:*",

infra/relay/src/agentActivity/AgentActivityPublisher.test.ts+1

...	@@ -95,6 +95,7 @@ function makeEnvironmentLinks(	...	@@ -95,6 +95,7 @@ function makeEnvironmentLinks(
95	listForUser: () => Effect.succeed([]),	95	listForUser: () => Effect.succeed([]),
96	getForUser: () => Effect.succeed(null),	96	getForUser: () => Effect.succeed(null),
97	findActiveManagedForEnvironment: () => Effect.succeed([]),	97	findActiveManagedForEnvironment: () => Effect.succeed([]),
		98	setHoldWebhooksWhileOffline: () => Effect.void,
98	revokeForUser: () => Effect.succeed(false),	99	revokeForUser: () => Effect.succeed(false),
99	...overrides,	100	...overrides,
100	};	101	};

infra/relay/src/agentActivity/FcmDeliveries.test.ts+1

...	@@ -172,6 +172,7 @@ function harness() {	...	@@ -172,6 +172,7 @@ function harness() {
172	),	172	),
173	listForUser: () => Effect.succeed([]),	173	listForUser: () => Effect.succeed([]),
174	findActiveManagedForEnvironment: () => Effect.succeed([]),	174	findActiveManagedForEnvironment: () => Effect.succeed([]),
		175	setHoldWebhooksWhileOffline: () => Effect.void,
175	revokeForUser: () => Effect.succeed(false),	176	revokeForUser: () => Effect.succeed(false),
176	getForUser: (input) =>	177	getForUser: (input) =>
177	Effect.sync(() =>	178	Effect.sync(() =>

infra/relay/src/agentActivity/MobileRegistrations.test.ts+1

...	@@ -117,6 +117,7 @@ function makeEnvironmentLinks(	...	@@ -117,6 +117,7 @@ function makeEnvironmentLinks(
117	listForUser: () => Effect.succeed([]),	117	listForUser: () => Effect.succeed([]),
118	getForUser: () => Effect.succeed(null),	118	getForUser: () => Effect.succeed(null),
119	findActiveManagedForEnvironment: () => Effect.succeed([]),	119	findActiveManagedForEnvironment: () => Effect.succeed([]),
		120	setHoldWebhooksWhileOffline: () => Effect.void,
120	revokeForUser: () => Effect.succeed(false),	121	revokeForUser: () => Effect.succeed(false),
121	...overrides,	122	...overrides,
122	};	123	};

infra/relay/src/deploymentConfig.ts+13

...	@@ -124,3 +124,16 @@ export function managedEndpointTunnelNamePrefix(stage: string): string {	...	@@ -124,3 +124,16 @@ export function managedEndpointTunnelNamePrefix(stage: string): string {
124	export function managedEndpointTunnelName(stage: string, hash: string): string {	124	export function managedEndpointTunnelName(stage: string, hash: string): string {
125	return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`;	125	return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`;
126	}	126	}
		127
		128	/**
		129	* A managed endpoint's public key in webhook URLs: the hash suffix its tunnel
		130	* name ends with. The hash covers user and environment, so one key names
		131	* exactly one link, unlike the environment id, which any account can claim.
		132	*/
		133	export const MANAGED_ENDPOINT_KEY_PATTERN = new RegExp(
		134	`^[0-9a-f]{${MANAGED_ENDPOINT_HASH_LENGTH}}$`,
		135	);
		136
		137	export function managedEndpointTunnelNameForKey(stage: string, endpointKey: string): string {
		138	return `${managedEndpointTunnelNamePrefix(stage)}${endpointKey}`;
		139	}

infra/relay/src/environments/EnvironmentConnector.test.ts+2

...	@@ -194,6 +194,7 @@ function makeAllocations(	...	@@ -194,6 +194,7 @@ function makeAllocations(
194	},	194	},
195	): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] {	195	): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] {
196	return {	196	return {
		197	getByTunnelName: () => Effect.die("unused getByTunnelName"),
197	get: () => Effect.succeed(allocation),	198	get: () => Effect.succeed(allocation),
198	reserve: () => Effect.die("unused"),	199	reserve: () => Effect.die("unused"),
199	recordTunnel: () => Effect.die("unused"),	200	recordTunnel: () => Effect.die("unused"),
...	@@ -230,6 +231,7 @@ function makeLinks(	...	@@ -230,6 +231,7 @@ function makeLinks(
230	...overrides,	231	...overrides,
231	}),	232	}),
232	findActiveManagedForEnvironment: () => Effect.succeed([]),	233	findActiveManagedForEnvironment: () => Effect.succeed([]),
		234	setHoldWebhooksWhileOffline: () => Effect.void,
233	revokeForUser: () => Effect.succeed(false),	235	revokeForUser: () => Effect.succeed(false),
234	};	236	};
235	}	237	}

infra/relay/src/environments/EnvironmentLinker.test.ts+1

...	@@ -127,6 +127,7 @@ function testLayer(input?: {	...	@@ -127,6 +127,7 @@ function testLayer(input?: {
127	listForUser: () => Effect.succeed([]),	127	listForUser: () => Effect.succeed([]),
128	getForUser: () => Effect.succeed(null),	128	getForUser: () => Effect.succeed(null),
129	findActiveManagedForEnvironment: () => Effect.succeed([]),	129	findActiveManagedForEnvironment: () => Effect.succeed([]),
		130	setHoldWebhooksWhileOffline: () => Effect.void,
130	revokeForUser: () => Effect.succeed(false),	131	revokeForUser: () => Effect.succeed(false),
131	}),	132	}),
132	Layer.succeed(EnvironmentCredentials.EnvironmentCredentials, {	133	Layer.succeed(EnvironmentCredentials.EnvironmentCredentials, {

infra/relay/src/environments/EnvironmentLinks.test.ts+39

...	@@ -122,6 +122,45 @@ describe("EnvironmentLinks", () => {	...	@@ -122,6 +122,45 @@ describe("EnvironmentLinks", () => {
122	);	122	);
123	});	123	});
124		124
		125	it.effect("carries the webhook-hold opt-in over only from links with the same key", () => {
		126	const inserted: Array<Record<string, unknown>> = [];
		127	const fakeDb = {
		128	insert: () => ({
		129	values: (values: Record<string, unknown>) => {
		130	inserted.push(values);
		131	return { onConflictDoUpdate: () => Effect.void };
		132	},
		133	}),
		134	} as unknown as RelayDb.RelayDb["Service"];
		135
		136	return Effect.gen(function* () {
		137	const links = yield* EnvironmentLinks.EnvironmentLinks;
		138	yield* links.upsert({
		139	userId: "user-1",
		140	request: {
		141	notificationsEnabled: false,
		142	liveActivitiesEnabled: false,
		143	managedTunnelsEnabled: true,
		144	} as never,
		145	proof: {
		146	environmentId: "env-1",
		147	environmentPublicKey: "public-key-1",
		148	descriptor: { label: "Laptop" },
		149	} as never,
		150	endpoint: { httpBaseUrl: "https://a.example", wsBaseUrl: "wss://a.example" } as never,
		151	});
		152	const query = new PgDialect().sqlToQuery(inserted[0]?.holdWebhooksWhileOffline as never);
		153	// An environment id is public: another account linking it with its own
		154	// key must not switch the opt-in on for this one.
		155	expect(query.sql).toContain("environment_public_key");
		156	expect(query.params).toEqual(["env-1", "public-key-1"]);
		157	}).pipe(
		158	Effect.provide(
		159	EnvironmentLinks.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, fakeDb))),
		160	),
		161	);
		162	});
		163
125	it.effect("revokes only the active link owned by the requesting user", () => {	164	it.effect("revokes only the active link owned by the requesting user", () => {
126	const updateValues: Array<Record<string, unknown>> = [];	165	const updateValues: Array<Record<string, unknown>> = [];
127	const whereConditions: Array<unknown> = [];	166	const whereConditions: Array<unknown> = [];

infra/relay/src/environments/EnvironmentLinks.ts+65-5

...	@@ -9,7 +9,7 @@ import * as DateTime from "effect/DateTime";	...	@@ -9,7 +9,7 @@ import * as DateTime from "effect/DateTime";
9	import * as Effect from "effect/Effect";	9	import * as Effect from "effect/Effect";
10	import * as Layer from "effect/Layer";	10	import * as Layer from "effect/Layer";
11	import * as Schema from "effect/Schema";	11	import * as Schema from "effect/Schema";
12	import { and, eq, isNull, or } from "drizzle-orm";	12	import { and, eq, isNull, or, sql } from "drizzle-orm";
13		13
14	import * as RelayDb from "../db.ts";	14	import * as RelayDb from "../db.ts";
15	import { relayEnvironmentLinks } from "../persistence/schema.ts";	15	import { relayEnvironmentLinks } from "../persistence/schema.ts";
...	@@ -126,13 +126,30 @@ export class EnvironmentLinks extends Context.Service<	...	@@ -126,13 +126,30 @@ export class EnvironmentLinks extends Context.Service<
126	readonly userId: string;	126	readonly userId: string;
127	readonly environmentId: string;	127	readonly environmentId: string;
128	}) => Effect.Effect<RelayLinkedEnvironmentRecord \| null, EnvironmentLinkLookupPersistenceError>;	128	}) => Effect.Effect<RelayLinkedEnvironmentRecord \| null, EnvironmentLinkLookupPersistenceError>;
129	/** Active relay-managed links for an environment, across all users (webhook forwarding). */	129	/**
		130	* Active relay-managed links for an environment, narrowed to one user or to
		131	* links proven by one environment key. The environment id alone is public
		132	* and any account can link it, so callers acting on it must narrow.
		133	*/
130	readonly findActiveManagedForEnvironment: (input: {	134	readonly findActiveManagedForEnvironment: (input: {
131	readonly environmentId: string;	135	readonly environmentId: string;
		136	readonly userId?: string;
		137	readonly environmentPublicKey?: string;
132	}) => Effect.Effect<	138	}) => Effect.Effect<
133	ReadonlyArray<RelayLinkedEnvironmentRecord & { readonly userId: string }>,	139	ReadonlyArray<
		140	RelayLinkedEnvironmentRecord & {
		141	readonly userId: string;
		142	readonly holdWebhooksWhileOffline: boolean;
		143	}
		144	>,
134	EnvironmentLinkEnvironmentLookupPersistenceError	145	EnvironmentLinkEnvironmentLookupPersistenceError
135	>;	146	>;
		147	/** Sets the webhook-hold opt-in on the active links proven by one environment key. */
		148	readonly setHoldWebhooksWhileOffline: (input: {
		149	readonly environmentId: string;
		150	readonly environmentPublicKey: string;
		151	readonly holdWebhooksWhileOffline: boolean;
		152	}) => Effect.Effect<void, EnvironmentLinkEnvironmentLookupPersistenceError>;
136	readonly revokeForUser: (input: {	153	readonly revokeForUser: (input: {
137	readonly userId: string;	154	readonly userId: string;
138	readonly environmentId: string;	155	readonly environmentId: string;
...	@@ -166,12 +183,24 @@ const make = Effect.gen(function* () {	...	@@ -166,12 +183,24 @@ const make = Effect.gen(function* () {
166	const now = DateTime.formatIso(yield* DateTime.now);	183	const now = DateTime.formatIso(yield* DateTime.now);
167	const { request, proof } = input;	184	const { request, proof } = input;
168	const environmentId = proof.environmentId;	185	const environmentId = proof.environmentId;
		186	// The webhook-hold opt-in belongs to the environment: a new or re-made
		187	// link carries it over from the environment's other active links. Only
		188	// links proven by the same key count; an environment id is public, so
		189	// anyone can link one and switch the opt-in on for their own link.
		190	const inheritedHoldWebhooks = sql<boolean>`EXISTS (
		191	SELECT 1 FROM ${relayEnvironmentLinks} AS other
		192	WHERE other.environment_id = ${environmentId}
		193	AND other.environment_public_key = ${proof.environmentPublicKey}
		194	AND other.revoked_at IS NULL
		195	AND other.hold_webhooks_while_offline
		196	)`;
169	const { endpoint } = input;	197	const { endpoint } = input;
170	yield* db	198	yield* db
171	.insert(relayEnvironmentLinks)	199	.insert(relayEnvironmentLinks)
172	.values({	200	.values({
173	userId: input.userId,	201	userId: input.userId,
174	environmentId,	202	environmentId,
		203	holdWebhooksWhileOffline: inheritedHoldWebhooks,
175	environmentLabel: proof.descriptor.label,	204	environmentLabel: proof.descriptor.label,
176	environmentPublicKey: proof.environmentPublicKey,	205	environmentPublicKey: proof.environmentPublicKey,
177	endpointHttpBaseUrl: endpoint.httpBaseUrl,	206	endpointHttpBaseUrl: endpoint.httpBaseUrl,
...	@@ -199,6 +228,7 @@ const make = Effect.gen(function* () {	...	@@ -199,6 +228,7 @@ const make = Effect.gen(function* () {
199	createdByDeviceId: request.deviceId ?? null,	228	createdByDeviceId: request.deviceId ?? null,
200	revokedAt: null,	229	revokedAt: null,
201	updatedAt: now,	230	updatedAt: now,
		231	holdWebhooksWhileOffline: inheritedHoldWebhooks,
202	},	232	},
203	})	233	})
204	.pipe(	234	.pipe(
...	@@ -355,6 +385,7 @@ const make = Effect.gen(function* () {	...	@@ -355,6 +385,7 @@ const make = Effect.gen(function* () {
355	endpointWsBaseUrl: relayEnvironmentLinks.endpointWsBaseUrl,	385	endpointWsBaseUrl: relayEnvironmentLinks.endpointWsBaseUrl,
356	endpointProviderKind: relayEnvironmentLinks.endpointProviderKind,	386	endpointProviderKind: relayEnvironmentLinks.endpointProviderKind,
357	createdAt: relayEnvironmentLinks.createdAt,	387	createdAt: relayEnvironmentLinks.createdAt,
		388	holdWebhooksWhileOffline: relayEnvironmentLinks.holdWebhooksWhileOffline,
358	})	389	})
359	.from(relayEnvironmentLinks)	390	.from(relayEnvironmentLinks)
360	.where(	391	.where(
...	@@ -363,10 +394,13 @@ const make = Effect.gen(function* () {	...	@@ -363,10 +394,13 @@ const make = Effect.gen(function* () {
363	isNull(relayEnvironmentLinks.revokedAt),	394	isNull(relayEnvironmentLinks.revokedAt),
364	eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"),	395	eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"),
365	eq(relayEnvironmentLinks.managedTunnelsEnabled, true),	396	eq(relayEnvironmentLinks.managedTunnelsEnabled, true),
		397	input.userId === undefined ? undefined : eq(relayEnvironmentLinks.userId, input.userId),
		398	input.environmentPublicKey === undefined
		399	? undefined
		400	: eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey),
366	),	401	),
367	)	402	)
368	// One row per user who linked this environment; every row is checked	403	// At most one row per user who linked this environment.
369	// until one has a ready endpoint, so none may be cut off.
370	.pipe(	404	.pipe(
371	Effect.map((rows) =>	405	Effect.map((rows) =>
372	rows.map((row) => ({	406	rows.map((row) => ({
...	@@ -382,6 +416,7 @@ const make = Effect.gen(function* () {	...	@@ -382,6 +416,7 @@ const make = Effect.gen(function* () {
382	},	416	},
383	environmentPublicKey: row.environmentPublicKey,	417	environmentPublicKey: row.environmentPublicKey,
384	linkedAt: row.createdAt,	418	linkedAt: row.createdAt,
		419	holdWebhooksWhileOffline: row.holdWebhooksWhileOffline,
385	})),	420	})),
386	),	421	),
387	Effect.mapError(	422	Effect.mapError(
...	@@ -394,6 +429,31 @@ const make = Effect.gen(function* () {	...	@@ -394,6 +429,31 @@ const make = Effect.gen(function* () {
394	);	429	);
395	}),	430	}),
396		431
		432	setHoldWebhooksWhileOffline: Effect.fn(
		433	"relay.environment_links.set_hold_webhooks_while_offline",
		434	)(function* (input) {
		435	yield* Effect.annotateCurrentSpan({ "relay.environment_id": input.environmentId });
		436	yield* db
		437	.update(relayEnvironmentLinks)
		438	.set({ holdWebhooksWhileOffline: input.holdWebhooksWhileOffline })
		439	.where(
		440	and(
		441	eq(relayEnvironmentLinks.environmentId, input.environmentId),
		442	eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey),
		443	isNull(relayEnvironmentLinks.revokedAt),
		444	),
		445	)
		446	.pipe(
		447	Effect.mapError(
		448	(cause) =>
		449	new EnvironmentLinkEnvironmentLookupPersistenceError({
		450	environmentId: input.environmentId,
		451	cause,
		452	}),
		453	),
		454	);
		455	}),
		456
397	revokeForUser: Effect.fn("relay.environment_links.revoke_for_user")(function* (input) {	457	revokeForUser: Effect.fn("relay.environment_links.revoke_for_user")(function* (input) {
398	yield* Effect.annotateCurrentSpan({	458	yield* Effect.annotateCurrentSpan({
399	"relay.environment_id": input.environmentId,	459	"relay.environment_id": input.environmentId,

infra/relay/src/environments/ManagedEndpointAllocations.ts+28

...	@@ -64,6 +64,7 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro	...	@@ -64,6 +64,7 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro
64	"claim-deprovision",	64	"claim-deprovision",
65	"remove",	65	"remove",
66	"remove-claimed",	66	"remove-claimed",
		67	"get-by-tunnel-name",
67	]),	68	]),
68	stage: Schema.Literals(["database-request", "resolve-reservation"]),	69	stage: Schema.Literals(["database-request", "resolve-reservation"]),
69	userId: Schema.String,	70	userId: Schema.String,
...	@@ -132,6 +133,10 @@ export class ManagedEndpointAllocations extends Context.Service<	...	@@ -132,6 +133,10 @@ export class ManagedEndpointAllocations extends Context.Service<
132	readonly get: (	133	readonly get: (
133	input: ManagedEndpointAllocationKey,	134	input: ManagedEndpointAllocationKey,
134	) => Effect.Effect<ManagedEndpointAllocation \| null, ManagedEndpointAllocationPersistenceError>;	135	) => Effect.Effect<ManagedEndpointAllocation \| null, ManagedEndpointAllocationPersistenceError>;
		136	/** The allocation that owns a tunnel name; tunnel names are unique. */
		137	readonly getByTunnelName: (
		138	tunnelName: string,
		139	) => Effect.Effect<ManagedEndpointAllocation \| null, ManagedEndpointAllocationPersistenceError>;
135	readonly reserve: (	140	readonly reserve: (
136	input: ReserveManagedEndpointAllocationInput,	141	input: ReserveManagedEndpointAllocationInput,
137	) => Effect.Effect<ManagedEndpointAllocation, ManagedEndpointAllocationPersistenceError>;	142	) => Effect.Effect<ManagedEndpointAllocation, ManagedEndpointAllocationPersistenceError>;
...	@@ -230,6 +235,29 @@ export const make = Effect.gen(function* () {	...	@@ -230,6 +235,29 @@ export const make = Effect.gen(function* () {
230	),	235	),
231	);	236	);
232	}),	237	}),
		238	getByTunnelName: Effect.fn("relay.managed_endpoint_allocations.get_by_tunnel_name")(function* (
		239	tunnelName: string,
		240	) {
		241	return yield* db
		242	.select(allocationSelection)
		243	.from(relayManagedEndpointAllocations)
		244	.where(eq(relayManagedEndpointAllocations.tunnelName, tunnelName))
		245	.limit(1)
		246	.pipe(
		247	Effect.map((rows) => rows[0] ?? null),
		248	Effect.mapError(
		249	(cause) =>
		250	new ManagedEndpointAllocationPersistenceError({
		251	operation: "get-by-tunnel-name",
		252	stage: "database-request",
		253	userId: "",
		254	environmentId: "",
		255	tunnelName,
		256	cause,
		257	}),
		258	),
		259	);
		260	}),
233	reserve: Effect.fn("relay.managed_endpoint_allocations.reserve")(function* (	261	reserve: Effect.fn("relay.managed_endpoint_allocations.reserve")(function* (
234	input: ReserveManagedEndpointAllocationInput,	262	input: ReserveManagedEndpointAllocationInput,
235	) {	263	) {

infra/relay/src/environments/ManagedEndpointProvider.test.ts+1

...	@@ -204,6 +204,7 @@ function makeAllocations(calls: AllocationCall[] = []) {	...	@@ -204,6 +204,7 @@ function makeAllocations(calls: AllocationCall[] = []) {
204	}	204	}
205	};	205	};
206	return ManagedEndpointAllocations.ManagedEndpointAllocations.of({	206	return ManagedEndpointAllocations.ManagedEndpointAllocations.of({
		207	getByTunnelName: () => Effect.die("unused getByTunnelName"),
207	get: (input) =>	208	get: (input) =>
208	Effect.sync(() => {	209	Effect.sync(() => {
209	calls.push({ operation: "get", input });	210	calls.push({ operation: "get", input });

infra/relay/src/environments/ManagedEndpointReaper.test.ts+1

...	@@ -166,6 +166,7 @@ function harness(input?: {	...	@@ -166,6 +166,7 @@ function harness(input?: {
166	}),	166	}),
167	});	167	});
168	const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({	168	const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
		169	getByTunnelName: () => Effect.die("unused"),
169	get: () => Effect.die("unused"),	170	get: () => Effect.die("unused"),
170	reserve: () => Effect.die("unused"),	171	reserve: () => Effect.die("unused"),
171	recordTunnel: () => Effect.die("unused"),	172	recordTunnel: () => Effect.die("unused"),

infra/relay/src/hooks/HookForwarder.test.ts+364-19

...	@@ -1,9 +1,13 @@	...	@@ -1,9 +1,13 @@
1	import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";	1	import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform";
2	import * as NodeServices from "@effect/platform-node/NodeServices";	2	import * as NodeServices from "@effect/platform-node/NodeServices";
		3	import * as NodeCrypto from "node:crypto";
		4	import * as EffectNodeCrypto from "@effect/platform-node/NodeCrypto";
3	import { describe, expect, it } from "@effect/vitest";	5	import { describe, expect, it } from "@effect/vitest";
4	import { RelayApi } from "@t3tools/contracts/relay";	6	import { RelayApi } from "@t3tools/contracts/relay";
		7	import * as DateTime from "effect/DateTime";
5	import * as Deferred from "effect/Deferred";	8	import * as Deferred from "effect/Deferred";
6	import * as Duration from "effect/Duration";	9	import * as Duration from "effect/Duration";
		10	import * as Clock from "effect/Clock";
7	import * as Effect from "effect/Effect";	11	import * as Effect from "effect/Effect";
8	import * as Fiber from "effect/Fiber";	12	import * as Fiber from "effect/Fiber";
9	import * as Layer from "effect/Layer";	13	import * as Layer from "effect/Layer";
...	@@ -18,6 +22,7 @@ import * as Etag from "effect/http/Etag";	...	@@ -18,6 +22,7 @@ import * as Etag from "effect/http/Etag";
18	import * as HttpRouter from "effect/http/HttpRouter";	22	import * as HttpRouter from "effect/http/HttpRouter";
19	import * as HttpApi from "effect/http-api/HttpApi";	23	import * as HttpApi from "effect/http-api/HttpApi";
20	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";	24	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
		25	import * as HttpMiddleware from "effect/http/HttpMiddleware";
21	import * as HttpServerRequest from "effect/http/HttpServerRequest";	26	import * as HttpServerRequest from "effect/http/HttpServerRequest";
22	import * as HttpServerResponse from "effect/http/HttpServerResponse";	27	import * as HttpServerResponse from "effect/http/HttpServerResponse";
23		28
...	@@ -31,6 +36,15 @@ import {	...	@@ -31,6 +36,15 @@ import {
31	traceRelayHttpRequestWith,	36	traceRelayHttpRequestWith,
32	} from "../http/Api.ts";	37	} from "../http/Api.ts";
33	import * as HookForwarder from "./HookForwarder.ts";	38	import * as HookForwarder from "./HookForwarder.ts";
		39	import { RELAY_HOOK_DELIVERY_TYP, verifyRelayJwt } from "@t3tools/shared/relayJwt";
		40	import * as HookInbox from "./HookInbox.ts";
		41	import type { HeldHook } from "./HookInboxStore.ts";
		42	import { RELAY_HOOK_UPSTREAM_TIMEOUT_MS } from "./upstream.ts";
		43
		44	const mintKeys = NodeCrypto.generateKeyPairSync("ed25519", {
		45	privateKeyEncoding: { format: "pem", type: "pkcs8" },
		46	publicKeyEncoding: { format: "pem", type: "spki" },
		47	});
34		48
35	const settings: RelayConfiguration.RelayConfiguration["Service"] = {	49	const settings: RelayConfiguration.RelayConfiguration["Service"] = {
36	relayIssuer: "https://relay.example.test",	50	relayIssuer: "https://relay.example.test",
...	@@ -39,19 +53,20 @@ const settings: RelayConfiguration.RelayConfiguration["Service"] = {	...	@@ -39,19 +53,20 @@ const settings: RelayConfiguration.RelayConfiguration["Service"] = {
39	clerkPublishableKey: "pk_test_test",	53	clerkPublishableKey: "pk_test_test",
40	clerkJwtAudience: "t3-code-relay",	54	clerkJwtAudience: "t3-code-relay",
41	apnsDeliveryJobSigningSecret: Redacted.make("apns-delivery-secret"),	55	apnsDeliveryJobSigningSecret: Redacted.make("apns-delivery-secret"),
42	cloudMintPrivateKey: Redacted.make("cloud-mint-private-key"),	56	cloudMintPrivateKey: Redacted.make(mintKeys.privateKey),
43	cloudMintPublicKey: "cloud-mint-public-key",	57	cloudMintPublicKey: mintKeys.publicKey,
44	managedEndpointBaseDomain: "example.test",	58	managedEndpointBaseDomain: "example.test",
45	managedEndpointNamespace: undefined,	59	managedEndpointNamespace: "dev",
46	};	60	};
47		61
48	const environmentId = "env-hook";	62	const environmentId = "env-hook";
		63	const endpointKey = "0123456789abcdef";
49	const readyAllocation: ManagedEndpointAllocations.ManagedEndpointAllocation = {	64	const readyAllocation: ManagedEndpointAllocations.ManagedEndpointAllocation = {
50	userId: "user_1",	65	userId: "user_1",
51	environmentId,	66	environmentId,
52	hostname: "env.example.test",	67	hostname: "env.example.test",
53	tunnelId: "tunnel-id",	68	tunnelId: "tunnel-id",
54	tunnelName: "tunnel-name",	69	tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`,
55	dnsRecordId: "dns-record-id",	70	dnsRecordId: "dns-record-id",
56	readyAt: "2026-05-25T00:00:00.000Z",	71	readyAt: "2026-05-25T00:00:00.000Z",
57	origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },	72	origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
...	@@ -70,6 +85,7 @@ const managedLink = {	...	@@ -70,6 +85,7 @@ const managedLink = {
70	},	85	},
71	environmentPublicKey: "public-key",	86	environmentPublicKey: "public-key",
72	linkedAt: "2026-05-25T00:00:00.000Z",	87	linkedAt: "2026-05-25T00:00:00.000Z",
		88	holdWebhooksWhileOffline: false,
73	};	89	};
74		90
75	interface Harness {	91	interface Harness {
...	@@ -79,11 +95,15 @@ interface Harness {	...	@@ -79,11 +95,15 @@ interface Harness {
79	readonly links?: ReadonlyArray<typeof managedLink>;	95	readonly links?: ReadonlyArray<typeof managedLink>;
80	readonly allocation?: ManagedEndpointAllocations.ManagedEndpointAllocation \| null;	96	readonly allocation?: ManagedEndpointAllocations.ManagedEndpointAllocation \| null;
81	readonly allow?: (key: string) => boolean;	97	readonly allow?: (key: string) => boolean;
		98	readonly allowEndpoint?: (endpointKey: string) => boolean;
		99	/** Inbox capacity; hold reports full once this many requests are held. */
		100	readonly inboxCapacity?: number;
82	}	101	}
83		102
84	function makeHarness(options: Harness = {}) {	103	function makeHarness(options: Harness = {}) {
85	const sent: Array<HttpClientRequest.HttpClientRequest> = [];	104	const sent: Array<HttpClientRequest.HttpClientRequest> = [];
86	const rateLimitKeys: Array<string> = [];	105	const rateLimitKeys: Array<string> = [];
		106	const held: Array<HeldHook & { readonly baseUrl: string }> = [];
87	const execute =	107	const execute =
88	options.execute ??	108	options.execute ??
89	((request: HttpClientRequest.HttpClientRequest) =>	109	((request: HttpClientRequest.HttpClientRequest) =>
...	@@ -100,12 +120,19 @@ function makeHarness(options: Harness = {}) {	...	@@ -100,12 +120,19 @@ function makeHarness(options: Harness = {}) {
100	Layer.mock(EnvironmentLinks.EnvironmentLinks, {	120	Layer.mock(EnvironmentLinks.EnvironmentLinks, {
101	findActiveManagedForEnvironment: (input) =>	121	findActiveManagedForEnvironment: (input) =>
102	Effect.succeed(	122	Effect.succeed(
103	input.environmentId === environmentId ? (options.links ?? [managedLink]) : [],	123	(options.links ?? [managedLink]).filter(
		124	(link) =>
		125	link.environmentId === input.environmentId &&
		126	(input.userId === undefined \|\| link.userId === input.userId),
		127	),
104	),	128	),
105	}),	129	}),
106	Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {	130	Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {
107	get: () =>	131	getByTunnelName: (tunnelName) => {
108	Effect.succeed(options.allocation === undefined ? readyAllocation : options.allocation),	132	const allocation =
		133	options.allocation === undefined ? readyAllocation : options.allocation;
		134	return Effect.succeed(allocation?.tunnelName === tunnelName ? allocation : null);
		135	},
109	}),	136	}),
110	Layer.succeed(	137	Layer.succeed(
111	HttpClient.HttpClient,	138	HttpClient.HttpClient,
...	@@ -114,12 +141,23 @@ function makeHarness(options: Harness = {}) {	...	@@ -114,12 +141,23 @@ function makeHarness(options: Harness = {}) {
114	return execute(request);	141	return execute(request);
115	}),	142	}),
116	),	143	),
		144	Layer.mock(HookInbox.HookInbox, {
		145	hold: ({ hook, baseUrl }) =>
		146	Effect.sync(() => {
		147	if (held.length >= (options.inboxCapacity ?? Infinity)) return false;
		148	held.push({ ...hook, baseUrl });
		149	return true;
		150	}),
		151	}),
		152	EffectNodeCrypto.layer,
117	Layer.succeed(HookForwarder.HookRateLimiter, {	153	Layer.succeed(HookForwarder.HookRateLimiter, {
118	allow: (key) =>	154	allowHook: (key) =>
119	Effect.sync(() => {	155	Effect.sync(() => {
120	rateLimitKeys.push(key);	156	rateLimitKeys.push(key);
121	return options.allow ? options.allow(key) : true;	157	return options.allow ? options.allow(key) : true;
122	}),	158	}),
		159	allowEndpoint: (key) =>
		160	Effect.sync(() => (options.allowEndpoint ? options.allowEndpoint(key) : true)),
123	}),	161	}),
124	),	162	),
125	),	163	),
...	@@ -144,11 +182,11 @@ function makeHarness(options: Harness = {}) {	...	@@ -144,11 +182,11 @@ function makeHarness(options: Harness = {}) {
144	),	182	),
145	);	183	);
146	});	184	});
147	return { sent, rateLimitKeys, send, httpEffect };	185	return { sent, rateLimitKeys, held, send, httpEffect };
148	}	186	}
149		187
150	const hookUrl = (path = "hook-1/secret-token", query = "") =>	188	const hookUrl = (path = "hook-1/secret-token", query = "") =>
151	`https://relay.test/v1/hooks/${environmentId}/${path}${query}`;	189	`https://relay.test/v1/hooks/${endpointKey}/${path}${query}`;
152		190
153	const readBody = (response: HttpServerResponse.HttpServerResponse) =>	191	const readBody = (response: HttpServerResponse.HttpServerResponse) =>
154	Effect.promise(() => HttpServerResponse.toWeb(response).arrayBuffer()).pipe(	192	Effect.promise(() => HttpServerResponse.toWeb(response).arrayBuffer()).pipe(
...	@@ -210,6 +248,36 @@ describe("HookForwarder", () => {	...	@@ -210,6 +248,36 @@ describe("HookForwarder", () => {
210	}),	248	}),
211	);	249	);
212		250
		251	it.effect("signs each forward for the environment, replacing any copy a sender sent", () =>
		252	Effect.gen(function* () {
		253	const harness = makeHarness();
		254	yield* harness.send(
		255	new Request(hookUrl("hook-1/tok"), {
		256	method: "POST",
		257	headers: { "x-t3-relay-delivery": "forged", "x-t3-relay-delivery-id": "forged" },
		258	body: "{}",
		259	}),
		260	);
		261	const sent = harness.sent[0]!;
		262	const payload = yield* verifyRelayJwt({
		263	publicKey: mintKeys.publicKey,
		264	token: sent.headers["x-t3-relay-delivery"]!,
		265	typ: RELAY_HOOK_DELIVERY_TYP,
		266	issuer: settings.relayIssuer,
		267	audience: `t3-env:${environmentId}`,
		268	nowEpochSeconds: Math.floor((yield* Clock.currentTimeMillis) / 1_000),
		269	});
		270	// The proof names exactly the delivery the environment receives.
		271	expect(payload).toMatchObject({
		272	environmentId,
		273	hookId: "hook-1",
		274	deliveryId: sent.headers["x-t3-relay-delivery-id"],
		275	receivedAt: sent.headers["x-t3-relay-received-at"],
		276	});
		277	expect(sent.headers["x-t3-relay-delivery-id"]).not.toBe("forged");
		278	}),
		279	);
		280
213	it.effect("passes upstream status, body and content-type through", () =>	281	it.effect("passes upstream status, body and content-type through", () =>
214	Effect.gen(function* () {	282	Effect.gen(function* () {
215	const harness = makeHarness({	283	const harness = makeHarness({
...	@@ -229,11 +297,24 @@ describe("HookForwarder", () => {	...	@@ -229,11 +297,24 @@ describe("HookForwarder", () => {
229	expect(response.headers["content-type"]).toBe("application/json");	297	expect(response.headers["content-type"]).toBe("application/json");
230	expect(response.headers["set-cookie"]).toBeUndefined();	298	expect(response.headers["set-cookie"]).toBeUndefined();
231	expect(response.headers["access-control-allow-origin"]).toBeUndefined();	299	expect(response.headers["access-control-allow-origin"]).toBeUndefined();
		300	// Served from the relay's origin, so it may never render or run there.
		301	expect(response.headers["x-content-type-options"]).toBe("nosniff");
		302	expect(response.headers["content-security-policy"]).toBe("sandbox; default-src 'none'");
232	expect(new TextDecoder().decode(yield* readBody(response))).toBe('{"error":"bad_signature"}');	303	expect(new TextDecoder().decode(yield* readBody(response))).toBe('{"error":"bad_signature"}');
233	expect(harness.sent[0]?.method).toBe("GET");	304	expect(harness.sent[0]?.method).toBe("GET");
234	}),	305	}),
235	);	306	);
236		307
		308	it.effect("never forwards or holds HEAD, which can carry no body", () =>
		309	Effect.gen(function* () {
		310	const harness = makeHarness({ execute: () => Effect.die("must not be sent") });
		311	const response = yield* harness.send(new Request(hookUrl(), { method: "HEAD" }));
		312	expect(response.status).toBeGreaterThanOrEqual(400);
		313	expect(response.status).toBeLessThan(500);
		314	expect(harness.held).toHaveLength(0);
		315	}),
		316	);
		317
237	it.effect("does not follow or relay upstream redirects", () =>	318	it.effect("does not follow or relay upstream redirects", () =>
238	Effect.gen(function* () {	319	Effect.gen(function* () {
239	const harness = makeHarness({	320	const harness = makeHarness({
...	@@ -284,14 +365,17 @@ describe("HookForwarder", () => {	...	@@ -284,14 +365,17 @@ describe("HookForwarder", () => {
284	}),	365	}),
285	);	366	);
286		367
287	it.effect("returns 404 for unknown environments and unready endpoints", () =>	368	it.effect("returns 404 for unknown endpoints and unready endpoints", () =>
288	Effect.gen(function* () {	369	Effect.gen(function* () {
289	const unknown = makeHarness();	370	const unknown = makeHarness();
290	const response = yield* unknown.send(	371	for (const key of ["fedcba9876543210", environmentId]) {
291	new Request("https://relay.test/v1/hooks/other-env/hook-1/token", { method: "POST" }),	372	const response = yield* unknown.send(
292	);	373	new Request(`https://relay.test/v1/hooks/${key}/hook-1/token`, { method: "POST" }),
293	expect(response.status).toBe(404);	374	);
294	expect(yield* readJson(response)).toEqual({ error: "hook_not_found" });	375	expect(response.status).toBe(404);
		376	expect(yield* readJson(response)).toEqual({ error: "hook_not_found" });
		377	}
		378	expect(unknown.sent).toHaveLength(0);
295		379
296	const unready = makeHarness({ allocation: { ...readyAllocation, readyAt: null } });	380	const unready = makeHarness({ allocation: { ...readyAllocation, readyAt: null } });
297	const unreadyResponse = yield* unready.send(new Request(hookUrl(), { method: "POST" }));	381	const unreadyResponse = yield* unready.send(new Request(hookUrl(), { method: "POST" }));
...	@@ -300,6 +384,42 @@ describe("HookForwarder", () => {	...	@@ -300,6 +384,42 @@ describe("HookForwarder", () => {
300	}),	384	}),
301	);	385	);
302		386
		387	it.effect("forwards only to the link that owns the endpoint key", () =>
		388	Effect.gen(function* () {
		389	// Another account linked the same environment id under its own key; its
		390	// link must not receive this endpoint's hooks, whatever order rows come in.
		391	const intruder = {
		392	...managedLink,
		393	userId: "user_attacker",
		394	environmentPublicKey: "attacker-key",
		395	endpoint: { ...managedLink.endpoint, httpBaseUrl: "https://attacker.example.test/" },
		396	};
		397	const harness = makeHarness({ links: [intruder, managedLink] });
		398	const response = yield* harness.send(new Request(hookUrl(), { method: "POST", body: "{}" }));
		399	expect(response.status).toBe(200);
		400	expect(harness.sent.map((request) => new URL(request.url).host)).toEqual([
		401	"env.example.test",
		402	]);
		403
		404	// Without the owner's link, the key resolves to nothing at all.
		405	const orphaned = makeHarness({ links: [intruder] });
		406	const orphanedResponse = yield* orphaned.send(
		407	new Request(hookUrl(), { method: "POST", body: "{}" }),
		408	);
		409	expect(orphanedResponse.status).toBe(404);
		410	expect(orphaned.sent).toHaveLength(0);
		411	}),
		412	);
		413
		414	it.effect("returns 429 when the endpoint's overall budget is spent", () =>
		415	Effect.gen(function* () {
		416	const harness = makeHarness({ allowEndpoint: () => false });
		417	const response = yield* harness.send(new Request(hookUrl(), { method: "POST" }));
		418	expect(response.status).toBe(429);
		419	expect(harness.sent).toHaveLength(0);
		420	}),
		421	);
		422
303	it.effect("maps tunnel-offline and network failures to 503", () =>	423	it.effect("maps tunnel-offline and network failures to 503", () =>
304	Effect.gen(function* () {	424	Effect.gen(function* () {
305	const offline = makeHarness({	425	const offline = makeHarness({
...	@@ -353,13 +473,32 @@ describe("HookForwarder", () => {	...	@@ -353,13 +473,32 @@ describe("HookForwarder", () => {
353	.send(new Request(hookUrl(), { method: "POST", body: "{}" }))	473	.send(new Request(hookUrl(), { method: "POST", body: "{}" }))
354	.pipe(Effect.forkChild);	474	.pipe(Effect.forkChild);
355	yield* Deferred.await(reachedUpstream);	475	yield* Deferred.await(reachedUpstream);
356	yield* TestClock.adjust(Duration.millis(HookForwarder.RELAY_HOOK_UPSTREAM_TIMEOUT_MS));	476	yield* TestClock.adjust(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS));
357	const response = yield* Fiber.join(fiber);	477	const response = yield* Fiber.join(fiber);
358	expect(response.status).toBe(504);	478	expect(response.status).toBe(504);
359	expect(yield* readJson(response)).toEqual({ error: "environment_timeout" });	479	expect(yield* readJson(response)).toEqual({ error: "environment_timeout" });
360	}),	480	}),
361	);	481	);
362		482
		483	it.effect("holds a timed-out request with the time it arrived", () =>
		484	Effect.gen(function* () {
		485	const reachedUpstream = yield* Deferred.make<void>();
		486	const harness = makeHarness({
		487	links: [{ ...managedLink, holdWebhooksWhileOffline: true }],
		488	execute: () =>
		489	Deferred.succeed(reachedUpstream, undefined).pipe(Effect.andThen(Effect.never)),
		490	});
		491	const arrivedAt = DateTime.formatIso(yield* DateTime.now);
		492	const fiber = yield* harness
		493	.send(new Request(hookUrl(), { method: "POST", body: "{}" }))
		494	.pipe(Effect.forkChild);
		495	yield* Deferred.await(reachedUpstream);
		496	yield* TestClock.adjust(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS));
		497	expect((yield* Fiber.join(fiber)).status).toBe(202);
		498	expect(harness.held[0]?.receivedAt).toBe(arrivedAt);
		499	}),
		500	);
		501
363	it.effect("answers OPTIONS without a CORS preflight or forwarding", () =>	502	it.effect("answers OPTIONS without a CORS preflight or forwarding", () =>
364	Effect.gen(function* () {	503	Effect.gen(function* () {
365	const harness = makeHarness();	504	const harness = makeHarness();
...	@@ -454,9 +593,215 @@ describe("HookForwarder", () => {	...	@@ -454,9 +593,215 @@ describe("HookForwarder", () => {
454	expect(serialized).not.toContain("also-secret");	593	expect(serialized).not.toContain("also-secret");
455	expect(serialized).not.toContain("header-secret");	594	expect(serialized).not.toContain("header-secret");
456	const server = spans.find((span) => span.kind === "server");	595	const server = spans.find((span) => span.kind === "server");
457	expect(server?.attributes.get("url.path")).toBe(	596	expect(server?.attributes.get("url.path")).toBe(`/v1/hooks/${endpointKey}/hook-1/<redacted>`);
458	`/v1/hooks/${environmentId}/hook-1/<redacted>`,	597	}),
		598	);
		599
		600	it.effect("records one redacted server span even inside the worker's own HTTP tracer", () =>
		601	Effect.gen(function* () {
		602	const spans: Array<Tracer.NativeSpan> = [];
		603	const tracer = Tracer.make({
		604	span: (options) => {
		605	const span = new Tracer.NativeSpan(options);
		606	spans.push(span);
		607	return span;
		608	},
		609	});
		610	const harness = makeHarness();
		611	const handler = yield* harness.httpEffect;
		612	// As the worker runtime runs it: its own tracer around ours, off for hook
		613	// paths. This checks the predicate; whether alchemy applies it per event
		614	// is only visible on a deployed worker (see worker.ts).
		615	yield* HttpMiddleware.tracer(
		616	traceRelayHttpRequestWith(handler, Layer.succeed(Tracer.Tracer, tracer)),
		617	).pipe(
		618	Effect.provideService(HttpMiddleware.TracerDisabledWhen, (request) =>
		619	HookForwarder.isRelayHookPath(request.url),
		620	),
		621	Effect.withTracer(tracer),
		622	Effect.provideService(
		623	HttpServerRequest.HttpServerRequest,
		624	HttpServerRequest.fromWeb(
		625	new Request(hookUrl("hook-1/super-secret-token"), {
		626	method: "POST",
		627	headers: { traceparent: "00-11111111111111111111111111111111-2222222222222222-01" },
		628	body: "{}",
		629	}),
		630	),
		631	),
		632	);
		633	yield* Effect.yieldNow;
		634	const servers = spans.filter((span) => span.kind === "server");
		635	expect(servers).toHaveLength(1);
		636	expect(servers[0]?.attributes.get("url.path")).toBe(
		637	`/v1/hooks/${endpointKey}/hook-1/<redacted>`,
459	);	638	);
		639	expect(spans.every((span) => span.traceId !== "11111111111111111111111111111111")).toBe(true);
		640	}),
		641	);
		642
		643	it.effect("joins the environment to its trace and records what the environment did", () =>
		644	Effect.gen(function* () {
		645	const spans: Array<Tracer.NativeSpan> = [];
		646	const tracer = Tracer.make({
		647	span: (options) => {
		648	const span = new Tracer.NativeSpan(options);
		649	spans.push(span);
		650	return span;
		651	},
		652	});
		653	const respondWith = (outcome: string) =>
		654	makeHarness({
		655	execute: (request) =>
		656	Effect.succeed(
		657	HttpClientResponse.fromWeb(
		658	request,
		659	new Response('{"deliveryId":"d"}', {
		660	status: 202,
		661	headers: { "x-t3-hook-outcome": outcome },
		662	}),
		663	),
		664	),
		665	});
		666	const forward = (harness: ReturnType<typeof makeHarness>) =>
		667	Effect.gen(function* () {
		668	const handler = yield* harness.httpEffect;
		669	return yield* traceRelayHttpRequestWith(
		670	handler,
		671	Layer.succeed(Tracer.Tracer, tracer),
		672	).pipe(
		673	Effect.provideService(
		674	HttpServerRequest.HttpServerRequest,
		675	HttpServerRequest.fromWeb(
		676	new Request(hookUrl(), {
		677	method: "POST",
		678	// A sender's own trace context never reaches the environment.
		679	headers: {
		680	traceparent: "00-11111111111111111111111111111111-2222222222222222-01",
		681	"x-b3-traceid": "33333333333333333333333333333333",
		682	},
		683	body: "{}",
		684	}),
		685	),
		686	),
		687	);
		688	});
		689
		690	const duplicate = respondWith("duplicate");
		691	expect((yield* forward(duplicate)).status).toBe(202);
		692	yield* Effect.yieldNow;
		693	const forwardSpan = spans.find((span) => span.name === "relay.hooks.forward");
		694	expect(forwardSpan?.attributes.get("relay.hook.upstream_outcome")).toBe("duplicate");
		695	const sent = duplicate.sent[0]!;
		696	expect(sent.headers.traceparent).toContain(forwardSpan!.traceId);
		697	expect(sent.headers.traceparent).not.toContain("1111111111");
		698	expect(sent.headers["x-b3-traceid"]).toBeUndefined();
		699
		700	// Only a plain outcome name is recorded.
		701	spans.length = 0;
		702	yield* forward(respondWith("<script>alert(1)</script>"));
		703	yield* Effect.yieldNow;
		704	expect(
		705	spans
		706	.find((span) => span.name === "relay.hooks.forward")
		707	?.attributes.has("relay.hook.upstream_outcome"),
		708	).toBe(false);
460	}),	709	}),
461	);	710	);
		711
		712	describe("holding requests while the environment is offline", () => {
		713	const offline = (request: HttpClientRequest.HttpClientRequest) =>
		714	Effect.fail(
		715	new HttpClientError.HttpClientError({
		716	reason: new HttpClientError.TransportError({ request, cause: new Error("offline") }),
		717	}),
		718	);
		719
		720	it.effect("holds when cloudflared answers that the local server is down", () =>
		721	Effect.gen(function* () {
		722	for (const status of [502, 503, 504, 530]) {
		723	const harness = makeHarness({
		724	links: [{ ...managedLink, holdWebhooksWhileOffline: true }],
		725	execute: (request) =>
		726	Effect.succeed(HttpClientResponse.fromWeb(request, new Response("", { status }))),
		727	});
		728	const response = yield* harness.send(
		729	new Request(hookUrl(), { method: "POST", body: "{}" }),
		730	);
		731	expect(response.status).toBe(202);
		732	expect(harness.held).toHaveLength(1);
		733	}
		734	}),
		735	);
		736
		737	it.effect("stays a plain proxy when the environment has not opted in", () =>
		738	Effect.gen(function* () {
		739	const harness = makeHarness({ execute: offline });
		740	const response = yield* harness.send(
		741	new Request(hookUrl(), { method: "POST", body: "{}" }),
		742	);
		743	expect(response.status).toBe(503);
		744	expect(harness.held).toHaveLength(0);
		745	}),
		746	);
		747
		748	it.effect("holds the exact request and answers 202 once opted in", () =>
		749	Effect.gen(function* () {
		750	const harness = makeHarness({
		751	execute: offline,
		752	links: [{ ...managedLink, holdWebhooksWhileOffline: true }],
		753	});
		754	const body = new Uint8Array([0, 255, 10]);
		755	const response = yield* harness.send(
		756	new Request(hookUrl("%68ook-1/tok%2Fen", "?a=1"), {
		757	method: "POST",
		758	body,
		759	headers: { "x-t3-relay-delivery-id": "forged", "x-sig": "s" },
		760	}),
		761	);
		762	expect(response.status).toBe(202);
		763	const [hook] = harness.held;
		764	expect(hook?.baseUrl).toBe("https://env.example.test/");
		765	expect(hook?.rawToken).toBe("tok%2Fen");
		766	expect(hook?.query).toBe("a=1");
		767	expect([...(hook?.body ?? [])]).toEqual([0, 255, 10]);
		768	expect(hook?.headers["x-sig"]).toBe("s");
		769	// Every spelling of the hook id shares one per-hook cap.
		770	expect(hook?.hookKey).toBe("hook-1");
		771	// The sender cannot choose the delivery id.
		772	expect(hook?.headers["x-t3-relay-delivery-id"]).toBeUndefined();
		773	expect(hook?.id).not.toBe("forged");
		774	}),
		775	);
		776
		777	it.effect("answers 503 inbox_full when the environment's inbox is full", () =>
		778	Effect.gen(function* () {
		779	const harness = makeHarness({
		780	execute: offline,
		781	links: [{ ...managedLink, holdWebhooksWhileOffline: true }],
		782	inboxCapacity: 0,
		783	});
		784	const response = yield* harness.send(
		785	new Request(hookUrl(), { method: "POST", body: "{}" }),
		786	);
		787	expect(response.status).toBe(503);
		788	expect(yield* readJson(response)).toEqual({ error: "inbox_full" });
		789	}),
		790	);
		791
		792	it.effect("tags every forward with a relay delivery id", () =>
		793	Effect.gen(function* () {
		794	const harness = makeHarness();
		795	yield* harness.send(
		796	new Request(hookUrl(), {
		797	method: "POST",
		798	body: "{}",
		799	headers: { "x-t3-relay-delivery-id": "forged" },
		800	}),
		801	);
		802	const id = harness.sent[0]?.headers["x-t3-relay-delivery-id"];
		803	expect(id).toMatch(/^[0-9a-f-]{36}$/);
		804	}),
		805	);
		806	});
462	});	807	});

infra/relay/src/hooks/HookForwarder.ts+253-105

...	@@ -1,28 +1,57 @@	...	@@ -1,28 +1,57 @@
		1	import * as Clock from "effect/Clock";
1	import * as Context from "effect/Context";	2	import * as Context from "effect/Context";
2	import * as Duration from "effect/Duration";	3	import * as Crypto from "effect/Crypto";
		4	import * as DateTime from "effect/DateTime";
3	import * as Effect from "effect/Effect";	5	import * as Effect from "effect/Effect";
4	import * as Layer from "effect/Layer";	6	import * as Layer from "effect/Layer";
5	import * as Option from "effect/Option";	7	import * as Option from "effect/Option";
		8	import * as Redacted from "effect/Redacted";
6	import * as Result from "effect/Result";	9	import * as Result from "effect/Result";
7	import * as Schema from "effect/Schema";	10	import * as Schema from "effect/Schema";
8	import * as Stream from "effect/Stream";	11	import * as Stream from "effect/Stream";
9	import * as HttpClient from "effect/http/HttpClient";	12	import * as HttpClient from "effect/http/HttpClient";
10	import * as HttpClientRequest from "effect/http/HttpClientRequest";
11	import type * as HttpClientResponse from "effect/http/HttpClientResponse";
12	import type * as HttpServerRequest from "effect/http/HttpServerRequest";	13	import type * as HttpServerRequest from "effect/http/HttpServerRequest";
13	import * as HttpServerResponse from "effect/http/HttpServerResponse";	14	import * as HttpServerResponse from "effect/http/HttpServerResponse";
14	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";	15	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
15	import { RelayApi } from "@t3tools/contracts/relay";	16	import { EnvironmentId } from "@t3tools/contracts";
		17	import { RelayApi, type RelayHookDeliveryProofPayload } from "@t3tools/contracts/relay";
		18	import {
		19	normalizeRelayIssuer,
		20	RELAY_HOOK_DELIVERY_HEADER,
		21	RELAY_HOOK_DELIVERY_TYP,
		22	signRelayJwt,
		23	} from "@t3tools/shared/relayJwt";
16		24
17	import * as RelayConfiguration from "../Config.ts";	25	import * as RelayConfiguration from "../Config.ts";
18	import { validateManagedEndpoint, withoutRedirects } from "../environments/EnvironmentConnector.ts";	26	import {
		27	MANAGED_ENDPOINT_KEY_PATTERN,
		28	managedEndpointTunnelNameForKey,
		29	} from "../deploymentConfig.ts";
		30	import { validateManagedEndpoint } from "../environments/EnvironmentConnector.ts";
19	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";	31	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";
20	import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts";	32	import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts";
		33	import * as HookInbox from "./HookInbox.ts";
		34	import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts";
21		35
22	export const RELAY_HOOK_PATH_PREFIX = "/v1/hooks/";	36	export const RELAY_HOOK_PATH_PREFIX = "/v1/hooks/";
23	export const RELAY_HOOK_MAX_BODY_BYTES = 1_048_576;	37	export const RELAY_HOOK_MAX_BODY_BYTES = 1_048_576;
24	export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000;
25	export const RELAY_HOOK_RATE_LIMIT = { limit: 60, periodSeconds: 60 } as const;	38	export const RELAY_HOOK_RATE_LIMIT = { limit: 60, periodSeconds: 60 } as const;
		39	/**
		40	* Hook budgets are per URL, and a sender who knows an endpoint key can mint
		41	* new URLs for free, so every endpoint also has one overall budget.
		42	*/
		43	export const RELAY_HOOK_ENDPOINT_RATE_LIMIT = { limit: 600, periodSeconds: 60 } as const;
		44	/**
		45	* Upstream statuses that mean the environment did not take the request: the
		46	* tunnel has no origin (530) or cloudflared cannot reach the local server
		47	* (502, 503, 504) while it restarts.
		48	*/
		49	export const ENVIRONMENT_UNREACHABLE_STATUSES: ReadonlySet<number> = new Set([
		50	502,
		51	503,
		52	504,
		53	TUNNEL_OFFLINE_STATUS,
		54	]);
26		55
27	const DROPPED_REQUEST_HEADERS = new Set([	56	const DROPPED_REQUEST_HEADERS = new Set([
28	"host",	57	"host",
...	@@ -34,12 +63,16 @@ const DROPPED_REQUEST_HEADERS = new Set([	...	@@ -34,12 +63,16 @@ const DROPPED_REQUEST_HEADERS = new Set([
34	"content-length",	63	"content-length",
35	"cookie",	64	"cookie",
36	"x-real-ip",	65	"x-real-ip",
		66	// Only the relay may set this; a sender could otherwise collide delivery ids.
		67	"x-t3-relay-delivery-id",
		68	"x-t3-relay-received-at",
		69	RELAY_HOOK_DELIVERY_HEADER,
		70	// The environment trusts trace context only from the relay, which sets its own.
		71	"traceparent",
		72	"tracestate",
		73	"b3",
37	]);	74	]);
38	const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-"];	75	const DROPPED_REQUEST_HEADER_PREFIXES = ["proxy-", "cf-", "x-forwarded-", "x-b3-"];
39	// Cloudflare answers 530 when the tunnel for a hostname has no connected origin.
40	const TUNNEL_OFFLINE_STATUS = 530;
41	/** The environment answers with a small JSON status; anything past this is cut off. */
42	const MAX_RESPONSE_BYTES = 64 * 1024;
43		76
44	export const isRelayHookPath = (url: string): boolean => url.startsWith(RELAY_HOOK_PATH_PREFIX);	77	export const isRelayHookPath = (url: string): boolean => url.startsWith(RELAY_HOOK_PATH_PREFIX);
45		78
...	@@ -55,13 +88,13 @@ export const redactRelayHookUrl = (url: string): string => {	...	@@ -55,13 +88,13 @@ export const redactRelayHookUrl = (url: string): string => {
55		88
56	/**	89	/**
57	* Request budget for public hook forwarding, keyed by a hash of the hook URL	90	* Request budget for public hook forwarding, keyed by a hash of the hook URL
58	* (environment, hook and token). Requests with a wrong token get their own	91	* (endpoint, hook and token). Requests with a wrong token get their own
59	* budget, so they cannot use up a real sender's; the environment rejects them.	92	* budget, so they cannot use up a real sender's; the environment rejects them.
60	* Built from decoded segments, because the environment decodes them too: two	93	* Built from decoded segments, because the environment decodes them too: two
61	* spellings of one token (`token`, `%74oken`) must share one budget.	94	* spellings of one token (`token`, `%74oken`) must share one budget.
62	*/	95	*/
63	const hookBudgetKey = (hook: {	96	const hookBudgetKey = (hook: {
64	readonly environmentId: string;	97	readonly endpointKey: string;
65	readonly hookId: string;	98	readonly hookId: string;
66	readonly token: string;	99	readonly token: string;
67	}) =>	100	}) =>
...	@@ -70,7 +103,7 @@ const hookBudgetKey = (hook: {	...	@@ -70,7 +103,7 @@ const hookBudgetKey = (hook: {
70	"SHA-256",	103	"SHA-256",
71	// Length-prefixed, so no segment contents can make two keys collide.	104	// Length-prefixed, so no segment contents can make two keys collide.
72	new TextEncoder().encode(	105	new TextEncoder().encode(
73	[hook.environmentId, hook.hookId, hook.token]	106	[hook.endpointKey, hook.hookId, hook.token]
74	.map((part) => `${part.length}:${part}`)	107	.map((part) => `${part.length}:${part}`)
75	.join(""),	108	.join(""),
76	),	109	),
...	@@ -83,7 +116,12 @@ const hookBudgetKey = (hook: {	...	@@ -83,7 +116,12 @@ const hookBudgetKey = (hook: {
83		116
84	export class HookRateLimiter extends Context.Service<	117	export class HookRateLimiter extends Context.Service<
85	HookRateLimiter,	118	HookRateLimiter,
86	{ readonly allow: (key: string) => Effect.Effect<boolean> }	119	{
		120	/** One hook URL's budget, keyed by `hookBudgetKey`. */
		121	readonly allowHook: (key: string) => Effect.Effect<boolean>;
		122	/** One endpoint's overall budget, keyed by its endpoint key. */
		123	readonly allowEndpoint: (endpointKey: string) => Effect.Effect<boolean>;
		124	}
87	>()("t3code-relay/hooks/HookForwarder/HookRateLimiter") {}	125	>()("t3code-relay/hooks/HookForwarder/HookRateLimiter") {}
88		126
89	export class HookForwarder extends Context.Service<	127	export class HookForwarder extends Context.Service<
...	@@ -96,25 +134,69 @@ export class HookForwarder extends Context.Service<	...	@@ -96,25 +134,69 @@ export class HookForwarder extends Context.Service<
96	>()("t3code-relay/hooks/HookForwarder") {}	134	>()("t3code-relay/hooks/HookForwarder") {}
97		135
98	class HookBodyTooLarge extends Schema.TaggedError<HookBodyTooLarge>()("HookBodyTooLarge", {}) {}	136	class HookBodyTooLarge extends Schema.TaggedError<HookBodyTooLarge>()("HookBodyTooLarge", {}) {}
99	class ResponseTooLarge extends Schema.TaggedError<ResponseTooLarge>()("ResponseTooLarge", {}) {}
100		137
101	const errorResponse = (status: number, error: string, headers?: Record<string, string>) =>	138	const errorResponse = (status: number, error: string, headers?: Record<string, string>) =>
102	HttpServerResponse.jsonUnsafe({ error }, { status, ...(headers ? { headers } : {}) });	139	HttpServerResponse.jsonUnsafe({ error }, { status, ...(headers ? { headers } : {}) });
103		140
104	const hookNotFound = () => errorResponse(404, "hook_not_found");	141	const hookNotFound = () => errorResponse(404, "hook_not_found");
105		142
		143	/** Longer than the inbox holds a request, so a held delivery's proof still verifies. */
		144	const DELIVERY_PROOF_LIFETIME_SECONDS = 25 * 60 * 60;
		145
		146	const signDeliveryProof = (input: {
		147	readonly settings: RelayConfiguration.RelayConfiguration["Service"];
		148	readonly environmentId: string;
		149	readonly deliveryId: string;
		150	readonly receivedAt: string;
		151	readonly hookId: string;
		152	readonly jti: string;
		153	}) =>
		154	Effect.gen(function* () {
		155	const now = Math.floor((yield* Clock.currentTimeMillis) / 1_000);
		156	return yield* signRelayJwt({
		157	privateKey: Redacted.value(input.settings.cloudMintPrivateKey),
		158	typ: RELAY_HOOK_DELIVERY_TYP,
		159	payload: {
		160	iss: normalizeRelayIssuer(input.settings.relayIssuer),
		161	aud: `t3-env:${input.environmentId}`,
		162	sub: input.environmentId,
		163	jti: input.jti,
		164	iat: now,
		165	exp: now + DELIVERY_PROOF_LIFETIME_SECONDS,
		166	environmentId: EnvironmentId.make(input.environmentId),
		167	deliveryId: input.deliveryId,
		168	receivedAt: input.receivedAt,
		169	hookId: input.hookId,
		170	} satisfies RelayHookDeliveryProofPayload,
		171	});
		172	}).pipe(Effect.orDie);
		173
		174	/** Methods a webhook can arrive with; HEAD reaches the GET route and is refused. */
		175	const FORWARDED_METHODS = new Set(["GET", "POST", "PUT", "PATCH"]);
		176
		177	/**
		178	* Whatever the environment answers is served from the relay's own origin, so
		179	* a body must never render or run there.
		180	*/
		181	const SANDBOXED_RESPONSE_HEADERS = {
		182	"x-content-type-options": "nosniff",
		183	"content-security-policy": "sandbox; default-src 'none'",
		184	} as const;
		185
106	function parseHookPath(url: string) {	186	function parseHookPath(url: string) {
107	const queryIndex = url.indexOf("?");	187	const queryIndex = url.indexOf("?");
108	const path = queryIndex === -1 ? url : url.slice(0, queryIndex);	188	const path = queryIndex === -1 ? url : url.slice(0, queryIndex);
109	const search = queryIndex === -1 ? "" : url.slice(queryIndex);	189	const search = queryIndex === -1 ? "" : url.slice(queryIndex);
110	const segments = path.split("/");	190	const segments = path.split("/");
111	// ["", "v1", "hooks", environmentId, hookId, token]	191	// ["", "v1", "hooks", endpointKey, hookId, token]
112	if (segments.length !== 6) return null;	192	if (segments.length !== 6) return null;
113	const [, , , rawEnvironmentId, rawHookId, rawToken] = segments;	193	const [, , , endpointKey, rawHookId, rawToken] = segments;
114	if (!rawEnvironmentId \|\| !rawHookId \|\| !rawToken) return null;	194	if (!endpointKey \|\| !MANAGED_ENDPOINT_KEY_PATTERN.test(endpointKey) \|\| !rawHookId \|\| !rawToken) {
		195	return null;
		196	}
115	try {	197	try {
116	return {	198	return {
117	environmentId: decodeURIComponent(rawEnvironmentId),	199	endpointKey,
118	hookId: decodeURIComponent(rawHookId),	200	hookId: decodeURIComponent(rawHookId),
119	token: decodeURIComponent(rawToken),	201	token: decodeURIComponent(rawToken),
120	// Forward the encoded segments byte-for-byte; the environment decodes them.	202	// Forward the encoded segments byte-for-byte; the environment decodes them.
...	@@ -156,20 +238,18 @@ function forwardedHeaders(headers: Readonly<Record<string, string>>): Record<str	...	@@ -156,20 +238,18 @@ function forwardedHeaders(headers: Readonly<Record<string, string>>): Record<str
156	const hasNoBody = (request: HttpServerRequest.HttpServerRequest) =>	238	const hasNoBody = (request: HttpServerRequest.HttpServerRequest) =>
157	request.source instanceof Request && request.source.body === null;	239	request.source instanceof Request && request.source.body === null;
158		240
159	/** Collects a byte stream, failing with `tooLarge` once it passes `maxBytes` rather than buffering it all. */	241	const readCappedBody = (request: HttpServerRequest.HttpServerRequest) =>
160	const collectCapped = <E, R, E2>(
161	stream: Stream.Stream<Uint8Array, E, R>,
162	maxBytes: number,
163	tooLarge: () => E2,
164	) =>
165	Effect.suspend(() => {	242	Effect.suspend(() => {
		243	if (hasNoBody(request)) {
		244	return Effect.succeed(new Uint8Array(0));
		245	}
166	const chunks: Array<Uint8Array> = [];	246	const chunks: Array<Uint8Array> = [];
167	let total = 0;	247	let total = 0;
168	return stream.pipe(	248	return request.stream.pipe(
169	Stream.runForEach((chunk) => {	249	Stream.runForEach((chunk) => {
170	total += chunk.length;	250	total += chunk.length;
171	if (total > maxBytes) {	251	if (total > RELAY_HOOK_MAX_BODY_BYTES) {
172	return Effect.fail(tooLarge());	252	return Effect.fail(new HookBodyTooLarge());
173	}	253	}
174	chunks.push(chunk);	254	chunks.push(chunk);
175	return Effect.void;	255	return Effect.void;
...	@@ -186,19 +266,47 @@ const collectCapped = <E, R, E2>(	...	@@ -186,19 +266,47 @@ const collectCapped = <E, R, E2>(
186	);	266	);
187	});	267	});
188		268
189	const readCappedBody = (request: HttpServerRequest.HttpServerRequest) =>	269	/**
190	hasNoBody(request)	270	* The ready managed endpoint a webhook URL's endpoint key names, with whether
191	? Effect.succeed(new Uint8Array(0))	271	* its link opted in to holding webhooks while offline. The key is the tunnel
192	: collectCapped(request.stream, RELAY_HOOK_MAX_BODY_BYTES, () => new HookBodyTooLarge());	272	* name's hash of user and environment, so it names exactly one allocation and
193		273	* at most one active link; nobody else can link their way onto it.
194	const readCappedResponse = (response: HttpClientResponse.HttpClientResponse) =>	274	*/
195	collectCapped(response.stream, MAX_RESPONSE_BYTES, () => new ResponseTooLarge()).pipe(	275	export const resolveHookEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* (
196	// A response without a body, such as a redirect, has no stream at all.	276	endpointKey: string,
197	Effect.catchIf(	277	) {
198	(error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError",	278	const links = yield* EnvironmentLinks.EnvironmentLinks;
199	() => Effect.succeed(new Uint8Array(0)),	279	const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
200	),	280	const settings = yield* RelayConfiguration.RelayConfiguration;
		281	if (!settings.managedEndpointNamespace) return null;
		282	const allocation = yield* allocations.getByTunnelName(
		283	managedEndpointTunnelNameForKey(settings.managedEndpointNamespace, endpointKey),
201	);	284	);
		285	if (allocation === null) return null;
		286	const [link] = yield* links.findActiveManagedForEnvironment({
		287	environmentId: allocation.environmentId,
		288	userId: allocation.userId,
		289	});
		290	if (!link) return null;
		291	const result = validateManagedEndpoint({
		292	link,
		293	allocation,
		294	baseDomain: settings.managedEndpointBaseDomain,
		295	});
		296	if (Result.isFailure(result)) return null;
		297	return {
		298	...result.success,
		299	environmentId: allocation.environmentId,
		300	holdWhileOffline: link.holdWebhooksWhileOffline,
		301	};
		302	});
		303
		304	/** The endpoint key of an environment's own managed endpoint, for authenticated callers. */
		305	export const endpointKeyForTunnelName = (namespace: string, tunnelName: string): string \| null => {
		306	const prefix = managedEndpointTunnelNameForKey(namespace, "");
		307	const key = tunnelName.startsWith(prefix) ? tunnelName.slice(prefix.length) : "";
		308	return MANAGED_ENDPOINT_KEY_PATTERN.test(key) ? key : null;
		309	};
202		310
203	const make = Effect.gen(function* () {	311	const make = Effect.gen(function* () {
204	const links = yield* EnvironmentLinks.EnvironmentLinks;	312	const links = yield* EnvironmentLinks.EnvironmentLinks;
...	@@ -206,39 +314,38 @@ const make = Effect.gen(function* () {	...	@@ -206,39 +314,38 @@ const make = Effect.gen(function* () {
206	const settings = yield* RelayConfiguration.RelayConfiguration;	314	const settings = yield* RelayConfiguration.RelayConfiguration;
207	const httpClient = yield* HttpClient.HttpClient;	315	const httpClient = yield* HttpClient.HttpClient;
208	const rateLimiter = yield* HookRateLimiter;	316	const rateLimiter = yield* HookRateLimiter;
209		317	const inbox = yield* HookInbox.HookInbox;
210	const resolveEndpoint = Effect.fn("relay.hooks.resolve_endpoint")(function* (	318	const crypto = yield* Crypto.Crypto;
211	environmentId: string,
212	) {
213	const candidates = yield* links.findActiveManagedForEnvironment({ environmentId });
214	for (const link of candidates) {
215	const allocation = yield* allocations.get({ userId: link.userId, environmentId });
216	const result = validateManagedEndpoint({
217	link,
218	allocation,
219	baseDomain: settings.managedEndpointBaseDomain,
220	});
221	if (Result.isSuccess(result)) {
222	return result.success;
223	}
224	}
225	return null;
226	});
227		319
228	const handle = Effect.fn("relay.hooks.forward")(function* (	320	const handle = Effect.fn("relay.hooks.forward")(function* (
229	request: HttpServerRequest.HttpServerRequest,	321	request: HttpServerRequest.HttpServerRequest,
230	) {	322	) {
231	const outcome = (value: string) => Effect.annotateCurrentSpan({ "relay.hook.outcome": value });	323	const outcome = (value: string) => Effect.annotateCurrentSpan({ "relay.hook.outcome": value });
		324	if (!FORWARDED_METHODS.has(request.method)) {
		325	yield* outcome("method_not_allowed");
		326	return errorResponse(405, "method_not_allowed", { allow: "GET, POST, PUT, PATCH" });
		327	}
		328	// When the sender called, not when the environment failed to answer.
		329	const receivedAt = DateTime.formatIso(yield* DateTime.now);
232	const parsed = parseHookPath(request.url);	330	const parsed = parseHookPath(request.url);
233	if (!parsed) {	331	if (!parsed) {
234	yield* outcome("invalid_path");	332	yield* outcome("invalid_path");
235	return hookNotFound();	333	return hookNotFound();
236	}	334	}
237	yield* Effect.annotateCurrentSpan({	335	yield* Effect.annotateCurrentSpan({
238	"relay.environment_id": parsed.environmentId,	336	"relay.hook.endpoint_key": parsed.endpointKey,
239	"relay.hook_id": parsed.hookId,	337	"relay.hook_id": parsed.hookId,
240	});	338	});
241	if (!(yield* rateLimiter.allow(yield* hookBudgetKey(parsed)))) {	339	// A coarse budget per endpoint first, so minting new hook ids or tokens
		340	// cannot buy unlimited lookups and forwards, or fill the inbox.
		341	const endpointAllowed = yield* rateLimiter.allowEndpoint(parsed.endpointKey);
		342	const hookAllowed =
		343	endpointAllowed && (yield* rateLimiter.allowHook(yield* hookBudgetKey(parsed)));
		344	if (!endpointAllowed \|\| !hookAllowed) {
		345	// Which budget ran out: the whole endpoint's, or this one hook URL's.
		346	yield* Effect.annotateCurrentSpan({
		347	"relay.hook.rate_limit": endpointAllowed ? "hook" : "endpoint",
		348	});
242	yield* outcome("rate_limited");	349	yield* outcome("rate_limited");
243	return errorResponse(429, "rate_limited", {	350	return errorResponse(429, "rate_limited", {
244	"retry-after": String(RELAY_HOOK_RATE_LIMIT.periodSeconds),	351	"retry-after": String(RELAY_HOOK_RATE_LIMIT.periodSeconds),
...	@@ -250,10 +357,13 @@ const make = Effect.gen(function* () {	...	@@ -250,10 +357,13 @@ const make = Effect.gen(function* () {
250	return errorResponse(413, "payload_too_large");	357	return errorResponse(413, "payload_too_large");
251	}	358	}
252		359
253	const endpoint = yield* resolveEndpoint(parsed.environmentId).pipe(	360	const endpoint = yield* resolveHookEndpoint(parsed.endpointKey).pipe(
		361	Effect.provideService(EnvironmentLinks.EnvironmentLinks, links),
		362	Effect.provideService(ManagedEndpointAllocations.ManagedEndpointAllocations, allocations),
		363	Effect.provideService(RelayConfiguration.RelayConfiguration, settings),
254	Effect.catch((error) =>	364	Effect.catch((error) =>
255	Effect.logWarning("Failed to resolve hook endpoint", {	365	Effect.logWarning("Failed to resolve hook endpoint", {
256	environmentId: parsed.environmentId,	366	endpointKey: parsed.endpointKey,
257	errorTag: error._tag,	367	errorTag: error._tag,
258	}).pipe(Effect.as(null)),	368	}).pipe(Effect.as(null)),
259	),	369	),
...	@@ -262,6 +372,10 @@ const make = Effect.gen(function* () {	...	@@ -262,6 +372,10 @@ const make = Effect.gen(function* () {
262	yield* outcome("not_found");	372	yield* outcome("not_found");
263	return hookNotFound();	373	return hookNotFound();
264	}	374	}
		375	yield* Effect.annotateCurrentSpan({
		376	"relay.environment_id": endpoint.environmentId,
		377	"relay.hook.hold_while_offline": endpoint.holdWhileOffline,
		378	});
265		379
266	const body =	380	const body =
267	request.method === "GET"	381	request.method === "GET"
...	@@ -276,67 +390,101 @@ const make = Effect.gen(function* () {	...	@@ -276,67 +390,101 @@ const make = Effect.gen(function* () {
276	return errorResponse(400, "invalid_body");	390	return errorResponse(400, "invalid_body");
277	}	391	}
278		392
279	const baseUrl = endpoint.httpBaseUrl.endsWith("/")	393	// One id per request, so a request that reached the environment before a
280	? endpoint.httpBaseUrl	394	// timeout and is later delivered from the inbox runs only once.
281	: `${endpoint.httpBaseUrl}/`;	395	yield* Effect.annotateCurrentSpan({ "relay.hook.body_bytes": body.success.byteLength });
282	const headers = forwardedHeaders(request.headers);	396	const deliveryId = yield* crypto.randomUUIDv4.pipe(Effect.orDie);
283	let upstreamRequest = HttpClientRequest.make(	397	// Proves to the environment that this delivery id, receive time and
284	request.method as "GET" \| "POST" \| "PUT" \| "PATCH",	398	// trace context came from the relay. Signed once here and stored with a
285	)(`${baseUrl}api/hooks/${parsed.rawHookId}/${parsed.rawToken}${parsed.search}`, { headers });	399	// held request, so the inbox never needs the signing key.
286	if (request.method !== "GET") {	400	const proof = yield* signDeliveryProof({
287	upstreamRequest = HttpClientRequest.bodyUint8Array(	401	settings,
288	upstreamRequest,	402	environmentId: endpoint.environmentId,
289	body.success,	403	deliveryId,
290	headers["content-type"],	404	receivedAt,
291	);	405	hookId: parsed.hookId,
292	}	406	jti: yield* crypto.randomUUIDv4.pipe(Effect.orDie),
		407	});
		408	const hook = {
		409	id: deliveryId,
		410	receivedAt,
		411	method: request.method,
		412	rawHookId: parsed.rawHookId,
		413	rawToken: parsed.rawToken,
		414	hookKey: parsed.hookId,
		415	query: parsed.search.replace(/^\?/, ""),
		416	headers: { ...forwardedHeaders(request.headers), [RELAY_HOOK_DELIVERY_HEADER]: proof },
		417	body: body.success,
		418	};
		419	// Held only for environments that opted in; otherwise the relay is a plain proxy.
		420	const holdOrFail = (status: 503 \| 504, error: string) =>
		421	Effect.gen(function* () {
		422	if (!endpoint.holdWhileOffline) {
		423	yield* outcome(error);
		424	return errorResponse(status, error);
		425	}
		426	const stored = yield* inbox
		427	.hold({
		428	endpointKey: parsed.endpointKey,
		429	baseUrl: endpoint.httpBaseUrl,
		430	hook,
		431	})
		432	.pipe(
		433	Effect.catch((cause) =>
		434	Effect.logWarning("Could not hold webhook request", {
		435	environmentId: endpoint.environmentId,
		436	errorTag: cause._tag,
		437	}).pipe(Effect.as(null)),
		438	),
		439	);
		440	if (stored === null) {
		441	yield* outcome(error);
		442	return errorResponse(status, error);
		443	}
		444	if (!stored) {
		445	// The inbox span carries which cap refused it (relay.inbox.refused).
		446	yield* outcome("inbox_full");
		447	return errorResponse(503, "inbox_full");
		448	}
		449	yield* outcome("held");
		450	return HttpServerResponse.jsonUnsafe({ queued: true }, { status: 202 });
		451	});
293		452
294	const upstream = yield* httpClient.execute(upstreamRequest).pipe(	453	const upstream = yield* sendUpstream(endpoint.httpBaseUrl, hook).pipe(
295	Effect.flatMap((response) =>	454	Effect.provideService(HttpClient.HttpClient, httpClient),
296	readCappedResponse(response).pipe(
297	Effect.map((body) => ({
298	status: response.status,
299	contentType: response.headers["content-type"],
300	body,
301	})),
302	),
303	),
304	withoutRedirects,
305	// The client span would record url.full, which carries the token.
306	Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
307	Effect.timeoutOption(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)),
308	Effect.result,	455	Effect.result,
309	);	456	);
310	if (Result.isFailure(upstream)) {	457	if (Result.isFailure(upstream)) {
311	yield* outcome("environment_unavailable");	458	yield* Effect.annotateCurrentSpan({ "relay.hook.upstream_error": upstream.failure._tag });
312	return errorResponse(503, "environment_unavailable");	459	return yield* holdOrFail(503, "environment_unavailable");
313	}	460	}
314	if (Option.isNone(upstream.success)) {	461	if (Option.isNone(upstream.success)) {
315	yield* outcome("environment_timeout");	462	return yield* holdOrFail(504, "environment_timeout");
316	return errorResponse(504, "environment_timeout");
317	}	463	}
318	const response = upstream.success.value;	464	const response = upstream.success.value;
319	if (response.status === TUNNEL_OFFLINE_STATUS) {	465	if (ENVIRONMENT_UNREACHABLE_STATUSES.has(response.status)) {
320	yield* outcome("environment_unavailable");	466	yield* Effect.annotateCurrentSpan({ "relay.hook.upstream_status": response.status });
321	return errorResponse(503, "environment_unavailable");	467	return yield* holdOrFail(503, "environment_unavailable");
322	}	468	}
323	yield* Effect.annotateCurrentSpan({	469	yield* Effect.annotateCurrentSpan({
324	"relay.hook.outcome": "forwarded",	470	"relay.hook.outcome": "forwarded",
325	"relay.hook.upstream_status": response.status,	471	"relay.hook.upstream_status": response.status,
		472	...(response.outcome === undefined
		473	? {}
		474	: { "relay.hook.upstream_outcome": response.outcome }),
326	});	475	});
327	// Only content-type is passed through: no location (redirects are never	476	// Only content-type is passed through: no location (redirects are never
328	// followed or relayed), no cookies, no upstream infrastructure headers.	477	// followed or relayed), no cookies, no upstream infrastructure headers.
329	const contentTypeHeaders = response.contentType	478	const headers = {
330	? { "content-type": response.contentType }	479	...SANDBOXED_RESPONSE_HEADERS,
331	: undefined;	480	...(response.contentType ? { "content-type": response.contentType } : {}),
		481	};
332	if (response.body.length === 0) {	482	if (response.body.length === 0) {
333	return HttpServerResponse.empty({	483	return HttpServerResponse.empty({ status: response.status, headers });
334	status: response.status,
335	...(contentTypeHeaders ? { headers: contentTypeHeaders } : {}),
336	});
337	}	484	}
338	return HttpServerResponse.uint8Array(response.body, {	485	return HttpServerResponse.uint8Array(response.body, {
339	status: response.status,	486	status: response.status,
		487	headers,
340	...(response.contentType ? { contentType: response.contentType } : {}),	488	...(response.contentType ? { contentType: response.contentType } : {}),
341	});	489	});
342	});	490	});

infra/relay/src/hooks/HookInbox.ts created+40

...	@@ -0,0 +1,40 @@
1	import * as Context from "effect/Context";
2	import * as Effect from "effect/Effect";
3	import * as Schema from "effect/Schema";
4
5	import type { HeldHook } from "./HookInboxStore.ts";
6
7	export class HookInboxError extends Schema.TaggedError<HookInboxError>()("HookInboxError", {
8	operation: Schema.Literals(["hold", "wake", "clear"]),
9	endpointKey: Schema.String,
10	cause: Schema.Defect(),
11	}) {
12	override get message(): string {
13	return `Hook inbox '${this.operation}' failed for endpoint '${this.endpointKey}'`;
14	}
15	}
16
17	/**
18	* Webhook requests held for environments that opted in, while they are
19	* offline. Each managed endpoint's requests live in its own `HookInboxObject`,
20	* named by endpoint key, which delivers them itself once the environment is back.
21	*/
22	export class HookInbox extends Context.Service<
23	HookInbox,
24	{
25	/** False when the environment's inbox is full and nothing was stored. */
26	readonly hold: (input: {
27	readonly endpointKey: string;
28	readonly baseUrl: string;
29	readonly hook: HeldHook;
30	}) => Effect.Effect<boolean, HookInboxError>;
31	/** Delivers what is waiting now; true when anything was waiting. */
32	readonly wake: (input: {
33	readonly endpointKey: string;
34	readonly baseUrl: string;
35	}) => Effect.Effect<boolean, HookInboxError>;
36	readonly clear: (input: {
37	readonly endpointKey: string;
38	}) => Effect.Effect<void, HookInboxError>;
39	}
40	>()("t3code-relay/hooks/HookInbox") {}

infra/relay/src/hooks/HookInboxObject.ts created+154

...	@@ -0,0 +1,154 @@
1	import * as SqliteClient from "@effect/sql-sqlite-do/SqliteClient";
2	import type * as Alchemy from "alchemy";
3	import * as Cloudflare from "alchemy/Cloudflare";
4	import * as Clock from "effect/Clock";
5	import * as Effect from "effect/Effect";
6	import * as Option from "effect/Option";
7	import * as Result from "effect/Result";
8	import * as FetchHttpClient from "effect/http/FetchHttpClient";
9
10	import * as HookInboxStore from "./HookInboxStore.ts";
11	import { sendUpstream, TUNNEL_OFFLINE_STATUS } from "./upstream.ts";
12
13	/** Statuses that mean the environment is not there; the whole inbox waits and backs off. */
14	const UNREACHABLE_STATUSES = new Set([502, 503, 504, TUNNEL_OFFLINE_STATUS]);
15	/**
16	* Statuses that mean the environment is there but did not take this request
17	* yet: its task's queue is full (429) or it failed while handling it (500).
18	* The environment drops a delivery id it has already run, so retrying is safe.
19	*/
20	const BUSY_STATUSES = new Set([429, 500]);
21	/** When a run itself fails, the next one is tried after this long. */
22	const RUN_FAILURE_RETRY_MS = 60_000;
23
24	type Call<A> = Effect.Effect<A, never, Alchemy.RuntimeContext>;
25
26	export interface HookInboxObjectShape {
27	/** Holds a request for `baseUrl`; false when the inbox is full and nothing was stored. */
28	readonly hold: (hook: HookInboxStore.HeldHook, baseUrl: string) => Call<boolean>;
29	/** The environment is back at `baseUrl`: deliver what is waiting now. */
30	readonly wake: (baseUrl: string) => Call<boolean>;
31	readonly clear: () => Call<void>;
32	}
33
34	/**
35	* One per managed endpoint, addressed by endpoint key. Holds webhook requests
36	* the environment could not take, in SQLite, and pushes them back through
37	* its tunnel from the alarm, oldest first, backing off while it stays away.
38	*/
39	export class HookInboxObject extends Cloudflare.DurableObject<
40	HookInboxObject,
41	HookInboxObjectShape
42	>()("HookInboxObject") {}
43
44	/**
45	* Each call into an inbox is its own trace: the alarm has no parent, and a
46	* hold arrives over Durable Object RPC without the forwarding span. The
47	* object's name is the endpoint key, which the forward spans carry too.
48	*/
49	const withInboxSpan =
50	(name: string, inboxId: string) =>
51	<A, E, R>(effect: Effect.Effect<A, E, R>) =>
52	effect.pipe(
53	Effect.withSpan(name, { root: true, attributes: { "relay.hook.endpoint_key": inboxId } }),
54	);
55
56	const deliver = (baseUrl: string, hook: HookInboxStore.HeldHook) =>
57	sendUpstream(baseUrl, hook).pipe(
58	Effect.result,
59	Effect.map((result) => {
60	// Unreachable or timed out: a timeout may still have run it, and the
61	// environment drops a delivery id it has already seen.
62	if (Result.isFailure(result)) {
63	return { outcome: "unreachable" as const, reason: result.failure._tag };
64	}
65	if (Option.isNone(result.success))
66	return { outcome: "unreachable" as const, reason: "timeout" };
67	const status = result.success.value.status;
68	const upstreamOutcome = result.success.value.outcome;
69	const outcome: HookInboxStore.DeliveryOutcome = UNREACHABLE_STATUSES.has(status)
70	? "unreachable"
71	: BUSY_STATUSES.has(status)
72	? "busy"
73	: "delivered";
74	return { outcome, reason: `status ${status}`, upstreamOutcome };
75	}),
76	Effect.tap(({ upstreamOutcome }) =>
77	upstreamOutcome === undefined
78	? Effect.void
79	: Effect.annotateCurrentSpan({ "relay.hook.upstream_outcome": upstreamOutcome }),
80	),
81	Effect.tap(({ outcome, reason }) =>
82	outcome === "delivered"
83	? Effect.void
84	: Effect.logInfo("Held webhook not delivered yet", {
85	outcome,
86	reason,
87	deliveryId: hook.id,
88	}),
89	),
90	Effect.map(({ outcome }) => outcome),
91	Effect.provide(FetchHttpClient.layer),
92	);
93
94	export const HookInboxObjectLive = HookInboxObject.make(
95	Effect.gen(function* () {
96	const state = yield* Cloudflare.DurableObjectState;
97	// The init phase returns the per-instance Effect, which alchemy runs once
98	// per object; only that inner Effect may touch storage.
99	// @effect-diagnostics-next-line returnEffectInGen:off
100	return Effect.gen(function* () {
101	const sql = SqliteClient.layer({ storage: state.raw.storage });
102	// Inboxes are opened by endpoint key, so spans line up with the
103	// forward spans that held their requests.
104	const inboxId = state.raw.id.name ?? state.raw.id.toString();
105	const run = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
106	effect.pipe(Effect.provide(sql), Effect.orDie);
107	yield* run(HookInboxStore.migrate);
108
109	/** Moves the alarm to `at`, unless one is already due sooner. */
110	const scheduleBy = (at: number) =>
111	Effect.gen(function* () {
112	const current = yield* state.storage.getAlarm();
113	if (current === null \|\| current > at) yield* state.storage.setAlarm(at);
114	});
115
116	return {
117	hold: (hook: HookInboxStore.HeldHook, baseUrl: string) =>
118	Effect.gen(function* () {
119	const dueAt = yield* run(HookInboxStore.hold(hook, baseUrl));
120	yield* Effect.annotateCurrentSpan({ "relay.inbox.stored": dueAt !== null });
121	if (dueAt === null) return false;
122	yield* scheduleBy(dueAt);
123	return true;
124	}).pipe(withInboxSpan("relay.inbox.hold", inboxId)),
125	wake: (baseUrl: string) =>
126	Effect.gen(function* () {
127	const pending = yield* run(HookInboxStore.wake(baseUrl));
128	yield* Effect.annotateCurrentSpan({ "relay.inbox.pending": pending });
129	if (pending) yield* state.storage.setAlarm(yield* Clock.currentTimeMillis);
130	return pending;
131	}).pipe(withInboxSpan("relay.inbox.wake", inboxId)),
132	clear: () =>
133	Effect.gen(function* () {
134	yield* run(HookInboxStore.clear);
135	yield* state.storage.deleteAlarm();
136	}).pipe(withInboxSpan("relay.inbox.clear", inboxId)),
137	alarm: () =>
138	run(HookInboxStore.deliverDue(deliver)).pipe(
139	// A wake during this run may already have asked for an earlier
140	// run; a backoff must not push it out.
141	Effect.flatMap((nextAt) => (nextAt === null ? Effect.void : scheduleBy(nextAt))),
142	Effect.catchCause((cause) =>
143	Effect.logWarning("Held webhook delivery run failed", { cause }).pipe(
144	Effect.andThen(Effect.annotateCurrentSpan({ "relay.inbox.run_result": "failed" })),
145	Effect.andThen(Clock.currentTimeMillis),
146	Effect.flatMap((now) => scheduleBy(now + RUN_FAILURE_RETRY_MS)),
147	),
148	),
149	withInboxSpan("relay.inbox.deliver", inboxId),
150	),
151	};
152	});
153	}),
154	);

infra/relay/src/hooks/HookInboxStore.test.ts created+260

...	@@ -0,0 +1,260 @@
1	import { describe, expect, it } from "@effect/vitest";
2	import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
3	import * as Clock from "effect/Clock";
4	import * as DateTime from "effect/DateTime";
5	import * as Duration from "effect/Duration";
6	import * as Effect from "effect/Effect";
7	import * as SqlClient from "effect/sql/SqlClient";
8	import * as TestClock from "effect/testing/TestClock";
9
10	import * as HookInboxStore from "./HookInboxStore.ts";
11
12	const BASE_URL = "https://env.example.test/";
13
14	const hook = (id: string, overrides: Partial<HookInboxStore.HeldHook> = {}) =>
15	Effect.map(Clock.currentTimeMillis, (now): HookInboxStore.HeldHook => ({
16	id,
17	receivedAt: DateTime.formatIso(DateTime.makeUnsafe(now)),
18	method: "POST",
19	rawHookId: "hook-1",
20	hookKey: "hook-1",
21	rawToken: "tok%2Fen",
22	query: "a=1",
23	headers: { "content-type": "application/json", "x-sig": "s" },
24	body: new Uint8Array([0, 255, 10]),
25	...overrides,
26	}));
27
28	/** Delivers with `outcome` per request and records what the environment was sent. */
29	const deliverer = (outcome: (hook: HookInboxStore.HeldHook) => HookInboxStore.DeliveryOutcome) => {
30	const sent: Array<{ readonly baseUrl: string; readonly hook: HookInboxStore.HeldHook }> = [];
31	const send = (baseUrl: string, held: HookInboxStore.HeldHook) =>
32	Effect.sync(() => {
33	sent.push({ baseUrl, hook: held });
34	return outcome(held);
35	});
36	return { sent, send };
37	};
38
39	const withInbox = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
40	HookInboxStore.migrate.pipe(
41	Effect.andThen(effect),
42	Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" })),
43	);
44
45	describe("HookInboxStore", () => {
46	it.effect("delivers held requests oldest first, byte for byte", () =>
47	withInbox(
48	Effect.gen(function* () {
49	const first = yield* hook("first");
50	expect(yield* HookInboxStore.hold(first, BASE_URL)).not.toBeNull();
51	yield* HookInboxStore.hold(yield* hook("second"), BASE_URL);
52	const { sent, send } = deliverer(() => "delivered");
53	expect(yield* HookInboxStore.deliverDue(send)).toBeNull();
54	expect(sent.map((entry) => entry.hook.id)).toEqual(["first", "second"]);
55	expect(sent[0]).toEqual({ baseUrl: BASE_URL, hook: first });
56	// Delivered requests are gone.
57	expect(yield* HookInboxStore.deliverDue(send)).toBeNull();
58	expect(sent).toHaveLength(2);
59	}),
60	),
61	);
62
63	it.effect("keeps a request the environment did not take and backs off", () =>
64	withInbox(
65	Effect.gen(function* () {
66	yield* HookInboxStore.hold(yield* hook("first"), BASE_URL);
67	yield* HookInboxStore.hold(yield* hook("second"), BASE_URL);
68	const offline = deliverer(() => "unreachable");
69	const now = yield* Clock.currentTimeMillis;
70	// Stops at the first failure, so order is kept.
71	expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
72	expect(offline.sent.map((entry) => entry.hook.id)).toEqual(["first"]);
73	expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
74
75	const online = deliverer(() => "delivered");
76	expect(yield* HookInboxStore.deliverDue(online.send)).toBeNull();
77	expect(online.sent.map((entry) => entry.hook.id)).toEqual(["first", "second"]);
78	}),
79	),
80	);
81
82	it.effect("lets other hooks through while one hook's environment is busy", () =>
83	withInbox(
84	Effect.gen(function* () {
85	const stuck = { rawHookId: "stuck", hookKey: "stuck" };
86	yield* HookInboxStore.hold(yield* hook("stuck-1", stuck), BASE_URL);
87	yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL);
88	yield* HookInboxStore.hold(yield* hook("stuck-2", stuck), BASE_URL);
89	const busy = deliverer((held) => (held.hookKey === "stuck" ? "busy" : "delivered"));
90	const now = yield* Clock.currentTimeMillis;
91	// The other hook is delivered; the busy hook keeps its order and runs again soon.
92	expect(yield* HookInboxStore.deliverDue(busy.send)).toBe(now);
93	expect(busy.sent.map((entry) => entry.hook.id)).toEqual(["stuck-1", "other-1"]);
94	// Only the busy hook is left, so the next run waits rather than spinning.
95	expect(yield* HookInboxStore.deliverDue(busy.send)).toBe(now + 30_000);
96
97	const drained = deliverer(() => "delivered");
98	expect(yield* HookInboxStore.deliverDue(drained.send)).toBeNull();
99	expect(drained.sent.map((entry) => entry.hook.id)).toEqual(["stuck-1", "stuck-2"]);
100	}),
101	),
102	);
103
104	it.effect("reaches a hook queued behind busy hooks' full backlogs in the same run", () =>
105	withInbox(
106	Effect.gen(function* () {
107	for (const name of ["stuck-a", "stuck-b"]) {
108	const stuck = { rawHookId: name, hookKey: name };
109	for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) {
110	yield* HookInboxStore.hold(yield* hook(`${name}-${index}`, stuck), BASE_URL);
111	}
112	}
113	yield* HookInboxStore.hold(yield* hook("other-1"), BASE_URL);
114	const busy = deliverer((held) => (held.hookKey.startsWith("stuck") ? "busy" : "delivered"));
115	yield* HookInboxStore.deliverDue(busy.send);
116	expect(busy.sent.map((entry) => entry.hook.id)).toEqual([
117	"stuck-a-0",
118	"stuck-b-0",
119	"other-1",
120	]);
121	}),
122	),
123	);
124
125	it.effect("drops a held request it cannot read instead of stalling on it", () =>
126	withInbox(
127	Effect.gen(function* () {
128	yield* HookInboxStore.hold(yield* hook("broken"), BASE_URL);
129	yield* HookInboxStore.hold(yield* hook("fine"), BASE_URL);
130	const sql = yield* SqlClient.SqlClient;
131	yield* sql`UPDATE held_hooks SET headers = 'not json' WHERE id = 'broken'`;
132	const { sent, send } = deliverer(() => "delivered");
133	expect(yield* HookInboxStore.deliverDue(send)).toBeNull();
134	expect(sent.map((entry) => entry.hook.id)).toEqual(["fine"]);
135	}),
136	),
137	);
138
139	it.effect("retries every 10 s for 3 minutes, then backs off to 10 minutes", () =>
140	Effect.gen(function* () {
141	const delays = yield* Effect.forEach(
142	[1, 18, 19, 20, 21, 23, 24, 40],
143	HookInboxStore.retryDelayMs,
144	);
145	expect(delays).toEqual([10_000, 10_000, 30_000, 60_000, 120_000, 480_000, 600_000, 600_000]);
146	}),
147	);
148
149	it.effect("waking resets the backoff and reports whether anything waits", () =>
150	withInbox(
151	Effect.gen(function* () {
152	expect(yield* HookInboxStore.wake(BASE_URL)).toBe(false);
153	yield* HookInboxStore.hold(yield* hook("first"), "https://old.example.test/");
154	const offline = deliverer(() => "unreachable");
155	yield* HookInboxStore.deliverDue(offline.send);
156	yield* HookInboxStore.deliverDue(offline.send);
157
158	// Past the fast phase: the environment was away a while.
159	for (let failure = 0; failure < 20; failure++) {
160	yield* HookInboxStore.deliverDue(offline.send);
161	}
162	const now = yield* Clock.currentTimeMillis;
163	expect(yield* HookInboxStore.deliverDue(offline.send)).toBeGreaterThan(now + 60_000);
164
165	// A wake means its tunnel just connected, which Cloudflare may not
166	// route to for a few minutes: back to retrying every 10 s, sent to
167	// where the environment is now.
168	expect(yield* HookInboxStore.wake(BASE_URL)).toBe(true);
169	expect(yield* HookInboxStore.deliverDue(offline.send)).toBe(now + 10_000);
170	expect(offline.sent.at(-1)?.baseUrl).toBe(BASE_URL);
171	}),
172	),
173	);
174
175	it.effect("starts the schedule over once the inbox drains", () =>
176	withInbox(
177	Effect.gen(function* () {
178	yield* HookInboxStore.hold(yield* hook("old"), BASE_URL);
179	const offline = deliverer(() => "unreachable");
180	for (let failure = 0; failure < 30; failure++) {
181	yield* HookInboxStore.deliverDue(offline.send);
182	}
183	// Nothing got through before the request expired.
184	yield* TestClock.adjust(Duration.hours(25));
185	expect(yield* HookInboxStore.deliverDue(offline.send)).toBeNull();
186
187	const now = yield* Clock.currentTimeMillis;
188	expect(yield* HookInboxStore.hold(yield* hook("new"), BASE_URL)).toBe(now + 10_000);
189	}),
190	),
191	);
192
193	it.effect("drops requests older than 24 hours", () =>
194	withInbox(
195	Effect.gen(function* () {
196	yield* HookInboxStore.hold(yield* hook("old"), BASE_URL);
197	yield* TestClock.adjust(Duration.hours(1));
198	yield* HookInboxStore.hold(yield* hook("new"), BASE_URL);
199	yield* TestClock.adjust(Duration.minutes(23 * 60 + 1));
200	const { sent, send } = deliverer(() => "delivered");
201	yield* HookInboxStore.deliverDue(send);
202	expect(sent.map((entry) => entry.hook.id)).toEqual(["new"]);
203	}),
204	),
205	);
206
207	it.effect("refuses requests past the per-hook cap", () =>
208	withInbox(
209	Effect.gen(function* () {
210	for (let index = 0; index < HookInboxStore.HOOK_INBOX_MAX_PER_HOOK; index++) {
211	expect(yield* HookInboxStore.hold(yield* hook(`a-${index}`), BASE_URL)).not.toBeNull();
212	}
213	expect(yield* HookInboxStore.hold(yield* hook("one-too-many"), BASE_URL)).toBeNull();
214	// Another hook still has room.
215	const other = yield* hook("other", { rawHookId: "hook-2", hookKey: "hook-2" });
216	expect(yield* HookInboxStore.hold(other, BASE_URL)).not.toBeNull();
217	}),
218	),
219	);
220
221	it.effect("refuses a request that would pass the byte cap", () =>
222	withInbox(
223	Effect.gen(function* () {
224	const big = new Uint8Array(HookInboxStore.HOOK_INBOX_MAX_BYTES - 10);
225	expect(
226	yield* HookInboxStore.hold(yield* hook("big", { body: big }), BASE_URL),
227	).not.toBeNull();
228	const small = new Uint8Array(11);
229	expect(
230	yield* HookInboxStore.hold(
231	yield* hook("small", { rawHookId: "x", hookKey: "x", body: small }),
232	BASE_URL,
233	),
234	).toBeNull();
235	}),
236	),
237	);
238
239	it.effect("stores a request id once", () =>
240	withInbox(
241	Effect.gen(function* () {
242	const first = yield* hook("same");
243	expect(yield* HookInboxStore.hold(first, BASE_URL)).not.toBeNull();
244	expect(yield* HookInboxStore.hold(first, BASE_URL)).toBeNull();
245	}),
246	),
247	);
248
249	it.effect("clear drops everything held", () =>
250	withInbox(
251	Effect.gen(function* () {
252	yield* HookInboxStore.hold(yield* hook("first"), BASE_URL);
253	yield* HookInboxStore.clear;
254	const { sent, send } = deliverer(() => "delivered");
255	expect(yield* HookInboxStore.deliverDue(send)).toBeNull();
256	expect(sent).toHaveLength(0);
257	}),
258	),
259	);
260	});

infra/relay/src/hooks/HookInboxStore.ts created+380

...	@@ -0,0 +1,380 @@
1	import * as Clock from "effect/Clock";
2	import * as DateTime from "effect/DateTime";
3	import * as Duration from "effect/Duration";
4	import * as Effect from "effect/Effect";
5	import * as Option from "effect/Option";
6	import * as Pull from "effect/Pull";
7	import * as Schedule from "effect/Schedule";
8	import * as Schema from "effect/Schema";
9	import * as SqlClient from "effect/sql/SqlClient";
10
11	/**
12	* The storage and delivery rules of one environment's held webhook requests.
13	* Runs inside the environment's `HookInboxObject` Durable Object against its SQLite
14	* storage; written against `SqlClient` so tests can run it on any SQLite.
15	*/
16
17	/** How long a held request waits for its environment. */
18	const HOOK_INBOX_TTL_MS = 24 * 60 * 60 * 1000;
19	/** Requests one environment may have waiting at once. */
20	const HOOK_INBOX_MAX_REQUESTS = 1_000;
21	/** Body bytes one environment may have waiting at once. */
22	export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576;
23	/**
24	* Requests one hook may have waiting at once. The relay cannot check tokens,
25	* so this keeps junk sent to one hook id from crowding out the others.
26	*/
27	export const HOOK_INBOX_MAX_PER_HOOK = 100;
28	/** Requests pushed per alarm run; the next run starts right away while more wait. */
29	const DELIVERIES_PER_RUN = 20;
30	/** Requests read at a time; a run keeps reading past busy hooks' backlogs. */
31	const ROWS_READ_PER_PAGE = 50;
32	/** Wait before trying a hook whose environment answered busy again. */
33	const BUSY_RETRY_MS = 30_000;
34	const MAX_RETRY_DELAY = Duration.minutes(10);
35	/** Past this many failures the schedule is at its cap, so stepping further changes nothing. */
36	const MAX_COUNTED_FAILURES = 25;
37
38	/**
39	* Delays between delivery attempts while the environment is unreachable,
40	* indexed by consecutive failures since it was last reached or woke us.
41	* A wake means its tunnel just connected, but Cloudflare can take a few
42	* minutes to route the hostname to it, so the first 3 minutes retry every
43	* 10 s. After that the environment is likely gone again: 30 s, 1 min, 2 min,
44	* ... up to 10 min.
45	*/
46	export const retrySchedule = Schedule.spaced("10 seconds").pipe(
47	Schedule.upTo({ times: 18 }),
48	Schedule.concat(
49	Schedule.exponential("30 seconds").pipe(
50	Schedule.modifyDelay(({ duration }) =>
51	Effect.succeed(Duration.min(Duration.fromInputUnsafe(duration), MAX_RETRY_DELAY)),
52	),
53	),
54	),
55	);
56
57	export interface HeldHook {
58	readonly id: string;
59	readonly receivedAt: string;
60	readonly method: string;
61	/** Path segments exactly as the sender sent them; the environment decodes them. */
62	readonly rawHookId: string;
63	readonly rawToken: string;
64	/** The decoded hook id, so every spelling of one hook shares its cap. */
65	readonly hookKey: string;
66	/** Without the leading `?`. */
67	readonly query: string;
68	readonly headers: Readonly<Record<string, string>>;
69	readonly body: Uint8Array;
70	}
71
72	/**
73	* `delivered` deletes the request. `unreachable` keeps it and backs off the
74	* whole inbox, since nothing else will get through either. `busy` keeps it
75	* and its hook's later requests for the next run, but lets other hooks'
76	* requests go ahead, so one stuck task cannot hold up the rest.
77	*/
78	export type DeliveryOutcome = "delivered" \| "busy" \| "unreachable";
79
80	/** The `retrySchedule` delay after `failures` consecutive unreachable attempts. */
81	export const retryDelayMs = Effect.fn("HookInboxStore.retryDelayMs")(function* (failures: number) {
82	const step = yield* Schedule.toStep(retrySchedule);
83	let delay = MAX_RETRY_DELAY;
84	for (
85	let attempt = 0;
86	attempt < Math.min(Math.max(1, failures), MAX_COUNTED_FAILURES);
87	attempt++
88	) {
89	const next = yield* step(0, undefined).pipe(
90	Effect.map(([, duration]) => Option.some(duration)),
91	// The schedule never ends; stay at the cap if it ever does.
92	Pull.catchDone(() => Effect.succeedNone),
93	);
94	if (Option.isNone(next)) break;
95	delay = next.value;
96	}
97	return Duration.toMillis(delay);
98	});
99
100	const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String));
101	const encodeHeaders = Schema.encodeSync(HeadersJson);
102	const decodeHeaders = Schema.decodeUnknownOption(HeadersJson);
103
104	interface HeldHookRow {
105	readonly seq: number;
106	readonly id: string;
107	readonly received_at: string;
108	readonly method: string;
109	readonly raw_hook_id: string;
110	readonly raw_token: string;
111	readonly hook_key: string;
112	readonly query: string;
113	readonly headers: string;
114	readonly body: Uint8Array;
115	}
116
117	interface TargetRow {
118	readonly base_url: string;
119	readonly failures: number;
120	}
121
122	const iso = (epochMillis: number) => DateTime.formatIso(DateTime.makeUnsafe(epochMillis));
123
124	export const migrate = Effect.gen(function* () {
125	const sql = yield* SqlClient.SqlClient;
126	yield* sql`
127	CREATE TABLE IF NOT EXISTS held_hooks (
128	seq INTEGER PRIMARY KEY AUTOINCREMENT,
129	id TEXT NOT NULL UNIQUE,
130	received_at TEXT NOT NULL,
131	method TEXT NOT NULL,
132	raw_hook_id TEXT NOT NULL,
133	raw_token TEXT NOT NULL,
134	hook_key TEXT NOT NULL,
135	query TEXT NOT NULL,
136	headers TEXT NOT NULL,
137	body BLOB NOT NULL
138	)
139	`;
140	yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (hook_key)`;
141	// Where to push, and how many attempts in a row have failed. One row.
142	yield* sql`
143	CREATE TABLE IF NOT EXISTS held_hooks_target (
144	id INTEGER PRIMARY KEY CHECK (id = 1),
145	base_url TEXT NOT NULL,
146	failures INTEGER NOT NULL DEFAULT 0
147	)
148	`;
149	});
150
151	const setTarget = (baseUrl: string, options: { readonly resetFailures: boolean }) =>
152	Effect.gen(function* () {
153	const sql = yield* SqlClient.SqlClient;
154	yield* options.resetFailures
155	? sql`
156	INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl})
157	ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url, failures = 0
158	`
159	: sql`
160	INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl})
161	ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url
162	`;
163	});
164
165	const readTarget = Effect.gen(function* () {
166	const sql = yield* SqlClient.SqlClient;
167	const rows = yield* sql<TargetRow>`SELECT base_url, failures FROM held_hooks_target WHERE id = 1`;
168	return rows[0] ?? null;
169	});
170
171	const resetFailures = Effect.gen(function* () {
172	const sql = yield* SqlClient.SqlClient;
173	yield* sql`UPDATE held_hooks_target SET failures = 0 WHERE id = 1`;
174	});
175
176	/**
177	* Counts one more unreachable attempt. Incremented in place, so a wake that
178	* reset the count while this run's request was in flight is not overwritten.
179	*/
180	const countFailure = Effect.gen(function* () {
181	const sql = yield* SqlClient.SqlClient;
182	const rows = yield* sql<{ readonly failures: number }>`
183	UPDATE held_hooks_target SET failures = min(failures + 1, ${MAX_COUNTED_FAILURES})
184	WHERE id = 1 RETURNING failures
185	`;
186	return rows[0]?.failures ?? 1;
187	});
188
189	const hasPending = Effect.gen(function* () {
190	const sql = yield* SqlClient.SqlClient;
191	const rows = yield* sql<{ readonly id: string }>`SELECT id FROM held_hooks LIMIT 1`;
192	return rows.length > 0;
193	});
194
195	/**
196	* Stores a request for later delivery to `baseUrl`. Returns the time the
197	* first attempt is due, or null when the inbox is full and nothing was stored.
198	*/
199	export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook, baseUrl: string) {
200	const sql = yield* SqlClient.SqlClient;
201	// One statement, so the caps hold however many requests arrive at once.
202	const inserted = yield* sql<{ readonly id: string }>`
203	INSERT INTO held_hooks
204	(id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body)
205	SELECT ${hook.id}, ${hook.receivedAt}, ${hook.method}, ${hook.rawHookId}, ${hook.rawToken},
206	${hook.hookKey}, ${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body}
207	WHERE (SELECT count(*) FROM held_hooks) < ${HOOK_INBOX_MAX_REQUESTS}
208	AND (SELECT count(*) FROM held_hooks WHERE hook_key = ${hook.hookKey})
209	< ${HOOK_INBOX_MAX_PER_HOOK}
210	AND (SELECT coalesce(sum(length(body)), 0) FROM held_hooks) + ${hook.body.byteLength}
211	<= ${HOOK_INBOX_MAX_BYTES}
212	ON CONFLICT (id) DO NOTHING
213	RETURNING id
214	`;
215	if (inserted.length === 0) {
216	yield* Effect.annotateCurrentSpan({ "relay.inbox.refused": yield* refusalReason(hook) });
217	return null;
218	}
219	yield* setTarget(baseUrl, { resetFailures: false });
220	const target = yield* readTarget;
221	const backlog = yield* backlogSize;
222	yield* Effect.annotateCurrentSpan({
223	"relay.inbox.held_count": backlog.count,
224	"relay.inbox.held_bytes": backlog.bytes,
225	});
226	return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(target?.failures ?? 0));
227	});
228
229	const backlogSize = Effect.gen(function* () {
230	const sql = yield* SqlClient.SqlClient;
231	const rows = yield* sql<{ readonly count: number; readonly bytes: number }>`
232	SELECT count(*) AS count, coalesce(sum(length(body)), 0) AS bytes FROM held_hooks
233	`;
234	return rows[0] ?? { count: 0, bytes: 0 };
235	});
236
237	/** Which cap refused a request, or that it was already held, for traces. */
238	const refusalReason = (hook: HeldHook) =>
239	Effect.gen(function* () {
240	const sql = yield* SqlClient.SqlClient;
241	const existing = yield* sql<{ readonly id: string }>`
242	SELECT id FROM held_hooks WHERE id = ${hook.id}
243	`;
244	if (existing.length > 0) return "already_held";
245	const backlog = yield* backlogSize;
246	if (backlog.count >= HOOK_INBOX_MAX_REQUESTS) return "max_requests";
247	if (backlog.bytes + hook.body.byteLength > HOOK_INBOX_MAX_BYTES) return "max_bytes";
248	return "max_per_hook";
249	});
250
251	/**
252	* The environment is reachable again at `baseUrl`. Returns whether anything
253	* is waiting, so the caller can deliver right away.
254	*/
255	export const wake = Effect.fn("HookInboxStore.wake")(function* (baseUrl: string) {
256	if (!(yield* hasPending)) return false;
257	yield* setTarget(baseUrl, { resetFailures: true });
258	return true;
259	});
260
261	export const clear = Effect.gen(function* () {
262	const sql = yield* SqlClient.SqlClient;
263	yield* sql`DELETE FROM held_hooks`;
264	yield* sql`DELETE FROM held_hooks_target`;
265	});
266
267	/**
268	* Pushes the oldest held requests to the environment, one at a time, in
269	* order per hook. Stops at the first sign the environment is unreachable;
270	* skips past a hook whose environment answered busy. Drops requests older
271	* than the TTL. Returns when the next run is due, or null when nothing is left.
272	*/
273	export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
274	send: (baseUrl: string, hook: HeldHook) => Effect.Effect<DeliveryOutcome, never, R>,
275	) {
276	const sql = yield* SqlClient.SqlClient;
277	const startedAt = yield* Clock.currentTimeMillis;
278	const expired = yield* sql<{ readonly id: string }>`
279	DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)}
280	RETURNING id
281	`;
282	const target = yield* readTarget;
283	/** Oldest held requests after `afterSeq`, skipping hooks already found busy. */
284	const readPage = (afterSeq: number, skip: ReadonlySet<string>) =>
285	sql<HeldHookRow>`
286	SELECT seq, id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body
287	FROM held_hooks
288	WHERE seq > ${afterSeq}
289	${skip.size === 0 ? sql`` : sql`AND hook_key NOT IN ${sql.in([...skip])}`}
290	ORDER BY seq LIMIT ${ROWS_READ_PER_PAGE}
291	`;
292	let batch = yield* readPage(0, new Set());
293	if (batch.length === 0 \|\| target === null) {
294	if (target === null) yield* sql`DELETE FROM held_hooks`;
295	// Empty, so the next request held starts the schedule from the top.
296	else yield* resetFailures;
297	yield* Effect.annotateCurrentSpan({ "relay.inbox.expired": expired.length });
298	return null;
299	}
300
301	let failures = target.failures;
302	let sent = 0;
303	let delivered = 0;
304	let unreadable = 0;
305	let longestWaitMs = 0;
306	const busyHooks = new Set<string>();
307	/** What this run did, for the alarm's span; never request contents. */
308	const annotateRun = (result: string) =>
309	Effect.gen(function* () {
310	const backlog = yield* backlogSize;
311	yield* Effect.annotateCurrentSpan({
312	"relay.inbox.run_result": result,
313	"relay.inbox.sent": sent,
314	"relay.inbox.delivered": delivered,
315	"relay.inbox.busy_hooks": busyHooks.size,
316	"relay.inbox.expired": expired.length,
317	"relay.inbox.unreadable": unreadable,
318	"relay.inbox.consecutive_failures": failures,
319	"relay.inbox.longest_wait_ms": longestWaitMs,
320	"relay.inbox.held_count": backlog.count,
321	"relay.inbox.held_bytes": backlog.bytes,
322	});
323	});
324	// Pages run out only once every non-busy request has been tried, so a
325	// backlog behind busy hooks never hides another hook's requests.
326	pages: while (batch.length > 0) {
327	for (const row of batch) {
328	if (sent === DELIVERIES_PER_RUN) break pages;
329	// Later requests to a busy hook wait their turn, so its order is kept.
330	if (busyHooks.has(row.hook_key)) continue;
331	const headers = decodeHeaders(row.headers);
332	if (Option.isNone(headers)) {
333	// Unreadable: it can never be delivered, and must not block the rest.
334	yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
335	unreadable += 1;
336	continue;
337	}
338	sent += 1;
339	const outcome = yield* send(target.base_url, {
340	id: row.id,
341	receivedAt: row.received_at,
342	method: row.method,
343	rawHookId: row.raw_hook_id,
344	rawToken: row.raw_token,
345	hookKey: row.hook_key,
346	query: row.query,
347	headers: headers.value,
348	body: row.body,
349	});
350	if (outcome === "unreachable") {
351	failures = yield* countFailure;
352	yield* annotateRun("unreachable");
353	return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures));
354	}
355	if (failures !== 0) {
356	failures = 0;
357	yield* resetFailures;
358	}
359	if (outcome === "busy") {
360	busyHooks.add(row.hook_key);
361	continue;
362	}
363	yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
364	delivered += 1;
365	const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at));
366	longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs);
367	}
368	batch = yield* readPage(batch.at(-1)!.seq, busyHooks);
369	}
370	if (!(yield* hasPending)) {
371	yield* resetFailures;
372	yield* annotateRun("drained");
373	return null;
374	}
375	// Everything left this run was busy: give those tasks a moment to drain.
376	const now = yield* Clock.currentTimeMillis;
377	const idle = delivered === 0 && busyHooks.size > 0;
378	yield* annotateRun(idle ? "busy" : "more_pending");
379	return idle ? now + BUSY_RETRY_MS : now;
380	});

infra/relay/src/hooks/upstream.ts created+125

...	@@ -0,0 +1,125 @@
1	import * as Duration from "effect/Duration";
2	import * as Effect from "effect/Effect";
3	import * as Option from "effect/Option";
4	import * as Schema from "effect/Schema";
5	import * as Stream from "effect/Stream";
6	import * as HttpClient from "effect/http/HttpClient";
7	import * as HttpClientRequest from "effect/http/HttpClientRequest";
8	import * as HttpMethod from "effect/http/HttpMethod";
9	import * as HttpTraceContext from "effect/http/HttpTraceContext";
10	import type * as HttpClientResponse from "effect/http/HttpClientResponse";
11
12	import { withoutRedirects } from "../environments/EnvironmentConnector.ts";
13
14	/** Set by the relay on every forward; the environment uses it as the delivery id. */
15	const RELAY_DELIVERY_ID_HEADER = "x-t3-relay-delivery-id";
16	/** When the relay received the request; the environment trusts it only with the delivery id. */
17	const RELAY_RECEIVED_AT_HEADER = "x-t3-relay-received-at";
18	/** The environment answers with a small JSON status; anything past this is cut off. */
19	const MAX_RESPONSE_BYTES = 64 * 1024;
20	export const RELAY_HOOK_UPSTREAM_TIMEOUT_MS = 8_000;
21	// Cloudflare answers 530 when the tunnel for a hostname has no connected origin.
22	export const TUNNEL_OFFLINE_STATUS = 530;
23
24	/** A webhook request as the relay sends it on to the environment. */
25	export interface UpstreamHook {
26	readonly id: string;
27	readonly receivedAt: string;
28	readonly method: string;
29	/** Path segments exactly as the sender sent them; the environment decodes them. */
30	readonly rawHookId: string;
31	readonly rawToken: string;
32	/** Without the leading `?`. */
33	readonly query: string;
34	readonly headers: Readonly<Record<string, string>>;
35	readonly body: Uint8Array;
36	}
37
38	export interface UpstreamResponse {
39	readonly status: number;
40	readonly contentType: string \| undefined;
41	/** What the environment did with the request (`x-t3-hook-outcome`). */
42	readonly outcome: string \| undefined;
43	readonly body: Uint8Array;
44	}
45
46	/** Outcome names are short identifiers; anything else is not recorded. */
47	const OUTCOME_PATTERN = /^[a-z_]{1,32}$/;
48
49	class ResponseTooLarge extends Schema.TaggedError<ResponseTooLarge>()("ResponseTooLarge", {}) {}
50
51	/** Reads a response body, failing once it passes the cap rather than buffering it all. */
52	const readCapped = (response: HttpClientResponse.HttpClientResponse) =>
53	Effect.suspend(() => {
54	const chunks: Array<Uint8Array> = [];
55	let total = 0;
56	return response.stream.pipe(
57	Stream.runForEach((chunk) => {
58	total += chunk.length;
59	if (total > MAX_RESPONSE_BYTES) return Effect.fail(new ResponseTooLarge());
60	chunks.push(chunk);
61	return Effect.void;
62	}),
63	Effect.map(() => {
64	const body = new Uint8Array(total);
65	let offset = 0;
66	for (const chunk of chunks) {
67	body.set(chunk, offset);
68	offset += chunk.length;
69	}
70	return body;
71	}),
72	// A response without a body, such as a redirect, has no stream at all.
73	Effect.catchIf(
74	(error) => error._tag === "HttpClientError" && error.reason._tag === "EmptyBodyError",
75	() => Effect.succeed(new Uint8Array(0)),
76	),
77	);
78	});
79
80	/**
81	* Sends a webhook request through the environment's tunnel. Fails when the
82	* environment cannot be reached, and succeeds with None on timeout.
83	*/
84	export const sendUpstream = (baseUrl: string, hook: UpstreamHook) =>
85	Effect.gen(function* () {
86	const httpClient = yield* HttpClient.HttpClient;
87	const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
88	// The environment's span joins this trace. Set by hand: the client span
89	// that would propagate it is off, because it records the token in url.full.
90	const parent = yield* Effect.currentSpan.pipe(Effect.option);
91	// `hook.headers` carries the signed delivery proof, set once when the
92	// relay received the request, so a held request sends the same proof.
93	const headers: Record<string, string> = {
94	...hook.headers,
95	...(Option.isSome(parent) ? HttpTraceContext.toHeaders(parent.value) : {}),
96	[RELAY_DELIVERY_ID_HEADER]: hook.id,
97	[RELAY_RECEIVED_AT_HEADER]: hook.receivedAt,
98	};
99	let request = HttpClientRequest.make(hook.method as "GET" \| "POST" \| "PUT" \| "PATCH")(
100	`${base}api/hooks/${hook.rawHookId}/${hook.rawToken}${hook.query ? `?${hook.query}` : ""}`,
101	{ headers },
102	);
103	if (HttpMethod.hasBody(request.method)) {
104	request = HttpClientRequest.bodyUint8Array(request, hook.body, headers["content-type"]);
105	}
106	return yield* httpClient.execute(request).pipe(
107	Effect.flatMap((response) =>
108	readCapped(response).pipe(
109	Effect.map((body): UpstreamResponse => {
110	const outcome = response.headers["x-t3-hook-outcome"];
111	return {
112	status: response.status,
113	contentType: response.headers["content-type"],
114	outcome: outcome !== undefined && OUTCOME_PATTERN.test(outcome) ? outcome : undefined,
115	body,
116	};
117	}),
118	),
119	),
120	withoutRedirects,
121	// The client span would record url.full, which carries the token.
122	Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
123	Effect.timeoutOption(Duration.millis(RELAY_HOOK_UPSTREAM_TIMEOUT_MS)),
124	);
125	});

infra/relay/src/http/Api.test.ts+63-2

...	@@ -61,6 +61,7 @@ import {	...	@@ -61,6 +61,7 @@ import {
61	import * as RelayConfiguration from "../Config.ts";	61	import * as RelayConfiguration from "../Config.ts";
62	import * as RelayDb from "../db.ts";	62	import * as RelayDb from "../db.ts";
63	import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";	63	import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";
		64	import * as HookInbox from "../hooks/HookInbox.ts";
64	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";	65	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";
65	import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts";	66	import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts";
66	import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts";	67	import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts";
...	@@ -124,6 +125,7 @@ describe("device listing compatibility", () => {	...	@@ -124,6 +125,7 @@ describe("device listing compatibility", () => {
124	Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),	125	Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
125	Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),	126	Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),
126	Layer.mock(RelayDb.RelayTransactions, {}),	127	Layer.mock(RelayDb.RelayTransactions, {}),
		128	Layer.mock(HookInbox.HookInbox, {}),
127	),	129	),
128	),	130	),
129	Layer.provide(	131	Layer.provide(
...	@@ -317,8 +319,12 @@ function relayUnlinkTestLayer(input?: {	...	@@ -317,8 +319,12 @@ function relayUnlinkTestLayer(input?: {
317	readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"];	319	readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"];
318	readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"];	320	readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"];
319	readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"];	321	readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"];
		322	readonly clearInbox?: HookInbox.HookInbox["Service"]["clear"];
320	}) {	323	}) {
321	return Layer.mergeAll(	324	return Layer.mergeAll(
		325	Layer.mock(HookInbox.HookInbox, {
		326	clear: input?.clearInbox ?? (() => Effect.void),
		327	}),
322	Layer.succeed(	328	Layer.succeed(
323	RelayDb.RelayTransactions,	329	RelayDb.RelayTransactions,
324	RelayDb.RelayTransactions.of({	330	RelayDb.RelayTransactions.of({
...	@@ -332,7 +338,8 @@ function relayUnlinkTestLayer(input?: {	...	@@ -332,7 +338,8 @@ function relayUnlinkTestLayer(input?: {
332	listDeliveryUsersForEnvironment: () => Effect.die("unused listDeliveryUsersForEnvironment"),	338	listDeliveryUsersForEnvironment: () => Effect.die("unused listDeliveryUsersForEnvironment"),
333	listForUser: () => Effect.die("unused listForUser"),	339	listForUser: () => Effect.die("unused listForUser"),
334	getForUser: input?.getForUser ?? (() => Effect.succeed(null)),	340	getForUser: input?.getForUser ?? (() => Effect.succeed(null)),
335	findActiveManagedForEnvironment: () => Effect.succeed([]),	341	findActiveManagedForEnvironment: () => Effect.die("unused findActiveManagedForEnvironment"),
		342	setHoldWebhooksWhileOffline: () => Effect.die("unused setHoldWebhooksWhileOffline"),
336	revokeForUser: input?.revokeForUser ?? (() => Effect.succeed(false)),	343	revokeForUser: input?.revokeForUser ?? (() => Effect.succeed(false)),
337	}),	344	}),
338	),	345	),
...	@@ -899,6 +906,53 @@ describe("relay environment unlink", () => {	...	@@ -899,6 +906,53 @@ describe("relay environment unlink", () => {
899	);	906	);
900	});	907	});
901		908
		909	it.effect("drops the unlinked endpoint's held webhooks, even if clearing fails", () => {
		910	const cleared: Array<string> = [];
		911	const endpointKey = "0123456789abcdef";
		912	const unlink = (clearFails: boolean) =>
		913	unlinkEnvironmentRecord({
		914	userId: "user-1",
		915	environmentId: "environment-1",
		916	managedEndpointNamespace: "dev",
		917	}).pipe(
		918	Effect.provide(
		919	relayUnlinkTestLayer({
		920	getForUser: () => Effect.succeed(linkedEnvironmentRecord),
		921	revokeForUser: () => Effect.succeed(true),
		922	prepareDeprovision: () =>
		923	Effect.succeed({
		924	userId: "user-1",
		925	environmentId: "environment-1",
		926	hostname: "dev-0123456789abcdef.example.test",
		927	tunnelId: "tunnel-1",
		928	tunnelName: `t3coderelay-managedendpoint-dev-${endpointKey}`,
		929	dnsRecordId: "dns-1",
		930	readyAt: "2026-07-28T00:00:00.000Z",
		931	origin: null,
		932	updatedAt: "2026-07-28T00:00:00.000Z",
		933	generation: 1,
		934	}),
		935	clearInbox: (input) =>
		936	clearFails
		937	? Effect.fail(
		938	new HookInbox.HookInboxError({
		939	operation: "clear",
		940	endpointKey: input.endpointKey,
		941	cause: new Error("unavailable"),
		942	}),
		943	)
		944	: Effect.sync(() => void cleared.push(input.endpointKey)),
		945	}),
		946	),
		947	);
		948	return Effect.gen(function* () {
		949	expect(yield* unlink(false)).toBe(true);
		950	expect(cleared).toEqual([endpointKey]);
		951	// The link is already revoked; a failed clear must not fail the unlink.
		952	expect(yield* unlink(true)).toBe(true);
		953	});
		954	});
		955
902	it.effect("commits database revocation before deprovisioning the managed endpoint", () => {	956	it.effect("commits database revocation before deprovisioning the managed endpoint", () => {
903	const calls: Array<string> = [];	957	const calls: Array<string> = [];
904	const deprovisionTarget = {	958	const deprovisionTarget = {
...	@@ -1215,7 +1269,14 @@ describe("relay routing fallback", () => {	...	@@ -1215,7 +1269,14 @@ describe("relay routing fallback", () => {
1215	Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),	1269	Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}),
1216	),	1270	),
1217	),	1271	),
1218	Layer.provide([publisher, signatures]),	1272	Layer.provide([
		1273	publisher,
		1274	signatures,
		1275	Layer.mock(EnvironmentLinks.EnvironmentLinks, {}),
		1276	Layer.mock(HookInbox.HookInbox, {}),
		1277	Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {}),
		1278	Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings),
		1279	]),
1219	),	1280	),
1220	),	1281	),
1221	Layer.provide(auth),	1282	Layer.provide(auth),

infra/relay/src/http/Api.ts+136-9

...	@@ -66,6 +66,8 @@ import * as DpopProofs from "../auth/DpopProofs.ts";	...	@@ -66,6 +66,8 @@ import * as DpopProofs from "../auth/DpopProofs.ts";
66	import * as RelayTokens from "../auth/RelayTokens.ts";	66	import * as RelayTokens from "../auth/RelayTokens.ts";
67	import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";	67	import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts";
68	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";	68	import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts";
		69	import * as HookForwarder from "../hooks/HookForwarder.ts";
		70	import * as HookInbox from "../hooks/HookInbox.ts";
69	import * as LiveActivities from "../agentActivity/LiveActivities.ts";	71	import * as LiveActivities from "../agentActivity/LiveActivities.ts";
70	import * as RelayConfiguration from "../Config.ts";	72	import * as RelayConfiguration from "../Config.ts";
71	import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts";	73	import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts";
...	@@ -188,6 +190,12 @@ export const relayDocsRedirectRoute = HttpRouter.add(	...	@@ -188,6 +190,12 @@ export const relayDocsRedirectRoute = HttpRouter.add(
188	// contains the exact child span that stalled, and the response still carries	190	// contains the exact child span that stalled, and the response still carries
189	// the traceparent back to the client.	191	// the traceparent back to the client.
190	export const RELAY_REQUEST_DEADLINE_MS = 9_000;	192	export const RELAY_REQUEST_DEADLINE_MS = 9_000;
		193	/**
		194	* Webhook forwarding reads a body of up to 1 MiB, waits up to the upstream
		195	* timeout, and may then hold the request, so it needs more room than an API
		196	* call; cutting it off before the hold would drop a request it should keep.
		197	*/
		198	export const RELAY_HOOK_REQUEST_DEADLINE_MS = 25_000;
191		199
192	const relayRequestDeadline = <E, R>(	200	const relayRequestDeadline = <E, R>(
193	httpEffect: Effect.Effect<	201	httpEffect: Effect.Effect<
...	@@ -195,9 +203,10 @@ const relayRequestDeadline = <E, R>(	...	@@ -195,9 +203,10 @@ const relayRequestDeadline = <E, R>(
195	E,	203	E,
196	HttpServerRequest.HttpServerRequest \| R	204	HttpServerRequest.HttpServerRequest \| R
197	>,	205	>,
		206	deadlineMs = RELAY_REQUEST_DEADLINE_MS,
198	) =>	207	) =>
199	httpEffect.pipe(	208	httpEffect.pipe(
200	Effect.timeoutOption(Duration.millis(RELAY_REQUEST_DEADLINE_MS)),	209	Effect.timeoutOption(Duration.millis(deadlineMs)),
201	Effect.flatMap(	210	Effect.flatMap(
202	Option.match({	211	Option.match({
203	onNone: () =>	212	onNone: () =>
...	@@ -206,7 +215,7 @@ const relayRequestDeadline = <E, R>(	...	@@ -206,7 +215,7 @@ const relayRequestDeadline = <E, R>(
206	yield* Effect.logError("relay request exceeded deadline", {	215	yield* Effect.logError("relay request exceeded deadline", {
207	"http.method": request.method,	216	"http.method": request.method,
208	"http.url": request.url,	217	"http.url": request.url,
209	"relay.request.deadline_ms": RELAY_REQUEST_DEADLINE_MS,	218	"relay.request.deadline_ms": deadlineMs,
210	});	219	});
211	yield* Effect.annotateCurrentSpan({	220	yield* Effect.annotateCurrentSpan({
212	"relay.request.deadline_exceeded": true,	221	"relay.request.deadline_exceeded": true,
...	@@ -221,6 +230,9 @@ const relayRequestDeadline = <E, R>(	...	@@ -221,6 +230,9 @@ const relayRequestDeadline = <E, R>(
221	),	230	),
222	);	231	);
223		232
		233	/** Trace context headers in every format the tracer reads (W3C and B3). */
		234	const SENDER_TRACE_HEADER = /^(traceparent\|tracestate\|b3\|x-b3-.*)$/i;
		235
224	export const traceRelayHttpRequest = <E, R>(	236	export const traceRelayHttpRequest = <E, R>(
225	httpEffect: Effect.Effect<	237	httpEffect: Effect.Effect<
226	HttpServerResponse.HttpServerResponse,	238	HttpServerResponse.HttpServerResponse,
...	@@ -238,18 +250,29 @@ export const traceRelayHttpRequest = <E, R>(	...	@@ -238,18 +250,29 @@ export const traceRelayHttpRequest = <E, R>(
238	return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow));	250	return yield* HttpMiddleware.tracer(traced).pipe(Effect.ensuring(Effect.yieldNow));
239	}	251	}
240	// Hook URLs carry a secret token: the tracer and deadline log see a redacted	252	// Hook URLs carry a secret token: the tracer and deadline log see a redacted
241	// request, while the route itself still receives the original.	253	// request, while the route itself still receives the original. A webhook
242	const redacted = request.modify({ url: redactRelayHookUrl(request.url) });	254	// sender's trace context is dropped, so it cannot pick the trace our relay
		255	// and environment spans land in.
		256	const redacted = request.modify({
		257	url: redactRelayHookUrl(request.url),
		258	headers: Headers.removeMany(
		259	request.headers,
		260	Object.keys(request.headers).filter((name) => SENDER_TRACE_HEADER.test(name)),
		261	),
		262	});
243	return yield* HttpMiddleware.tracer(	263	return yield* HttpMiddleware.tracer(
244	appendRelayTraceContextResponseHeader.pipe(	264	appendRelayTraceContextResponseHeader.pipe(
245	Effect.andThen(	265	Effect.andThen(
246	relayRequestDeadline(	266	relayRequestDeadline(
247	httpEffect.pipe(Effect.provideService(HttpServerRequest.HttpServerRequest, request)),	267	httpEffect.pipe(Effect.provideService(HttpServerRequest.HttpServerRequest, request)),
		268	RELAY_HOOK_REQUEST_DEADLINE_MS,
248	),	269	),
249	),	270	),
250	),	271	),
251	).pipe(	272	).pipe(
252	Effect.provideService(HttpServerRequest.HttpServerRequest, redacted),	273	Effect.provideService(HttpServerRequest.HttpServerRequest, redacted),
		274	// The worker disables its own span for hook paths; this one is ours.
		275	Effect.provideService(HttpMiddleware.TracerDisabledWhen, () => false),
253	Effect.ensuring(Effect.yieldNow),	276	Effect.ensuring(Effect.yieldNow),
254	);	277	);
255	});	278	});
...	@@ -494,7 +517,12 @@ export const revokeEnvironmentLinkRecord = Effect.fn(	...	@@ -494,7 +517,12 @@ export const revokeEnvironmentLinkRecord = Effect.fn(
494	});	517	});
495		518
496	export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnvironmentRecord")(	519	export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnvironmentRecord")(
497	function* (input: { readonly userId: string; readonly environmentId: string }) {	520	function* (input: {
		521	readonly userId: string;
		522	readonly environmentId: string;
		523	/** The stage's tunnel-name namespace, to find this link's held webhook requests. */
		524	readonly managedEndpointNamespace?: string \| undefined;
		525	}) {
498	const links = yield* EnvironmentLinks.EnvironmentLinks;	526	const links = yield* EnvironmentLinks.EnvironmentLinks;
499	const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider;	527	const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
500	const deprovisionTarget = yield* managedEndpointProvider.prepareDeprovision({	528	const deprovisionTarget = yield* managedEndpointProvider.prepareDeprovision({
...	@@ -513,7 +541,6 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron	...	@@ -513,7 +541,6 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron
513	environmentId: link.environmentId,	541	environmentId: link.environmentId,
514	environmentPublicKey: link.environmentPublicKey,	542	environmentPublicKey: link.environmentPublicKey,
515	});	543	});
516
517	// External teardown cannot share the SQL transaction. Run it only after	544	// External teardown cannot share the SQL transaction. Run it only after
518	// revocation commits so a database failure leaves a fully usable active	545	// revocation commits so a database failure leaves a fully usable active
519	// link. Still run teardown when the link is already revoked, allowing a	546	// link. Still run teardown when the link is already revoked, allowing a
...	@@ -523,10 +550,31 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron	...	@@ -523,10 +550,31 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron
523	environmentId: input.environmentId,	550	environmentId: input.environmentId,
524	target: deprovisionTarget,	551	target: deprovisionTarget,
525	});	552	});
		553	// Requests held for this link's endpoint go with it. Best effort: the link
		554	// is already gone, and its inbox drops anything left after its TTL.
		555	const endpointKey =
		556	deprovisionTarget && input.managedEndpointNamespace
		557	? HookForwarder.endpointKeyForTunnelName(
		558	input.managedEndpointNamespace,
		559	deprovisionTarget.tunnelName,
		560	)
		561	: null;
		562	if (endpointKey !== null) {
		563	const inbox = yield* HookInbox.HookInbox;
		564	yield* inbox.clear({ endpointKey }).pipe(
		565	Effect.catch((error) =>
		566	Effect.logWarning("Could not clear held webhook requests", {
		567	environmentId: input.environmentId,
		568	errorTag: error._tag,
		569	}),
		570	),
		571	);
		572	}
526	if (!deprovisioned) {	573	if (!deprovisioned) {
527	const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input);	574	const key = { userId: input.userId, environmentId: input.environmentId };
528	if (retryTarget !== null && (yield* links.getForUser(input)) === null) {	575	const retryTarget = yield* managedEndpointProvider.prepareDeprovision(key);
529	yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget });	576	if (retryTarget !== null && (yield* links.getForUser(key)) === null) {
		577	yield* managedEndpointProvider.deprovision({ ...key, target: retryTarget });
530	}	578	}
531	}	579	}
532	return unlinked;	580	return unlinked;
...	@@ -926,6 +974,7 @@ export const clientApi = HttpApiBuilder.group(	...	@@ -926,6 +974,7 @@ export const clientApi = HttpApiBuilder.group(
926	const unlinked = yield* unlinkEnvironmentRecord({	974	const unlinked = yield* unlinkEnvironmentRecord({
927	userId,	975	userId,
928	environmentId: params.environmentId,	976	environmentId: params.environmentId,
		977	managedEndpointNamespace: config.managedEndpointNamespace,
929	}).pipe(	978	}).pipe(
930	Effect.catchTags({	979	Effect.catchTags({
931	SqlError: () => relayInternalErrorResponse("internal_error"),	980	SqlError: () => relayInternalErrorResponse("internal_error"),
...	@@ -1127,6 +1176,40 @@ export const serverApi = HttpApiBuilder.group(	...	@@ -1127,6 +1176,40 @@ export const serverApi = HttpApiBuilder.group(
1127	Effect.fnUntraced(function* (handlers) {	1176	Effect.fnUntraced(function* (handlers) {
1128	const publisher = yield* AgentActivityPublisher.AgentActivityPublisher;	1177	const publisher = yield* AgentActivityPublisher.AgentActivityPublisher;
1129	const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures;	1178	const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures;
		1179	const links = yield* EnvironmentLinks.EnvironmentLinks;
		1180	const inbox = yield* HookInbox.HookInbox;
		1181	const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
		1182	const settings = yield* RelayConfiguration.RelayConfiguration;
		1183	const requireOwnEnvironment = (environmentId: string) =>
		1184	Effect.gen(function* () {
		1185	const principal = yield* RelayEnvironmentPrincipal;
		1186	if (principal.environmentId !== environmentId) {
		1187	return yield* new HttpApiError.Unauthorized({});
		1188	}
		1189	});
		1190	/**
		1191	* Endpoint keys of the managed links the calling environment key proved.
		1192	* The environment id alone would also match other accounts' links of it.
		1193	*/
		1194	const ownEndpointKeys = (environmentId: string) =>
		1195	Effect.gen(function* () {
		1196	const principal = yield* RelayEnvironmentPrincipal;
		1197	const namespace = settings.managedEndpointNamespace;
		1198	if (!namespace) return [];
		1199	const ownLinks = yield* links.findActiveManagedForEnvironment({
		1200	environmentId,
		1201	environmentPublicKey: principal.environmentPublicKey,
		1202	});
		1203	const keys: Array<string> = [];
		1204	for (const link of ownLinks) {
		1205	const allocation = yield* allocations.get({ userId: link.userId, environmentId });
		1206	const key = allocation
		1207	? HookForwarder.endpointKeyForTunnelName(namespace, allocation.tunnelName)
		1208	: null;
		1209	if (key !== null) keys.push(key);
		1210	}
		1211	return keys;
		1212	});
1130	const activityHandlers = handlers.handle(	1213	const activityHandlers = handlers.handle(
1131	"publishAgentActivity",	1214	"publishAgentActivity",
1132	Effect.fn("relay.api.server.publishAgentActivity")(	1215	Effect.fn("relay.api.server.publishAgentActivity")(
...	@@ -1336,6 +1419,48 @@ export const serverApi = HttpApiBuilder.group(	...	@@ -1336,6 +1419,48 @@ export const serverApi = HttpApiBuilder.group(
1336	}),	1419	}),
1337	mapRelayCommonApiErrors("not_authorized"),	1420	mapRelayCommonApiErrors("not_authorized"),
1338	),	1421	),
		1422	)
		1423	.handle(
		1424	"updateLinkPreferences",
		1425	Effect.fn("relay.api.server.updateLinkPreferences")(function* ({ params, payload }) {
		1426	yield* requireOwnEnvironment(params.environmentId);
		1427	const principal = yield* RelayEnvironmentPrincipal;
		1428	yield* links.setHoldWebhooksWhileOffline({
		1429	environmentId: params.environmentId,
		1430	environmentPublicKey: principal.environmentPublicKey,
		1431	holdWebhooksWhileOffline: payload.holdWebhooksWhileOffline,
		1432	});
		1433	// Opting out also drops what is already held, rather than delivering
		1434	// it later to an environment that said it does not want it.
		1435	if (!payload.holdWebhooksWhileOffline) {
		1436	for (const endpointKey of yield* ownEndpointKeys(params.environmentId)) {
		1437	yield* inbox.clear({ endpointKey });
		1438	}
		1439	}
		1440	return payload;
		1441	}, mapRelayCommonApiErrors("not_authorized")),
		1442	)
		1443	.handle(
		1444	"wakeHeldHooks",
		1445	Effect.fn("relay.api.server.wakeHeldHooks")(function* ({ params }) {
		1446	yield* requireOwnEnvironment(params.environmentId);
		1447	let pending = false;
		1448	for (const endpointKey of yield* ownEndpointKeys(params.environmentId)) {
		1449	const endpoint = yield* HookForwarder.resolveHookEndpoint(endpointKey).pipe(
		1450	Effect.provideService(EnvironmentLinks.EnvironmentLinks, links),
		1451	Effect.provideService(
		1452	ManagedEndpointAllocations.ManagedEndpointAllocations,
		1453	allocations,
		1454	),
		1455	Effect.provideService(RelayConfiguration.RelayConfiguration, settings),
		1456	);
		1457	if (endpoint === null) continue;
		1458	if (yield* inbox.wake({ endpointKey, baseUrl: endpoint.httpBaseUrl })) {
		1459	pending = true;
		1460	}
		1461	}
		1462	return { pending };
		1463	}, mapRelayCommonApiErrors("not_authorized")),
1339	);	1464	);
1340	}),	1465	}),
1341	);	1466	);
...	@@ -1378,6 +1503,8 @@ const RelayCommonPersistenceError = Schema.Union([	...	@@ -1378,6 +1503,8 @@ const RelayCommonPersistenceError = Schema.Union([
1378	AgentActivityRows.AgentActivityRowListPersistenceError,	1503	AgentActivityRows.AgentActivityRowListPersistenceError,
1379	LiveActivities.LiveActivityDeliveryMarkPersistenceError,	1504	LiveActivities.LiveActivityDeliveryMarkPersistenceError,
1380	DeliveryAttempts.DeliveryAttemptRecordPersistenceError,	1505	DeliveryAttempts.DeliveryAttemptRecordPersistenceError,
		1506	EnvironmentLinks.EnvironmentLinkEnvironmentLookupPersistenceError,
		1507	HookInbox.HookInboxError,
1381	]);	1508	]);
1382	type RelayCommonPersistenceError = typeof RelayCommonPersistenceError.Type;	1509	type RelayCommonPersistenceError = typeof RelayCommonPersistenceError.Type;
1383	const isRelayCommonPersistenceError = Schema.is(RelayCommonPersistenceError);	1510	const isRelayCommonPersistenceError = Schema.is(RelayCommonPersistenceError);

infra/relay/src/persistence/schema.ts+2

...	@@ -74,6 +74,8 @@ export const relayEnvironmentLinks = pgTable(	...	@@ -74,6 +74,8 @@ export const relayEnvironmentLinks = pgTable(
74	notificationsEnabled: boolean("notifications_enabled").notNull().default(true),	74	notificationsEnabled: boolean("notifications_enabled").notNull().default(true),
75	liveActivitiesEnabled: boolean("live_activities_enabled").notNull().default(true),	75	liveActivitiesEnabled: boolean("live_activities_enabled").notNull().default(true),
76	managedTunnelsEnabled: boolean("managed_tunnels_enabled").notNull().default(false),	76	managedTunnelsEnabled: boolean("managed_tunnels_enabled").notNull().default(false),
		77	// Opt-in: hold webhook requests while the environment is offline.
		78	holdWebhooksWhileOffline: boolean("hold_webhooks_while_offline").notNull().default(false),
77	createdByDeviceId: varchar("created_by_device_id", { length: 191 }),	79	createdByDeviceId: varchar("created_by_device_id", { length: 191 }),
78	revokedAt: varchar("revoked_at", { length: 64 }),	80	revokedAt: varchar("revoked_at", { length: 64 }),
79	createdAt: varchar("created_at", { length: 64 }).notNull(),	81	createdAt: varchar("created_at", { length: 64 }).notNull(),

infra/relay/src/worker.ts+78-5

...	@@ -1,4 +1,5 @@	...	@@ -1,4 +1,5 @@
1	import * as Alchemy from "alchemy";	1	import * as Alchemy from "alchemy";
		2	import * as Axiom from "alchemy/Axiom";
2	import * as Cloudflare from "alchemy/Cloudflare";	3	import * as Cloudflare from "alchemy/Cloudflare";
3	import * as Drizzle from "alchemy/Drizzle/Postgres";	4	import * as Drizzle from "alchemy/Drizzle/Postgres";
4	import * as Config from "effect/Config";	5	import * as Config from "effect/Config";
...	@@ -11,6 +12,7 @@ import * as Option from "effect/Option";	...	@@ -11,6 +12,7 @@ import * as Option from "effect/Option";
11	import * as Redacted from "effect/Redacted";	12	import * as Redacted from "effect/Redacted";
12	import * as Stream from "effect/Stream";	13	import * as Stream from "effect/Stream";
13	import * as Etag from "effect/http/Etag";	14	import * as Etag from "effect/http/Etag";
		15	import * as HttpMiddleware from "effect/http/HttpMiddleware";
14	import * as HttpPlatform from "effect/http/HttpPlatform";	16	import * as HttpPlatform from "effect/http/HttpPlatform";
15	import * as HttpRouter from "effect/http/HttpRouter";	17	import * as HttpRouter from "effect/http/HttpRouter";
16	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";	18	import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
...	@@ -74,6 +76,8 @@ import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts"	...	@@ -74,6 +76,8 @@ import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts"
74	import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts";	76	import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts";
75	import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts";	77	import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts";
76	import * as HookForwarder from "./hooks/HookForwarder.ts";	78	import * as HookForwarder from "./hooks/HookForwarder.ts";
		79	import * as HookInbox from "./hooks/HookInbox.ts";
		80	import { HookInboxObject, HookInboxObjectLive } from "./hooks/HookInboxObject.ts";
77		81
78	const webcryptoLayer = Layer.succeed(	82	const webcryptoLayer = Layer.succeed(
79	Crypto.Crypto,	83	Crypto.Crypto,
...	@@ -190,6 +194,8 @@ export const ApiLive = Api.make(	...	@@ -190,6 +194,8 @@ export const ApiLive = Api.make(
190	const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);	194	const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);
191	const managedEndpointZoneName = yield* managedEndpointZone.name;	195	const managedEndpointZoneName = yield* managedEndpointZone.name;
192	const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig;	196	const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig;
		197	// Keys are endpoint keys or hashes over them, which already differ per
		198	// stage, so stages sharing an account cannot collide in these namespaces.
193	const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", {	199	const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", {
194	namespaceId: 1001,	200	namespaceId: 1001,
195	simple: {	201	simple: {
...	@@ -197,6 +203,14 @@ export const ApiLive = Api.make(	...	@@ -197,6 +203,14 @@ export const ApiLive = Api.make(
197	period: HookForwarder.RELAY_HOOK_RATE_LIMIT.periodSeconds,	203	period: HookForwarder.RELAY_HOOK_RATE_LIMIT.periodSeconds,
198	},	204	},
199	});	205	});
		206	const hookEndpointRateLimit = yield* Cloudflare.RateLimit("HOOK_ENDPOINT_RATE_LIMIT", {
		207	namespaceId: 1002,
		208	simple: {
		209	limit: HookForwarder.RELAY_HOOK_ENDPOINT_RATE_LIMIT.limit,
		210	period: HookForwarder.RELAY_HOOK_ENDPOINT_RATE_LIMIT.periodSeconds,
		211	},
		212	});
		213	const hookInboxes = yield* HookInboxObject;
200		214
201	//	215	//
202	// 3. Runtime layers and app construction	216	// 3. Runtime layers and app construction
...	@@ -228,6 +242,31 @@ export const ApiLive = Api.make(	...	@@ -228,6 +242,31 @@ export const ApiLive = Api.make(
228	}).pipe(Effect.map(makeRelayTraceLayer)),	242	}).pipe(Effect.map(makeRelayTraceLayer)),
229	);	243	);
230		244
		245	// Each managed endpoint's held webhook requests live in its own Durable Object.
		246	const inboxCall =
		247	<A>(operation: HookInbox.HookInboxError["operation"], endpointKey: string) =>
		248	(effect: Effect.Effect<A, never, Alchemy.RuntimeContext>) =>
		249	effect.pipe(
		250	Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
		251	Effect.catchCause((cause) =>
		252	Effect.fail(
		253	new HookInbox.HookInboxError({
		254	operation,
		255	endpointKey,
		256	cause: Cause.squash(cause),
		257	}),
		258	),
		259	),
		260	);
		261	const hookInboxLayer = Layer.succeed(HookInbox.HookInbox, {
		262	hold: ({ endpointKey, baseUrl, hook }) =>
		263	hookInboxes.getByName(endpointKey).hold(hook, baseUrl).pipe(inboxCall("hold", endpointKey)),
		264	wake: ({ endpointKey, baseUrl }) =>
		265	hookInboxes.getByName(endpointKey).wake(baseUrl).pipe(inboxCall("wake", endpointKey)),
		266	clear: ({ endpointKey }) =>
		267	hookInboxes.getByName(endpointKey).clear().pipe(inboxCall("clear", endpointKey)),
		268	});
		269
231	const runtimeLayer = Layer.empty.pipe(	270	const runtimeLayer = Layer.empty.pipe(
232	Layer.provideMerge(MobileRegistrations.layer),	271	Layer.provideMerge(MobileRegistrations.layer),
233	Layer.provideMerge(AgentActivityPublisher.layer),	272	Layer.provideMerge(AgentActivityPublisher.layer),
...	@@ -269,7 +308,7 @@ export const ApiLive = Api.make(	...	@@ -269,7 +308,7 @@ export const ApiLive = Api.make(
269	Layer.provideMerge(	308	Layer.provideMerge(
270	ApnsDeliveryQueue.layerCloudflareQueues(apnsDeliveryQueueSender, alchemyRuntimeContext),	309	ApnsDeliveryQueue.layerCloudflareQueues(apnsDeliveryQueueSender, alchemyRuntimeContext),
271	),	310	),
272	Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer)),	311	Layer.provideMerge(Layer.mergeAll(AgentActivityRows.layer, Devices.layer, hookInboxLayer)),
273	Layer.provideMerge(EnvironmentCredentials.layer),	312	Layer.provideMerge(EnvironmentCredentials.layer),
274	Layer.provideMerge(	313	Layer.provideMerge(
275	Layer.mergeAll(	314	Layer.mergeAll(
...	@@ -291,17 +330,25 @@ export const ApiLive = Api.make(	...	@@ -291,17 +330,25 @@ export const ApiLive = Api.make(
291	);	330	);
292		331
293	// Fails open: a limiter outage must not drop webhooks the environment would accept.	332	// Fails open: a limiter outage must not drop webhooks the environment would accept.
294	const hookRateLimiterLayer = Layer.succeed(HookForwarder.HookRateLimiter, {	333	const allowWith =
295	allow: (key) =>	334	(limiter: typeof hookRateLimit) =>
296	hookRateLimit.limit({ key }).pipe(	335	(key: string): Effect.Effect<boolean> =>
		336	limiter.limit({ key }).pipe(
297	Effect.map((result) => result.success),	337	Effect.map((result) => result.success),
298	Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),	338	Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext),
299	Effect.catch((error) =>	339	Effect.catch((error) =>
300	Effect.logWarning("Hook rate limiter unavailable", { error: error.message }).pipe(	340	Effect.logWarning("Hook rate limiter unavailable", { error: error.message }).pipe(
		341	// Visible on the forward span, so an outage that disables limits shows up.
		342	Effect.andThen(
		343	Effect.annotateCurrentSpan({ "relay.hook.rate_limiter_failed_open": true }),
		344	),
301	Effect.as(true),	345	Effect.as(true),
302	),	346	),
303	),	347	),
304	),	348	);
		349	const hookRateLimiterLayer = Layer.succeed(HookForwarder.HookRateLimiter, {
		350	allowHook: allowWith(hookRateLimit),
		351	allowEndpoint: allowWith(hookEndpointRateLimit),
305	});	352	});
306		353
307	const appLayer = Layer.merge(	354	const appLayer = Layer.merge(
...	@@ -426,6 +473,32 @@ export const ApiLive = Api.make(	...	@@ -426,6 +473,32 @@ export const ApiLive = Api.make(
426	Layer.provideMerge(Cloudflare.Tunnel.ReadWriteTunnelBinding),	473	Layer.provideMerge(Cloudflare.Tunnel.ReadWriteTunnelBinding),
427	Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),	474	Layer.provideMerge(Cloudflare.DNS.ReadWriteDnsHttp),
428	Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),	475	Layer.provideMerge(Cloudflare.Workers.RateLimitBinding),
		476	Layer.provideMerge(HookInboxObjectLive),
		477	// The worker runtime opens its own HTTP span around ours. For webhook
		478	// paths it would record the raw URL, token included, and adopt the
		479	// sender's traceparent, so only our redacted span covers those.
		480	// Registered as telemetry: request-time context is assembled per
		481	// event, and only these layers are built into it.
		482	Layer.provideMerge(
		483	Alchemy.Telemetry.layer(
		484	Layer.succeed(HttpMiddleware.TracerDisabledWhen)((request) =>
		485	HookForwarder.isRelayHookPath(request.url),
		486	),
		487	),
		488	),
		489	// Exports spans from events the HTTP tracer does not wrap, notably
		490	// HookInboxObject calls and alarms, to the same Axiom dataset.
		491	Layer.provideMerge(
		492	Layer.unwrap(
		493	Effect.map(RelayObservability, (observability) =>
		494	Axiom.Telemetry({
		495	serviceName: "t3code-relay",
		496	token: observability.workerIngestToken,
		497	traces: observability.traces,
		498	}),
		499	),
		500	),
		501	),
429	),	502	),
430	),	503	),
431	),	504	),

packages/client-runtime/package.json+4

 @@ -35,6 +35,10 @@
       "types": "./src/userMessage.ts",
       "default": "./src/userMessage.ts"
     },
+    "./scheduled-task-webhook": {
+      "types": "./src/scheduledTaskWebhook.ts",
+      "default": "./src/scheduledTaskWebhook.ts"
+    },
     "./connection": {
       "types": "./src/connection/index.ts",
       "default": "./src/connection/index.ts"

packages/client-runtime/src/scheduledTaskWebhook.test.ts created+13

...	@@ -0,0 +1,13 @@
1	import { describe, expect, it } from "vite-plus/test";
2
3	import { parseMaxDeliveryAge } from "./scheduledTaskWebhook.ts";
4
5	describe("parseMaxDeliveryAge", () => {
6	it("treats blank as no limit and rejects values the server would not accept", () => {
7	expect(parseMaxDeliveryAge("")).toBeNull();
8	expect(parseMaxDeliveryAge(" 45 ")).toBe(45);
9	for (const invalid of ["0", "1.5", "-3", "abc", "1441"]) {
10	expect(parseMaxDeliveryAge(invalid)).toBeUndefined();
11	}
12	});
13	});

packages/client-runtime/src/scheduledTaskWebhook.ts created+13

...	@@ -0,0 +1,13 @@
1	import { MAX_WEBHOOK_DELIVERY_AGE_MINUTES } from "@t3tools/contracts";
2
3	/** Prompt a new webhook task starts with: just the body, so request headers stay out unless the user adds them. */
4	export const DEFAULT_WEBHOOK_PROMPT = "Handle this webhook:\n{{body}}";
5
6	/** Blank means "no limit"; undefined means the input is not a valid limit, which blocks saving. */
7	export function parseMaxDeliveryAge(value: string): number \| null \| undefined {
8	if (value.trim() === "") return null;
9	const minutes = Number(value.trim());
10	return Number.isInteger(minutes) && minutes > 0 && minutes <= MAX_WEBHOOK_DELIVERY_AGE_MINUTES
11	? minutes
12	: undefined;
13	}

packages/contracts/src/environmentHttp.ts+5

 @@ -404,11 +404,16 @@ export const EnvironmentCloudLinkStateResult = Schema.Struct({
   // Optional so newer clients tolerate older environment servers.
   managedTunnelActive: Schema.optional(Schema.Boolean),
   publishAgentActivity: Schema.Boolean,
+  // Opt-in: T3 Connect holds webhook requests while this environment is
+  // offline. Optional so newer clients tolerate older environment servers.
+  holdWebhooksWhileOffline: Schema.optional(Schema.Boolean),
 });
 export type EnvironmentCloudLinkStateResult = typeof EnvironmentCloudLinkStateResult.Type;
 export const EnvironmentCloudPreferencesRequest = Schema.Struct({
   publishAgentActivity: Schema.Boolean,
+  // Omit to leave the current value unchanged.
+  holdWebhooksWhileOffline: Schema.optional(Schema.Boolean),
 });
 export type EnvironmentCloudPreferencesRequest = typeof EnvironmentCloudPreferencesRequest.Type;

packages/contracts/src/relay.ts+50-3

...	@@ -864,6 +864,21 @@ export const RelayCloudEnvironmentHealthProofPayload = Schema.Struct({	...	@@ -864,6 +864,21 @@ export const RelayCloudEnvironmentHealthProofPayload = Schema.Struct({
864	export type RelayCloudEnvironmentHealthProofPayload =	864	export type RelayCloudEnvironmentHealthProofPayload =
865	typeof RelayCloudEnvironmentHealthProofPayload.Type;	865	typeof RelayCloudEnvironmentHealthProofPayload.Type;
866		866
		867	/**
		868	* Sent with every webhook the relay forwards, signed with the relay's mint key
		869	* (`x-t3-relay-delivery`). The environment trusts the relay's delivery id,
		870	* receive time and trace context only when this verifies, since its webhook
		871	* URL can also be called directly.
		872	*/
		873	export const RelayHookDeliveryProofPayload = Schema.Struct({
		874	...RelaySignedJwtRegisteredClaims,
		875	environmentId: EnvironmentId,
		876	deliveryId: TrimmedNonEmptyString,
		877	receivedAt: TrimmedNonEmptyString,
		878	hookId: TrimmedNonEmptyString,
		879	});
		880	export type RelayHookDeliveryProofPayload = typeof RelayHookDeliveryProofPayload.Type;
		881
867	export const RelayCloudEnvironmentHealthProof = TrimmedNonEmptyString;	882	export const RelayCloudEnvironmentHealthProof = TrimmedNonEmptyString;
868	export type RelayCloudEnvironmentHealthProof = typeof RelayCloudEnvironmentHealthProof.Type;	883	export type RelayCloudEnvironmentHealthProof = typeof RelayCloudEnvironmentHealthProof.Type;
869		884
...	@@ -1136,6 +1151,18 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient")	...	@@ -1136,6 +1151,18 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient")
1136	.annotate(OpenApi.Description, "DPoP-authenticated client access to linked environments.")	1151	.annotate(OpenApi.Description, "DPoP-authenticated client access to linked environments.")
1137	.middleware(RelayDpopClientAuth);	1152	.middleware(RelayDpopClientAuth);
1138		1153
		1154	export const RelayEnvironmentLinkPreferencesRequest = Schema.Struct({
		1155	holdWebhooksWhileOffline: Schema.Boolean,
		1156	});
		1157	export type RelayEnvironmentLinkPreferencesRequest =
		1158	typeof RelayEnvironmentLinkPreferencesRequest.Type;
		1159
		1160	export const RelayWakeHeldHooksResponse = Schema.Struct({
		1161	/** True when the relay was holding requests and has started delivering them. */
		1162	pending: Schema.Boolean,
		1163	});
		1164	export type RelayWakeHeldHooksResponse = typeof RelayWakeHeldHooksResponse.Type;
		1165
1139	const RelayServerGroup = HttpApiGroup.make("server")	1166	const RelayServerGroup = HttpApiGroup.make("server")
1140	.add(	1167	.add(
1141	HttpApiEndpoint.post(	1168	HttpApiEndpoint.post(
...	@@ -1171,6 +1198,24 @@ const RelayServerGroup = HttpApiGroup.make("server")	...	@@ -1171,6 +1198,24 @@ const RelayServerGroup = HttpApiGroup.make("server")
1171	error: RelayAgentActivityPublishErrors,	1198	error: RelayAgentActivityPublishErrors,
1172	},	1199	},
1173	).annotate(OpenApi.Summary, "Publish agent activity"),	1200	).annotate(OpenApi.Summary, "Publish agent activity"),
		1201	HttpApiEndpoint.post(
		1202	"updateLinkPreferences",
		1203	"/v1/environments/:environmentId/link-preferences",
		1204	{
		1205	params: Schema.Struct({ environmentId: EnvironmentId }),
		1206	payload: RelayEnvironmentLinkPreferencesRequest,
		1207	success: RelayEnvironmentLinkPreferencesRequest,
		1208	error: RelayAuthAndInternalErrors,
		1209	},
		1210	).annotate(OpenApi.Summary, "Update an environment's link preferences"),
		1211	HttpApiEndpoint.post("wakeHeldHooks", "/v1/environments/:environmentId/hooks/wake", {
		1212	params: Schema.Struct({ environmentId: EnvironmentId }),
		1213	success: RelayWakeHeldHooksResponse,
		1214	error: RelayAuthAndInternalErrors,
		1215	}).annotate(
		1216	OpenApi.Summary,
		1217	"Deliver webhook requests held while the environment was offline now",
		1218	),
1174	)	1219	)
1175	.annotate(OpenApi.Description, "Environment-authenticated activity publication.")	1220	.annotate(OpenApi.Description, "Environment-authenticated activity publication.")
1176	.middleware(RelayEnvironmentAuth);	1221	.middleware(RelayEnvironmentAuth);
...	@@ -1179,14 +1224,16 @@ const RelayServerGroup = HttpApiGroup.make("server")	...	@@ -1179,14 +1224,16 @@ const RelayServerGroup = HttpApiGroup.make("server")
1179	* Public, stateless webhook forwarding to an environment's managed tunnel.	1224	* Public, stateless webhook forwarding to an environment's managed tunnel.
1180	* Unauthenticated: the token in the path is the environment's credential, and	1225	* Unauthenticated: the token in the path is the environment's credential, and
1181	* the relay only routes and forwards. Raw, so the body reaches the	1226	* the relay only routes and forwards. Raw, so the body reaches the
1182	* environment byte for byte and signatures still verify there.	1227	* environment byte for byte and signatures still verify there. `endpointKey`
		1228	* names one managed endpoint (its tunnel's hash of user and environment), not
		1229	* the environment id, which any account can link.
1183	*/	1230	*/
1184	const RelayHookParams = Schema.Struct({	1231	const RelayHookParams = Schema.Struct({
1185	environmentId: Schema.String,	1232	endpointKey: Schema.String,
1186	hookId: Schema.String,	1233	hookId: Schema.String,
1187	token: Schema.String,	1234	token: Schema.String,
1188	});	1235	});
1189	const RELAY_HOOK_PATH = "/v1/hooks/:environmentId/:hookId/:token";	1236	const RELAY_HOOK_PATH = "/v1/hooks/:endpointKey/:hookId/:token";
1190	const relayHookEndpoint = { params: RelayHookParams } as const;	1237	const relayHookEndpoint = { params: RelayHookParams } as const;
1191	const RelayHooksGroup = HttpApiGroup.make("hooks")	1238	const RelayHooksGroup = HttpApiGroup.make("hooks")
1192	.add(	1239	.add(

packages/contracts/src/scheduledTask.ts+14

 @@ -73,11 +73,23 @@ export const ScheduledTaskWebhookSignature = Schema.Struct(
 ).annotate({ description: "Optional HMAC-SHA256 signature check over the raw request body." });
 export type ScheduledTaskWebhookSignature = typeof ScheduledTaskWebhookSignature.Type;
+/** Matches how long the relay holds a request for an offline environment. */
+export const MAX_WEBHOOK_DELIVERY_AGE_MINUTES = 24 * 60;
+const WebhookMaxDeliveryAgeMinutes = Schema.Int.check(
+  Schema.isBetween({ minimum: 1, maximum: MAX_WEBHOOK_DELIVERY_AGE_MINUTES }),
+).annotate({
+  description:
+    "Skip requests the relay held longer than this many minutes while the environment was offline. Null runs every request.",
+});
 const ScheduledTaskWebhookSchedule = Schema.Struct({
   type: Schema.Literal("webhook").annotate({
     description: "Run when the task's webhook URL receives a request.",
   }),
   signature: Schema.NullOr(ScheduledTaskWebhookSignature),
+  // Optional so rows saved before this setting existed still decode.
+  maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)),
 }).annotate({
   description:
     "Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.",
 @@ -99,6 +111,7 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({
   ).annotate({
     description: "Signature check; omit or null to accept requests by URL token only.",
   }),
+  maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)),
 }).annotate({
   description:
     "Run on each request to the task's webhook URL. The prompt may use {{body.path}}, {{headers.name}}, {{query.name}}, {{body}} and {{request}} placeholders.",
 @@ -241,6 +254,7 @@ export const ScheduledTaskWebhookDeliveryOutcome = Schema.Literals([
   "rejected_signature",
   "disabled",
   "rate_limited",
+  "expired",
 ]);
 export type ScheduledTaskWebhookDeliveryOutcome = typeof ScheduledTaskWebhookDeliveryOutcome.Type;

packages/shared/src/relayJwt.ts+3

 @@ -11,6 +11,9 @@ export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt";
 export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt";
 export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt";
 export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt";
+export const RELAY_HOOK_DELIVERY_TYP = "t3-relay-hook-delivery+jwt";
+/** Header carrying the signed proof that a webhook request came from the relay. */
+export const RELAY_HOOK_DELIVERY_HEADER = "x-t3-relay-delivery";
 export class RelayJwtError extends Schema.TaggedError<RelayJwtError>()("RelayJwtError", {
   operation: Schema.Literals(["sign", "verify"]),

pnpm-lock.yaml+16-10

...	@@ -9,6 +9,9 @@ catalogs:	...	@@ -9,6 +9,9 @@ catalogs:
9	'@effect/openapi-generator':	9	'@effect/openapi-generator':
10	specifier: 4.0.1	10	specifier: 4.0.1
11	version: 4.0.1	11	version: 4.0.1
		12	'@effect/sql-sqlite-do':
		13	specifier: 4.0.1
		14	version: 4.0.1
12	'@effect/tsgo':	15	'@effect/tsgo':
13	specifier: 0.41.0	16	specifier: 0.41.0
14	version: 0.41.0	17	version: 0.41.0
...	@@ -837,6 +840,9 @@ importers:	...	@@ -837,6 +840,9 @@ importers:
837	'@effect/sql-pg':	840	'@effect/sql-pg':
838	specifier: 4.0.1	841	specifier: 4.0.1
839	version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	842	version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
		843	'@effect/sql-sqlite-do':
		844	specifier: 'catalog:'
		845	version: 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
840	'@noble/curves':	846	'@noble/curves':
841	specifier: 'catalog:'	847	specifier: 'catalog:'
842	version: 1.9.1	848	version: 1.9.1
...	@@ -854,10 +860,10 @@ importers:	...	@@ -854,10 +860,10 @@ importers:
854	version: link:../../packages/shared	860	version: link:../../packages/shared
855	alchemy:	861	alchemy:
856	specifier: 2.0.0-beta.80	862	specifier: 2.0.0-beta.80
857	version: 2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(824b7c95c4257cf54626a40f00f69c9b)	863	version: 2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(d02b4e60a6a3c6b542dfa700209cf5ce)
858	drizzle-orm:	864	drizzle-orm:
859	specifier: 1.0.0-rc.5-ab785fc	865	specifier: 1.0.0-rc.5-ab785fc
860	version: 1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d)	866	version: 1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c)
861	effect:	867	effect:
862	specifier: 4.0.1	868	specifier: 4.0.1
863	version: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)	869	version: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)
...	@@ -2359,8 +2365,8 @@ packages:	...	@@ -2359,8 +2365,8 @@ packages:
2359	peerDependencies:	2365	peerDependencies:
2360	effect: 4.0.1	2366	effect: 4.0.1
2361		2367
2362	'@effect/sql-sqlite-do@4.0.0':	2368	'@effect/sql-sqlite-do@4.0.1':
2363	resolution: {integrity: sha512-XkRZmQOPblzwjmAJ0RWSOiLlTsKkf1EigZsxUNuIuk4uZ/pnufSqvDpXubCwv8iyRTbOLFMmM+EyOEH5V5XqAw==}	2369	resolution: {integrity: sha512-tS+qQok3xKsycnnhvIDX+6nvlM4Y2ouvru3voOzxtDQqX96NSOMX2xReLf4Feb6GN3jnzgEEtVGDDWk3YLrpsg==}
2364	peerDependencies:	2370	peerDependencies:
2365	effect: 4.0.1	2371	effect: 4.0.1
2366		2372
...	@@ -13036,7 +13042,7 @@ snapshots:	...	@@ -13036,7 +13042,7 @@ snapshots:
13036	effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)	13042	effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)
13037	optional: true	13043	optional: true
13038		13044
13039	'@effect/sql-sqlite-do@4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))':	13045	'@effect/sql-sqlite-do@4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))':
13040	dependencies:	13046	dependencies:
13041	effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)	13047	effect: 4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)
13042		13048
...	@@ -16619,7 +16625,7 @@ snapshots:	...	@@ -16619,7 +16625,7 @@ snapshots:
16619	json-schema-traverse: 1.0.0	16625	json-schema-traverse: 1.0.0
16620	require-from-string: 2.0.2	16626	require-from-string: 2.0.2
16621		16627
16622	alchemy@2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(824b7c95c4257cf54626a40f00f69c9b):	16628	alchemy@2.0.0-beta.80(patch_hash=bea1b6c0c0b23157fd5f439a1d7feff430f28ac1621ba135d8a13eeec60ccacd)(d02b4e60a6a3c6b542dfa700209cf5ce):
16623	dependencies:	16629	dependencies:
16624	'@alchemy.run/cloudflare-runtime': 2.0.0-beta.80(@distilled.cloud/cloudflare@1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)))(@effect/platform-node@4.0.1(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(redis@6.2.1)(utf-8-validate@6.0.6))(@types/node@24.12.4)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(rolldown@1.2.5)(typescript@7.0.2)	16630	'@alchemy.run/cloudflare-runtime': 2.0.0-beta.80(@distilled.cloud/cloudflare@1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f)))(@effect/platform-node@4.0.1(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(redis@6.2.1)(utf-8-validate@6.0.6))(@types/node@24.12.4)(@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0))(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(rolldown@1.2.5)(typescript@7.0.2)
16625	'@alchemy.run/floci': 2.0.0-beta.80(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	16631	'@alchemy.run/floci': 2.0.0-beta.80(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
...	@@ -16642,7 +16648,7 @@ snapshots:	...	@@ -16642,7 +16648,7 @@ snapshots:
16642	'@distilled.cloud/stripe': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	16648	'@distilled.cloud/stripe': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16643	'@distilled.cloud/zerossl': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	16649	'@distilled.cloud/zerossl': 1.0.0-rc.13(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16644	'@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	16650	'@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16645	'@effect/sql-sqlite-do': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	16651	'@effect/sql-sqlite-do': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16646	'@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6)	16652	'@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6)
16647	'@neon/functions': 0.11.0(hono@4.13.7)	16653	'@neon/functions': 0.11.0(hono@4.13.7)
16648	'@octokit/rest': 22.0.1	16654	'@octokit/rest': 22.0.1
...	@@ -16662,7 +16668,7 @@ snapshots:	...	@@ -16662,7 +16668,7 @@ snapshots:
16662	'@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	16668	'@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
16663	'@effect/vitest': 4.0.1(patch_hash=359f6fb2f7b3ec145bb72208edb9034f02489791aa2491a55cdbd69bd56ee0d2)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0)	16669	'@effect/vitest': 4.0.1(patch_hash=359f6fb2f7b3ec145bb72208edb9034f02489791aa2491a55cdbd69bd56ee0d2)(@types/node@24.12.4)(@vitest/ui@5.0.1)(bufferutil@4.1.0)(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))(esbuild@0.28.2)(jiti@2.7.0)(jsdom@30.1.0(@noble/hashes@1.8.0))(msw@2.12.11(@types/node@24.12.4)(typescript@7.0.2))(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(utf-8-validate@6.0.6)(yaml@2.9.0)
16664	drizzle-kit: 1.0.0-rc.5-ab785fc	16670	drizzle-kit: 1.0.0-rc.5-ab785fc
16665	drizzle-orm: 1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d)	16671	drizzle-orm: 1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c)
16666	mongodb: 6.21.0(@aws-sdk/credential-providers@3.1062.0)(socks@2.8.9)	16672	mongodb: 6.21.0(@aws-sdk/credential-providers@3.1062.0)(socks@2.8.9)
16667	pg: 8.23.0	16673	pg: 8.23.0
16668	vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0)'	16674	vite: '@voidzero-dev/vite-plus-core@1.0.0(@types/node@24.12.4)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.48.0)(typescript@7.0.2)(unrun@0.2.39)(yaml@2.9.0)'
...	@@ -17817,13 +17823,13 @@ snapshots:	...	@@ -17817,13 +17823,13 @@ snapshots:
17817	get-tsconfig: 4.14.3	17823	get-tsconfig: 4.14.3
17818	jiti: 2.7.0	17824	jiti: 2.7.0
17819		17825
17820	drizzle-orm@1.0.0-rc.5-ab785fc(0a256ab99caa2c53eee62f60929ef39d):	17826	drizzle-orm@1.0.0-rc.5-ab785fc(74e00612579436038985093f7f46ef5c):
17821	optionalDependencies:	17827	optionalDependencies:
17822	'@cloudflare/workers-types': 4.20260604.1	17828	'@cloudflare/workers-types': 4.20260604.1
17823	'@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	17829	'@effect/sql-d1': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
17824	'@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	17830	'@effect/sql-pg': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
17825	'@effect/sql-sqlite-bun': 4.0.0-rc.112(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	17831	'@effect/sql-sqlite-bun': 4.0.0-rc.112(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
17826	'@effect/sql-sqlite-do': 4.0.0(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))	17832	'@effect/sql-sqlite-do': 4.0.1(effect@4.0.1(patch_hash=a33cba07c41f32374c2aaa86ea4a84d3ab9a872a78c94b65e929b19f8361856f))
17827	'@electric-sql/pglite': 0.3.15	17833	'@electric-sql/pglite': 0.3.15
17828	'@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6)	17834	'@libsql/client': 0.17.3(bufferutil@4.1.0)(utf-8-validate@6.0.6)
17829	bun-types: 1.3.14	17835	bun-types: 1.3.14

pnpm-workspace.yaml+2

...	@@ -35,6 +35,7 @@ catalog:	...	@@ -35,6 +35,7 @@ catalog:
35	"@effect/platform-node": 4.0.1	35	"@effect/platform-node": 4.0.1
36	"@effect/platform-node-shared": 4.0.1	36	"@effect/platform-node-shared": 4.0.1
37	"@effect/sql-pg": 4.0.1	37	"@effect/sql-pg": 4.0.1
		38	"@effect/sql-sqlite-do": 4.0.1
38	"@effect/tsgo": 0.41.0	39	"@effect/tsgo": 0.41.0
39	"@effect/vitest": 4.0.1	40	"@effect/vitest": 4.0.1
40	"@legendapp/list": 3.3.5	41	"@legendapp/list": 3.3.5
...	@@ -75,6 +76,7 @@ minimumReleaseAgeExclude:	...	@@ -75,6 +76,7 @@ minimumReleaseAgeExclude:
75	- "@effect/platform-node-shared@4.0.1"	76	- "@effect/platform-node-shared@4.0.1"
76	- "@effect/platform-node@4.0.1"	77	- "@effect/platform-node@4.0.1"
77	- "@effect/sql-pg@4.0.1"	78	- "@effect/sql-pg@4.0.1"
		79	- "@effect/sql-sqlite-do@4.0.1"
78	- "@effect/vitest@4.0.1"	80	- "@effect/vitest@4.0.1"
79	- alchemy@2.0.0-beta.80	81	- alchemy@2.0.0-beta.80
80	- effect@4.0.1	82	- effect@4.0.1

author	Julius Marminge <julius0216@outlook.com> 2026-10-05 15:14:53-07:00
committer	GitHub <noreply@github.com> 2026-10-05 15:14:53-07:00
log	f33b060cafc96d7c3229744315c230a3b8d1f631
tree	0d84f75282264d93612e1103ebc56c2cfd5ad8cf
parent	ea4b44343d3c8f71cf49a72a266d00eac7389435
signature	Signed by PGP key B5690EEEBB952194

...	@@ -47,6 +47,8 @@ export class CloudManagedEndpointRuntime extends Context.Service<	...	@@ -47,6 +47,8 @@ export class CloudManagedEndpointRuntime extends Context.Service<
47	) => Effect.Effect<CloudManagedEndpointRuntimeStatus>;	47	) => Effect.Effect<CloudManagedEndpointRuntimeStatus>;
48	readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;	48	readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
49	readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;	49	readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
		50	/** Emits when the connector registers a tunnel connection, i.e. the relay can reach us again. */
		51	readonly tunnelConnected: Stream.Stream<void>;
50	readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;	52	readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
51	}	53	}
52	>()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {}	54	>()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {}
...	@@ -134,6 +136,7 @@ export const make = Effect.gen(function* () {	...	@@ -134,6 +136,7 @@ export const make = Effect.gen(function* () {
134	const activeRef = yield* Ref.make<ActiveConnector \| null>(null);	136	const activeRef = yield* Ref.make<ActiveConnector \| null>(null);
135	const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig \| null>(null);	137	const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig \| null>(null);
136	const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1);	138	const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1);
		139	const tunnelConnections = yield* Queue.sliding<void>(1);
137	const reconcileSemaphore = yield* Semaphore.make(1);	140	const reconcileSemaphore = yield* Semaphore.make(1);
138	const restartDelayRef = yield* Ref.make(0);	141	const restartDelayRef = yield* Ref.make(0);
139	const linkStateSemaphore = yield* Semaphore.make(1);	142	const linkStateSemaphore = yield* Semaphore.make(1);
...	@@ -236,7 +239,10 @@ export const make = Effect.gen(function* () {	...	@@ -236,7 +239,10 @@ export const make = Effect.gen(function* () {
236	switch (classifyRelayClientOutput(line)) {	239	switch (classifyRelayClientOutput(line)) {
237	case "connected":	240	case "connected":
238	rejectedRegistrations = 0;	241	rejectedRegistrations = 0;
239	return Effect.logInfo("Relay client tunnel connection registered", attributes);	242	return Effect.logInfo("Relay client tunnel connection registered", attributes).pipe(
		243	Effect.andThen(Queue.offer(tunnelConnections, undefined)),
		244	Effect.asVoid,
		245	);
240	case "warning":	246	case "warning":
241	if (isRejectedRelayClientTunnelOutput(line)) {	247	if (isRejectedRelayClientTunnelOutput(line)) {
242	rejectedRegistrations += 1;	248	rejectedRegistrations += 1;
...	@@ -412,6 +418,7 @@ export const make = Effect.gen(function* () {	...	@@ -412,6 +418,7 @@ export const make = Effect.gen(function* () {
412	applyConfig,	418	applyConfig,
413	recoveryRequests: Stream.fromQueue(recoveryRequests),	419	recoveryRequests: Stream.fromQueue(recoveryRequests),
414	requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid),	420	requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid),
		421	tunnelConnected: Stream.fromQueue(tunnelConnections),
415	withLinkStateLock: linkStateSemaphore.withPermits(1),	422	withLinkStateLock: linkStateSemaphore.withPermits(1),
416	});	423	});
417		424

...	@@ -13,7 +13,7 @@ layer("055_OrchestrationV2", (it) => {	...	@@ -13,7 +13,7 @@ layer("055_OrchestrationV2", (it) => {
13	Effect.sync(() => {	13	Effect.sync(() => {
14	assert.deepStrictEqual(	14	assert.deepStrictEqual(
15	migrationEntries.map(([id]) => id),	15	migrationEntries.map(([id]) => id),
16	Array.from({ length: 57 }, (_, index) => index + 1),	16	Array.from({ length: 58 }, (_, index) => index + 1),
17	);	17	);
18	}),	18	}),
19	);	19	);
...	@@ -29,6 +29,7 @@ layer("055_OrchestrationV2", (it) => {	...	@@ -29,6 +29,7 @@ layer("055_OrchestrationV2", (it) => {
29	[55, "OrchestrationV2"],	29	[55, "OrchestrationV2"],
30	[56, "RemoveRedundantProjectionIndexes"],	30	[56, "RemoveRedundantProjectionIndexes"],
31	[57, "ScheduledTaskWebhooks"],	31	[57, "ScheduledTaskWebhooks"],
		32	[58, "WebhookRelayDeliveries"],
32	]);	33	]);
33	assert.deepStrictEqual(yield* runMigrations(), []);	34	assert.deepStrictEqual(yield* runMigrations(), []);
34		35
...	@@ -52,6 +53,7 @@ layer("055_OrchestrationV2", (it) => {	...	@@ -52,6 +53,7 @@ layer("055_OrchestrationV2", (it) => {
52	{ migration_id: 55, name: "OrchestrationV2" },	53	{ migration_id: 55, name: "OrchestrationV2" },
53	{ migration_id: 56, name: "RemoveRedundantProjectionIndexes" },	54	{ migration_id: 56, name: "RemoveRedundantProjectionIndexes" },
54	{ migration_id: 57, name: "ScheduledTaskWebhooks" },	55	{ migration_id: 57, name: "ScheduledTaskWebhooks" },
		56	{ migration_id: 58, name: "WebhookRelayDeliveries" },
55	]);	57	]);
56		58
57	const tables = yield* sql<{ readonly name: string }>`	59	const tables = yield* sql<{ readonly name: string }>`

feat(relay,server,web,mobile): opt-in to hold webhooks while offline (#15487)