authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-07-30 07:31:53-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-07-30 07:31:53-07:00
log1877237c7ae99ecf2163b4c5bb7d3e14bebcca4d
tree5bb28c9ce2f991cc90ab17aded77e4a7e3c6f515
parente0513d29839f934df1d5a13551e6ccd76d11006b
signature Signed by PGP key B5690EEEBB952194

fix(connect): reboots no longer strand the relay link, 403s now say why (#4988)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

6 files changed, 63 insertions(+), 21 deletions(-)

apps/mobile/src/features/cloud/linkEnvironment.ts+8-1
...@@ -134,7 +134,14 @@ function relayProtectedErrorMessage(error: RelayProtectedErrorType): string {...@@ -134,7 +134,14 @@ function relayProtectedErrorMessage(error: RelayProtectedErrorType): string {
134 case "RelayEnvironmentLinkProofInvalidError":134 case "RelayEnvironmentLinkProofInvalidError":
135 return `Relay rejected the environment link proof (${error.reason}).`;135 return `Relay rejected the environment link proof (${error.reason}).`;
136 case "RelayEnvironmentConnectNotAuthorizedError":136 case "RelayEnvironmentConnectNotAuthorizedError":
137 return "Relay rejected the environment connection request.";137 // "Not authorized" covers non-auth causes too; surface the reason so a
138 // missing link doesn't read as a credential problem.
139 if (error.reason === "environment_link_not_found") {
140 return "Relay has no active link for this environment. The environment server may not have re-established its link yet.";
141 }
142 return error.reason
143 ? `Relay rejected the environment connection request (${error.reason}).`
144 : "Relay rejected the environment connection request.";
138 case "RelayEnvironmentEndpointUnavailableError":145 case "RelayEnvironmentEndpointUnavailableError":
139 return `Relay could not reach the environment endpoint (${error.reason}).`;146 return `Relay could not reach the environment endpoint (${error.reason}).`;
140 case "RelayEnvironmentEndpointTimedOutError":147 case "RelayEnvironmentEndpointTimedOutError":
apps/server/src/server.ts+21-1
...@@ -1,6 +1,8 @@...@@ -1,6 +1,8 @@
1import { EnvironmentHttpApi } from "@t3tools/contracts";1import { EnvironmentHttpApi } from "@t3tools/contracts";
2import * as Duration from "effect/Duration";
2import * as Effect from "effect/Effect";3import * as Effect from "effect/Effect";
3import * as Layer from "effect/Layer";4import * as Layer from "effect/Layer";
5import * as Schedule from "effect/Schedule";
4import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http";6import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http";
5import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";7import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
68
...@@ -535,7 +537,25 @@ export const makeServerLayer = Layer.unwrap(...@@ -535,7 +537,25 @@ export const makeServerLayer = Layer.unwrap(
535 yield* Effect.forkScoped(537 yield* Effect.forkScoped(
536 Effect.sleep("250 millis").pipe(538 Effect.sleep("250 millis").pipe(
537 Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)),539 Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)),
538 Effect.retry({ times: 4 }),540 // On reboot this races NIC/DNS bring-up, so back off exponentially
541 // (capped at 30s) instead of burning all retries in a second.
542 // Bounded overall so a permanently broken setup still surfaces the
543 // warning below. Bad-request/unauthorized/conflict are
544 // deterministic failures (malformed origin, not linked yet, linked
545 // to a different cloud account) that no amount of retrying
546 // converges.
547 Effect.retry({
548 while: (error) =>
549 error._tag !== "EnvironmentHttpBadRequestError" &&
550 error._tag !== "EnvironmentHttpUnauthorizedError" &&
551 error._tag !== "EnvironmentHttpConflictError",
552 schedule: Schedule.exponential("1 second").pipe(
553 Schedule.modifyDelay(({ duration }) =>
554 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
555 ),
556 Schedule.upTo({ duration: "10 minutes" }),
557 ),
558 }),
539 Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")),559 Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")),
540 Effect.catch((cause) =>560 Effect.catch((cause) =>
541 Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {561 Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", {
apps/web/src/cloud/linkEnvironment.ts+8-1
...@@ -145,7 +145,14 @@ function relayProtectedErrorMessage(error: RelayProtectedErrorType): string {...@@ -145,7 +145,14 @@ function relayProtectedErrorMessage(error: RelayProtectedErrorType): string {
145 case "RelayEnvironmentLinkProofInvalidError":145 case "RelayEnvironmentLinkProofInvalidError":
146 return `Relay rejected the environment link proof (${error.reason}).`;146 return `Relay rejected the environment link proof (${error.reason}).`;
147 case "RelayEnvironmentConnectNotAuthorizedError":147 case "RelayEnvironmentConnectNotAuthorizedError":
148 return "Relay rejected the environment connection request.";148 // "Not authorized" covers non-auth causes too; surface the reason so a
149 // missing link doesn't read as a credential problem.
150 if (error.reason === "environment_link_not_found") {
151 return "Relay has no active link for this environment. The environment server may not have re-established its link yet.";
152 }
153 return error.reason
154 ? `Relay rejected the environment connection request (${error.reason}).`
155 : "Relay rejected the environment connection request.";
149 case "RelayEnvironmentEndpointUnavailableError":156 case "RelayEnvironmentEndpointUnavailableError":
150 return `Relay could not reach the environment endpoint (${error.reason}).`;157 return `Relay could not reach the environment endpoint (${error.reason}).`;
151 case "RelayEnvironmentEndpointTimedOutError":158 case "RelayEnvironmentEndpointTimedOutError":
infra/relay/src/environments/EnvironmentConnector.ts+3-15
...@@ -13,6 +13,7 @@ import {...@@ -13,6 +13,7 @@ import {
13 RelayEnvironmentMintResponse,13 RelayEnvironmentMintResponse,
14 RelayEnvironmentMintResponseProofPayload,14 RelayEnvironmentMintResponseProofPayload,
15 RelayCloudMintCredentialProofPayload,15 RelayCloudMintCredentialProofPayload,
16 RelayEnvironmentConnectNotAuthorizedReason,
16 type RelayEnvironmentConnectResponse,17 type RelayEnvironmentConnectResponse,
17 type RelayEnvironmentStatusResponse,18 type RelayEnvironmentStatusResponse,
18} from "@t3tools/contracts/relay";19} from "@t3tools/contracts/relay";
...@@ -44,21 +45,8 @@ import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";...@@ -44,21 +45,8 @@ import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
44import * as RelayConfiguration from "../Config.ts";45import * as RelayConfiguration from "../Config.ts";
45import { isManagedEndpointHostname } from "../deploymentConfig.ts";46import { isManagedEndpointHostname } from "../deploymentConfig.ts";
4647
47export const EnvironmentConnectNotAuthorizedReason = Schema.Literals([
48 "client_proof_key_thumbprint_missing",
49 "environment_link_not_found",
50 "endpoint_provider_not_managed",
51 "managed_endpoint_allocation_not_found",
52 "managed_endpoint_base_domain_not_configured",
53 "managed_endpoint_allocation_not_ready",
54 "managed_endpoint_hostname_invalid",
55 "managed_endpoint_mismatch",
56]);
57export type EnvironmentConnectNotAuthorizedReason =
58 typeof EnvironmentConnectNotAuthorizedReason.Type;
59
60function environmentConnectNotAuthorizedReasonMessage(48function environmentConnectNotAuthorizedReasonMessage(
61 reason: EnvironmentConnectNotAuthorizedReason,49 reason: RelayEnvironmentConnectNotAuthorizedReason,
62): string {50): string {
63 switch (reason) {51 switch (reason) {
64 case "client_proof_key_thumbprint_missing":52 case "client_proof_key_thumbprint_missing":
...@@ -85,7 +73,7 @@ export class EnvironmentConnectNotAuthorized extends Schema.TaggedErrorClass<Env...@@ -85,7 +73,7 @@ export class EnvironmentConnectNotAuthorized extends Schema.TaggedErrorClass<Env
85 {73 {
86 environmentId: Schema.String,74 environmentId: Schema.String,
87 operation: Schema.Literals(["connect", "status"]),75 operation: Schema.Literals(["connect", "status"]),
88 reason: EnvironmentConnectNotAuthorizedReason,76 reason: RelayEnvironmentConnectNotAuthorizedReason,
89 },77 },
90) {78) {
91 override get message(): string {79 override get message(): string {
infra/relay/src/http/Api.ts+4-2
...@@ -778,9 +778,10 @@ export const dpopClientApi = HttpApiBuilder.group(...@@ -778,9 +778,10 @@ export const dpopClientApi = HttpApiBuilder.group(
778 },778 },
779 mapRelayCommonApiErrors("invalid_dpop"),779 mapRelayCommonApiErrors("invalid_dpop"),
780 mapErrorTags({780 mapErrorTags({
781 EnvironmentConnectNotAuthorized: (_error, traceId) =>781 EnvironmentConnectNotAuthorized: (error, traceId) =>
782 new RelayEnvironmentConnectNotAuthorizedError({782 new RelayEnvironmentConnectNotAuthorizedError({
783 code: "environment_connect_not_authorized",783 code: "environment_connect_not_authorized",
784 reason: error.reason,
784 traceId,785 traceId,
785 }),786 }),
786 EnvironmentMintRequestFailed: (_error, traceId) =>787 EnvironmentMintRequestFailed: (_error, traceId) =>
...@@ -820,9 +821,10 @@ export const dpopClientApi = HttpApiBuilder.group(...@@ -820,9 +821,10 @@ export const dpopClientApi = HttpApiBuilder.group(
820 },821 },
821 mapRelayCommonApiErrors("invalid_dpop"),822 mapRelayCommonApiErrors("invalid_dpop"),
822 mapErrorTags({823 mapErrorTags({
823 EnvironmentConnectNotAuthorized: (_error, traceId) =>824 EnvironmentConnectNotAuthorized: (error, traceId) =>
824 new RelayEnvironmentConnectNotAuthorizedError({825 new RelayEnvironmentConnectNotAuthorizedError({
825 code: "environment_connect_not_authorized",826 code: "environment_connect_not_authorized",
827 reason: error.reason,
826 traceId,828 traceId,
827 }),829 }),
828 EnvironmentMintRequestFailed: (_error, traceId) =>830 EnvironmentMintRequestFailed: (_error, traceId) =>
packages/contracts/src/relay.ts+19-1
...@@ -371,16 +371,34 @@ export class RelayEnvironmentLinkProofInvalidError extends Schema.TaggedErrorCla...@@ -371,16 +371,34 @@ export class RelayEnvironmentLinkProofInvalidError extends Schema.TaggedErrorCla
371 }371 }
372}372}
373373
374export const RelayEnvironmentConnectNotAuthorizedReason = Schema.Literals([
375 "client_proof_key_thumbprint_missing",
376 "environment_link_not_found",
377 "endpoint_provider_not_managed",
378 "managed_endpoint_allocation_not_found",
379 "managed_endpoint_base_domain_not_configured",
380 "managed_endpoint_allocation_not_ready",
381 "managed_endpoint_hostname_invalid",
382 "managed_endpoint_mismatch",
383]);
384export type RelayEnvironmentConnectNotAuthorizedReason =
385 typeof RelayEnvironmentConnectNotAuthorizedReason.Type;
386
374export class RelayEnvironmentConnectNotAuthorizedError extends Schema.TaggedErrorClass<RelayEnvironmentConnectNotAuthorizedError>()(387export class RelayEnvironmentConnectNotAuthorizedError extends Schema.TaggedErrorClass<RelayEnvironmentConnectNotAuthorizedError>()(
375 "RelayEnvironmentConnectNotAuthorizedError",388 "RelayEnvironmentConnectNotAuthorizedError",
376 {389 {
377 code: Schema.Literal("environment_connect_not_authorized"),390 code: Schema.Literal("environment_connect_not_authorized"),
391 // Optional so responses from relays deployed before the reason was
392 // threaded through still decode.
393 reason: Schema.optional(RelayEnvironmentConnectNotAuthorizedReason),
378 traceId: TrimmedNonEmptyString,394 traceId: TrimmedNonEmptyString,
379 },395 },
380 { httpApiStatus: 403 },396 { httpApiStatus: 403 },
381) {397) {
382 override get message(): string {398 override get message(): string {
383 return "Relay environment connection is not authorized";399 return this.reason
400 ? `Relay environment connection is not authorized: ${this.reason}`
401 : "Relay environment connection is not authorized";
384 }402 }
385}403}
386404