From a21b42cec478b093cdc50cc2105b5368ea8b3546 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 25 Sep 2026 21:58:26 -0700 Subject: [PATCH] fix(server): prune expired replay-protection files from the secrets directory (#13695) Co-authored-by: Claude Opus 5.5 (1M context) --- apps/server/src/auth/dpop.ts | 5 +- apps/server/src/auth/replayMarkers.test.ts | 92 +++++++++++++++++ apps/server/src/auth/replayMarkers.ts | 77 ++++++++++++++ apps/server/src/cloud/http.ts | 7 ++ apps/server/src/server.test.ts | 111 +++++++++++++++++++++ apps/server/src/server.ts | 2 + 6 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/auth/replayMarkers.test.ts create mode 100644 apps/server/src/auth/replayMarkers.ts diff --git a/apps/server/src/auth/dpop.ts b/apps/server/src/auth/dpop.ts index a841f4fd444c7630180f3a9e40d45587c4462e6f..5e026f584273f011650b334cd6ba5652d7153aa8 100644 --- a/apps/server/src/auth/dpop.ts +++ b/apps/server/src/auth/dpop.ts @@ -18,6 +18,9 @@ import { } from "./EnvironmentAuth.ts"; import * as ServerSecretStore from "./ServerSecretStore.ts"; +/** Secret store name prefix of DPoP replay markers. The server prunes expired ones. */ +export const DPOP_REPLAY_MARKER_PREFIX = "dpop-proof-"; + export const mapDpopFailureReason = (code: DpopVerificationFailureCodeType): DpopFailureReason => { switch (code) { case "time_window": @@ -96,7 +99,7 @@ export const verifyRequestDpopProof = (input: { ); yield* secretStore .create( - `dpop-proof-${replayKey}`, + `${DPOP_REPLAY_MARKER_PREFIX}${replayKey}`, new TextEncoder().encode( [ `thumbprint=${result.thumbprint}`, diff --git a/apps/server/src/auth/replayMarkers.test.ts b/apps/server/src/auth/replayMarkers.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..76c31b0162bd666f23a8bb86233e01d9a633b4aa --- /dev/null +++ b/apps/server/src/auth/replayMarkers.test.ts @@ -0,0 +1,92 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerConfig from "../config.ts"; +import { pruneExpiredReplayMarkers, REPLAY_MARKER_MAX_AGE } from "./replayMarkers.ts"; +import * as ServerSecretStore from "./ServerSecretStore.ts"; + +// Every secret name the server stores today. The last three stand for names +// built from an id at runtime. +const REAL_SECRET_NAMES = [ + "server-signing-key", + "asset-access-signing-key", + "cloud-cli-oauth-token", + "cloud-cli-desired-link", + "cloud-link-ed25519-key-pair", + "cloud-link-ed25519-private-key", + "cloud-link-ed25519-public-key", + "cloud-mint-ed25519-public-key", + "cloud-endpoint-runtime-config", + "cloud-endpoint-confirmed-origin", + "cloud-linked-user-id", + "cloud-relay-url", + "cloud-relay-issuer", + "cloud-relay-environment-credential", + "cloud-publish-agent-activity", + "provider-env-Y29kZXg-T1BFTkFJX0FQSV9LRVk", + "provider-auth-0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0", + "usage-limit-source-aHVi", +]; + +it.layer(NodeServices.layer)("replayMarkers", (it) => { + it.effect("prunes only replay markers older than the max age", () => + Effect.gen(function* () { + const secretStore = yield* ServerSecretStore.ServerSecretStore; + const { secretsDir } = yield* ServerConfig.ServerConfig; + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const now = DateTime.makeUnsafe("2026-01-01T00:00:00Z"); + const setAge = (fileName: string, age: Duration.Duration) => { + const mtime = DateTime.toDateUtc(DateTime.subtractDuration(now, age)); + return fileSystem.utimes(path.join(secretsDir, fileName), mtime, mtime); + }; + const writeAged = (name: string, age: Duration.Duration) => + secretStore + .create(name, Uint8Array.from([1])) + .pipe(Effect.andThen(setAge(`${name}.bin`, age))); + + const justExpired = Duration.sum(REPLAY_MARKER_MAX_AGE, Duration.seconds(1)); + const expiredMarkers = [ + "dpop-proof-old", + "cloud-mint-jti-old", + "cloud-mint-nonce-old", + "cloud-health-jti-old", + "cloud-health-nonce-old", + ]; + for (const name of expiredMarkers) yield* writeAged(name, justExpired); + yield* writeAged("dpop-proof-at-max-age", REPLAY_MARKER_MAX_AGE); + for (const name of REAL_SECRET_NAMES) yield* writeAged(name, Duration.days(30)); + const pendingSetFile = "dpop-proof-pending.bin.0000.tmp"; + yield* fileSystem.writeFile(path.join(secretsDir, pendingSetFile), Uint8Array.from([1])); + yield* setAge(pendingSetFile, Duration.days(30)); + yield* TestClock.setTime(DateTime.toEpochMillis(now)); + + yield* pruneExpiredReplayMarkers(); + + const remaining = yield* fileSystem.readDirectory(secretsDir); + assert.deepStrictEqual( + remaining.toSorted(), + [ + ...REAL_SECRET_NAMES.map((name) => `${name}.bin`), + "dpop-proof-at-max-age.bin", + pendingSetFile, + ].toSorted(), + ); + }).pipe( + Effect.provide( + ServerSecretStore.layer.pipe( + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-replay-markers-test-" }), + ), + ), + ), + ), + ); +}); diff --git a/apps/server/src/auth/replayMarkers.ts b/apps/server/src/auth/replayMarkers.ts new file mode 100644 index 0000000000000000000000000000000000000000..67b56b722334192bc2b4d1d585bb0d6301e5c140 --- /dev/null +++ b/apps/server/src/auth/replayMarkers.ts @@ -0,0 +1,77 @@ +import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schedule from "effect/Schedule"; + +import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/http.ts"; +import * as ServerConfig from "../config.ts"; +import { forkParked } from "../serverActivation.ts"; +import { DPOP_REPLAY_MARKER_PREFIX } from "./dpop.ts"; + +const REPLAY_MARKER_PREFIXES = [DPOP_REPLAY_MARKER_PREFIX, ...CLOUD_REPLAY_MARKER_PREFIXES]; + +/** + * How long a replay marker stays on disk. A marker only matters while its proof + * can pass the time check (about 5 minutes for DPoP, 7 for cloud proofs). After + * that, the time check rejects a replay by itself. The sweep and the time check + * both use the wall clock, so a pruned marker can let a replay through only if + * the clock moves back by almost a day, or if the filesystem stamps mtimes almost + * a day behind. Markers are files, so a restart does not reset them. + */ +export const REPLAY_MARKER_MAX_AGE = Duration.days(1); + +/** + * Deletes replay markers whose mtime is older than `REPLAY_MARKER_MAX_AGE`. + * `ServerSecretStore` saves each secret as `.bin`, so only + * `*.bin` names match. Other secrets and the `*.bin..tmp` + * files that `set` writes are never touched. + */ +export const pruneExpiredReplayMarkers = Effect.fn("replayMarkers.pruneExpired")(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { secretsDir } = yield* ServerConfig.ServerConfig; + const cutoff = (yield* Clock.currentTimeMillis) - Duration.toMillis(REPLAY_MARKER_MAX_AGE); + const markers = (yield* fileSystem.readDirectory(secretsDir)).filter( + (name) => + name.endsWith(".bin") && REPLAY_MARKER_PREFIXES.some((prefix) => name.startsWith(prefix)), + ); + // `partition` visits every marker, so one locked file does not stop the sweep. + const [failures, removed] = yield* Effect.partition(markers, (name) => { + const markerPath = path.join(secretsDir, name); + return fileSystem.stat(markerPath).pipe( + Effect.flatMap((info) => + Option.exists(info.mtime, (mtime) => mtime.getTime() < cutoff) + ? fileSystem.remove(markerPath).pipe(Effect.as(true)) + : Effect.succeed(false), + ), + Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)), + ); + }); + yield* Effect.annotateCurrentSpan({ + "replay_markers.matched": markers.length, + "replay_markers.removed": removed.filter(Boolean).length, + "replay_markers.failed": failures.length, + }); + if (failures.length > 0) { + yield* Effect.logWarning("Failed to prune some replay markers", { + failed: failures.length, + cause: failures[0], + }); + } +}); + +/** Prunes expired replay markers after server activation, then every hour. */ +export const layer = Layer.effectDiscard( + forkParked( + pruneExpiredReplayMarkers().pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to prune expired replay markers", { cause }), + ), + Effect.repeat(Schedule.spaced(Duration.hours(1))), + ), + ), +); diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index b4c366a500c1865658c7a328da7d0836c3715f91..8e2d0e3614582e77da9d74e6e957aef05941afcb 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -104,6 +104,13 @@ const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-"; const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-"; const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-"; const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-"; +/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */ +export const CLOUD_REPLAY_MARKER_PREFIXES = [ + CLOUD_MINT_NONCE_PREFIX, + CLOUD_MINT_JTI_PREFIX, + CLOUD_HEALTH_NONCE_PREFIX, + CLOUD_HEALTH_JTI_PREFIX, +] as const; const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60; const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60; // The desktop app stops its backends within seconds of writing the marker. diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 6836d28e25c1bdf24bde4e1459d61eee5c91549d..5dc793b5cdf1ed54c79599ba9982404b0d9a3c52 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -176,6 +176,7 @@ import * as VcsProcess from "./vcs/VcsProcess.ts"; import * as GitWorkflowService from "./git/GitWorkflowService.ts"; import * as ReviewService from "./review/ReviewService.ts"; import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts"; +import { REPLAY_MARKER_MAX_AGE } from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import * as PairingGrantStore from "./auth/PairingGrantStore.ts"; @@ -2643,6 +2644,40 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("rejects a DPoP replay by time alone once its marker can be pruned", () => + Effect.gen(function* () { + yield* buildAppUnderTest(); + + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const credentialResponse = yield* HttpClient.post("/api/auth/pairing-token", { + headers: { cookie: ownerCookie }, + body: yield* HttpBody.json({}), + }); + const credential = (yield* credentialResponse.json) as { readonly credential: string }; + const tokenUrl = yield* getHttpServerUrl("/oauth/token"); + const acceptedAt = yield* DateTime.now; + // The longest-lived proof: `iat` at the 5 s future skew the verifier allows. + const dpop = makeDpopProof({ + method: "POST", + url: tokenUrl, + iat: Math.floor(acceptedAt.epochMilliseconds / 1_000) + 5, + }); + const exchange = exchangeAccessToken(credential.credential, { + headers: { dpop: dpop.proof }, + scope: "orchestration:read orchestration:operate terminal:operate review:write", + }); + + assert.equal((yield* exchange).response.status, 200); + // While the proof is fresh, only the replay marker rejects it. + assert.equal((yield* exchange).body.dpopFailureReason, "replay"); + // Once the marker can be pruned, the time check rejects the proof by itself. + yield* TestClock.setTime( + acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE), + ); + assert.equal((yield* exchange).body.dpopFailureReason, "time_window"); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("ignores forwarded host headers when validating token exchange DPoP URLs", () => Effect.gen(function* () { yield* buildAppUnderTest(); @@ -3710,6 +3745,82 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(NodeHttpServer.layerTest)), ); + it.effect("rejects cloud replays by time alone once their markers can be pruned", () => + Effect.gen(function* () { + yield* buildAppUnderTest(); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigResponse = yield* fetchEffect( + yield* getHttpServerUrl("/api/connect/relay-config"), + { + method: "POST", + headers: { cookie: ownerCookie, "content-type": "application/json" }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: null, + }), + }, + ); + assert.equal(relayConfigResponse.status, 200); + + const acceptedAt = yield* DateTime.now; + // The longest-lived proofs: `iat` at the 60 s future skew the handlers + // allow, and the 5 minute maximum lifetime. + const issuedAt = DateTime.add(acceptedAt, { minutes: 1 }); + const proofTimes = { + issuedAt: DateTime.formatIso(issuedAt), + expiresAt: DateTime.formatIso(DateTime.add(issuedAt, { minutes: 5 })), + }; + const requests = [ + [ + "/api/t3-connect/health", + makeCloudEnvironmentHealthRequest({ + privateKey: cloudKeyPair.privateKey, + environmentId: testEnvironmentDescriptor.environmentId, + nonce: "cloud-health-nonce-pruned", + ...proofTimes, + }), + ], + [ + "/api/t3-connect/mint-credential", + makeCloudMintCredentialRequest({ + privateKey: cloudKeyPair.privateKey, + environmentId: testEnvironmentDescriptor.environmentId, + clientProofKeyThumbprint: "client-proof-key-thumbprint", + nonce: "cloud-mint-nonce-pruned", + ...proofTimes, + }), + ], + ] as const; + const postAll = Effect.forEach(requests, ([pathname, request]) => + Effect.gen(function* () { + const response = yield* fetchEffect(yield* getHttpServerUrl(pathname), { + method: "POST", + headers: { "content-type": "application/json" }, + body: jsonRequestBody(request), + }); + return response.status; + }), + ); + + assert.deepStrictEqual(yield* postAll, [200, 200]); + // While the proofs are fresh, only the replay markers reject them (409). + assert.deepStrictEqual(yield* postAll, [409, 409]); + // Once the markers can be pruned, the time checks reject the proofs by themselves (401). + yield* TestClock.setTime( + acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE), + ); + assert.deepStrictEqual(yield* postAll, [401, 401]); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect( "validates cloud proofs against the configured relay issuer, not the transport URL", () => diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index c22b48519da2265136c7695bc8b7a06d88055901..b7020458160de64a1ac14c1451656b50436c11a4 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -124,6 +124,7 @@ import * as EventLoopMonitor from "./observability/EventLoopMonitor.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts"; import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts"; +import * as ReplayMarkers from "./auth/replayMarkers.ts"; import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import { @@ -502,6 +503,7 @@ const AntigravityInstallationRefreshLive = Layer.effectDiscard( const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( Layer.provideMerge(AntigravityInstallationRefreshLive), + Layer.provideMerge(ReplayMarkers.layer), Layer.provideMerge(ProviderAuthServiceLive), // Core Services Layer.provideMerge(ServerSettingsLayerLive), -- 2.54.0