| author |  Theo Browne <me@t3.gg> 2026-08-06 04:51:08-07:00 |
| committer |  Theo Browne <me@theo.li> 2026-08-06 08:09:51-04:00 |
| log | df2f1273e9e1f1c6f342eecbf3539a5e992163e1 |
| tree | a14961fd87e03549cd2264c55a3f6c21f6ceaab3 |
| parent | f9e8236899f25653ce95d688fac798cd070afbc1 |
fix(server): clean up interrupted update trials
3 files changed, 28 insertions(+), 41 deletions(-)
apps
server
apps/server/src/cloud/http.ts+1-1
| ... | @@ -645,7 +645,7 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD | ... | @@ -645,7 +645,7 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD |
| 645 | // the pending window also tears this server down, with nothing starting | 645 | // the pending window also tears this server down, with nothing starting |
| 646 | // afterwards. The launcher marks that case with a stop-marker file just | 646 | // afterwards. The launcher marks that case with a stop-marker file just |
| 647 | // before it signals the child, so pending + no marker is the update handoff. | 647 | // before it signals the child, so pending + no marker is the update handoff. |
| 648 | const pendingUpdateHandoffExists = Effect.gen(function* () { | 648 | export const pendingUpdateHandoffExists = Effect.gen(function* () { |
| 649 | const config = yield* ServerConfig.ServerConfig; | 649 | const config = yield* ServerConfig.ServerConfig; |
| 650 | const fs = yield* FileSystem.FileSystem; | 650 | const fs = yield* FileSystem.FileSystem; |
| 651 | const path = yield* Path.Path; | 651 | const path = yield* Path.Path; |
apps/server/src/server.ts+27-18
| ... | @@ -83,6 +83,7 @@ import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; | ... | @@ -83,6 +83,7 @@ import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; |
| 83 | import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; | 83 | import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; |
| 84 | import { | 84 | import { |
| 85 | connectHttpApiLayer, | 85 | connectHttpApiLayer, |
| 86 | pendingUpdateHandoffExists, | ||
| 86 | reconcileDesiredCloudLink, | 87 | reconcileDesiredCloudLink, |
| 87 | releaseManagedTunnelOnShutdown, | 88 | releaseManagedTunnelOnShutdown, |
| 88 | } from "./cloud/http.ts"; | 89 | } from "./cloud/http.ts"; |
| ... | @@ -559,26 +560,34 @@ export const makeServerLayer = Layer.unwrap( | ... | @@ -559,26 +560,34 @@ export const makeServerLayer = Layer.unwrap( |
| 559 | yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); | 560 | yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); |
| 560 | return; | 561 | return; |
| 561 | } | 562 | } |
| 563 | const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe( | ||
| 564 | Effect.timeout("10 seconds"), | ||
| 565 | Effect.tap((released) => | ||
| 566 | released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void, | ||
| 567 | ), | ||
| 568 | Effect.catchCause((cause) => | ||
| 569 | Effect.logWarning( | ||
| 570 | "Failed to release the managed tunnel on shutdown; the next link reuses it", | ||
| 571 | { cause }, | ||
| 572 | ), | ||
| 573 | ), | ||
| 574 | Effect.asVoid, | ||
| 575 | ); | ||
| 576 | // A launcher trial can be stopped before activation. The previous | ||
| 577 | // server is already gone, so the trial owns cleanup immediately; the | ||
| 578 | // pending-state check keeps the tunnel for normal commit or rollback, | ||
| 579 | // while the launcher's explicit-stop marker allows it to be released. | ||
| 580 | // Other runtimes wait for activation so a failed standby cannot tear | ||
| 581 | // down the active runtime's tunnel. | ||
| 582 | const cleanupBeforeActivation = yield* pendingUpdateHandoffExists; | ||
| 583 | if (cleanupBeforeActivation) { | ||
| 584 | yield* Effect.addFinalizer(() => releaseManagedTunnel); | ||
| 585 | } | ||
| 562 | yield* forkParked( | 586 | yield* forkParked( |
| 563 | Effect.gen(function* () { | 587 | Effect.gen(function* () { |
| 564 | // Only an activated runtime owns the tunnel cleanup finalizer. | 588 | if (!cleanupBeforeActivation) { |
| 565 | yield* Effect.addFinalizer(() => | 589 | yield* Effect.addFinalizer(() => releaseManagedTunnel); |
| 566 | releaseManagedTunnelOnShutdown().pipe( | 590 | } |
| 567 | Effect.timeout("10 seconds"), | ||
| 568 | Effect.tap((released) => | ||
| 569 | released | ||
| 570 | ? Effect.logInfo("Released the managed tunnel on shutdown") | ||
| 571 | : Effect.void, | ||
| 572 | ), | ||
| 573 | Effect.catchCause((cause) => | ||
| 574 | Effect.logWarning( | ||
| 575 | "Failed to release the managed tunnel on shutdown; the next link reuses it", | ||
| 576 | { cause }, | ||
| 577 | ), | ||
| 578 | ), | ||
| 579 | Effect.asVoid, | ||
| 580 | ), | ||
| 581 | ); | ||
| 582 | if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; | 591 | if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; |
| 583 | const server = yield* HttpServer.HttpServer; | 592 | const server = yield* HttpServer.HttpServer; |
| 584 | const address = server.address; | 593 | const address = server.address; |
apps/server/src/serviceLauncher.test.ts-22
| ... | @@ -1,6 +1,3 @@ | ... | @@ -1,6 +1,3 @@ |
| 1 | // @effect-diagnostics-next-line nodeBuiltinImport:off - sync marker poll outside the test clock. | ||
| 2 | import * as NodeFS from "node:fs"; | ||
| 3 | |||
| 4 | import * as NodeServices from "@effect/platform-node/NodeServices"; | 1 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 5 | import { assert, it } from "@effect/vitest"; | 2 | import { assert, it } from "@effect/vitest"; |
| 6 | import * as Effect from "effect/Effect"; | 3 | import * as Effect from "effect/Effect"; |
| ... | @@ -120,25 +117,6 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { | ... | @@ -120,25 +117,6 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { |
| 120 | // An explicit stop leaves the marker that tells a child shutting down | 117 | // An explicit stop leaves the marker that tells a child shutting down |
| 121 | // mid-update that no replacement server is coming. | 118 | // mid-update that no replacement server is coming. |
| 122 | assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); | 119 | assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); |
| 123 | |||
| 124 | // The next launcher start clears the stale marker so a later update | ||
| 125 | // handoff keeps its tunnel again. Recovery runs before any stop | ||
| 126 | // transition, so polling until the marker disappears observes it; the | ||
| 127 | // poll uses real timers because it.effect runs under the test clock. | ||
| 128 | const restarted = new Launcher( | ||
| 129 | root, | ||
| 130 | yield* Effect.promise(() => readServiceState(statePath)), | ||
| 131 | ); | ||
| 132 | const runningAgain = restarted.run(); | ||
| 133 | const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE); | ||
| 134 | yield* Effect.promise(async () => { | ||
| 135 | while (NodeFS.existsSync(markerPath)) { | ||
| 136 | // @effect-diagnostics-next-line globalTimers:off - real delay under the test clock. | ||
| 137 | await new Promise((resolve) => setTimeout(resolve, 25)); | ||
| 138 | } | ||
| 139 | }); | ||
| 140 | yield* Effect.promise(() => restarted.stop("SIGTERM")); | ||
| 141 | yield* Effect.promise(() => runningAgain); | ||
| 142 | }), | 120 | }), |
| 143 | ); | 121 | ); |
| 144 | 122 |