| author |  Theo Browne <me@t3.gg> 2026-08-06 04:51:08-07:00 |
| committer |  Theo Browne <me@theo.li> 2026-08-06 08:09:51-04:00 |
| log | df2f1273e9e1f1c6f342eecbf3539a5e992163e1 |
| tree | a14961fd87e03549cd2264c55a3f6c21f6ceaab3 |
| parent | f9e8236899f25653ce95d688fac798cd070afbc1 |
fix(server): clean up interrupted update trials
3 files changed, 28 insertions(+), 41 deletions(-)
apps
server
apps/server/src/cloud/http.ts+1-1
| ... | ... | @@ -645,7 +645,7 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD |
| 645 | 645 | // the pending window also tears this server down, with nothing starting |
| 646 | 646 | // afterwards. The launcher marks that case with a stop-marker file just |
| 647 | 647 | // before it signals the child, so pending + no marker is the update handoff. |
| 648 | const pendingUpdateHandoffExists = Effect.gen(function* () { | |
| 648 | export const pendingUpdateHandoffExists = Effect.gen(function* () { | |
| 649 | 649 | const config = yield* ServerConfig.ServerConfig; |
| 650 | 650 | const fs = yield* FileSystem.FileSystem; |
| 651 | 651 | const path = yield* Path.Path; |
apps/server/src/server.ts+27-18
| ... | ... | @@ -83,6 +83,7 @@ import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; |
| 83 | 83 | import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; |
| 84 | 84 | import { |
| 85 | 85 | connectHttpApiLayer, |
| 86 | pendingUpdateHandoffExists, | |
| 86 | 87 | reconcileDesiredCloudLink, |
| 87 | 88 | releaseManagedTunnelOnShutdown, |
| 88 | 89 | } from "./cloud/http.ts"; |
| ... | ... | @@ -559,26 +560,34 @@ export const makeServerLayer = Layer.unwrap( |
| 559 | 560 | yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); |
| 560 | 561 | return; |
| 561 | 562 | } |
| 563 | const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe( | |
| 564 | Effect.timeout("10 seconds"), | |
| 565 | Effect.tap((released) => | |
| 566 | released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void, | |
| 567 | ), | |
| 568 | Effect.catchCause((cause) => | |
| 569 | Effect.logWarning( | |
| 570 | "Failed to release the managed tunnel on shutdown; the next link reuses it", | |
| 571 | { cause }, | |
| 572 | ), | |
| 573 | ), | |
| 574 | Effect.asVoid, | |
| 575 | ); | |
| 576 | // A launcher trial can be stopped before activation. The previous | |
| 577 | // server is already gone, so the trial owns cleanup immediately; the | |
| 578 | // pending-state check keeps the tunnel for normal commit or rollback, | |
| 579 | // while the launcher's explicit-stop marker allows it to be released. | |
| 580 | // Other runtimes wait for activation so a failed standby cannot tear | |
| 581 | // down the active runtime's tunnel. | |
| 582 | const cleanupBeforeActivation = yield* pendingUpdateHandoffExists; | |
| 583 | if (cleanupBeforeActivation) { | |
| 584 | yield* Effect.addFinalizer(() => releaseManagedTunnel); | |
| 585 | } | |
| 562 | 586 | yield* forkParked( |
| 563 | 587 | Effect.gen(function* () { |
| 564 | // Only an activated runtime owns the tunnel cleanup finalizer. | |
| 565 | yield* Effect.addFinalizer(() => | |
| 566 | releaseManagedTunnelOnShutdown().pipe( | |
| 567 | Effect.timeout("10 seconds"), | |
| 568 | Effect.tap((released) => | |
| 569 | released | |
| 570 | ? Effect.logInfo("Released the managed tunnel on shutdown") | |
| 571 | : Effect.void, | |
| 572 | ), | |
| 573 | Effect.catchCause((cause) => | |
| 574 | Effect.logWarning( | |
| 575 | "Failed to release the managed tunnel on shutdown; the next link reuses it", | |
| 576 | { cause }, | |
| 577 | ), | |
| 578 | ), | |
| 579 | Effect.asVoid, | |
| 580 | ), | |
| 581 | ); | |
| 588 | if (!cleanupBeforeActivation) { | |
| 589 | yield* Effect.addFinalizer(() => releaseManagedTunnel); | |
| 590 | } | |
| 582 | 591 | if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; |
| 583 | 592 | const server = yield* HttpServer.HttpServer; |
| 584 | 593 | const address = server.address; |
apps/server/src/serviceLauncher.test.ts-22
| ... | ... | @@ -1,6 +1,3 @@ |
| 1 | // @effect-diagnostics-next-line nodeBuiltinImport:off - sync marker poll outside the test clock. | |
| 2 | import * as NodeFS from "node:fs"; | |
| 3 | ||
| 4 | 1 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 5 | 2 | import { assert, it } from "@effect/vitest"; |
| 6 | 3 | import * as Effect from "effect/Effect"; |
| ... | ... | @@ -120,25 +117,6 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { |
| 120 | 117 | // An explicit stop leaves the marker that tells a child shutting down |
| 121 | 118 | // mid-update that no replacement server is coming. |
| 122 | 119 | assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); |
| 123 | ||
| 124 | // The next launcher start clears the stale marker so a later update | |
| 125 | // handoff keeps its tunnel again. Recovery runs before any stop | |
| 126 | // transition, so polling until the marker disappears observes it; the | |
| 127 | // poll uses real timers because it.effect runs under the test clock. | |
| 128 | const restarted = new Launcher( | |
| 129 | root, | |
| 130 | yield* Effect.promise(() => readServiceState(statePath)), | |
| 131 | ); | |
| 132 | const runningAgain = restarted.run(); | |
| 133 | const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE); | |
| 134 | yield* Effect.promise(async () => { | |
| 135 | while (NodeFS.existsSync(markerPath)) { | |
| 136 | // @effect-diagnostics-next-line globalTimers:off - real delay under the test clock. | |
| 137 | await new Promise((resolve) => setTimeout(resolve, 25)); | |
| 138 | } | |
| 139 | }); | |
| 140 | yield* Effect.promise(() => restarted.stop("SIGTERM")); | |
| 141 | yield* Effect.promise(() => runningAgain); | |
| 142 | 120 | }), |
| 143 | 121 | ); |
| 144 | 122 |