1import * as machines from "@distilled.cloud/fly-io/machines";
2import * as Retry from "@distilled.cloud/fly-io/Retry";
3import * as Fly from "@/Fly";
4import { checksPassing, configuredCheckNames } from "@/Fly/replicas";
5import type { ScratchStack } from "@/Test/Alchemy";
6import { expect } from "alchemy-test";
7import * as Cause from "effect/Cause";
8import * as Effect from "effect/Effect";
9import { FileSystem } from "effect/FileSystem";
10import * as Path from "effect/Path";
11import * as Redacted from "effect/Redacted";
12import * as Schedule from "effect/Schedule";
13import * as Schema from "effect/Schema";
14import * as FetchHttpClient from "effect/http/FetchHttpClient";
15import * as HttpClient from "effect/http/HttpClient";
16import * as HttpClientRequest from "effect/http/HttpClientRequest";
17import { randomBytes } from "node:crypto";
18
19const Receipt = Schema.Struct({
20 machineId: Schema.String,
21 ready: Schema.Boolean,
22});
23
24// HTTP failures can retain the authenticated request, including on defects.
25const privateFailure = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
26 effect.pipe(
27 Effect.catchCause((cause) =>
28 Effect.failCause(
29 Cause.fromReasons(
30 cause.reasons.map((reason) =>
31 Cause.isFailReason(reason)
32 ? Cause.makeFailReason(
33 new Error("Readiness control request failed"),
34 )
35 : Cause.isDieReason(reason)
36 ? Cause.makeDieReason(
37 new Error("Readiness control request defect"),
38 )
39 : Cause.makeInterruptReason(reason.fiberId),
40 ),
41 ),
42 ),
43 ),
44 );
45
46export const makeReadinessControl = () =>
47 Effect.gen(function* () {
48 const token = yield* Effect.sync(() =>
49 Redacted.make(
50 Array.from(randomBytes(32), (byte) =>
51 byte.toString(16).padStart(2, "0"),
52 ).join(""),
53 ),
54 );
55 const fs = yield* FileSystem;
56 const path = yield* Path.Path;
57 const script = yield* fs.readFileString(
58 yield* path.fromFileUrl(
59 new URL("./http-readiness-control.mjs", import.meta.url),
60 ),
61 );
62 const infrastructure = Effect.gen(function* () {
63 const app = yield* Fly.App("Site");
64 const secret = yield* Fly.Secret("ReadinessControl", {
65 app,
66 name: "READINESS_CONTROL_TOKEN",
67 value: token,
68 });
69 yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
70 return { app, secret };
71 });
72 const ready = {
73 type: "http" as const,
74 port: 3000,
75 path: "/ready",
76 interval: "2s",
77 timeout: "1s",
78 };
79 const deployApp = (stack: ScratchStack) =>
80 stack.deploy(infrastructure.pipe(Effect.map(({ app }) => app)));
81 const deployWorker = (stack: ScratchStack, version: string) =>
82 stack.deploy(
83 Effect.gen(function* () {
84 const { app, secret } = yield* infrastructure;
85 return yield* Fly.Machine("Worker", {
86 app,
87 image: "node:22-alpine",
88 init: { exec: ["node", "--input-type=module", "-e", script] },
89 env: {
90 VERSION: version,
91 READINESS_CONTROL_SECRET: secret.name,
92 },
93 checks: { ready },
94 services: [
95 {
96 protocol: "tcp",
97 internalPort: 3000,
98 ports: [{ port: 443, handlers: ["tls", "http"] }],
99 autostop: "off",
100 autostart: true,
101 checks: [ready],
102 },
103 ],
104 deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
105 shutdown: { signal: "SIGTERM", timeout: "5 seconds" },
106 });
107 }),
108 );
109 const request = (appName: string, machineId: string, off: boolean) =>
110 Effect.gen(function* () {
111 const url = `https://${appName}.fly.dev`;
112 const response = yield* HttpClient.execute(
113 (off
114 ? HttpClientRequest.post(`${url}/readiness/off`)
115 : HttpClientRequest.get(`${url}/ready`)
116 ).pipe(
117 HttpClientRequest.bearerToken(token),
118 HttpClientRequest.setHeader("fly-force-instance-id", machineId),
119 HttpClientRequest.setHeader("x-readiness-machine-id", machineId),
120 ),
121 );
122 if (response.status !== 200)
123 return yield* Effect.fail(new Error("Unexpected readiness status"));
124 return yield* response.json.pipe(
125 Effect.flatMap(Schema.decodeUnknownEffect(Receipt)),
126 );
127 }).pipe(
128 Effect.timeout("10 seconds"),
129 Effect.provide(FetchHttpClient.layer),
130 privateFailure,
131 );
132 const turnOff = (appName: string, machineId: string) =>
133 Effect.gen(function* () {
134 const observe = machines
135 .getMachine({ app_name: appName, machine_id: machineId })
136 .pipe(Retry.none, Effect.timeout("10 seconds"));
137 const before = yield* observe.pipe(
138 Effect.repeat({
139 until: (machine) => checksPassing(machine, machine.config),
140 times: 8,
141 schedule: Schedule.spaced("1 second"),
142 }),
143 Effect.timeout("10 seconds"),
144 );
145 expect(before.id).toBe(machineId);
146 expect(before.cordoned).toBe(false);
147 const checkNames = configuredCheckNames(before.config);
148 expect(checkNames).toEqual(["ready", "servicecheck-00-http-3000"]);
149 expect(checksPassing(before, before.config)).toBe(true);
150 const healthy = yield* request(appName, machineId, false).pipe(
151 Effect.retry({ times: 8, schedule: Schedule.spaced("1 second") }),
152 Effect.timeout("30 seconds"),
153 );
154 expect(healthy).toEqual({ machineId, ready: true });
155 const receipt = yield* request(appName, machineId, true);
156 expect(receipt).toEqual({ machineId, ready: false });
157 const failed = (machine: machines.Machine) =>
158 checkNames.every((name) =>
159 machine.checks?.some(
160 (check) => check.name === name && check.status === "critical",
161 ),
162 );
163 // Hold the provider barrier until Fly has observed the real file change.
164 const after = yield* observe.pipe(
165 Effect.repeat({
166 until: failed,
167 times: 8,
168 schedule: Schedule.spaced("1 second"),
169 }),
170 Effect.timeout("20 seconds"),
171 );
172 expect(after.id).toBe(machineId);
173 expect(after.cordoned).toBe(false);
174 expect(failed(after)).toBe(true);
175 }).pipe(
176 Effect.provide(FetchHttpClient.layer),
177 Effect.timeout("25 seconds"),
178 );
179 return { deployApp, deployWorker, turnOff };
180 });
181
182export const repairReadiness = (appName: string, machineId: string) =>
183 Effect.gen(function* () {
184 const unleased = yield* machines
185 .getMachineLease({ app_name: appName, machine_id: machineId })
186 .pipe(
187 Retry.none,
188 Effect.timeout("15 seconds"),
189 Effect.as(false),
190 Effect.catchTag("NotFound", () => Effect.succeed(true)),
191 );
192 expect(unleased).toBe(true);
193 const repaired = yield* machines
194 .execMachine({
195 app_name: appName,
196 machine_id: machineId,
197 command: ["touch", "/tmp/ready"],
198 timeout: 5,
199 })
200 .pipe(Retry.none, Effect.timeout("15 seconds"));
201 expect(repaired.exit_code).toBe(0);
202 }).pipe(Effect.provide(FetchHttpClient.layer));