1import { sanitizeLockKey, withLock } from "@/Auth/Lock.ts";
2import { rootDir } from "@/Auth/Paths.ts";
3import * as NodeServices from "@effect/platform-node/NodeServices";
4import { spawn } from "node:child_process";
5import { resolve } from "node:path";
6import { assert, describe, expect, it, layer } from "alchemy-test";
7import * as Clock from "effect/Clock";
8import * as Console from "effect/Console";
9import * as Deferred from "effect/Deferred";
10import * as Effect from "effect/Effect";
11import * as Exit from "effect/Exit";
12import * as Fiber from "effect/Fiber";
13import * as FileSystem from "effect/FileSystem";
14import * as Option from "effect/Option";
15import * as Path from "effect/Path";
16import * as Schedule from "effect/Schedule";
17
18describe("sanitizeLockKey", { tags: ["unit", "local"] }, () => {
19 it("leaves conventional keys untouched", () => {
20 expect(sanitizeLockKey("default-Cloudflare")).toBe("default-Cloudflare");
21 expect(sanitizeLockKey("my_profile.v2-AWS")).toBe("my_profile.v2-AWS");
22 });
23
24 it("neutralises unexpanded shell placeholders", () => {
25 // Seen verbatim in production: EINVAL mkdir
26 // '...\${ALCHEMY_PROFILE:-default}-Cloudflare.lock.lock' on Windows,
27 // where `:`/`$`/`{`/`}` are invalid in file names.
28 const sanitized = sanitizeLockKey("${ALCHEMY_PROFILE:-default}-Cloudflare");
29 expect(sanitized).toBe("__ALCHEMY_PROFILE_-default_-Cloudflare");
30 expect(sanitized).not.toMatch(/[<>:"/\\|?*${}]/);
31 });
32
33 it("replaces path separators so keys cannot escape the lock dir", () => {
34 expect(sanitizeLockKey("../../etc/passwd")).toBe(".._.._etc_passwd");
35 });
36});
37
38// Scenarios adapted from proper-lockfile's suite
39// (https://github.com/moxystudio/node-proper-lockfile/tree/master/test) to
40// the Effect lock: withLock is scoped acquire+release, the lockfile is a
41// directory with an owner marker, staleness is 30s, the heartbeat is 10s.
42layer(NodeServices.layer, { excludeTestServices: true })("withLock", (it) => {
43 const lockPathOf = Effect.fn(function* (key: string) {
44 const path = yield* Path.Path;
45 return path.join(rootDir(), "lock", `${sanitizeLockKey(key)}.lock`);
46 });
47
48 /** Create a foreign lock as another process would have left it. */
49 const plantForeignLock = Effect.fn(function* (
50 lockPath: string,
51 options?: { readonly ageMillis?: number },
52 ) {
53 const fs = yield* FileSystem.FileSystem;
54 const path = yield* Path.Path;
55 yield* fs.makeDirectory(lockPath, { recursive: true });
56 yield* fs.writeFileString(path.join(lockPath, "owner"), "foreign-owner");
57 if (options?.ageMillis !== undefined) {
58 const stamp = new Date(
59 (yield* Clock.currentTimeMillis) - options.ageMillis,
60 );
61 yield* fs.utimes(lockPath, stamp, stamp);
62 }
63 });
64
65 const removeLock = Effect.fn(function* (lockPath: string) {
66 const fs = yield* FileSystem.FileSystem;
67 yield* fs.remove(lockPath, { recursive: true, force: true });
68 });
69
70 it.effect(
71 "acquires and releases a lock whose key contains characters invalid in file names",
72 () =>
73 Effect.gen(function* () {
74 // Regression for the production EINVAL: this key is unusable as a
75 // raw Windows file name without sanitisation.
76 const result = yield* withLock(
77 "${ALCHEMY_PROFILE:-default}-LockTest",
78 Effect.succeed("ran"),
79 );
80 expect(result).toBe("ran");
81 }),
82 { tags: ["unit", "local"] },
83 );
84
85 it.effect(
86 "creates the lock directory while held and removes it on release",
87 () =>
88 Effect.gen(function* () {
89 const fs = yield* FileSystem.FileSystem;
90 const key = "lock-test-lifecycle";
91 const lockPath = yield* lockPathOf(key);
92 const seen = yield* withLock(
93 key,
94 Effect.all({
95 lock: fs.exists(lockPath),
96 owner: fs
97 .readFileString(`${lockPath}/owner`)
98 .pipe(Effect.map((owner) => owner.length > 0)),
99 }),
100 );
101 expect(seen).toEqual({ lock: true, owner: true });
102 expect(yield* fs.exists(lockPath)).toBe(false);
103 }),
104 { tags: ["unit", "local"] },
105 );
106
107 it.effect(
108 "serialises same-key critical sections in-process",
109 () =>
110 Effect.gen(function* () {
111 const order: number[] = [];
112 const critical = (i: number) =>
113 withLock(
114 "lock-test-serialise",
115 Effect.gen(function* () {
116 order.push(i);
117 yield* Effect.sleep("50 millis");
118 order.push(i);
119 }),
120 );
121 yield* Effect.all([critical(1), critical(2)], {
122 concurrency: "unbounded",
123 });
124 // Each critical section's two entries must be adjacent — no
125 // interleaving between holders.
126 expect(order.slice(0, 2)).toEqual([order[0], order[0]]);
127 expect(order.slice(2)).toEqual([order[2], order[2]]);
128 }),
129 { tags: ["unit", "local"] },
130 );
131
132 it.effect(
133 "allows different-key critical sections to overlap",
134 () =>
135 Effect.gen(function* () {
136 const firstEntered = yield* Deferred.make<void>();
137 const releaseFirst = yield* Deferred.make<void>();
138 const first = yield* withLock(
139 "lock-test-independent-a",
140 Deferred.succeed(firstEntered, undefined).pipe(
141 Effect.andThen(Deferred.await(releaseFirst)),
142 ),
143 ).pipe(Effect.forkScoped);
144
145 yield* Deferred.await(firstEntered);
146 const secondRan = yield* withLock(
147 "lock-test-independent-b",
148 Effect.succeed(true),
149 );
150 expect(secondRan).toBe(true);
151
152 yield* Deferred.succeed(releaseFirst, undefined);
153 yield* Fiber.await(first);
154 }),
155 { tags: ["unit", "local"] },
156 );
157
158 it.effect(
159 "excludes another process and recovers its lock after a crash",
160 () =>
161 Effect.gen(function* () {
162 const fs = yield* FileSystem.FileSystem;
163 const key = `lock-test-process-${crypto.randomUUID()}`;
164 const lockPath = yield* lockPathOf(key);
165 const child = spawn(
166 process.execPath,
167 ["test/Auth/fixtures/lock-holder.ts", key],
168 {
169 cwd: resolve(import.meta.dir, "../.."),
170 stdio: ["ignore", "pipe", "inherit"],
171 },
172 );
173
174 yield* Effect.addFinalizer(() =>
175 Effect.sync(() => {
176 if (child.exitCode === null) child.kill("SIGKILL");
177 }),
178 );
179
180 yield* Effect.tryPromise(
181 () =>
182 new Promise<void>((resolve, reject) => {
183 child.once("error", reject);
184 child.once("exit", (code) =>
185 reject(new Error(`lock holder exited before ready: ${code}`)),
186 );
187 child.stdout.once("data", (chunk) => {
188 if (chunk.toString().includes("ready")) resolve();
189 });
190 }),
191 );
192
193 const blocked = yield* withLock(key, Effect.void, {
194 timeout: "250 millis",
195 }).pipe(Effect.exit);
196 assert(Exit.isFailure(blocked));
197 expect(String(blocked.cause)).toContain("Timed out waiting");
198
199 child.kill("SIGKILL");
200 yield* Effect.tryPromise(
201 () =>
202 new Promise<void>((resolve) => {
203 if (child.exitCode !== null) resolve();
204 else child.once("exit", () => resolve());
205 }),
206 );
207 expect(yield* fs.exists(lockPath)).toBe(true);
208
209 const stale = new Date((yield* Clock.currentTimeMillis) - 60_000);
210 yield* fs.utimes(lockPath, stale, stale);
211 expect(
212 yield* withLock(key, Effect.succeed("recovered"), {
213 timeout: "2 seconds",
214 }),
215 ).toBe("recovered");
216 expect(yield* fs.exists(lockPath)).toBe(false);
217 }).pipe(Effect.scoped),
218 { tags: ["unit", "local"], timeout: 10_000 },
219 );
220
221 it.effect(
222 "releases the lock when the critical section fails",
223 () =>
224 Effect.gen(function* () {
225 const fs = yield* FileSystem.FileSystem;
226 const key = "lock-test-release-on-failure";
227 const lockPath = yield* lockPathOf(key);
228 const exit = yield* withLock(key, Effect.fail("boom")).pipe(
229 Effect.exit,
230 );
231 assert(Exit.isFailure(exit));
232 expect(String(exit.cause)).toContain("boom");
233 expect(yield* fs.exists(lockPath)).toBe(false);
234 }),
235 { tags: ["unit", "local"] },
236 );
237
238 it.effect(
239 "releases the lock when the holder is interrupted",
240 () =>
241 Effect.gen(function* () {
242 const fs = yield* FileSystem.FileSystem;
243 const key = "lock-test-release-on-interrupt";
244 const lockPath = yield* lockPathOf(key);
245 const entered = yield* Deferred.make<void>();
246 const fiber = yield* withLock(
247 key,
248 Deferred.succeed(entered, undefined).pipe(
249 Effect.andThen(Effect.never),
250 ),
251 ).pipe(Effect.forkScoped);
252 yield* Deferred.await(entered);
253 yield* Fiber.interrupt(fiber);
254 expect(yield* fs.exists(lockPath)).toBe(false);
255 }),
256 { tags: ["unit", "local"] },
257 );
258
259 it.effect(
260 "releases the lock when interrupted during acquisition",
261 () =>
262 Effect.gen(function* () {
263 const fs = yield* FileSystem.FileSystem;
264 const key = "lock-test-interrupt-acquisition";
265 const lockPath = yield* lockPathOf(key);
266 const created = yield* Deferred.make<void>();
267 const resume = yield* Deferred.make<void>();
268 yield* Effect.addFinalizer(() =>
269 removeLock(lockPath).pipe(Effect.orDie),
270 );
271 const acquisitionFs: FileSystem.FileSystem = {
272 ...fs,
273 makeDirectory: (directory, options) =>
274 fs
275 .makeDirectory(directory, options)
276 .pipe(
277 Effect.andThen(
278 directory === lockPath
279 ? Deferred.succeed(created, undefined).pipe(
280 Effect.andThen(Deferred.await(resume)),
281 )
282 : Effect.void,
283 ),
284 ),
285 };
286 const holder = yield* withLock(key, Effect.never).pipe(
287 Effect.provideService(FileSystem.FileSystem, acquisitionFs),
288 Effect.forkScoped,
289 );
290 yield* Deferred.await(created);
291 const interrupted = yield* Fiber.interrupt(holder).pipe(
292 Effect.forkScoped({ startImmediately: true }),
293 );
294 yield* Deferred.succeed(resume, undefined);
295 yield* Fiber.join(interrupted);
296 expect(yield* fs.exists(lockPath)).toBe(false);
297 }).pipe(Effect.scoped),
298 { tags: ["unit", "local"] },
299 );
300
301 it.effect(
302 "dies with a timeout while a live foreign lock is held",
303 () =>
304 Effect.gen(function* () {
305 const fs = yield* FileSystem.FileSystem;
306 const key = "lock-test-foreign-holder";
307 const lockPath = yield* lockPathOf(key);
308 yield* plantForeignLock(lockPath);
309
310 const exit = yield* withLock(key, Effect.void, {
311 timeout: "500 millis",
312 }).pipe(Effect.exit);
313 assert(Exit.isFailure(exit));
314 expect(String(exit.cause)).toMatch(/Timed out waiting/);
315 // A fresh (non-stale) foreign lock must survive our failed attempt.
316 expect(yield* fs.readFileString(`${lockPath}/owner`)).toBe(
317 "foreign-owner",
318 );
319 yield* removeLock(lockPath);
320 }),
321 { tags: ["unit", "local"] },
322 );
323
324 it.effect(
325 "reaps and acquires over a stale lock",
326 () =>
327 Effect.gen(function* () {
328 const fs = yield* FileSystem.FileSystem;
329 const key = "lock-test-stale";
330 const lockPath = yield* lockPathOf(key);
331 // A holder that crashed 60s ago: owner file present, mtime past the
332 // 30s stale threshold, nobody refreshing.
333 yield* plantForeignLock(lockPath, { ageMillis: 60_000 });
334
335 const result = yield* withLock(key, Effect.succeed("recovered"), {
336 timeout: "5 seconds",
337 });
338 expect(result).toBe("recovered");
339 expect(yield* fs.exists(lockPath)).toBe(false);
340 }),
341 { tags: ["unit", "local"] },
342 );
343
344 it.effect(
345 "retries until a foreign lock is released",
346 () =>
347 Effect.gen(function* () {
348 const key = "lock-test-wait-for-release";
349 const lockPath = yield* lockPathOf(key);
350 yield* plantForeignLock(lockPath);
351 yield* removeLock(lockPath).pipe(
352 Effect.delay("300 millis"),
353 Effect.forkScoped,
354 );
355 const result = yield* withLock(key, Effect.succeed("ran"), {
356 timeout: "5 seconds",
357 });
358 expect(result).toBe("ran");
359 }),
360 { tags: ["unit", "local"] },
361 );
362
363 it.effect(
364 "does not remove a lock it no longer owns on release",
365 () =>
366 Effect.gen(function* () {
367 const fs = yield* FileSystem.FileSystem;
368 const key = "lock-test-compromised-release";
369 const lockPath = yield* lockPathOf(key);
370 const release = yield* Deferred.make<void>();
371 const fiber = yield* withLock(key, Deferred.await(release)).pipe(
372 Effect.forkScoped,
373 );
374 yield* fs.exists(lockPath).pipe(
375 Effect.repeat({
376 schedule: Schedule.spaced("25 millis"),
377 until: (exists) => exists,
378 times: 100,
379 }),
380 );
381 // Another process reaped our lock as stale and took it over.
382 yield* fs.writeFileString(`${lockPath}/owner`, "foreign-owner");
383 yield* Deferred.succeed(release, undefined);
384 yield* Fiber.await(fiber);
385 // Release must leave the usurper's lock in place.
386 expect(yield* fs.readFileString(`${lockPath}/owner`)).toBe(
387 "foreign-owner",
388 );
389 yield* removeLock(lockPath);
390 }),
391 { tags: ["unit", "local"] },
392 );
393
394 it.effect.skipIf(process.env.FAST)(
395 "refreshes the lock mtime while held",
396 () =>
397 Effect.gen(function* () {
398 const fs = yield* FileSystem.FileSystem;
399 const key = "lock-test-heartbeat";
400 const lockPath = yield* lockPathOf(key);
401 const release = yield* Deferred.make<void>();
402 const fiber = yield* withLock(key, Deferred.await(release)).pipe(
403 Effect.forkScoped,
404 );
405 yield* fs.exists(lockPath).pipe(
406 Effect.repeat({
407 schedule: Schedule.spaced("25 millis"),
408 until: (exists) => exists,
409 times: 100,
410 }),
411 );
412 // Age the lock past the heartbeat interval; the next refresh must
413 // bump the mtime back to the present so it never appears stale.
414 const before = new Date((yield* Clock.currentTimeMillis) - 25_000);
415 yield* fs.utimes(lockPath, before, before);
416 const refreshed = yield* fs.stat(lockPath).pipe(
417 Effect.map((info) =>
418 Option.exists(
419 info.mtime,
420 (mtime) => mtime.getTime() > before.getTime() + 20_000,
421 ),
422 ),
423 Effect.repeat({
424 schedule: Schedule.spaced("500 millis"),
425 until: (bumped) => bumped,
426 times: 25,
427 }),
428 );
429 expect(refreshed).toBe(true);
430 yield* Deferred.succeed(release, undefined);
431 yield* Fiber.await(fiber);
432 }),
433 { tags: ["unit", "local"], timeout: 30_000 },
434 );
435 /**
436 * A locked operation that never completes used to be entirely invisible:
437 * the deploy sat there with no output at any log level (#1231). The stall
438 * notice is the backstop that makes a hang self-describing, so it has to
439 * name the operation and it must stay quiet for flows that legitimately
440 * block on the user.
441 */
442 const captureConsole = () => {
443 const messages: Array<string> = [];
444 return {
445 messages,
446 provide: <A, E, R>(self: Effect.Effect<A, E, R>) =>
447 Console.consoleWith((base) =>
448 Effect.provideService(self, Console.Console, {
449 ...base,
450 error: (...args: ReadonlyArray<any>) => {
451 messages.push(args.join(" "));
452 },
453 }),
454 ),
455 };
456 };
457
458 it.effect(
459 "names the stalled operation on the console",
460 () =>
461 Effect.gen(function* () {
462 const console = captureConsole();
463 yield* console.provide(
464 withLock("stall-notice-key", Effect.sleep("250 millis"), {
465 label: "AWS.read (profile 'probe')",
466 stallInterval: "50 millis",
467 }),
468 );
469 expect(console.messages.length).toBeGreaterThan(0);
470 expect(console.messages[0]).toContain("AWS.read (profile 'probe')");
471 expect(console.messages[0]).toContain("stall-notice-key.lock");
472 yield* removeLock(yield* lockPathOf("stall-notice-key"));
473 }),
474 { tags: ["unit", "local"] },
475 );
476
477 it.effect(
478 "stays silent when the watchdog is disabled",
479 () =>
480 Effect.gen(function* () {
481 const console = captureConsole();
482 yield* console.provide(
483 withLock("stall-quiet-key", Effect.sleep("250 millis"), {
484 watchdog: false,
485 stallInterval: "50 millis",
486 }),
487 );
488 expect(console.messages).toEqual([]);
489 yield* removeLock(yield* lockPathOf("stall-quiet-key"));
490 }),
491 { tags: ["unit", "local"] },
492 );
493});