1/**
2 * Single-process test runner.
3 *
4 * Discovers `*.test.ts` files, imports them concurrently with an AsyncLocalStorage
5 * per-file collector (see Registry.ts), then executes every collected
6 * test as an Effect — files run concurrently up to a limit, tests within a
7 * file run sequentially unless their suite is `describe.concurrent`. Each
8 * test gets a buffering Effect Logger + Console so its output can be shown
9 * in isolation.
10 */
11import * as Cause from "effect/Cause";
12import * as ConsoleModule from "effect/Console";
13import * as Duration from "effect/Duration";
14import * as Effect from "effect/Effect";
15import * as Exit from "effect/Exit";
16import * as Fiber from "effect/Fiber";
17import * as FileSystem from "effect/FileSystem";
18import * as Logger from "effect/Logger";
19import * as Option from "effect/Option";
20import * as Path from "effect/Path";
21import * as Semaphore from "effect/Semaphore";
22import { inspect } from "node:util";
23import { pathToFileURL } from "node:url";
24
25import { makeFileLog } from "./FileLog.ts";
26import type { FileSuite, Hook, LogEntry, Suite, TestCase } from "./Model.ts";
27import { containsOnly, forEachTest, titlePath } from "./Model.ts";
28import type { TestPlan } from "./Plan.ts";
29import * as Registry from "./Registry.ts";
30import { compileTagsFilter, type TagsFilter } from "./Tags.ts";
31import {
32 Reporter,
33 type RunController,
34 type RunSummary,
35 type TestEvent,
36 type TestMeta,
37 type TestResult,
38} from "./Reporter.ts";
39
40export interface RunOptions {
41 /** Directory the run is rooted at (usually `packages/alchemy`). */
42 readonly root: string;
43 /**
44 * Positional filters. Existing files/directories are used as-is; anything
45 * else is a case-insensitive substring filter on test file paths (like
46 * vitest's positional filters). Defaults to `test`.
47 */
48 readonly paths: ReadonlyArray<string>;
49 /**
50 * Exclusion filters (`--exclude`). Existing files/directories exclude by
51 * path prefix; anything else is a case-insensitive substring exclusion on
52 * test file paths. A positional root explicitly given inside an excluded
53 * path overrides the exclusion (so `alchemy-test test/Railway` still works
54 * when the package script bakes in `--exclude test/Railway`).
55 */
56 readonly exclude?: ReadonlyArray<string> | undefined;
57 /**
58 * `-t` test-name filter, applied to the full title
59 * (`file > describe chain > name`).
60 */
61 readonly filter?: ((fullTitle: string) => boolean) | undefined;
62 /** Match the combined suite and test tags, in addition to other filters. */
63 readonly tagsFilter?: ReadonlyArray<string>;
64 readonly plan?: TestPlan | undefined;
65 readonly dryRun?: boolean;
66 /** Default per-test timeout in ms. */
67 readonly timeout: number;
68 /** Times a failing test body is re-run before being reported as failed. */
69 readonly retry: number;
70 /** Maximum number of files executing concurrently (default unbounded). */
71 readonly concurrency: number | "unbounded";
72 /** Force sequential execution within every file. */
73 readonly sequential: boolean;
74 /** Absolute path of the persistent run log (test.log). */
75 readonly logFile: string;
76}
77
78// ---------------------------------------------------------------------------
79// Log capture
80// ---------------------------------------------------------------------------
81
82/**
83 * Strip ANSI escape sequences (SGR colors, cursor movement, OSC). Captured
84 * output is re-rendered by the reporters — embedded escapes corrupt the
85 * TUI's cell-based rendering and garble plain output.
86 */
87const ANSI_RE =
88 // eslint-disable-next-line no-control-regex
89 /\u001B(?:\[[0-9;?]*[ -/]*[@-~]|\][^\u0007\u001B]*(?:\u0007|\u001B\\)|[@-Z\\-_])/g;
90
91const stripAnsi = (text: string): string => text.replace(ANSI_RE, "");
92
93const formatArg = (value: unknown): string =>
94 stripAnsi(
95 typeof value === "string"
96 ? value
97 : inspect(value, { depth: 4, colors: false }),
98 );
99
100const formatArgs = (args: ReadonlyArray<unknown>): string =>
101 args.map(formatArg).join(" ");
102
103const bufferingConsole = (logs: Array<LogEntry>): ConsoleModule.Console => {
104 const push = (level: string, args: ReadonlyArray<unknown>) => {
105 logs.push({ level, message: formatArgs(args), time: new Date() });
106 };
107 const times = new Map<string, number>();
108 return {
109 assert: (condition, ...args) => {
110 if (!condition) push("error", ["Assertion failed:", ...args]);
111 },
112 clear: () => {},
113 count: (label) => push("info", [`count: ${label ?? "default"}`]),
114 countReset: () => {},
115 debug: (...args) => push("debug", args),
116 dir: (item) => push("info", [item]),
117 dirxml: (...args) => push("info", args),
118 error: (...args) => push("error", args),
119 group: (...args) => push("info", args),
120 groupCollapsed: (...args) => push("info", args),
121 groupEnd: () => {},
122 info: (...args) => push("info", args),
123 log: (...args) => push("info", args),
124 table: (data) => push("info", [data]),
125 time: (label) => {
126 times.set(label ?? "default", Date.now());
127 },
128 timeEnd: (label) => {
129 const start = times.get(label ?? "default");
130 push("info", [
131 `${label ?? "default"}: ${start === undefined ? "?" : Date.now() - start}ms`,
132 ]);
133 },
134 timeLog: (label, ...args) => push("info", [label, ...args]),
135 trace: (...args) => push("debug", args),
136 warn: (...args) => push("warn", args),
137 };
138};
139
140/** Provide a buffering Logger + Console around an effect. */
141const withCapture =
142 (logs: Array<LogEntry>) =>
143 <A, E>(effect: Effect.Effect<A, E>): Effect.Effect<A, E> =>
144 effect.pipe(
145 Effect.provide(
146 Logger.layer([
147 Logger.make((options) => {
148 // `Effect.log("a", "b")` delivers the message as an array —
149 // unwrap it so buffered output reads exactly like console output
150 // instead of `[ 'a', 'b' ]`.
151 const parts = Array.isArray(options.message)
152 ? options.message
153 : [options.message];
154 logs.push({
155 level: options.logLevel,
156 message:
157 parts.map(formatArg).join(" ") +
158 (options.cause.reasons.length === 0
159 ? ""
160 : `\n${Cause.pretty(options.cause)}`),
161 time: options.date,
162 });
163 }),
164 ]),
165 ),
166 Effect.provideService(ConsoleModule.Console, bufferingConsole(logs)),
167 );
168
169// ---------------------------------------------------------------------------
170// Discovery
171// ---------------------------------------------------------------------------
172
173const isTestFile = (name: string): boolean =>
174 name.endsWith(".test.ts") || name.endsWith(".test.tsx");
175
176export const discover = Effect.fn(function* (options: RunOptions) {
177 const fs = yield* FileSystem.FileSystem;
178 const path = yield* Path.Path;
179 const files: Array<string> = [];
180
181 const walk: (
182 dir: string,
183 ) => Effect.Effect<void, unknown, FileSystem.FileSystem> = Effect.fn(
184 function* (dir: string) {
185 const entries = yield* fs.readDirectory(dir);
186 entries.sort();
187 for (const entry of entries) {
188 if (entry === "node_modules" || entry.startsWith(".")) continue;
189 const full = path.join(dir, entry);
190 const stat = yield* fs.stat(full);
191 if (stat.type === "Directory") {
192 yield* walk(full);
193 } else if (isTestFile(entry)) {
194 files.push(full);
195 }
196 }
197 },
198 );
199
200 // Positional args that exist on disk are roots; anything else is a
201 // case-insensitive substring filter on discovered file paths (vitest-style:
202 // `alchemy-test Bucket` runs every *Bucket* test file).
203 const roots: Array<string> = [];
204 const nameFilters: Array<string> = [];
205 for (const p of options.paths) {
206 const abs = path.isAbsolute(p) ? p : path.resolve(options.root, p);
207 const exists = yield* fs
208 .exists(abs)
209 .pipe(Effect.orElseSucceed(() => false));
210 if (exists) {
211 roots.push(abs);
212 } else {
213 nameFilters.push(p.toLowerCase());
214 }
215 }
216 // Roots the user asked for explicitly (before defaulting to `test`) are
217 // exempt from `--exclude` — a baked-in package-script exclusion must not
218 // make `alchemy-test test/Railway/Foo.test.ts` silently run nothing.
219 const explicitRoots = [...roots];
220 if (roots.length === 0) {
221 roots.push(path.resolve(options.root, "test"));
222 }
223
224 // Excludes mirror positional semantics: existing paths exclude by prefix,
225 // anything else is a case-insensitive substring exclusion.
226 const excludePrefixes: Array<string> = [];
227 const excludeSubstrings: Array<string> = [];
228 for (const e of options.exclude ?? []) {
229 const abs = path.isAbsolute(e) ? e : path.resolve(options.root, e);
230 const exists = yield* fs
231 .exists(abs)
232 .pipe(Effect.orElseSucceed(() => false));
233 if (exists) {
234 excludePrefixes.push(abs);
235 } else {
236 excludeSubstrings.push(e.toLowerCase());
237 }
238 }
239 const isWithin = (file: string, dir: string): boolean =>
240 file === dir || file.startsWith(dir + path.sep);
241 const isExcluded = (file: string): boolean => {
242 if (excludePrefixes.some((prefix) => isWithin(file, prefix))) return true;
243 const rel = path.relative(options.root, file).toLowerCase();
244 return excludeSubstrings.some((substring) => rel.includes(substring));
245 };
246 const exemptRoots = explicitRoots.filter(isExcluded);
247
248 for (const abs of roots) {
249 const stat = yield* fs
250 .stat(abs)
251 .pipe(
252 Effect.mapError(
253 () => new Error(`alchemy-test: path not found: ${abs}`),
254 ),
255 );
256 if (stat.type === "Directory") {
257 yield* walk(abs);
258 } else {
259 files.push(abs);
260 }
261 }
262
263 let unique = [...new Set(files)];
264 if (nameFilters.length > 0) {
265 unique = unique.filter((file) => {
266 const rel = path.relative(options.root, file).toLowerCase();
267 return nameFilters.some((filter) => rel.includes(filter));
268 });
269 }
270 if (excludePrefixes.length > 0 || excludeSubstrings.length > 0) {
271 unique = unique.filter(
272 (file) =>
273 exemptRoots.some((root) => isWithin(file, root)) || !isExcluded(file),
274 );
275 }
276 return unique.sort();
277});
278
279// ---------------------------------------------------------------------------
280// Collection
281// ---------------------------------------------------------------------------
282
283export interface CollectedFile {
284 readonly file: string;
285 readonly suite: FileSuite | undefined;
286 readonly error?: string | undefined;
287}
288
289const collectFile = (
290 absolute: string,
291 relative: string,
292): Effect.Effect<CollectedFile> =>
293 Effect.promise(async (): Promise<CollectedFile> => {
294 try {
295 const suite = await Registry.collect(relative, async () => {
296 await import(pathToFileURL(absolute).href);
297 // Flush microtasks + one macrotask so registrations deferred with
298 // queueMicrotask (e.g. Test.make's fallback afterAll) land in the
299 // tree — their AsyncLocalStorage context resolves this file's collector.
300 await new Promise<void>((resolve) => setImmediate(resolve));
301 });
302 return { file: relative, suite };
303 } catch (error) {
304 return {
305 file: relative,
306 suite: undefined,
307 error:
308 error instanceof Error
309 ? (error.stack ?? error.message)
310 : String(error),
311 };
312 }
313 });
314
315// ---------------------------------------------------------------------------
316// Execution
317// ---------------------------------------------------------------------------
318
319/**
320 * Whole-process read/write lock. Normal tests hold a read permit; tests
321 * registered with `{ exclusive: true }` (they mutate process-global state
322 * like `process.env`) take every permit, so they never overlap with any
323 * other test in the run.
324 */
325const EXCLUSIVE_PERMITS = 100_000;
326
327const hookPermits = (hooks: ReadonlyArray<Hook>): number =>
328 hooks.some((hook) => hook.exclusive === true) ? EXCLUSIVE_PERMITS : 1;
329
330interface ExecContext {
331 readonly options: Omit<RunOptions, "tagsFilter"> & {
332 readonly tagsFilter: TagsFilter;
333 readonly selectedTests?: ReadonlySet<TestCase>;
334 };
335 readonly suiteStates?: Map<Suite, Exit.Exit<void, unknown>>;
336 readonly onlyMode: boolean;
337 readonly emit: (event: TestEvent) => Effect.Effect<void>;
338 readonly fileLogs: Array<LogEntry>;
339 /** File-level hook failures, counted once for the file in the run summary. */
340 readonly fileErrors: Array<string>;
341 readonly results: Array<{ meta: TestMeta; result: TestResult }>;
342 readonly file: string;
343 readonly lock: Semaphore.Semaphore;
344 /** Run-global registry of currently-executing test fibers (for kill). */
345 readonly running: Map<string, Fiber.Fiber<unknown, unknown>>;
346 /** Run-global set of tests that have finished at least once (for retry). */
347 readonly completed: Set<string>;
348}
349
350const metaOf = (file: string, test: TestCase): TestMeta => {
351 const parts = titlePath(test);
352 return {
353 id: `${file} > ${parts.join(" > ")}`,
354 file,
355 titlePath: parts,
356 name: test.name,
357 tags: test.tags,
358 optInTags: test.optInTags,
359 };
360};
361
362/** Should this test run at all given only-mode and the -t filter? */
363const included = (
364 test: TestCase,
365 ctx: Pick<ExecContext, "onlyMode" | "options" | "file">,
366): boolean => {
367 if (
368 ctx.options.selectedTests !== undefined &&
369 !ctx.options.selectedTests.has(test)
370 )
371 return false;
372 if (
373 ctx.options.tagsFilter !== undefined &&
374 !ctx.options.tagsFilter(test.tags, test.optInTags)
375 ) {
376 return false;
377 }
378 if (ctx.options.filter !== undefined) {
379 // Match against the full nested title, so `-t` finds a test by any
380 // fragment regardless of how it's nested in describe blocks.
381 const full = `${ctx.file} > ${titlePath(test).join(" > ")}`;
382 if (!ctx.options.filter(full)) {
383 return false;
384 }
385 }
386 if (ctx.onlyMode) {
387 let node: Suite | TestCase | undefined = test;
388 while (node !== undefined) {
389 if (node.mode === "only") return true;
390 node = node.parent;
391 }
392 return false;
393 }
394 return true;
395};
396
397const isSkipped = (test: TestCase): "skip" | "todo" | undefined => {
398 if (test.mode === "todo") return "todo";
399 let node: Suite | TestCase | undefined = test;
400 while (node !== undefined) {
401 if (node.mode === "skip") return "skip";
402 node = node.parent;
403 }
404 // Wrappers (e.g. alchemy's `test.skipIf`) drop the body of skipped tests,
405 // so a missing body only means "todo" once skip has been ruled out.
406 return test.body === undefined ? "todo" : undefined;
407};
408
409const hookChain = (
410 test: TestCase,
411 kind: "beforeEach" | "afterEach",
412): Array<Hook> => {
413 const chain: Array<Array<Hook>> = [];
414 let suite: Suite | undefined = test.parent;
415 while (suite !== undefined) {
416 chain.unshift(suite[kind]);
417 suite = suite.parent;
418 }
419 const flat = chain.flat();
420 return kind === "afterEach" ? flat.reverse() : flat;
421};
422
423const runHooks = (
424 hooks: ReadonlyArray<Hook>,
425 defaultTimeout: number,
426): Effect.Effect<void, unknown> =>
427 Effect.forEach(
428 hooks,
429 (hook) =>
430 Effect.suspend(hook.body).pipe(
431 Effect.timeout(Duration.millis(hook.timeout ?? defaultTimeout)),
432 ),
433 { discard: true },
434 );
435
436const prettyCause = (cause: Cause.Cause<unknown>): string => {
437 const rendered = Cause.pretty(cause);
438 return stripAnsi(
439 rendered.trim().length === 0 ? inspect(Cause.squash(cause)) : rendered,
440 );
441};
442
443const wasInterrupted = (exit: Exit.Exit<unknown, unknown>): boolean =>
444 Exit.isFailure(exit) &&
445 exit.cause.reasons.some((reason) => reason._tag === "Interrupt");
446
447interface TestAttempt {
448 readonly beforeEach: Exit.Exit<void, unknown>;
449 readonly body: Exit.Exit<unknown, unknown> | undefined;
450 readonly afterEach: Exit.Exit<void, unknown>;
451}
452
453const attemptNeedsRetry = (
454 exit: Exit.Exit<TestAttempt, unknown>,
455 expectsFailure: boolean | undefined,
456): boolean => {
457 if (Exit.isFailure(exit)) return !wasInterrupted(exit);
458 if (
459 Exit.isFailure(exit.value.beforeEach) ||
460 Exit.isFailure(exit.value.afterEach)
461 ) {
462 return true;
463 }
464 return (
465 !expectsFailure &&
466 exit.value.body !== undefined &&
467 Exit.isFailure(exit.value.body)
468 );
469};
470
471const hookError = (attempt: TestAttempt): string | undefined => {
472 const errors: Array<string> = [];
473 if (Exit.isFailure(attempt.beforeEach)) {
474 errors.push(
475 `beforeEach hook failed:\n${prettyCause(attempt.beforeEach.cause)}`,
476 );
477 }
478 if (Exit.isFailure(attempt.afterEach)) {
479 errors.push(
480 `afterEach hook failed:\n${prettyCause(attempt.afterEach.cause)}`,
481 );
482 }
483 return errors.length === 0 ? undefined : errors.join("\n\n");
484};
485
486/**
487 * How long a timed-out test body's interruption (finalizers included) may
488 * run before the runner abandons the fiber and reports the timeout anyway.
489 * Without this bound, a finalizer blocked on the same wedged machinery the
490 * timeout just interrupted (e.g. a `test.provider` scratch destroy against
491 * a hung dev deploy) swallows the report entirely — the test never finishes
492 * and the run dies at the wall clock with no error attribution.
493 */
494const INTERRUPT_GRACE_MS = 10_000;
495
496/**
497 * Run a test body with a timeout that cannot be swallowed by hung
498 * finalizers: on timeout the body fiber is interrupted, its finalizers get
499 * {@link INTERRUPT_GRACE_MS} to settle, and then the fiber is abandoned
500 * (it dies with the run) and the timeout is reported.
501 */
502const runBodyWithTimeout = Effect.fn(function* (
503 body: () => Effect.Effect<unknown, unknown>,
504 timeoutMs: number,
505) {
506 // Detached: a timed-out body whose teardown never settles must not block
507 // the attempt fiber's own completion (a supervised child would).
508 const fiber = yield* Effect.forkDetach(Effect.suspend(body), {
509 startImmediately: true,
510 });
511 const awaited = yield* Fiber.await(fiber).pipe(
512 Effect.timeoutOption(Duration.millis(timeoutMs)),
513 );
514 if (Option.isSome(awaited)) return awaited.value;
515 // Fire-and-forget interrupt: `Fiber.interrupt` AWAITS settlement, which a
516 // hung finalizer never provides. Fire it, then give teardown a bounded
517 // grace before abandoning the fiber (it dies with the process).
518 yield* Effect.sync(() => fiber.interruptUnsafe());
519 const settled = yield* Fiber.await(fiber).pipe(
520 Effect.timeoutOption(Duration.millis(INTERRUPT_GRACE_MS)),
521 );
522 return Exit.fail(
523 new Error(
524 Option.isNone(settled)
525 ? `test timed out after ${timeoutMs}ms (teardown did not settle within ${INTERRUPT_GRACE_MS}ms and was abandoned)`
526 : `test timed out after ${timeoutMs}ms`,
527 ),
528 ) as Exit.Exit<unknown, unknown>;
529});
530
531const runTest = Effect.fn(function* (test: TestCase, ctx: ExecContext) {
532 const meta = metaOf(ctx.file, test);
533 const skipped = isSkipped(test);
534 if (skipped !== undefined) {
535 const result: TestResult = {
536 status: skipped,
537 durationMs: 0,
538 logs: [],
539 retries: 0,
540 };
541 ctx.results.push({ meta, result });
542 yield* ctx.emit({ _tag: "TestEnd", test: meta, result });
543 return;
544 }
545
546 // One stable buffer for the whole runTest call (cleared in place between
547 // retry attempts) — TestStart shares the LIVE reference so the TUI can
548 // tail a running test's output.
549 const logs: Array<LogEntry> = [];
550 yield* ctx.emit({ _tag: "TestStart", test: meta, logs });
551
552 const timeoutMs = test.timeout ?? ctx.options.timeout;
553 const before = hookChain(test, "beforeEach");
554 const after = hookChain(test, "afterEach");
555
556 const attempt = (): Effect.Effect<TestAttempt> => {
557 // Store the finalizer's Exit separately so a teardown failure cannot be
558 // swallowed or mistaken for an expected (`test.fails`) body failure.
559 let afterExit: Exit.Exit<void, unknown> = Exit.succeed(undefined);
560 return Effect.gen(function* () {
561 const beforeExit = yield* runHooks(before, timeoutMs).pipe(Effect.exit);
562 const bodyExit = Exit.isSuccess(beforeExit)
563 ? yield* runBodyWithTimeout(test.body!, timeoutMs)
564 : undefined;
565 return { beforeEach: beforeExit, body: bodyExit };
566 }).pipe(
567 // afterEach must run on success, failure and interruption alike. Its
568 // own Exit is recorded without making the finalizer itself fail.
569 Effect.onExit(() =>
570 runHooks(after, timeoutMs).pipe(
571 Effect.exit,
572 Effect.tap((exit) =>
573 Effect.sync(() => {
574 afterExit = exit;
575 }),
576 ),
577 Effect.asVoid,
578 ),
579 ),
580 Effect.map(({ beforeEach, body }) => ({
581 beforeEach,
582 body,
583 afterEach: afterExit,
584 })),
585 withCapture(logs),
586 );
587 };
588
589 let durationMs = 0;
590 let retries = 0;
591 const withLock = ctx.lock.withPermits(test.exclusive ? EXCLUSIVE_PERMITS : 1);
592
593 // Each attempt runs in its own fiber, registered run-globally so the TUI's
594 // kill command can interrupt it.
595 const runAttempt = Effect.fn(function* (): Generator<
596 Effect.Effect<any>,
597 Exit.Exit<TestAttempt, unknown>
598 > {
599 const fiber = yield* Effect.forkChild(
600 withLock(
601 Effect.suspend(() => {
602 // Queue time behind an exclusive test is not execution time. Sum only
603 // attempts (including their hooks), retaining time spent on retries.
604 const start = Date.now();
605 return attempt().pipe(
606 Effect.ensuring(
607 Effect.sync(() => {
608 durationMs += Date.now() - start;
609 }),
610 ),
611 );
612 }),
613 ),
614 {
615 startImmediately: true,
616 },
617 );
618 ctx.running.set(meta.id, fiber);
619 const exit: Exit.Exit<TestAttempt, unknown> = yield* Fiber.await(fiber);
620 ctx.running.delete(meta.id);
621 return exit;
622 });
623
624 let exit = yield* runAttempt();
625 while (
626 attemptNeedsRetry(exit, test.fails) &&
627 retries < (test.retry ?? ctx.options.retry)
628 ) {
629 retries++;
630 // Clear IN PLACE — TestStart handed this array's reference out.
631 logs.length = 0;
632 exit = yield* runAttempt();
633 }
634
635 let status: TestResult["status"];
636 let error: string | undefined;
637 if (wasInterrupted(exit)) {
638 status = "fail";
639 error = "killed (interrupted by user)";
640 } else if (Exit.isFailure(exit)) {
641 status = "fail";
642 error = prettyCause(exit.cause);
643 } else {
644 const failedHook = hookError(exit.value);
645 const bodyExit = exit.value.body;
646 if (failedHook !== undefined) {
647 status = "fail";
648 error = failedHook;
649 } else if (test.fails && bodyExit !== undefined) {
650 if (Exit.isFailure(bodyExit)) {
651 status = "pass";
652 } else {
653 status = "fail";
654 error = "expected test to fail, but it passed";
655 }
656 } else if (bodyExit !== undefined && Exit.isSuccess(bodyExit)) {
657 status = "pass";
658 } else {
659 status = "fail";
660 error =
661 bodyExit === undefined
662 ? "test body did not run"
663 : prettyCause(bodyExit.cause);
664 }
665 }
666
667 ctx.completed.add(meta.id);
668 const result: TestResult = { status, durationMs, error, logs, retries };
669 ctx.results.push({ meta, result });
670 yield* ctx.emit({ _tag: "TestEnd", test: meta, result });
671});
672
673/** Fail every (non-skipped) test in a subtree without running it. */
674const failSubtree = Effect.fn(function* (
675 suite: Suite,
676 ctx: ExecContext,
677 error: string,
678) {
679 const tests: Array<TestCase> = [];
680 const collect = (s: Suite) => {
681 for (const child of s.children) {
682 if (child.type === "test") tests.push(child);
683 else collect(child);
684 }
685 };
686 collect(suite);
687 for (const test of tests) {
688 const meta = metaOf(ctx.file, test);
689 if (!included(test, ctx)) continue;
690 const skipped = isSkipped(test);
691 const result: TestResult =
692 skipped !== undefined
693 ? { status: skipped, durationMs: 0, logs: [], retries: 0 }
694 : {
695 status: "fail",
696 durationMs: 0,
697 error: `beforeAll hook failed:\n${error}`,
698 logs: [],
699 retries: 0,
700 };
701 ctx.results.push({ meta, result });
702 yield* ctx.emit({ _tag: "TestEnd", test: meta, result });
703 }
704});
705
706const runSuite: (suite: Suite, ctx: ExecContext) => Effect.Effect<void> =
707 Effect.fn(function* (suite: Suite, ctx: ExecContext) {
708 const runnable = suite.children.filter((child) =>
709 child.type === "test"
710 ? included(child, ctx)
711 : suiteHasIncludedTests(child, ctx),
712 );
713 if (runnable.length === 0) return;
714
715 // If every included test below is skipped (e.g. describe.skip), report
716 // them without running any hooks.
717 if (!suiteHasRunnableTests(suite, ctx)) {
718 yield* Effect.forEach(
719 runnable,
720 (child) =>
721 child.type === "test" ? runTest(child, ctx) : runSuite(child, ctx),
722 { discard: true },
723 );
724 return;
725 }
726
727 // beforeAll — captured into the file-level log buffer. Emits hook events
728 // so the TUI can show "setting up" instead of an unexplained queue.
729 const previousSetup = ctx.suiteStates?.get(suite);
730 if (previousSetup !== undefined && Exit.isFailure(previousSetup)) {
731 yield* failSubtree(suite, ctx, prettyCause(previousSetup.cause));
732 return;
733 }
734 if (previousSetup === undefined && suite.beforeAll.length > 0) {
735 yield* ctx.emit({ _tag: "HookStart", file: ctx.file, hook: "beforeAll" });
736 // Honor `{ exclusive: true }` on the hook (Hetzner quota, Railway
737 // plugin DBs). Non-exclusive beforeAll keeps the default 1-permit
738 // slot so unrelated files can still run concurrently.
739 const exit = yield* ctx.lock
740 .withPermits(hookPermits(suite.beforeAll))(
741 runHooks(suite.beforeAll, ctx.options.timeout),
742 )
743 .pipe(withCapture(ctx.fileLogs), Effect.exit);
744 yield* ctx.emit({ _tag: "HookEnd", file: ctx.file, hook: "beforeAll" });
745 ctx.suiteStates?.set(suite, exit);
746 if (Exit.isFailure(exit)) {
747 yield* failSubtree(suite, ctx, prettyCause(exit.cause));
748 if (ctx.suiteStates === undefined) yield* runAfterAll(suite, ctx);
749 return;
750 }
751 }
752
753 if (ctx.suiteStates !== undefined && !ctx.suiteStates.has(suite))
754 ctx.suiteStates.set(suite, Exit.succeed(undefined));
755
756 const sequential = suite.sequential || ctx.options.sequential;
757 yield* Effect.forEach(
758 runnable,
759 (child) =>
760 child.type === "test" ? runTest(child, ctx) : runSuite(child, ctx),
761 { concurrency: sequential ? 1 : "unbounded", discard: true },
762 );
763
764 if (ctx.suiteStates === undefined) yield* runAfterAll(suite, ctx);
765 });
766
767const runAfterAll = Effect.fn(function* (suite: Suite, ctx: ExecContext) {
768 if (suite.afterAll.length === 0) return;
769 yield* ctx.emit({ _tag: "HookStart", file: ctx.file, hook: "afterAll" });
770 // Unlike beforeAll (where a failure invalidates everything after it),
771 // every teardown hook runs even when an earlier one fails: a failing
772 // teardown assertion must not drop later cleanup — in particular
773 // Test.make's fallback hook that closes the shared scope and local
774 // provider sidecar, which registers last and would otherwise leak the
775 // sidecar for the rest of the process. Failures aggregate.
776 const afterAllRun = Effect.forEach(suite.afterAll, (hook) =>
777 Effect.suspend(hook.body).pipe(
778 Effect.timeout(Duration.millis(hook.timeout ?? ctx.options.timeout)),
779 Effect.exit,
780 ),
781 );
782 const exits = yield* ctx.lock
783 .withPermits(hookPermits(suite.afterAll))(afterAllRun)
784 .pipe(withCapture(ctx.fileLogs));
785 yield* ctx.emit({ _tag: "HookEnd", file: ctx.file, hook: "afterAll" });
786 const failures = exits.filter(Exit.isFailure);
787 if (failures.length > 0) {
788 const error = `afterAll hook failed:\n${failures
789 .map((exit) => prettyCause(exit.cause))
790 .join("\n")}`;
791 ctx.fileErrors.push(error);
792 }
793});
794
795const suiteHasIncludedTests = (
796 suite: Suite,
797 ctx: Pick<ExecContext, "onlyMode" | "options" | "file">,
798): boolean => {
799 for (const child of suite.children) {
800 if (child.type === "test" && included(child, ctx)) return true;
801 if (child.type === "suite" && suiteHasIncludedTests(child, ctx))
802 return true;
803 }
804 return false;
805};
806
807/** True if the subtree has at least one included test that will actually run. */
808const suiteHasRunnableTests = (
809 suite: Suite,
810 ctx: Pick<ExecContext, "onlyMode" | "options" | "file">,
811): boolean => {
812 for (const child of suite.children) {
813 if (
814 child.type === "test" &&
815 included(child, ctx) &&
816 isSkipped(child) === undefined
817 ) {
818 return true;
819 }
820 if (child.type === "suite" && suiteHasRunnableTests(child, ctx))
821 return true;
822 }
823 return false;
824};
825
826// ---------------------------------------------------------------------------
827// run
828// ---------------------------------------------------------------------------
829
830export const run = Effect.fn(function* (input: RunOptions) {
831 const options = {
832 ...input,
833 tagsFilter: compileTagsFilter(input.tagsFilter ?? []),
834 };
835 const reporter = yield* Reporter;
836 const path = yield* Path.Path;
837 const startedAt = Date.now();
838
839 // Every event is teed into the persistent run log (`.alchemy/log/test.log`)
840 // in addition to the active reporter.
841 const fileLog = yield* makeFileLog(options.logFile);
842 const emit = (event: TestEvent): Effect.Effect<void> =>
843 reporter.emit(event).pipe(Effect.andThen(fileLog.append(event)));
844
845 const absoluteFiles = yield* discover(input).pipe(Effect.orDie);
846 const relative = absoluteFiles.map((f) => path.relative(options.root, f));
847 yield* emit({ _tag: "CollectStart", files: relative });
848
849 // Import every file before running anything so `.only` applies across
850 // the whole run. AsyncLocalStorage keeps registrations attached to their
851 // file while imports overlap. Bound collection to avoid loader saturation.
852 const collectConcurrency =
853 options.concurrency === "unbounded"
854 ? 32
855 : Math.min(options.concurrency, 32);
856 // collectFile never fails — import errors are captured on the result.
857 const collected = yield* Effect.forEach(
858 absoluteFiles.map((absolute, i) => [absolute, relative[i]!] as const),
859 ([absolute, rel]) =>
860 collectFile(absolute, rel).pipe(
861 Effect.tap(() => emit({ _tag: "FileCollected", file: rel })),
862 ),
863 { concurrency: collectConcurrency },
864 );
865
866 const onlyMode = collected.some(
867 (c) => c.suite !== undefined && containsOnly(c.suite),
868 );
869
870 // Assign each case once, in phase/branch declaration order, before execution.
871 const assigned = new Set<TestCase>();
872 const remainingFileBranches = new Map<string, number>();
873 const candidates: Array<{ file: string; test: TestCase }> = [];
874 const candidateOptions = {
875 ...options,
876 tagsFilter: (
877 tags: ReadonlyArray<string>,
878 optInTags: ReadonlyArray<string> = [],
879 ) => options.tagsFilter([...tags, ...optInTags], []),
880 };
881 for (const c of collected) {
882 if (c.suite !== undefined)
883 forEachTest(c.suite, (test) => {
884 if (
885 included(test, { onlyMode, options: candidateOptions, file: c.file })
886 )
887 candidates.push({ file: c.file, test });
888 });
889 }
890 const phases = (input.plan ?? [{ tags: [] }]).map((phase) =>
891 (Array.isArray(phase) ? phase : [phase]).map((branch) => {
892 const tagsFilter = compileTagsFilter([
893 ...(input.tagsFilter ?? []),
894 ...branch.tags,
895 ]);
896 const selectedTests = new Set<TestCase>();
897 const selectedFiles = new Set<string>();
898 for (const { file, test } of candidates) {
899 if (!assigned.has(test) && tagsFilter(test.tags, test.optInTags)) {
900 assigned.add(test);
901 selectedFiles.add(file);
902 selectedTests.add(test);
903 }
904 }
905 for (const file of selectedFiles) {
906 remainingFileBranches.set(
907 file,
908 (remainingFileBranches.get(file) ?? 0) + 1,
909 );
910 }
911 return {
912 ...options,
913 tagsFilter,
914 expressions: [...(input.tagsFilter ?? []), ...branch.tags],
915 selectedTests,
916 concurrency: branch.concurrency ?? options.concurrency,
917 };
918 }),
919 );
920 const allMetas = candidates
921 .filter(({ test }) => assigned.has(test))
922 .map(({ file, test }) => metaOf(file, test));
923 if (input.dryRun) {
924 yield* emit({
925 _tag: "PlanPreview",
926 phases: phases.map((branches) =>
927 branches.map((branch) => {
928 const matched = candidates.filter(({ test }) =>
929 branch.selectedTests.has(test),
930 );
931 return {
932 tags: branch.expressions,
933 concurrency: branch.concurrency,
934 tests: matched.length,
935 files: new Set(matched.map(({ file }) => file)).size,
936 skipped: matched.filter(({ test }) => isSkipped(test) !== undefined)
937 .length,
938 };
939 }),
940 ),
941 });
942 const fileFailures = collected.flatMap((c) =>
943 c.error === undefined ? [] : [{ file: c.file, error: c.error }],
944 );
945 for (const failure of fileFailures)
946 yield* emit({ _tag: "FileEnd", ...failure, logs: [] });
947 const summary: RunSummary = {
948 dryRun: true,
949 files: collected.length,
950 plan: {
951 found: candidates.length,
952 selected: assigned.size,
953 excluded: candidates.length - assigned.size,
954 },
955 passed: 0,
956 failed: fileFailures.length,
957 skipped: 0,
958 todo: 0,
959 durationMs: Date.now() - startedAt,
960 failures: [],
961 fileFailures,
962 };
963 yield* emit({ _tag: "RunEnd", summary });
964 yield* fileLog.close;
965 return summary;
966 }
967 yield* emit({
968 _tag: "RunStart",
969 files: collected.length,
970 tests: allMetas,
971 });
972
973 // Phase 2 — run files concurrently.
974 const allResults: Array<{ meta: TestMeta; result: TestResult }> = [];
975 const fileFailures: Array<{ file: string; error: string }> = [];
976 const lock = yield* Semaphore.make(EXCLUSIVE_PERMITS);
977 const running = new Map<string, Fiber.Fiber<unknown, unknown>>();
978 const completed = new Set<string>();
979 const testIndex = new Map<string, { test: TestCase; ctx: ExecContext }>();
980
981 // Interactive control (TUI `r` retry / `x` kill). Retried tests re-run as
982 // standalone forked fibers and re-emit TestStart/TestEnd through the same
983 // reporter; the header counters simply update in place.
984 const controller: RunController = {
985 retryTest: (id) => {
986 const entry = testIndex.get(id);
987 if (entry === undefined || running.has(id) || !completed.has(id)) return;
988 completed.delete(id);
989 Effect.runFork(runTest(entry.test, entry.ctx));
990 },
991 retryFile: (file) => {
992 for (const [id, entry] of testIndex) {
993 if (entry.ctx.file === file) controller.retryTest(id);
994 }
995 },
996 killTest: (id) => {
997 const fiber = running.get(id);
998 if (fiber !== undefined) Effect.runFork(Fiber.interrupt(fiber));
999 },
1000 };
1001 if (reporter.attachController !== undefined) {
1002 yield* reporter.attachController(controller);
1003 }
1004
1005 // Array whose pushes are teed to `tee` as they happen. File-hook output is
1006 // captured into this buffer over the (potentially very long) life of a
1007 // deploy/destroy hook; teeing each entry into the run log keeps the log
1008 // live instead of silent-until-FileEnd (which reads as a deadlocked run).
1009 const liveHookLogBuffer = (
1010 tee: (entry: LogEntry) => void,
1011 ): Array<LogEntry> => {
1012 const buffer: Array<LogEntry> = [];
1013 const push = Array.prototype.push.bind(buffer);
1014 buffer.push = (...entries: Array<LogEntry>) => {
1015 for (const entry of entries) tee(entry);
1016 return push(...entries);
1017 };
1018 return buffer;
1019 };
1020
1021 const fileContexts = new Map<string, ExecContext>();
1022 const fileLocks = new Map<string, Semaphore.Semaphore>();
1023 for (const c of collected) fileLocks.set(c.file, yield* Semaphore.make(1));
1024 const runFile = Effect.fn(function* (
1025 c: CollectedFile,
1026 branchOptions: ExecContext["options"],
1027 ) {
1028 const previous = fileContexts.get(c.file);
1029 const fileLogs =
1030 previous?.fileLogs ??
1031 liveHookLogBuffer((entry) => fileLog.appendHookLine(c.file, entry));
1032 const fileErrors = previous?.fileErrors ?? [];
1033 // Shares the LIVE hook-log buffer so the TUI can tail deploys.
1034 if (previous === undefined)
1035 yield* emit({ _tag: "FileStart", file: c.file, logs: fileLogs });
1036 let fileError = c.error;
1037 if (c.suite !== undefined) {
1038 const ctx: ExecContext = {
1039 options: branchOptions,
1040 suiteStates:
1041 input.plan === undefined
1042 ? undefined
1043 : (previous?.suiteStates ?? new Map()),
1044 onlyMode,
1045 emit,
1046 fileLogs,
1047 fileErrors,
1048 results: allResults,
1049 file: c.file,
1050 lock,
1051 running,
1052 completed,
1053 };
1054 fileContexts.set(c.file, ctx);
1055 forEachTest(c.suite, (test) => {
1056 if (included(test, ctx) && isSkipped(test) === undefined) {
1057 testIndex.set(metaOf(c.file, test).id, { test, ctx });
1058 }
1059 });
1060 const exit = yield* runSuite(c.suite, ctx).pipe(Effect.exit);
1061 if (Exit.isFailure(exit)) {
1062 fileError = prettyCause(exit.cause);
1063 } else if (input.plan === undefined && fileErrors.length > 0) {
1064 fileError = fileErrors.join("\n\n");
1065 }
1066 }
1067 if (fileError !== undefined) {
1068 fileFailures.push({ file: c.file, error: fileError });
1069 }
1070 if (input.plan !== undefined && c.suite !== undefined) {
1071 // Count completed visits rather than declaration order: branches in a
1072 // phase run concurrently, and any of them may finish this file last.
1073 const remaining = remainingFileBranches.get(c.file)! - 1;
1074 remainingFileBranches.set(c.file, remaining);
1075 if (remaining > 0) return;
1076
1077 // Keep scopes only while another selected branch still needs the file.
1078 // Teardown stays inside its concurrency slot and per-file lock so new
1079 // files cannot accumulate resources retained until the end of a phase.
1080 const ctx = fileContexts.get(c.file)!;
1081 for (const suite of [...ctx.suiteStates!.keys()].reverse())
1082 yield* runAfterAll(suite, ctx);
1083 if (ctx.fileErrors.length > 0)
1084 fileFailures.push({
1085 file: ctx.file,
1086 error: ctx.fileErrors.join("\n\n"),
1087 });
1088 fileError =
1089 fileFailures
1090 .filter((failure) => failure.file === c.file)
1091 .map((failure) => failure.error)
1092 .join("\n\n") || undefined;
1093 }
1094 yield* emit({
1095 _tag: "FileEnd",
1096 file: c.file,
1097 logs: fileLogs,
1098 error: fileError,
1099 });
1100 });
1101
1102 // Import errors are reported once even if no branch selects that file.
1103 yield* Effect.forEach(
1104 collected.filter((c) => c.error !== undefined),
1105 (c) => runFile(c, options),
1106 );
1107 for (const [index, branches] of phases.entries()) {
1108 if (input.plan !== undefined)
1109 yield* emit({
1110 _tag: "PlanPhaseStart",
1111 phase: index + 1,
1112 phases: phases.length,
1113 tests: branches.reduce(
1114 (count, branch) => count + branch.selectedTests.size,
1115 0,
1116 ),
1117 });
1118 yield* Effect.forEach(
1119 branches,
1120 (branch) =>
1121 Effect.forEach(
1122 collected.filter(
1123 (c) =>
1124 c.suite !== undefined &&
1125 suiteHasIncludedTests(c.suite, {
1126 file: c.file,
1127 onlyMode,
1128 options: branch,
1129 }),
1130 ),
1131 (c) => fileLocks.get(c.file)!.withPermits(1)(runFile(c, branch)),
1132 { concurrency: branch.concurrency, discard: true },
1133 ),
1134 { concurrency: "unbounded", discard: true },
1135 );
1136 }
1137
1138 const failures = allResults.filter((r) => r.result.status === "fail");
1139 const summary: RunSummary = {
1140 files: collected.length,
1141 plan:
1142 input.plan === undefined
1143 ? undefined
1144 : {
1145 found: candidates.length,
1146 selected: assigned.size,
1147 excluded: candidates.length - assigned.size,
1148 },
1149 passed: allResults.filter((r) => r.result.status === "pass").length,
1150 failed: failures.length + fileFailures.length,
1151 skipped: allResults.filter((r) => r.result.status === "skip").length,
1152 todo: allResults.filter((r) => r.result.status === "todo").length,
1153 durationMs: Date.now() - startedAt,
1154 failures,
1155 fileFailures,
1156 };
1157 yield* emit({ _tag: "RunEnd", summary });
1158 // Drain the live hook-line queue so tail lines from the final file's
1159 // hooks are on disk before the process exits.
1160 yield* fileLog.close;
1161 return summary;
1162});