1/**
2 * Generates `public/llms.txt` and `public/llms-full.txt` — navigation
3 * indexes of the docs pages, grouped by section, with title + description
4 * pulled from each page's frontmatter.
5 *
6 * `llms.txt` covers the guides and concept docs and links to
7 * `llms-full.txt`, which additionally lists every generated service-level
8 * API reference page (the Providers section) and is much larger.
9 *
10 * Run with: `bun scripts/generate-llms-txt.ts`
11 *
12 * Section ordering, headings, and prose intros are configured here.
13 * Page metadata (title, description) comes from the source frontmatter,
14 * so editing a page's frontmatter is enough to update llms.txt.
15 */
16
17import { readdir, readFile, writeFile } from "node:fs/promises";
18import path from "node:path";
19import { fileURLToPath } from "node:url";
20
21const here = path.dirname(fileURLToPath(import.meta.url));
22const docsDir = path.resolve(here, "../src/content/docs");
23const outFile = path.resolve(here, "../public/llms.txt");
24const outFullFile = path.resolve(here, "../public/llms-full.txt");
25const siteUrl = "https://alchemy.run";
26
27interface Page {
28 /** URL path, e.g. "/infrastructure-as-effects/binding" */
29 href: string;
30 /** Path relative to docs dir without extension, e.g. "infrastructure-as-effects/binding" */
31 slug: string;
32 title: string;
33 description: string;
34 draft: boolean;
35 /** `sidebar.order` from frontmatter; `Infinity` when unset. */
36 order: number;
37}
38
39interface Section {
40 /** H2 heading */
41 heading: string;
42 /** Optional prose paragraph after the heading. */
43 intro?: string;
44 /**
45 * Pages to include. Either a list of explicit slugs (relative to docs dir,
46 * no extension) in the desired order, or a directory to enumerate
47 * alphabetically.
48 */
49 pages: { slugs: string[] } | { directory: string; exclude?: string[] };
50}
51
52const SECTIONS: Section[] = [
53 {
54 heading: "Start here",
55 pages: {
56 slugs: ["what-is-alchemy", "getting-started", "migrating-from-v1"],
57 },
58 },
59 {
60 heading: "CLI",
61 intro:
62 "Every command runs through your package manager: bun alchemy <command>. Command pages by role, plus the Adopting Resources and Inspecting State guides.",
63 pages: {
64 slugs: [
65 "cli/index",
66 "cli/deploy",
67 "cli/plan",
68 "cli/destroy",
69 "cli/drift",
70 "cli/nuke",
71 "cli/adopting-resources",
72 "cli/dev",
73 "cli/logs",
74 "cli/profile",
75 "cli/state",
76 "cli/inspecting-state",
77 "cli/aws",
78 "cli/cloudflare",
79 ],
80 },
81 },
82 {
83 heading: "Infrastructure as Code",
84 intro:
85 "The noun graph and its semantics: Stacks of Resources and Actions, data flow via Inputs & Outputs and References, deploy behavior via the Resource lifecycle and Providers.",
86 pages: {
87 slugs: [
88 "infrastructure-as-code/stack",
89 "infrastructure-as-code/resource",
90 "infrastructure-as-code/action",
91 "infrastructure-as-code/outputs",
92 "infrastructure-as-code/references",
93 "infrastructure-as-code/resource-lifecycle",
94 "infrastructure-as-code/provider",
95 "infrastructure-as-code/custom-provider",
96 ],
97 },
98 },
99 {
100 heading: "Infrastructure as Effects",
101 intro:
102 "How app code and infrastructure compose: Runtime, Bindings, the Construction and Runtime Phases, and Layers.",
103 pages: {
104 slugs: [
105 "infrastructure-as-effects/runtime",
106 "infrastructure-as-effects/binding",
107 "infrastructure-as-effects/layers",
108 "infrastructure-as-effects/event-sources",
109 "infrastructure-as-effects/sinks",
110 "infrastructure-as-effects/phases",
111 "infrastructure-as-effects/circular-bindings",
112 "infrastructure-as-effects/custom-runtime",
113 ],
114 },
115 },
116 {
117 heading: "APIs",
118 intro:
119 "Typed calls between Functions and Servers: schemaless RPC for internal communication, Effect RPC and Effect HTTP for trust boundaries.",
120 pages: {
121 slugs: [
122 "apis/index",
123 "apis/schemaless",
124 "apis/effect-rpc",
125 "apis/effect-http",
126 ],
127 },
128 },
129 {
130 heading: "Environments — the same app in many places",
131 intro:
132 "Stages, credential profiles, auth providers, secrets and config, local development, and CI.",
133 pages: {
134 slugs: [
135 "environments/stages",
136 "environments/profiles",
137 "environments/auth-providers",
138 "environments/custom-auth-provider",
139 "environments/secrets",
140 "environments/local-development",
141 "environments/ci",
142 ],
143 },
144 },
145 {
146 heading: "State Store",
147 pages: { slugs: ["state-store/index", "state-store/custom-state-store"] },
148 },
149 {
150 heading: "Project structure",
151 intro:
152 "Scaling the codebase: file conventions for one stack, monorepos for many — single Stack or one per package.",
153 pages: {
154 slugs: ["project-structure/file-layout", "project-structure/monorepo"],
155 },
156 },
157 {
158 heading: "Testing & observability",
159 intro:
160 "Tests run against real clouds: the model, the end-to-end walkthrough, provider-lifecycle testing, and the harness reference.",
161 pages: {
162 slugs: [
163 "testing/index",
164 "testing/testing-a-stack",
165 "testing/testing-providers",
166 "testing/test-harness",
167 "testing/observability",
168 ],
169 },
170 },
171 {
172 heading: "Cloudflare — start here",
173 intro:
174 "The Cloudflare hub: overview (resources + recipes) and setup (install, account, OAuth vs API token, profiles).",
175 pages: { slugs: ["cloudflare/index", "cloudflare/setup"] },
176 },
177 {
178 heading: "Cloudflare — tutorial",
179 intro:
180 "A linear five-part walkthrough from zero to a tested, locally-developed, CI-deployed Cloudflare project. Each part builds on the previous one.",
181 pages: {
182 slugs: [
183 "cloudflare/tutorial/part-1",
184 "cloudflare/tutorial/part-2",
185 "cloudflare/tutorial/part-3",
186 "cloudflare/tutorial/part-4",
187 "cloudflare/tutorial/part-5",
188 ],
189 },
190 },
191 {
192 heading: "Cloudflare — Compute",
193 pages: { directory: "cloudflare/compute" },
194 },
195 {
196 heading: "Cloudflare — Frontend",
197 pages: { directory: "cloudflare/frontend" },
198 },
199 {
200 heading: "Cloudflare — APIs",
201 pages: { directory: "cloudflare/apis" },
202 },
203 {
204 heading: "Cloudflare — Data",
205 pages: { directory: "cloudflare/data" },
206 },
207 {
208 heading: "Cloudflare — Messaging & events",
209 pages: { directory: "cloudflare/messaging" },
210 },
211 {
212 heading: "Cloudflare — Email",
213 pages: { directory: "cloudflare/email" },
214 },
215 {
216 heading: "Cloudflare — AI",
217 pages: { directory: "cloudflare/ai" },
218 },
219 {
220 heading: "Cloudflare — Security & secrets",
221 pages: { directory: "cloudflare/security" },
222 },
223 {
224 heading: "Cloudflare — Observability",
225 pages: { directory: "cloudflare/observability" },
226 },
227 {
228 heading: "Cloudflare — Networking",
229 pages: { directory: "cloudflare/networking" },
230 },
231 {
232 heading: "Prisma — start here",
233 pages: { slugs: ["prisma/index", "prisma/setup"] },
234 },
235 {
236 heading: "Prisma — tutorial",
237 intro:
238 "A four-part walkthrough: a Project, an Effect-native Compute API, Postgres, and a Vite frontend.",
239 pages: { directory: "prisma/tutorial" },
240 },
241 {
242 heading: "Prisma — Compute",
243 pages: { directory: "prisma/compute" },
244 },
245 {
246 heading: "Prisma — Data",
247 pages: { directory: "prisma/data" },
248 },
249 {
250 heading: "Prisma — Frontend",
251 pages: { directory: "prisma/frontend" },
252 },
253 {
254 heading: "Prisma — Guides",
255 pages: { directory: "prisma/guides" },
256 },
257 {
258 heading: "AWS — start here",
259 intro:
260 "The AWS hub: overview (runtimes + resources + recipes), setup (credentials, profiles, region), and the Lambda vs ECS vs EKS vs EC2 decision page.",
261 pages: {
262 slugs: ["aws/index", "aws/setup", "aws/compute/choosing-a-runtime"],
263 },
264 },
265 {
266 heading: "AWS — tutorial",
267 intro:
268 "A linear five-part walkthrough from zero to a tested, CI-deployed AWS project. Each part builds on the previous one.",
269 pages: {
270 slugs: [
271 "aws/tutorial/part-1",
272 "aws/tutorial/part-2",
273 "aws/tutorial/part-3",
274 "aws/tutorial/part-4",
275 "aws/tutorial/part-5",
276 ],
277 },
278 },
279 {
280 heading: "AWS — Compute",
281 pages: { directory: "aws/compute" },
282 },
283 {
284 heading: "AWS — Frontend",
285 pages: { directory: "aws/frontend" },
286 },
287 {
288 heading: "AWS — APIs",
289 pages: { directory: "aws/apis" },
290 },
291 {
292 heading: "AWS — Data",
293 pages: { directory: "aws/data" },
294 },
295 {
296 heading: "AWS — Messaging & events",
297 pages: { directory: "aws/messaging" },
298 },
299 {
300 heading: "AWS — Email",
301 pages: { directory: "aws/email" },
302 },
303 {
304 heading: "AWS — Security & secrets",
305 pages: { directory: "aws/security" },
306 },
307 {
308 heading: "AWS — Observability",
309 pages: { directory: "aws/observability" },
310 },
311 {
312 heading: "AWS — Networking",
313 pages: { directory: "aws/networking" },
314 },
315 {
316 heading: "GCP",
317 intro:
318 "Google Cloud as Stack resources — Cloud Run services and jobs, Firestore, Pub/Sub, BigQuery, Secret Manager, Memorystore. Bindings grant IAM onto a per-host service account.",
319 pages: {
320 slugs: [
321 "gcp/index",
322 "gcp/setup",
323 "gcp/guides/cloud-run-api",
324 "gcp/guides/event-pipeline",
325 "gcp/guides/memorystore",
326 "gcp/guides/bindings",
327 ],
328 },
329 },
330 {
331 heading: "Fly — start here",
332 intro:
333 "The Fly hub: overview (Apps, Machines, Services, Sprites) and setup (org, API token, profile).",
334 pages: {
335 slugs: ["fly/index", "fly/setup"],
336 },
337 },
338 {
339 heading: "Fly — SQL",
340 pages: { slugs: ["fly/data/postgres", "fly/data/drizzle-postgres"] },
341 },
342 {
343 heading: "Fly — Frontend",
344 pages: { directory: "fly/frontend" },
345 },
346 {
347 heading: "Hetzner — start here",
348 intro:
349 "The Hetzner hub: overview (Servers and Services) and setup (project, API token, profile).",
350 pages: {
351 slugs: ["hetzner/index", "hetzner/setup"],
352 },
353 },
354 {
355 heading: "Hetzner — SQL",
356 pages: { slugs: ["hetzner/data/drizzle-postgres"] },
357 },
358 {
359 heading: "Hetzner — Frontend",
360 pages: { directory: "hetzner/frontend" },
361 },
362 {
363 heading: "Railway — start here",
364 intro:
365 "The Railway hub: overview (Projects, Services, databases) and setup (workspace, API token, profile).",
366 pages: {
367 slugs: ["railway/index", "railway/setup"],
368 },
369 },
370 {
371 heading: "Railway — SQL",
372 pages: {
373 slugs: [
374 "railway/data/postgres",
375 "railway/data/mysql",
376 "railway/data/drizzle-postgres",
377 "railway/data/drizzle-mysql",
378 ],
379 },
380 },
381 {
382 heading: "Prisma — SQL",
383 pages: {
384 slugs: [
385 "prisma/data/postgres",
386 "prisma/data/branches",
387 "prisma/data/connections",
388 "prisma/data/drizzle-postgres",
389 ],
390 },
391 },
392 {
393 heading: "Railway — Frontend",
394 pages: { directory: "railway/frontend" },
395 },
396 {
397 heading: "PlanetScale",
398 intro:
399 "Serverless MySQL & Postgres as Stack resources. Composes with Cloudflare Hyperdrive and Drizzle — those guides are listed under Cloudflare.",
400 pages: {
401 slugs: [
402 "planetscale/index",
403 "planetscale/setup",
404 "planetscale/data/postgres",
405 "planetscale/data/mysql",
406 "planetscale/data/migrations",
407 "planetscale/data/credentials",
408 "planetscale/data/backups",
409 "planetscale/guides/preview-branches",
410 "planetscale/guides/drizzle",
411 ],
412 },
413 },
414 {
415 heading: "Neon",
416 intro:
417 "Neon backend resources: Postgres branches, Functions, private object storage, managed Auth, Data API, AI Gateway and Website adapters. The upload tutorial combines native and Effect handlers with authenticated browser uploads.",
418 pages: {
419 slugs: [
420 "neon/index",
421 "neon/setup",
422 "neon/governance",
423 "neon/guides/ai-gateway",
424 "neon/guides/production-auth",
425 "neon/guides/private-networking",
426 "neon/guides/custom-domains",
427 "neon/guides/state-recovery",
428 "neon/tutorial/index",
429 "neon/tutorial/backend",
430 "neon/tutorial/functions",
431 "neon/tutorial/frontend",
432 "neon/tutorial/previews",
433 "neon/data/branching",
434 "neon/data/connections",
435 "neon/data/migrations",
436 "neon/guides/preview-branches",
437 "neon/guides/drizzle",
438 ],
439 },
440 },
441 {
442 heading: "Axiom",
443 intro:
444 "Observability as Stack resources — datasets, monitors, notifiers. The exporter-Layer pattern is documented in Concepts → Observability.",
445 pages: {
446 slugs: [
447 "axiom/index",
448 "axiom/setup",
449 "axiom/data/ingest",
450 "axiom/guides/alerting",
451 "axiom/guides/dashboards",
452 "axiom/guides/annotations",
453 ],
454 },
455 },
456 {
457 heading: "GitHub",
458 intro:
459 "Repos, secrets, variables, and repository event sources as Stack resources; the CI/CD guides live under Guides and each cloud's tutorial part 5.",
460 pages: {
461 slugs: [
462 "github/index",
463 "github/setup",
464 "github/repository",
465 "github/actions-config",
466 "github/events",
467 ],
468 },
469 },
470 {
471 heading: "Git",
472 intro:
473 "A git server you build in one file, use like any remote, and reshape one block at a time: setup, a four-part tutorial (build it, repositories and tokens, your own rules, your app on the same domain), the REST and GitHub-compatible planes, the building blocks with their implementations, and recipes with the scaling model and measurements.",
474 pages: {
475 slugs: [
476 "git/index",
477 "git/getting-started",
478 "git/tutorial/part-1",
479 "git/tutorial/part-2",
480 "git/tutorial/part-3",
481 "git/tutorial/part-4",
482 "git/clone-and-push",
483 "git/repositories",
484 "git/pull-requests",
485 "git/github-api",
486 "git/blocks/index",
487 "git/blocks/server",
488 "git/blocks/repositories",
489 "git/blocks/registry",
490 "git/blocks/blob-store",
491 "git/blocks/hasher",
492 "git/blocks/auth",
493 "git/recipes/index",
494 "git/recipes/scaling",
495 "git/recipes/cloudflare",
496 "git/recipes/cloudflare-aws",
497 "git/recipes/your-own-store",
498 ],
499 },
500 },
501 {
502 heading: "Stripe",
503 intro:
504 "Products, prices, coupons, customers, payment links, and webhooks as Stack resources. HTTP bindings run inside a Worker or Lambda.",
505 pages: {
506 slugs: [
507 "stripe/index",
508 "stripe/setup",
509 "stripe/guides/subscriptions",
510 "stripe/guides/connect",
511 "stripe/guides/webhooks",
512 ],
513 },
514 },
515 {
516 heading: "Docker",
517 intro:
518 "Local and CI Docker as Stack resources — images, containers, networks, and volumes driven through the active Docker CLI context; cloud container runtimes (Cloudflare Containers, ECS) consume the pushed image refs from their own hubs.",
519 pages: {
520 slugs: [
521 "docker/index",
522 "docker/setup",
523 "docker/local-services",
524 "docker/build-and-push",
525 ],
526 },
527 },
528 {
529 heading: "Kubernetes",
530 intro:
531 "Cluster-agnostic Kubernetes workloads — Deployments, Jobs and CronJobs (as images or Effect programs), raw manifests, and Helm charts — on a local cluster or any cluster reachable through a kubeconfig, token, client certificate, or exec plugin.",
532 pages: {
533 slugs: [
534 "kubernetes/index",
535 "kubernetes/setup",
536 "kubernetes/tutorial/part-1",
537 "kubernetes/tutorial/part-2",
538 "kubernetes/tutorial/part-3",
539 "kubernetes/tutorial/part-4",
540 "kubernetes/tutorial/part-5",
541 "kubernetes/clusters/connecting",
542 "kubernetes/clusters/registries",
543 "kubernetes/clusters/local",
544 "kubernetes/clusters/eks",
545 "kubernetes/clusters/cluster-adapters",
546 "kubernetes/workloads/deployments",
547 "kubernetes/workloads/jobs",
548 "kubernetes/workloads/images",
549 "kubernetes/workloads/bindings",
550 "kubernetes/workloads/object-lifecycle",
551 "kubernetes/objects/manifests",
552 "kubernetes/objects/helm-charts",
553 ],
554 },
555 },
556 {
557 heading: "SQL",
558 intro:
559 "Database comparison, Effect SQL, Drizzle, Prisma ORM v8 (Postgres-only), committed migrations, and connection lifecycle. Deployment walkthroughs live with their runtime providers.",
560 pages: {
561 slugs: [
562 "sql/index",
563 "sql/databases",
564 "sql/effect-sql/postgres",
565 "sql/effect-sql/mysql",
566 "sql/effect-sql/d1",
567 "sql/effect-sql/migrations",
568 "sql/effect-sql/lifecycle",
569 "sql/drizzle/postgres",
570 "sql/drizzle/mysql",
571 "sql/drizzle/d1",
572 "sql/drizzle/migrations",
573 "sql/prisma/postgres",
574 "sql/prisma/contracts",
575 "sql/prisma/migrations",
576 ],
577 },
578 },
579 {
580 heading: "Command",
581 intro:
582 "Cloud-agnostic local process primitives — memoized builds, one-off commands, and dev servers; the static-site guides in each cloud hub consume them under the hood.",
583 pages: {
584 slugs: ["command/index", "command/memoization", "command/dev-servers"],
585 },
586 },
587];
588
589const PROVIDERS_INTRO_BASE = `Service-level API reference, generated from JSDoc on the source \`.ts\` files via \`bun scripts/generate-api-reference.ts\`. Each page groups resources and capabilities by service, with a heading per resource and examples extracted from source JSDoc.`;
590
591const PROVIDERS_INTRO = `${PROVIDERS_INTRO_BASE} Grouped by cloud below.`;
592
593/**
594 * Enumerates every generated provider page under `providers/{Cloud}/...`,
595 * grouped by cloud. These pages are produced by `build:reference` (which runs
596 * before this script in the build), so they exist on disk at generation time
597 * even though they are gitignored.
598 */
599async function renderProvidersSection(): Promise<string> {
600 const providersDir = path.join(docsDir, "providers");
601 let clouds: string[];
602 try {
603 const entries = await readdir(providersDir, { withFileTypes: true });
604 clouds = entries
605 .filter((e) => e.isDirectory())
606 .map((e) => e.name)
607 .sort();
608 } catch (err: any) {
609 if (err?.code === "ENOENT") return `## Providers\n\n${PROVIDERS_INTRO}`;
610 throw err;
611 }
612
613 const blocks: string[] = [`## Providers`, PROVIDERS_INTRO];
614 for (const cloud of clouds) {
615 const slugs = await listSlugs(`providers/${cloud}`);
616 const pages = (await Promise.all(slugs.map(loadPage)))
617 .filter((p) => !p.draft)
618 // Starlight serves provider routes lowercased (e.g. the CamelCase source
619 // `providers/AWS/S3/Bucket.md` is reachable at `/providers/aws/s3/bucket`).
620 .map((p) => ({ ...p, href: p.href.toLowerCase() }))
621 .sort((a, b) => a.title.localeCompare(b.title));
622 if (pages.length === 0) continue;
623 blocks.push(`### ${cloud}`);
624 blocks.push(pages.map(renderPage).join("\n"));
625 }
626 return blocks.join("\n\n");
627}
628
629const HEADER = `# Alchemy
630
631> Alchemy Effect is an Infrastructure-as-Effects (IaE) framework that combines cloud infrastructure and application logic into a single, type-safe program powered by [Effect](https://effect.website). Resources are declared as Effects; bindings wire IAM, env vars, and typed SDKs in one call; deploys and runtime share the same code.
632
633This file is a navigation index for the documentation site at ${siteUrl}. Guide and concept pages are listed below with their URL and a one-line summary, so an agent can pick the right page in one hop. The service-level API reference is indexed separately in [llms-full.txt](${siteUrl}/llms-full.txt).`;
634
635const FULL_HEADER = `# Alchemy
636
637> Alchemy Effect is an Infrastructure-as-Effects (IaE) framework that combines cloud infrastructure and application logic into a single, type-safe program powered by [Effect](https://effect.website). Resources are declared as Effects; bindings wire IAM, env vars, and typed SDKs in one call; deploys and runtime share the same code.
638
639This file is a navigation index for the documentation site at ${siteUrl}. Every page under \`/src/content/docs/\` is listed below with its URL and a one-line summary, so an agent can pick the right page in one hop. This is the full index including every service-level API reference page; the smaller [llms.txt](${siteUrl}/llms.txt) covers just the guides and concept docs.`;
640
641function parseFrontmatter(source: string): Record<string, string> {
642 if (!source.startsWith("---")) return {};
643 const end = source.indexOf("\n---", 3);
644 if (end === -1) return {};
645 const block = source.slice(3, end);
646 const out: Record<string, string> = {};
647 for (const rawLine of block.split("\n")) {
648 const line = rawLine.trimEnd();
649 const m = line.match(/^([A-Za-z_][A-Za-z0-9_-]*):\s*(.*)$/);
650 if (!m) continue;
651 let value = m[2].trim();
652 if (
653 (value.startsWith('"') && value.endsWith('"')) ||
654 (value.startsWith("'") && value.endsWith("'"))
655 ) {
656 value = value.slice(1, -1);
657 }
658 out[m[1]] = value;
659 }
660 return out;
661}
662
663/**
664 * Extracts the nested `sidebar.order` value from a frontmatter block.
665 * Starlight uses this to order autogenerated sidebar groups; we mirror it
666 * so llms.txt lists pages in the same order the sidebar shows them.
667 * Returns `Infinity` when unset, so unordered pages sort after ordered ones.
668 */
669function parseSidebarOrder(source: string): number {
670 if (!source.startsWith("---")) return Number.POSITIVE_INFINITY;
671 const end = source.indexOf("\n---", 3);
672 if (end === -1) return Number.POSITIVE_INFINITY;
673 const block = source.slice(3, end);
674 const lines = block.split("\n");
675 for (let i = 0; i < lines.length; i++) {
676 if (!/^sidebar:\s*$/.test(lines[i])) continue;
677 for (let j = i + 1; j < lines.length; j++) {
678 if (/^\S/.test(lines[j])) break; // dedented out of the sidebar block
679 const m = lines[j].match(/^\s+order:\s*(-?[\d.]+)\s*$/);
680 if (m) return Number.parseFloat(m[1]);
681 }
682 break;
683 }
684 return Number.POSITIVE_INFINITY;
685}
686
687async function loadPage(slug: string): Promise<Page> {
688 const candidates = [`${slug}.mdx`, `${slug}.md`];
689 for (const rel of candidates) {
690 const full = path.join(docsDir, rel);
691 try {
692 // Dev-generated reference pages carry copy-editor markers.
693 const source = (await readFile(full, "utf8")).replace(
694 /<!--copy:[^>]*?-->|^<div data-copy=[^\n]*>\n|^<\/div><!--\/copy-->\n/gm,
695 "",
696 );
697 const fm = parseFrontmatter(source);
698 const title = fm.title;
699 const description = fm.description ?? fm.excerpt ?? "";
700 if (!title) {
701 throw new Error(`Missing title in frontmatter: ${rel}`);
702 }
703 return {
704 // `foo/index.mdx` is served at `/foo`.
705 href: `/${slug}`.replace(/\/index$/, "") || "/",
706 slug,
707 title,
708 description,
709 draft: fm.draft === "true" || (fm.draft as unknown as boolean) === true,
710 order: parseSidebarOrder(source),
711 };
712 } catch (err: any) {
713 if (err?.code !== "ENOENT") throw err;
714 }
715 }
716 throw new Error(`Page not found: ${slug} (looked for .mdx and .md)`);
717}
718
719async function listSlugs(
720 directory: string,
721 exclude: string[] = [],
722): Promise<string[]> {
723 const dir = path.join(docsDir, directory);
724 const entries = await readdir(dir, { withFileTypes: true });
725 const slugs: string[] = [];
726 for (const entry of entries) {
727 const rel = `${directory}/${entry.name}`;
728 if (entry.isDirectory()) {
729 slugs.push(...(await listSlugs(rel, exclude)));
730 continue;
731 }
732 if (!entry.isFile()) continue;
733 const ext = path.extname(entry.name);
734 if (ext !== ".md" && ext !== ".mdx") continue;
735 const slug = `${directory}/${entry.name.slice(0, -ext.length)}`;
736 if (exclude.includes(slug)) continue;
737 slugs.push(slug);
738 }
739 slugs.sort();
740 return slugs;
741}
742
743function byOrderThenTitle(a: Page, b: Page): number {
744 if (a.order !== b.order) return a.order - b.order;
745 return a.title.localeCompare(b.title);
746}
747
748function renderPage(page: Page): string {
749 const url = `${siteUrl}${page.href}`;
750 const desc = page.description ? ` — ${page.description}` : "";
751 return `- [${page.title}](${url})${desc}`;
752}
753
754async function main() {
755 const sections: string[] = [];
756
757 for (const section of SECTIONS) {
758 const isDirectory = !("slugs" in section.pages);
759 const slugs =
760 "slugs" in section.pages
761 ? section.pages.slugs
762 : await listSlugs(section.pages.directory, section.pages.exclude);
763 const pages = (await Promise.all(slugs.map(loadPage))).filter(
764 (p) => !p.draft,
765 );
766 // Directory sections mirror the sidebar's `sidebar.order` ordering; slug
767 // sections keep the curated order they were declared in.
768 if (isDirectory) pages.sort(byOrderThenTitle);
769
770 sections.push(`## ${section.heading}`);
771 if (section.intro) sections.push(section.intro);
772 sections.push(pages.map(renderPage).join("\n"));
773 }
774
775 const fullBody =
776 [FULL_HEADER, ...sections, await renderProvidersSection()].join("\n\n") +
777 "\n";
778 await writeFile(outFullFile, fullBody, "utf8");
779 console.log(`Wrote ${outFullFile}`);
780
781 const fullSizeKb = Math.round(Buffer.byteLength(fullBody, "utf8") / 1024);
782 const referenceSection = [
783 `## API reference`,
784 `${PROVIDERS_INTRO_BASE}\n\nThe service-level pages are indexed in [llms-full.txt](${siteUrl}/llms-full.txt), which repeats this entire file plus one link per service. Warning: it is large (~${fullSizeKb} KB) — only fetch it when you need to locate a specific resource's reference page.`,
785 ];
786
787 const body = [HEADER, ...sections, ...referenceSection].join("\n\n") + "\n";
788 await writeFile(outFile, body, "utf8");
789 console.log(`Wrote ${outFile}`);
790}
791
792main().catch((err) => {
793 console.error(err);
794 process.exit(1);
795});