| 1 | import * as Cloudflare from "@/Cloudflare/index.ts"; |
| 2 | import * as Effect from "effect/Effect"; |
| 3 | import { HttpServerRequest } from "effect/http/HttpServerRequest"; |
| 4 | import * as HttpServerResponse from "effect/http/HttpServerResponse"; |
| 5 | |
| 6 | const page = (title: string, body: string) => |
| 7 | HttpServerResponse.html( |
| 8 | `<!doctype html><html><head><title>${title}</title></head>` + |
| 9 | `<body>${body}</body></html>`, |
| 10 | ); |
| 11 | |
| 12 | /** |
| 13 | * A tiny Worker that serves a small crawlable site. Its `workers.dev` URL is a |
| 14 | * domain the account owns, so it can seed an AI Search web-crawler instance |
| 15 | * (which rejects domains the account hasn't verified). |
| 16 | * |
| 17 | * It serves both a `robots.txt` (advertising the sitemap) and a `sitemap.xml` |
| 18 | * listing every page, plus inline links between pages. Cloudflare validates a |
| 19 | * web-crawler seed synchronously at create time and rejects with |
| 20 | * `missing_sitemap` if it finds no content — which happened intermittently |
| 21 | * when relying on link-discovery alone against a freshly-deployed |
| 22 | * `workers.dev` URL. Serving a real sitemap (and `robots.txt` pointing at it) |
| 23 | * makes the seed valid no matter which discovery path Cloudflare takes. |
| 24 | */ |
| 25 | export default class AiSearchCrawlTargetWorker extends Cloudflare.Worker<AiSearchCrawlTargetWorker>()( |
| 26 | "AiSearchCrawlTargetWorker", |
| 27 | { |
| 28 | main: import.meta.url, |
| 29 | }, |
| 30 | Effect.gen(function* () { |
| 31 | return { |
| 32 | fetch: Effect.gen(function* () { |
| 33 | const request = yield* HttpServerRequest; |
| 34 | // `HttpServerRequest.url` is the request path (e.g. `/docs`), not an |
| 35 | // absolute URL — `new URL(request.url)` would throw and 500. Match on |
| 36 | // the path prefix directly. Build absolute URLs from the `Host` |
| 37 | // header so the sitemap works behind the dynamic `workers.dev` host. |
| 38 | const host = request.headers.host ?? "localhost"; |
| 39 | const origin = `https://${host}`; |
| 40 | |
| 41 | if (request.url.startsWith("/robots.txt")) { |
| 42 | return HttpServerResponse.text( |
| 43 | `User-agent: *\nAllow: /\nSitemap: ${origin}/sitemap.xml\n`, |
| 44 | ); |
| 45 | } |
| 46 | |
| 47 | if (request.url.startsWith("/sitemap.xml")) { |
| 48 | return HttpServerResponse.text( |
| 49 | `<?xml version="1.0" encoding="UTF-8"?>` + |
| 50 | `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` + |
| 51 | `<url><loc>${origin}/</loc></url>` + |
| 52 | `<url><loc>${origin}/docs</loc></url>` + |
| 53 | `</urlset>`, |
| 54 | { headers: { "content-type": "application/xml" } }, |
| 55 | ); |
| 56 | } |
| 57 | |
| 58 | if (request.url.startsWith("/docs")) { |
| 59 | return page( |
| 60 | "Docs", |
| 61 | "<h1>Alchemy docs</h1><p>Indexable documentation content.</p>", |
| 62 | ); |
| 63 | } |
| 64 | |
| 65 | return page( |
| 66 | "Crawl Target", |
| 67 | "<h1>Alchemy AI Search crawl target</h1>" + |
| 68 | "<p>This page exists so AI Search has something to index.</p>" + |
| 69 | '<p><a href="/docs">Docs</a></p>', |
| 70 | ); |
| 71 | }), |
| 72 | }; |
| 73 | }), |
| 74 | ) {} |