1import { dotAlchemyDirectory } from "../../../AlchemyContext.ts";
2import { isPathWithin } from "../../../Util/isPathWithin.ts";
3import * as Effect from "effect/Effect";
4import * as FileSystem from "effect/FileSystem";
5import * as Stream from "effect/Stream";
6import * as ChildProcess from "effect/process/ChildProcess";
7import { convertPathToPattern, glob } from "tinyglobby";
8import path from "pathe";
9import * as Artifacts from "../../../Artifacts.ts";
10import * as Bundle from "../../../Bundle/Bundle.ts";
11import { exec } from "../../../Util/exec.ts";
12import { sha256 } from "../../../Util/sha256.ts";
13import type { SourceContext, SourceProvider } from "../Source.ts";
14import {
15 bundleSource,
16 resolveMainPath,
17 watchBundleDirectory,
18} from "./shared.ts";
19
20/**
21 * Whether a Worker `main` entry points at a Python module. Python Workers
22 * skip the rolldown pipeline entirely — Cloudflare interprets `.py` sources
23 * directly with Pyodide, so the "bundle" is just the source files plus the
24 * vendored `python_modules/` directory.
25 *
26 * The parameter is `unknown` on purpose. `getCompatibility` reaches this from
27 * `WorkerProvider.precreate`, which is handed *raw, unresolved* props so
28 * resources in a dependency cycle can signal early — and `main` is an
29 * `Input`, so it can legitimately be an `Output` there (e.g. an entry path
30 * derived from a `Command.Build` output). An unresolved Output is a callable
31 * Proxy: it passes a `!== undefined` guard, `.split` yields another proxy,
32 * and calling it returns `undefined` — so a value-shaped check explodes with
33 * a bare `TypeError`. Narrowing on `typeof === "string"` keeps the predicate
34 * total: an unresolved `main` is simply not Python yet, and `reconcile`
35 * re-derives compatibility from fully resolved props.
36 */
37export const isPythonMain = (main: unknown): main is string =>
38 typeof main === "string" && main.split("?")[0].endsWith(".py");
39
40/**
41 * The Pyodide cross-compilation targets uv understands, keyed by the Python
42 * version the Workers runtime selects from the compatibility date/flags.
43 * The wheel index does not have to match the exact Pyodide release the
44 * runtime uses — only the ABI (emscripten-wasm32) must be compatible.
45 * Mirrors pywrangler (cloudflare/workers-py `pywrangler/utils.py`).
46 */
47const PYODIDE_TARGETS = {
48 "3.12": {
49 interpreter: "cpython-3.12.7-emscripten-wasm32-musl",
50 index: "https://index.pyodide.org/0.27.7",
51 },
52 "3.13": {
53 interpreter: "cpython-3.13.2-emscripten-wasm32-musl",
54 index: "https://index.pyodide.org/0.28.3",
55 },
56} as const;
57
58// `python_workers_20250116` (Python 3.13 / Pyodide 0.28) is implied by
59// `python_workers` for compatibility dates on or after this day.
60const PYTHON_3_13_DEFAULT_ON = "2025-09-29";
61
62/**
63 * The Python version the Workers runtime will use for the given
64 * compatibility settings — the same selection pywrangler performs.
65 */
66export const pythonVersionForCompatibility = (compatibility: {
67 date: string;
68 flags: string[];
69}): keyof typeof PYODIDE_TARGETS => {
70 if (compatibility.flags.includes("no_python_workers_20250116")) {
71 return "3.12";
72 }
73 if (
74 compatibility.flags.includes("python_workers_20250116") ||
75 // ISO dates compare lexically.
76 compatibility.date >= PYTHON_3_13_DEFAULT_ON
77 ) {
78 return "3.13";
79 }
80 return "3.12";
81};
82
83/**
84 * The PyPI package pywrangler always adds to the vendored dependencies: it
85 * provides the `workers` Python module (Request/Response/WorkerEntrypoint
86 * wrappers) plus the JS shims under `python_modules/workers/` that
87 * `import_from_javascript()` resolves.
88 */
89const MANAGED_SDK_PACKAGE = "workers-runtime-sdk";
90
91export interface PythonWorkerBundleOptions {
92 id: string;
93 /** Namespace-qualified id — the display prefix for log lines. */
94 fqn: string;
95 /** Path (or `file://` URL) to the Worker's `.py` entry module. */
96 main: string;
97 compatibility: {
98 date: string;
99 flags: string[];
100 };
101}
102
103const uvError = (message: string) => (cause: unknown) =>
104 new Bundle.BundleError({ message, cause });
105
106/**
107 * Run a `uv` command, failing with a {@link Bundle.BundleError} that
108 * includes uv's output when the command exits non-zero, and with an
109 * actionable install hint when the binary is missing entirely.
110 */
111const runUv = Effect.fn(function* (
112 args: string[],
113 options: { cwd: string; env?: Record<string, string> },
114) {
115 const result = yield* exec(
116 ChildProcess.make("uv", args, {
117 cwd: options.cwd,
118 env: options.env,
119 extendEnv: true,
120 }),
121 ).pipe(
122 Effect.mapError(
123 uvError(
124 "Failed to run `uv` — Python Workers use uv (>= 0.8.10) to vendor " +
125 "dependencies for Pyodide. Install it from https://docs.astral.sh/uv/ " +
126 "and re-run.",
127 ),
128 ),
129 );
130 if (result.exitCode !== 0) {
131 return yield* new Bundle.BundleError({
132 message: `\`uv ${args.join(" ")}\` failed with exit code ${result.exitCode}:\n${result.stdout}\n${result.stderr}`,
133 });
134 }
135 return result;
136});
137
138/**
139 * Resolve the `python_modules/` directory to upload alongside a Python
140 * Worker's sources:
141 *
142 * - If the Worker's directory already contains `python_modules/` (the
143 * user vendored it themselves, e.g. with `pywrangler sync` or any other
144 * tool that produces Wrangler's vendored layout), it is used as-is.
145 * - Otherwise, the managed Workers runtime SDK and any dependencies from an
146 * optional `pyproject.toml` are vendored with uv into a staging directory
147 * under `.alchemy/`:
148 * resolve against the Pyodide wheel index for the runtime's Python
149 * version (`uv pip compile` → `pylock.toml`), install the prebuilt
150 * wheels into an emscripten cross-venv (`uv venv` + `uv pip install
151 * --no-build`), and copy the venv's `site-packages` out. Re-vendoring
152 * is skipped while the `pyproject.toml` hash is unchanged.
153 *
154 * workerd no longer bundles the `workers` Python module at current
155 * compatibility dates, so even a dependency-free Worker needs the managed
156 * SDK under `python_modules/`.
157 */
158const resolvePythonModulesDir = Effect.fn(function* (
159 options: PythonWorkerBundleOptions & { root: string },
160) {
161 const fs = yield* FileSystem.FileSystem;
162
163 const userManaged = path.join(options.root, "python_modules");
164 if (yield* orDefault(fs.exists(userManaged), false)) {
165 return userManaged;
166 }
167
168 const pyproject = path.join(options.root, "pyproject.toml");
169 const hasPyproject = yield* orDefault(fs.exists(pyproject), false);
170
171 const pythonVersion = pythonVersionForCompatibility(options.compatibility);
172 const target = PYODIDE_TARGETS[pythonVersion];
173 const pyprojectContent = hasPyproject
174 ? yield* fs
175 .readFileString(pyproject)
176 .pipe(Effect.mapError(uvError(`Failed to read "${pyproject}"`)))
177 : "";
178 const inputHash = yield* sha256(
179 `${pythonVersion}\0${target.index}\0${pyprojectContent}`,
180 );
181
182 const runtimeBase = process.cwd();
183 const dotAlchemy = yield* dotAlchemyDirectory;
184 // uv runs from the Worker root, so its arguments need absolute staging paths.
185 const staging = path.resolve(runtimeBase, dotAlchemy, "python", options.id);
186 const vendorDir = path.join(staging, "python_modules");
187 const tokenFile = path.join(staging, ".synced");
188
189 const token = yield* orDefault(fs.readFileString(tokenFile), undefined);
190 if (token === inputHash && (yield* orDefault(fs.exists(vendorDir), false))) {
191 return vendorDir;
192 }
193
194 yield* Effect.logDebug(
195 `[${options.fqn}] Vendoring Python dependencies for ${pythonVersion} (emscripten-wasm32) with uv`,
196 );
197 yield* fs
198 .makeDirectory(staging, { recursive: true })
199 .pipe(Effect.mapError(uvError(`Failed to create "${staging}"`)));
200
201 // pywrangler always vendors the managed SDK package alongside the
202 // project's own dependencies; passing it as a second requirements input
203 // avoids mutating the user's pyproject.toml.
204 const extraRequirements = path.join(staging, "requirements.extra.txt");
205 yield* fs
206 .writeFileString(extraRequirements, `${MANAGED_SDK_PACKAGE}\n`)
207 .pipe(Effect.mapError(uvError(`Failed to write "${extraRequirements}"`)));
208
209 // 1. Resolve to a pylock.toml against the Pyodide wheel index. `--no-build`
210 // restricts the resolution to prebuilt (pure-Python or emscripten) wheels
211 // — building a Pyodide-platformed wheel locally is not possible.
212 const lockfile = path.join(staging, "pylock.toml");
213 yield* runUv(
214 [
215 "pip",
216 "compile",
217 ...(hasPyproject ? [pyproject] : []),
218 extraRequirements,
219 "--python",
220 target.interpreter,
221 "--extra-index-url",
222 target.index,
223 "--index-strategy",
224 "unsafe-best-match",
225 "--no-header",
226 "--no-build",
227 "-o",
228 lockfile,
229 ],
230 { cwd: options.root },
231 );
232
233 // 2. Install the locked wheels into a fresh emscripten cross-venv.
234 const venv = path.join(staging, "venv");
235 yield* orDefault(fs.remove(venv, { recursive: true }), undefined);
236 yield* runUv(["venv", venv, "--python", target.interpreter], {
237 cwd: options.root,
238 });
239 yield* runUv(
240 [
241 "pip",
242 "install",
243 "--no-build",
244 "-r",
245 lockfile,
246 "--preview-features",
247 "pylock",
248 ],
249 { cwd: options.root, env: { VIRTUAL_ENV: venv } },
250 );
251
252 // 3. The venv's site-packages IS the vendored python_modules directory.
253 const sitePackages =
254 process.platform === "win32"
255 ? path.join(venv, "Lib", "site-packages")
256 : path.join(venv, "lib", `python${pythonVersion}`, "site-packages");
257 yield* orDefault(fs.remove(vendorDir, { recursive: true }), undefined);
258 yield* fs
259 .copy(sitePackages, vendorDir)
260 .pipe(
261 Effect.mapError(
262 uvError(`Failed to copy "${sitePackages}" to "${vendorDir}"`),
263 ),
264 );
265 // Mark the directory as a virtual environment (pywrangler parity).
266 yield* fs
267 .writeFileString(path.join(vendorDir, "pyvenv.cfg"), "")
268 .pipe(Effect.mapError(uvError("Failed to write pyvenv.cfg")));
269 yield* fs
270 .writeFileString(tokenFile, inputHash)
271 .pipe(Effect.mapError(uvError(`Failed to write "${tokenFile}"`)));
272
273 return vendorDir;
274});
275
276const orDefault = <A, B, R>(
277 effect: Effect.Effect<A, unknown, R>,
278 fallback: B,
279): Effect.Effect<A | B, never, R> =>
280 Effect.catchCause(effect, () => Effect.succeed(fallback));
281
282const globFiles = (
283 patterns: string[],
284 options: { cwd: string; ignore?: string[]; dot?: boolean },
285) =>
286 Effect.tryPromise({
287 try: () =>
288 glob(patterns, {
289 cwd: options.cwd,
290 onlyFiles: true,
291 expandDirectories: false,
292 dot: options.dot ?? false,
293 ignore: options.ignore,
294 }),
295 catch: (error) =>
296 new Bundle.BundleError({
297 message: `Failed to list Python worker files in "${options.cwd}"`,
298 cause: error,
299 }),
300 }).pipe(
301 Effect.map((names) =>
302 names.map((name) => name.replaceAll("\\", "/")).sort(),
303 ),
304 );
305
306/**
307 * Read a Python Worker "bundle" from disk. There is no compilation step —
308 * the output is:
309 *
310 * 1. the entry module (first, so it becomes `main_module`), then every
311 * other `.py` file under the entry's directory, read as text;
312 * 2. every file of the resolved `python_modules/` directory (vendored
313 * wheels), read as bytes and named `python_modules/<relpath>` —
314 * Wrangler's vendored-module layout, which both the upload API and
315 * workerd resolve at runtime.
316 */
317export const readPythonWorkerBundle = Effect.fn(function* (
318 options: PythonWorkerBundleOptions,
319) {
320 const fs = yield* FileSystem.FileSystem;
321 const main = yield* resolveMainPath(options.main);
322 const root = path.dirname(main);
323 const runtimeBase = process.cwd();
324 const dotAlchemy = yield* dotAlchemyDirectory;
325 const entryName = path.basename(main);
326
327 const readTextModule = Effect.fn(function* (name: string) {
328 const content = yield* fs
329 .readFileString(path.join(root, name))
330 .pipe(
331 Effect.mapError(
332 uvError(
333 `Failed to read Python worker module "${path.join(root, name)}"`,
334 ),
335 ),
336 );
337 const hash = yield* sha256(content);
338 return { path: name, content, hash } satisfies Bundle.BundleFile;
339 });
340
341 const sources = globFiles(["**/*.py"], {
342 cwd: root,
343 ignore: [
344 "**/__pycache__/**",
345 "python_modules/**",
346 "**/.venv*/**",
347 "**/node_modules/**",
348 "**/.alchemy/**",
349 `${convertPathToPattern(path.resolve(runtimeBase, dotAlchemy))}/**`,
350 ],
351 }).pipe(
352 Effect.map((names) =>
353 names.filter(
354 (name) =>
355 name !== entryName &&
356 !isPathWithin(dotAlchemy, path.join(root, name), runtimeBase),
357 ),
358 ),
359 Effect.flatMap(
360 Effect.forEach(readTextModule, { concurrency: "unbounded" }),
361 ),
362 );
363
364 const vendored = resolvePythonModulesDir({ ...options, root }).pipe(
365 Effect.flatMap((vendorDir) => {
366 if (!vendorDir) {
367 return Effect.succeed([]);
368 }
369 const readVendoredModule = Effect.fn(function* (name: string) {
370 const content = yield* fs
371 .readFile(path.join(vendorDir, name))
372 .pipe(
373 Effect.mapError(
374 uvError(
375 `Failed to read vendored module "${path.join(vendorDir, name)}"`,
376 ),
377 ),
378 );
379 const hash = yield* sha256(content);
380 return {
381 path: `python_modules/${name}`,
382 content,
383 hash,
384 } satisfies Bundle.BundleFile;
385 });
386 return globFiles(["**/*"], {
387 cwd: vendorDir,
388 dot: true,
389 // Compiled bytecode is host-specific and excluded by Wrangler too.
390 ignore: ["**/*.pyc", "**/__pycache__/**"],
391 }).pipe(
392 Effect.flatMap(
393 Effect.forEach(readVendoredModule, { concurrency: "unbounded" }),
394 ),
395 );
396 }),
397 );
398
399 const [entry, additional, vendoredFiles] = yield* Effect.all(
400 [readTextModule(entryName), sources, vendored],
401 { concurrency: "unbounded" },
402 );
403 return yield* Bundle.bundleOutputFromFiles([
404 entry,
405 ...additional,
406 ...vendoredFiles,
407 ]);
408});
409
410/**
411 * Watch a Python Worker for changes: emits the initial build, then
412 * rebuilds whenever a file under the entry's directory changes (debounced).
413 * Mirrors the {@link Bundle.BundleWatchEvent} protocol of the rolldown
414 * watcher so local dev can consume either stream interchangeably.
415 */
416export const watchPythonWorkerBundle = (options: PythonWorkerBundleOptions) =>
417 watchBundleDirectory({
418 main: options.main,
419 read: readPythonWorkerBundle(options),
420 ignore: (path) =>
421 path.includes("__pycache__") || path.includes("/python_modules/"),
422 });
423
424/**
425 * Source provider for Python workers (`.py` entry): no bundling — the
426 * entry plus sibling `.py` files upload as Python modules with
427 * dependencies vendored via uv. The read is cached per run under the
428 * shared `"build"` artifact key so diff and reconcile vendor once.
429 */
430export const makePythonSource = (main: string): SourceProvider => {
431 const pythonOptions = (ctx: SourceContext) => ({
432 id: ctx.id,
433 fqn: ctx.fqn,
434 main,
435 compatibility: ctx.compatibility,
436 });
437 return bundleSource({
438 build: (ctx) =>
439 readPythonWorkerBundle(pythonOptions(ctx)).pipe(
440 Artifacts.cached("build"),
441 ),
442 watch: (ctx) => Effect.succeed(watchPythonWorkerBundle(pythonOptions(ctx))),
443 });
444};