1import * as Config from "effect/Config";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as FileSystem from "effect/FileSystem";
5import * as Path from "effect/Path";
6import * as Stream from "effect/Stream";
7import * as ChildProcess from "effect/process/ChildProcess";
8import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
9import { Unowned } from "../AdoptPolicy.ts";
10import { deepEqual, isResolved } from "../Diff.ts";
11import { createPhysicalName } from "../PhysicalName.ts";
12import * as Provider from "../Provider.ts";
13import { Resource } from "../Resource.ts";
14import { createInternalTags, hasAlchemyTags } from "../Tags.ts";
15import type { Connection, ContainerRegistry } from "./Connection.ts";
16import { applyObject, connectCluster } from "./internal/client.ts";
17import { kindClusterConfig, kindConfigOf } from "./internal/kind.ts";
18import type { Providers } from "./Providers.ts";
19
20export interface LocalClusterProps extends KindClusterConfig {
21 /**
22 * Name of the kind cluster. Its kubeconfig context is `kind-<name>`.
23 * Changing it replaces the cluster.
24 * @default a name generated from the stack, stage, and logical ID
25 */
26 name?: string;
27 /**
28 * Host port of the cluster's image registry, reachable from the deploying
29 * machine as `localhost:<registryPort>`. Give each local cluster its own
30 * port.
31 * @default 5001
32 */
33 registryPort?: number;
34 /**
35 * The kind node image, which selects the Kubernetes version (e.g.
36 * `kindest/node:v1.33.1`). A node's own `image` overrides it. Changing it
37 * replaces the cluster.
38 * @default kind's default node image
39 */
40 nodeImage?: string;
41 /**
42 * The kubeconfig file kind writes the cluster's context into. Changing it
43 * replaces the cluster.
44 * @default `$KUBECONFIG` or `~/.kube/config`
45 */
46 kubeconfig?: string;
47}
48
49/**
50 * The fields of a [kind `Cluster` config](https://kind.sigs.k8s.io/docs/user/configuration/)
51 * (`kind.x-k8s.io/v1alpha4`), accepted directly on `LocalCluster`. kind
52 * can't reconfigure a running cluster, so changing any of them replaces the
53 * cluster.
54 */
55export interface KindClusterConfig {
56 /**
57 * The cluster's nodes.
58 * @default a single control-plane node
59 */
60 nodes?: KindNode[];
61 /** Cluster networking. */
62 networking?: KindNetworking;
63 /** Kubernetes feature gates, e.g. `{ InPlacePodVerticalScaling: true }`. */
64 featureGates?: Record<string, boolean>;
65 /** API server `--runtime-config` entries, e.g. `{ "api/alpha": "false" }`. */
66 runtimeConfig?: Record<string, string>;
67 /** kubeadm config patches (YAML strategic-merge) applied to every node. */
68 kubeadmConfigPatches?: string[];
69 /** kubeadm config patches (RFC 6902 JSON patches) applied to every node. */
70 kubeadmConfigPatchesJSON6902?: KindJsonPatch[];
71 /**
72 * containerd config patches (TOML). Alchemy appends its own for the image
73 * registry.
74 */
75 containerdConfigPatches?: string[];
76 /** containerd config patches (RFC 6902 JSON patches). */
77 containerdConfigPatchesJSON6902?: string[];
78}
79
80/** A node in a kind cluster. */
81export interface KindNode {
82 /** Whether the node runs the control plane or only workloads. */
83 role: "control-plane" | "worker";
84 /** Node image for this node, overriding `nodeImage`. */
85 image?: string;
86 /** Kubernetes labels for the node. */
87 labels?: Record<string, string>;
88 /** Host ports forwarded to the node container. */
89 extraPortMappings?: KindPortMapping[];
90 /** Host paths mounted into the node container. */
91 extraMounts?: KindMount[];
92 /** kubeadm config patches (YAML strategic-merge) for this node. */
93 kubeadmConfigPatches?: string[];
94 /** kubeadm config patches (RFC 6902 JSON patches) for this node. */
95 kubeadmConfigPatchesJSON6902?: KindJsonPatch[];
96}
97
98/** A host port forwarded to a kind node container. */
99export interface KindPortMapping {
100 /** Port on the node container, e.g. a NodePort or hostPort. */
101 containerPort: number;
102 /** Port on the host. */
103 hostPort: number;
104 /**
105 * Host address to listen on.
106 * @default "0.0.0.0"
107 */
108 listenAddress?: string;
109 /** @default "TCP" */
110 protocol?: "TCP" | "UDP" | "SCTP";
111}
112
113/** A host path mounted into a kind node container. */
114export interface KindMount {
115 /** Path on the host. */
116 hostPath: string;
117 /** Path in the node container. */
118 containerPath: string;
119 readOnly?: boolean;
120 /** Relabel the mount for SELinux hosts. */
121 selinuxRelabel?: boolean;
122 propagation?: "None" | "HostToContainer" | "Bidirectional";
123}
124
125/** Cluster networking settings for kind. */
126export interface KindNetworking {
127 ipFamily?: "ipv4" | "ipv6" | "dual";
128 /**
129 * Host port of the API server.
130 * @default a random port
131 */
132 apiServerPort?: number;
133 /**
134 * Host address the API server listens on.
135 * @default "127.0.0.1"
136 */
137 apiServerAddress?: string;
138 /** Pod network CIDR. */
139 podSubnet?: string;
140 /** Service network CIDR. */
141 serviceSubnet?: string;
142 /** Skip kind's default CNI so you can install your own. */
143 disableDefaultCNI?: boolean;
144 kubeProxyMode?: "iptables" | "ipvs" | "nftables" | "none";
145 /** DNS search domains for nodes. */
146 dnsSearch?: string[];
147}
148
149/** An RFC 6902 JSON patch targeting a kubeadm config object. */
150export interface KindJsonPatch {
151 group: string;
152 version: string;
153 kind: string;
154 /** The JSON patch document, as YAML or JSON. */
155 patch: string;
156}
157
158export interface LocalCluster extends Resource<
159 "Kubernetes.LocalCluster",
160 LocalClusterProps,
161 {
162 /** The kind cluster name. */
163 name: string;
164 /** The kubeconfig context of the cluster (`kind-<name>`). */
165 context: string;
166 /** The kubeconfig file holding the context, if not the default. */
167 kubeconfig: string | undefined;
168 /** The cluster's image registry (`localhost:<registryPort>`). */
169 registry: ContainerRegistry;
170 /** Name of the registry's Docker container. */
171 registryContainer: string;
172 /** CPU architecture of the cluster's nodes (the Docker host's). */
173 architecture: "amd64" | "arm64";
174 /**
175 * Connection for `Kubernetes.*` resources, including the registry and
176 * node architecture. Pass the whole resource as their `cluster`.
177 */
178 connection: Connection;
179 },
180 {},
181 Providers
182> {}
183
184/**
185 * A Kubernetes cluster on your machine, for trying Alchemy's Kubernetes
186 * resources and developing against them without a hosted cluster.
187 *
188 * `LocalCluster` runs a [kind](https://kind.sigs.k8s.io) cluster in Docker
189 * together with a local image registry wired into the cluster's nodes, so
190 * workloads built from a `main` Effect program or a `context` Dockerfile
191 * work the same way they do on a hosted cluster with a registry. Pass the
192 * resource as the `cluster` of any `Kubernetes.*` resource.
193 *
194 * Requires Docker and the `kind` CLI on the deploying machine (set
195 * `KIND_BIN` to use a specific binary). The cluster's kubeconfig context,
196 * `kind-<name>`, works with `kubectl`.
197 *
198 * ### Creating a Local Cluster
199 * **Example:** Cluster and a Deployment
200 * ```typescript
201 * const cluster = yield* Kubernetes.LocalCluster("Cluster", {
202 * name: "alchemy",
203 * });
204 *
205 * const web = yield* Kubernetes.Deployment("Web", {
206 * cluster,
207 * image: "ghcr.io/stefanprodan/podinfo:6.15.0",
208 * port: 9898,
209 * serviceType: "ClusterIP",
210 * });
211 * ```
212 *
213 * ### Effect Programs
214 * **Example:** Run an Effect program as a Job
215 * ```typescript
216 * const cluster = yield* Kubernetes.LocalCluster("Cluster", {
217 * name: "alchemy",
218 * });
219 *
220 * const hello = yield* Kubernetes.Job(
221 * "Hello",
222 * { cluster, main: import.meta.url },
223 * Effect.gen(function* () {
224 * return {
225 * run: Effect.log("hello from a Job"),
226 * };
227 * }),
228 * );
229 * ```
230 *
231 * ### Kind Configuration
232 * **Example:** Forward a host port to a NodePort
233 * ```typescript
234 * const cluster = yield* Kubernetes.LocalCluster("Cluster", {
235 * name: "alchemy",
236 * nodes: [
237 * {
238 * role: "control-plane",
239 * extraPortMappings: [{ containerPort: 30080, hostPort: 8080 }],
240 * },
241 * ],
242 * });
243 * ```
244 *
245 * ### Registry Port
246 * **Example:** Two local clusters
247 * ```typescript
248 * const blue = yield* Kubernetes.LocalCluster("Blue", { name: "blue" });
249 * const green = yield* Kubernetes.LocalCluster("Green", {
250 * name: "green",
251 * registryPort: 5002,
252 * });
253 * ```
254 *
255 * @resource
256 * @product LocalCluster
257 */
258export const LocalCluster = Resource<LocalCluster>("Kubernetes.LocalCluster");
259
260export class LocalClusterError extends Data.TaggedError(
261 "Kubernetes.LocalClusterError",
262)<{ message: string }> {}
263
264const KindBin = Config.String("KIND_BIN").pipe(
265 Effect.orElseSucceed(() => "kind"),
266);
267const DockerBin = Config.String("DOCKER_BIN").pipe(
268 Effect.orElseSucceed(() => "docker"),
269);
270
271const REGISTRY_IMAGE = "registry:3";
272const DEFAULT_REGISTRY_PORT = 5001;
273
274interface CommandResult {
275 exitCode: number;
276 stdout: string;
277 stderr: string;
278}
279
280const toArchitecture = (arch: string): "amd64" | "arm64" =>
281 /^(arm64|aarch64)$/i.test(arch.trim()) ? "arm64" : "amd64";
282
283const sanitizeName = (name: string) =>
284 name
285 .toLowerCase()
286 .replaceAll(/[^a-z0-9-]/g, "-")
287 .replaceAll(/^-+|-+$/g, "");
288
289interface RegistryInspect {
290 State?: { Running?: boolean };
291 Config?: { Labels?: Record<string, string> | null };
292 HostConfig?: {
293 PortBindings?: Record<string, Array<{ HostPort?: string }> | null> | null;
294 };
295 NetworkSettings?: { Networks?: Record<string, unknown> | null };
296}
297
298export const LocalClusterProvider = () =>
299 Provider.effect(
300 LocalCluster,
301 Effect.gen(function* () {
302 const fs = yield* FileSystem.FileSystem;
303 const path = yield* Path.Path;
304 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
305
306 /** Run a CLI, returning its exit code and output. */
307 const run = (bin: string, args: string[], stdin?: string) =>
308 ChildProcess.make(bin, args, {
309 stdin:
310 stdin === undefined
311 ? "ignore"
312 : Stream.succeed(new TextEncoder().encode(stdin)),
313 stdout: "pipe",
314 stderr: "pipe",
315 detached: false,
316 extendEnv: true,
317 }).pipe(
318 spawner.spawn,
319 Effect.flatMap((child) =>
320 Effect.all(
321 {
322 exitCode: child.exitCode,
323 stdout: child.stdout.pipe(Stream.decodeText, Stream.mkString),
324 stderr: child.stderr.pipe(Stream.decodeText, Stream.mkString),
325 },
326 { concurrency: "unbounded" },
327 ),
328 ),
329 Effect.map((result): CommandResult => ({
330 exitCode: Number(result.exitCode),
331 stdout: result.stdout.trim(),
332 stderr: result.stderr.trim(),
333 })),
334 Effect.scoped,
335 // A spawn failure (almost always ENOENT) means the CLI itself is
336 // missing — a machine-setup problem, not a resource error.
337 Effect.catchCause((cause) =>
338 Effect.die(
339 new Error(
340 `Failed to run '${bin}': ${String(cause)}. ` +
341 "Kubernetes.LocalCluster needs Docker and the kind CLI " +
342 "(https://kind.sigs.k8s.io/docs/user/quick-start/#installation). " +
343 "Set KIND_BIN or DOCKER_BIN to use specific binaries.",
344 ),
345 ),
346 ),
347 );
348
349 /** Run a CLI and fail with its stderr on a non-zero exit. */
350 const exec = Effect.fn(function* (
351 bin: string,
352 args: string[],
353 stdin?: string,
354 ) {
355 const result = yield* run(bin, args, stdin);
356 if (result.exitCode !== 0) {
357 return yield* new LocalClusterError({
358 message: `${bin} ${args.join(" ")} exited with code ${result.exitCode}: ${result.stderr}`,
359 });
360 }
361 return result;
362 });
363
364 const kind = Effect.fn(function* (args: string[]) {
365 return yield* exec(yield* KindBin, args);
366 });
367 const docker = Effect.fn(function* (args: string[], stdin?: string) {
368 return yield* exec(yield* DockerBin, args, stdin);
369 });
370
371 const kubeconfigArgs = (kubeconfig: string | undefined) =>
372 kubeconfig !== undefined ? ["--kubeconfig", kubeconfig] : [];
373
374 const clusterName = (
375 id: string,
376 props: LocalClusterProps | undefined,
377 output: LocalCluster["Attributes"] | undefined,
378 ) =>
379 output?.name !== undefined
380 ? Effect.succeed(output.name)
381 : props?.name !== undefined
382 ? Effect.succeed(sanitizeName(props.name))
383 : createPhysicalName({ id, maxLength: 40, lowercase: true }).pipe(
384 Effect.map(sanitizeName),
385 );
386
387 const clusterExists = Effect.fn(function* (name: string) {
388 const { stdout } = yield* kind(["get", "clusters"]);
389 return stdout.split(/\r?\n/).some((line) => line.trim() === name);
390 });
391
392 const inspectRegistry = Effect.fn(function* (container: string) {
393 const result = yield* run(yield* DockerBin, [
394 "container",
395 "inspect",
396 container,
397 ]);
398 if (result.exitCode !== 0) return undefined;
399 const [info] = JSON.parse(result.stdout) as RegistryInspect[];
400 return info;
401 });
402
403 const registryHostPort = (info: RegistryInspect | undefined) => {
404 const binding = info?.HostConfig?.PortBindings?.["5000/tcp"]?.[0];
405 return binding?.HostPort !== undefined
406 ? Number(binding.HostPort)
407 : undefined;
408 };
409
410 const hostArchitecture = Effect.gen(function* () {
411 const { stdout } = yield* docker([
412 "version",
413 "--format",
414 "{{.Server.Arch}}",
415 ]);
416 return toArchitecture(stdout);
417 });
418
419 const toAttributes = (options: {
420 name: string;
421 kubeconfig: string | undefined;
422 registryPort: number;
423 architecture: "amd64" | "arm64";
424 }): LocalCluster["Attributes"] => {
425 const context = `kind-${options.name}`;
426 const registry = { server: `localhost:${options.registryPort}` };
427 return {
428 name: options.name,
429 context,
430 kubeconfig: options.kubeconfig,
431 registry,
432 registryContainer: `${options.name}-registry`,
433 architecture: options.architecture,
434 connection: {
435 auth: {
436 kind: "kubeconfig",
437 path: options.kubeconfig,
438 context,
439 },
440 registry,
441 architecture: options.architecture,
442 },
443 };
444 };
445
446 /** Ensure the kind cluster exists and its context is in the kubeconfig. */
447 const ensureCluster = Effect.fn(function* (options: {
448 name: string;
449 nodeImage: string | undefined;
450 kubeconfig: string | undefined;
451 config: KindClusterConfig;
452 session: { note: (message: string) => Effect.Effect<void> };
453 }) {
454 if (!(yield* clusterExists(options.name))) {
455 yield* options.session.note(
456 `Creating kind cluster ${options.name} (about 30 seconds)...`,
457 );
458 const dir = yield* fs.makeTempDirectory({
459 prefix: "alchemy-kind-",
460 });
461 const configFile = path.join(dir, "kind.yaml");
462 yield* fs.writeFileString(
463 configFile,
464 JSON.stringify(kindClusterConfig(options.config), null, 2),
465 );
466 yield* kind([
467 "create",
468 "cluster",
469 "--name",
470 options.name,
471 "--config",
472 configFile,
473 "--wait",
474 "120s",
475 ...(options.nodeImage !== undefined
476 ? ["--image", options.nodeImage]
477 : []),
478 ...kubeconfigArgs(options.kubeconfig),
479 ]).pipe(
480 // A concurrent create of the same cluster is a race, not a failure.
481 Effect.catchTag("Kubernetes.LocalClusterError", (error) =>
482 /already exist/i.test(error.message)
483 ? Effect.void
484 : Effect.fail(error),
485 ),
486 );
487 yield* fs.remove(dir, { recursive: true });
488 }
489 // Restore the context if it was removed from the kubeconfig.
490 yield* kind([
491 "export",
492 "kubeconfig",
493 "--name",
494 options.name,
495 ...kubeconfigArgs(options.kubeconfig),
496 ]);
497 });
498
499 /**
500 * Ensure the registry container runs on the desired port, sits on the
501 * `kind` network, and every node's containerd resolves
502 * `localhost:<port>` to it.
503 */
504 const ensureRegistry = Effect.fn(function* (options: {
505 id: string;
506 name: string;
507 registryPort: number;
508 session: { note: (message: string) => Effect.Effect<void> };
509 }) {
510 const container = `${options.name}-registry`;
511 let info = yield* inspectRegistry(container);
512 if (
513 info !== undefined &&
514 registryHostPort(info) !== options.registryPort
515 ) {
516 yield* docker(["container", "rm", "--force", container]);
517 info = undefined;
518 }
519 if (info === undefined) {
520 yield* options.session.note(
521 `Starting image registry on localhost:${options.registryPort}...`,
522 );
523 const labels = yield* createInternalTags(options.id);
524 yield* docker([
525 "container",
526 "run",
527 "--detach",
528 "--restart=always",
529 "--publish",
530 `127.0.0.1:${options.registryPort}:5000`,
531 "--network",
532 "bridge",
533 "--name",
534 container,
535 ...Object.entries(labels).flatMap(([key, value]) => [
536 "--label",
537 `${key}=${value}`,
538 ]),
539 REGISTRY_IMAGE,
540 ]);
541 info = yield* inspectRegistry(container);
542 } else if (info.State?.Running !== true) {
543 yield* docker(["container", "start", container]);
544 }
545 if (info?.NetworkSettings?.Networks?.kind === undefined) {
546 yield* docker(["network", "connect", "kind", container]).pipe(
547 Effect.catchTag("Kubernetes.LocalClusterError", (error) =>
548 /already exists/i.test(error.message)
549 ? Effect.void
550 : Effect.fail(error),
551 ),
552 );
553 }
554
555 // containerd on each node: pull `localhost:<port>/…` from the
556 // registry container over the kind network.
557 const registryDir = `/etc/containerd/certs.d/localhost:${options.registryPort}`;
558 const { stdout } = yield* kind([
559 "get",
560 "nodes",
561 "--name",
562 options.name,
563 ]);
564 const nodes = stdout
565 .split(/\r?\n/)
566 .map((line) => line.trim())
567 .filter((line) => line.length > 0);
568 yield* Effect.forEach(
569 nodes,
570 (node) =>
571 Effect.gen(function* () {
572 yield* docker(["exec", node, "mkdir", "-p", registryDir]);
573 yield* docker(
574 [
575 "exec",
576 "-i",
577 node,
578 "cp",
579 "/dev/stdin",
580 `${registryDir}/hosts.toml`,
581 ],
582 `[host."http://${container}:5000"]\n`,
583 );
584 }),
585 { concurrency: "unbounded" },
586 );
587 });
588
589 return {
590 stables: ["name", "context", "kubeconfig", "registryContainer"],
591 list: () => Effect.succeed([] as LocalCluster["Attributes"][]),
592 diff: Effect.fn(function* ({ id, olds, news, output }) {
593 if (!isResolved(news)) return;
594 const oldName = yield* clusterName(id, olds, output);
595 const newName = yield* clusterName(id, news, undefined);
596 if (
597 (news.name !== undefined && oldName !== newName) ||
598 olds?.nodeImage !== news.nodeImage ||
599 olds?.kubeconfig !== news.kubeconfig ||
600 !deepEqual(kindConfigOf(olds ?? {}), kindConfigOf(news), {
601 stripNullish: true,
602 })
603 ) {
604 return { action: "replace", deleteFirst: true } as const;
605 }
606 }),
607 read: Effect.fn(function* ({ id, olds, output }) {
608 const name = yield* clusterName(id, olds, output);
609 if (!(yield* clusterExists(name))) return undefined;
610 const registryInfo = yield* inspectRegistry(`${name}-registry`);
611 const attrs = toAttributes({
612 name,
613 kubeconfig: output?.kubeconfig ?? olds?.kubeconfig,
614 registryPort:
615 registryHostPort(registryInfo) ??
616 olds?.registryPort ??
617 DEFAULT_REGISTRY_PORT,
618 architecture: yield* hostArchitecture,
619 });
620 if (output) return attrs;
621 // kind clusters carry no ownership marker; the registry container
622 // we start next to them does. Anything else is foreign.
623 const owned = yield* hasAlchemyTags(
624 id,
625 registryInfo?.Config?.Labels ?? undefined,
626 );
627 return owned ? attrs : Unowned(attrs);
628 }),
629 reconcile: Effect.fn(function* ({ id, news, output, session }) {
630 const name = yield* clusterName(id, news, output);
631 const registryPort = news.registryPort ?? DEFAULT_REGISTRY_PORT;
632 yield* ensureCluster({
633 name,
634 nodeImage: news.nodeImage,
635 kubeconfig: news.kubeconfig,
636 config: kindConfigOf(news),
637 session,
638 });
639 yield* ensureRegistry({ id, name, registryPort, session });
640 const attrs = toAttributes({
641 name,
642 kubeconfig: news.kubeconfig,
643 registryPort,
644 architecture: yield* hostArchitecture,
645 });
646
647 // Advertise the registry to in-cluster tooling (KEP-1755).
648 const transport = yield* connectCluster(attrs.connection);
649 yield* applyObject({
650 transport,
651 object: {
652 apiVersion: "v1",
653 kind: "ConfigMap",
654 metadata: {
655 name: "local-registry-hosting",
656 namespace: "kube-public",
657 },
658 data: {
659 "localRegistryHosting.v1": `host: "${attrs.registry.server}"\nhelp: "https://kind.sigs.k8s.io/docs/user/local-registry/"\n`,
660 },
661 },
662 });
663 yield* session.note(
664 `kind cluster ${name} ready (context ${attrs.context}, registry ${attrs.registry.server})`,
665 );
666 return attrs;
667 }),
668 delete: Effect.fn(function* ({ output }) {
669 // Both commands are idempotent: kind ignores a missing cluster.
670 yield* kind([
671 "delete",
672 "cluster",
673 "--name",
674 output.name,
675 ...kubeconfigArgs(output.kubeconfig),
676 ]);
677 yield* run(yield* DockerBin, [
678 "container",
679 "rm",
680 "--force",
681 output.registryContainer,
682 ]);
683 }),
684 };
685 }),
686 );