1import * as Hetzner from "@distilled.cloud/hetzner";
2import * as Data from "effect/Data";
3import * as Duration from "effect/Duration";
4import * as Effect from "effect/Effect";
5import * as FileSystem from "effect/FileSystem";
6import * as Path from "effect/Path";
7import * as Schedule from "effect/Schedule";
8import type * as rolldown from "rolldown";
9import * as Bundle from "../Bundle/Bundle.ts";
10import {
11 matchesPackageRoot,
12 normalizeInstallTargets,
13 resolvePackageInstallIdentity,
14 type PackageInstall,
15} from "../Bundle/InstalledPackages.ts";
16import { findCwdForBundle, resolveMainPath } from "../Bundle/TempRoot.ts";
17import type { ResourceBinding } from "../Resource.ts";
18import {
19 createHostRuntimeContext,
20 type HostRuntimeContext,
21} from "../Server/Process.ts";
22import {
23 contextRootOf,
24 extraFileDestination,
25 hashExtraFiles,
26 isContextRootDest,
27 posixRelUnder,
28 resolveExtraSource,
29} from "../Util/extraFiles.ts";
30import { sha256, sha256Object } from "../Util/sha256.ts";
31import { zipCode, zipFiles, type ZipFile } from "../Util/zip.ts";
32import { waitForAction } from "./actions.ts";
33import type { ServiceBinding } from "./MountVolume.ts";
34import { openSshClient, type SshClient } from "./Ssh.ts";
35
36export type HetznerHostRuntimeContext = HostRuntimeContext;
37
38class VolumeAttachTimeout extends Data.TaggedError(
39 "Hetzner.VolumeAttachTimeout",
40)<{
41 volumeId: number;
42 serverId: number;
43}> {}
44
45export const createHetznerHostRuntimeContext = createHostRuntimeContext;
46
47export interface HetznerBuildOptions extends Bundle.BundleConfig {
48 /**
49 * Native or Node-only packages to `npm install` into the unit instead
50 * of bundling them. Same shape as Fly/Lambda `build.install`.
51 */
52 readonly install?: PackageInstall;
53}
54
55export interface HostedProgramProps {
56 main: string;
57 handler?: string;
58 port?: number;
59 env?: Record<string, any>;
60 isExternal?: boolean;
61 build?: HetznerBuildOptions;
62 /**
63 * Extra host directories packed into the unit archive (framework
64 * client assets, Next.js `.next`, …). Hashed into `code.hash` so
65 * asset changes update the unit. Destination is relative to the
66 * unit root.
67 */
68 extraFiles?: ReadonlyArray<ExtraFile>;
69}
70
71export interface ExtraFile {
72 readonly source: string;
73 readonly destination: string;
74}
75
76const matchesConfiguredExternal = (
77 external: rolldown.InputOptions["external"],
78 moduleId: string,
79 parentId: string | undefined,
80 isResolved: boolean,
81): boolean => {
82 if (external === undefined) return false;
83 if (typeof external === "function") {
84 return external(moduleId, parentId, isResolved) === true;
85 }
86 const matchers = Array.isArray(external) ? external : [external];
87 return matchers.some((matcher) =>
88 typeof matcher === "string" ? matcher === moduleId : matcher.test(moduleId),
89 );
90};
91
92export { extraFileDestination };
93
94const quoteEnvValue = (value: unknown) => {
95 const text =
96 typeof value === "string" ? value : JSON.stringify(value ?? null);
97 return `'${text.replaceAll(/'/g, `'""'`).replaceAll(/\n/g, "\\n")}'`;
98};
99
100export const renderEnvFile = (env: Record<string, unknown>) =>
101 Object.entries(env)
102 .sort(([a], [b]) => a.localeCompare(b))
103 .map(([key, value]) => `${key}=${quoteEnvValue(value)}`)
104 .join("\n");
105
106export const collectBindingState = (
107 bindings: ResourceBinding<ServiceBinding>[],
108) => {
109 const active = bindings.filter(
110 (binding: ResourceBinding<ServiceBinding> & { action?: string }) =>
111 binding.action !== "delete",
112 );
113 const env = active
114 .map((binding) => binding?.data?.env)
115 .reduce<Record<string, any>>((acc, value) => ({ ...acc, ...value }), {});
116 const volumes: Array<{ volumeId: number; path: string }> = [];
117 const seen = new Set<string>();
118 for (const binding of active) {
119 for (const volume of binding?.data?.volumes ?? []) {
120 const key = `${volume.volumeId}:${volume.path}`;
121 if (seen.has(key)) continue;
122 seen.add(key);
123 volumes.push(volume);
124 }
125 }
126 return { env, volumes };
127};
128
129const skipExtraPath = (relative: string) =>
130 relative
131 .replaceAll("\\", "/")
132 .split("/")
133 .some(
134 (segment) =>
135 segment === "node_modules" ||
136 segment === ".git" ||
137 segment.startsWith(".alchemy-hetzner"),
138 );
139
140const toSharedExtraFiles = (extraFiles: ReadonlyArray<ExtraFile> | undefined) =>
141 extraFiles?.map((file) => ({
142 source: file.source,
143 dest: file.destination,
144 }));
145
146const collectExtraFiles = Effect.fn(function* (
147 extraFiles: ReadonlyArray<ExtraFile> | undefined,
148) {
149 if (extraFiles === undefined || extraFiles.length === 0) {
150 return [] as ZipFile[];
151 }
152 const fs = yield* FileSystem.FileSystem;
153 const path = yield* Path.Path;
154 const files: ZipFile[] = [];
155 for (const extra of extraFiles) {
156 const dest = extraFileDestination(extra.destination);
157 const source = resolveExtraSource(extra.source, path);
158 if (!(yield* fs.exists(source))) continue;
159 const rootStat = yield* fs.stat(source);
160 const skip = isContextRootDest(dest)
161 ? (relative: string) =>
162 relative
163 .replaceAll("\\", "/")
164 .split("/")
165 .some(
166 (segment) =>
167 segment === ".git" || segment.startsWith(".alchemy-hetzner"),
168 )
169 : skipExtraPath;
170 if (rootStat.type === "File") {
171 files.push({
172 path: isContextRootDest(dest) ? path.basename(source) : dest,
173 content: yield* fs.readFile(source),
174 });
175 continue;
176 }
177 const names = yield* fs.readDirectory(source, { recursive: true });
178 for (const name of names) {
179 if (skip(name)) continue;
180 const full = path.join(source, name);
181 const stat = yield* fs.stat(full);
182 if (stat.type !== "File") continue;
183 const posixName = name.replaceAll("\\", "/");
184 files.push({
185 path: isContextRootDest(dest)
186 ? posixName
187 : `${dest}/${posixName}`.replaceAll(/\/+/g, "/"),
188 content: yield* fs.readFile(full),
189 });
190 }
191 }
192 return files;
193});
194
195/**
196 * The generated entry for `Hetzner.Service` units: a shim importing only
197 * `alchemy/Runtime/Bootstrap/Hetzner` plus the user's `main` — see that
198 * module for why the entry never imports alchemy's own dependencies.
199 */
200const makeBunBootstrap =
201 (handler: string) =>
202 (importPath: string): string =>
203 `
204import { bootstrap } from "alchemy/Runtime/Bootstrap/Hetzner";
205import { ${handler} as entrypoint } from ${JSON.stringify(importPath)};
206
207await bootstrap(entrypoint);
208`;
209
210export const createHetznerHostedSupport = ({
211 stackName,
212 stage,
213 virtualEntryPlugin,
214}: {
215 stackName: string;
216 stage: string;
217 virtualEntryPlugin: (
218 content: (importPath: string) => string,
219 ) => rolldown.Plugin;
220}) => {
221 const alchemyEnv = {
222 ALCHEMY_STACK_NAME: stackName,
223 ALCHEMY_STAGE: stage,
224 ALCHEMY_PHASE: "runtime",
225 HOST: "0.0.0.0",
226 };
227
228 const bundleProgram = Effect.fn(function* (
229 _id: string,
230 props: HostedProgramProps,
231 ) {
232 const handler = props.handler ?? "default";
233 const realMain = yield* resolveMainPath(props.main);
234 const cwd = yield* findCwdForBundle(realMain);
235
236 const requested = yield* normalizeInstallTargets(props.build?.install);
237 const installRoots = new Set(Object.keys(requested));
238 const configuredExternal = props.build?.input?.external;
239
240 const buildBundle = Effect.fn(function* (
241 entry: string,
242 plugins?: rolldown.RolldownPluginOption,
243 ) {
244 return yield* Bundle.build(
245 {
246 ...props.build?.input,
247 input: entry,
248 cwd,
249 platform: "node",
250 external: (moduleId, parentId, isResolved) => {
251 if (moduleId === "bun" || moduleId.startsWith("bun:")) return true;
252 for (const root of installRoots) {
253 if (matchesPackageRoot(moduleId, root)) return true;
254 }
255 return matchesConfiguredExternal(
256 configuredExternal,
257 moduleId,
258 parentId,
259 isResolved,
260 );
261 },
262 resolve: {
263 conditionNames: [...Bundle.NODE_CONDITION_NAMES],
264 ...props.build?.input?.resolve,
265 },
266 plugins: [props.build?.input?.plugins, plugins],
267 },
268 {
269 ...props.build?.output,
270 format: "esm",
271 sourcemap: props.build?.output?.sourcemap ?? false,
272 minify: props.build?.output?.minify ?? false,
273 entryFileNames: "index.mjs",
274 // Preserve init order for framework SSR bundles.
275 strictExecutionOrder: true,
276 keepNames: true,
277 },
278 props.build,
279 );
280 });
281
282 const toBytes = (content: string | Uint8Array<ArrayBufferLike>) =>
283 typeof content === "string" ? new TextEncoder().encode(content) : content;
284 const extraFiles = yield* collectExtraFiles(props.extraFiles);
285 const identity =
286 Object.keys(requested).length > 0
287 ? yield* resolvePackageInstallIdentity({ cwd, requested })
288 : undefined;
289 const install =
290 identity !== undefined && Object.keys(identity.resolved).length > 0
291 ? identity.resolved
292 : undefined;
293 const packageJson =
294 install === undefined
295 ? undefined
296 : `${JSON.stringify(
297 { private: true, type: "module", dependencies: install },
298 null,
299 2,
300 )}\n`;
301 const installFiles =
302 packageJson !== undefined
303 ? [
304 {
305 path: "package.json",
306 content: new TextEncoder().encode(packageJson),
307 },
308 ]
309 : [];
310
311 // `isExternal` mains are already complete Node programs. Pack the
312 // dist tree as-built so relative imports (`../client`, nitro chunks)
313 // survive. Do not rolldown-flatten.
314 if (props.isExternal) {
315 const fs = yield* FileSystem.FileSystem;
316 const pathMod = yield* Path.Path;
317 const extras = toSharedExtraFiles(props.extraFiles) ?? [];
318 const root = contextRootOf(realMain, extras, pathMod, (source) =>
319 resolveExtraSource(source, pathMod),
320 );
321 const entryRel =
322 posixRelUnder(root, realMain, pathMod) ?? pathMod.basename(realMain);
323 const entryBytes = yield* fs.readFile(realMain);
324 const zipEntries: ZipFile[] = [
325 ...extraFiles.filter(
326 (file) =>
327 !(file.path === "package.json" && packageJson !== undefined),
328 ),
329 ...installFiles,
330 ];
331 if (!zipEntries.some((file) => file.path === entryRel)) {
332 zipEntries.push({ path: entryRel, content: entryBytes });
333 }
334 const archive = yield* zipFiles(zipEntries);
335 const extraHash = yield* hashExtraFiles(
336 toSharedExtraFiles(props.extraFiles),
337 );
338 const hash = yield* sha256Object({
339 bundle: yield* sha256(entryBytes),
340 extra: extraHash,
341 packageJson: packageJson ?? "",
342 entryRel,
343 });
344 return { archive, hash, entryRel };
345 }
346
347 const bundleOutput = yield* buildBundle(
348 realMain,
349 virtualEntryPlugin(makeBunBootstrap(handler)),
350 );
351 const [entryFile, ...chunkFiles] = bundleOutput.files;
352 const zipExtras = [
353 ...chunkFiles.map((file) => ({
354 path: file.path,
355 content: toBytes(file.content),
356 })),
357 ...extraFiles,
358 ...installFiles,
359 ];
360 const archive = yield* zipCode(toBytes(entryFile.content), zipExtras);
361 const extraHash = yield* hashExtraFiles(
362 toSharedExtraFiles(props.extraFiles),
363 );
364 const hash = yield* sha256Object({
365 bundle: bundleOutput.hash,
366 extra: extraHash,
367 packageJson: packageJson ?? "",
368 });
369 return { archive, hash, entryRel: "index.mjs" };
370 });
371
372 const renderUnit = (
373 unitName: string,
374 appDir: string,
375 entryRel: string,
376 ) => `[Unit]
377Description=Alchemy Hetzner Service ${unitName}
378After=network-online.target
379Wants=network-online.target
380
381[Service]
382Type=simple
383WorkingDirectory=${appDir}
384EnvironmentFile=-${appDir}/env
385Environment=HOST=0.0.0.0
386ExecStart=/usr/bin/env node ${appDir}/${entryRel}
387Restart=always
388RestartSec=5
389
390[Install]
391WantedBy=multi-user.target
392`;
393
394 const waitForSsh = (ssh: SshClient) =>
395 ssh.exec("true").pipe(
396 Effect.retry({
397 while: (e) => e._tag === "Hetzner.SshError",
398 times: 10,
399 schedule: Schedule.min([
400 Schedule.exponential(Duration.millis(500), 1.5),
401 Schedule.spaced(Duration.seconds(5)),
402 ]),
403 }),
404 );
405
406 const hetznerTransient = (e: { readonly _tag: string }): boolean =>
407 e._tag === "TooManyRequests" ||
408 e._tag === "ServiceUnavailable" ||
409 e._tag === "InternalServerError" ||
410 e._tag === "BadGateway" ||
411 e._tag === "GatewayTimeout" ||
412 e._tag === "Locked";
413
414 const attachAndMount = Effect.fn(function* (input: {
415 ssh: SshClient;
416 serverId: number;
417 volumes: Array<{ volumeId: number; path: string }>;
418 }) {
419 for (const { volumeId, path } of input.volumes) {
420 let volume = yield* Hetzner.volumes.getVolume({ id: volumeId }).pipe(
421 Effect.map(({ volume }) => volume),
422 Effect.retry({
423 while: hetznerTransient,
424 times: 8,
425 schedule: Schedule.min([
426 Schedule.exponential(Duration.millis(500), 1.5),
427 Schedule.spaced(Duration.seconds(5)),
428 ]),
429 }),
430 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
431 );
432 if (volume === undefined) continue;
433
434 if (volume.server !== input.serverId) {
435 if (volume.server !== null) {
436 yield* Hetzner.volumeActions.detachVolume({ id: volumeId }).pipe(
437 Effect.tap(({ action }) =>
438 waitForAction(action).pipe(
439 Effect.catchTag("ActionTimeout", () => Effect.void),
440 ),
441 ),
442 Effect.catchTag(
443 ["NotFound", "UnprocessableEntity", "Locked", "Conflict"],
444 () => Effect.void,
445 ),
446 );
447 }
448 yield* Hetzner.volumeActions
449 .attachVolume({
450 id: volumeId,
451 server: input.serverId,
452 // No automount: Hetzner would mount the device at
453 // `/mnt/HC_Volume_{id}`, and a device mounted elsewhere makes
454 // our own `mount` at the service path fail with
455 // "already mounted". We own the mount + fstab below.
456 automount: false,
457 })
458 .pipe(
459 Effect.retry({
460 while: hetznerTransient,
461 times: 8,
462 schedule: Schedule.min([
463 Schedule.exponential(Duration.millis(500), 1.5),
464 Schedule.spaced(Duration.seconds(5)),
465 ]),
466 }),
467 Effect.tap(({ action }) =>
468 waitForAction(action).pipe(
469 Effect.catchTag("ActionTimeout", () => Effect.void),
470 ),
471 ),
472 Effect.catchTag(
473 ["UnprocessableEntity", "Locked", "Conflict"],
474 () => Effect.void,
475 ),
476 );
477 volume = yield* Hetzner.volumes.getVolume({ id: volumeId }).pipe(
478 Effect.flatMap(({ volume }) =>
479 volume.server === input.serverId
480 ? Effect.succeed(volume)
481 : Effect.fail({ _tag: "AttachPending" as const }),
482 ),
483 Effect.retry({
484 while: (e) =>
485 e._tag === "AttachPending" ||
486 e._tag === "TooManyRequests" ||
487 e._tag === "Locked",
488 times: 10,
489 schedule: Schedule.min([
490 Schedule.exponential(Duration.millis(500), 1.5),
491 Schedule.spaced(Duration.seconds(5)),
492 ]),
493 }),
494 Effect.catchIf(
495 (e) => e._tag === "AttachPending",
496 () =>
497 Hetzner.volumes.getVolume({ id: volumeId }).pipe(
498 Effect.map(({ volume }) => volume),
499 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
500 ),
501 ),
502 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
503 );
504 }
505 if (volume === undefined || volume.server !== input.serverId) {
506 return yield* new VolumeAttachTimeout({
507 volumeId,
508 serverId: input.serverId,
509 });
510 }
511
512 const device = volume.linux_device;
513 const fsType = volume.format === "xfs" ? "xfs" : "ext4";
514 const pathLit = JSON.stringify(path);
515 const deviceLit = JSON.stringify(device);
516 const fsLit = JSON.stringify(fsType);
517 // Attach returns before udev creates `/dev/disk/by-id/scsi-0HC_Volume_*`.
518 yield* input.ssh.exec(
519 [
520 `set -euo pipefail`,
521 `mkdir -p ${pathLit}`,
522 `if ! findmnt -n ${pathLit} >/dev/null 2>&1; then`,
523 ` udevadm settle --timeout=10 2>/dev/null || true`,
524 ` mounted=0`,
525 ` for i in 1 2 3 4 5 6 7 8 9 10; do`,
526 // A sibling Service sharing the same (volume, path) may have
527 // mounted it between iterations — that IS the desired state.
528 ` if findmnt -n ${pathLit} >/dev/null 2>&1; then`,
529 ` mounted=1`,
530 ` break`,
531 ` fi`,
532 ` if [ -e ${deviceLit} ]; then`,
533 // A previous attach with automount (or a leaked mount from an
534 // interrupted run) may hold the device at /mnt/HC_Volume_* —
535 // "already mounted" would fail our mount at the service path.
536 ` current="$(findmnt -n -o TARGET ${deviceLit} 2>/dev/null || true)"`,
537 ` if [ -n "$current" ] && [ "$current" != ${pathLit} ]; then`,
538 ` umount ${deviceLit} 2>/dev/null || true`,
539 ` fi`,
540 ` if mount -t ${fsLit} ${deviceLit} ${pathLit}; then`,
541 ` mounted=1`,
542 ` break`,
543 ` fi`,
544 ` fi`,
545 ` sleep 2`,
546 ` done`,
547 ` if [ "$mounted" != 1 ]; then`,
548 ` echo "volume ${deviceLit} could not be mounted at ${pathLit}" >&2`,
549 ` ls -l /dev/disk/by-id 2>/dev/null || true`,
550 ` exit 1`,
551 ` fi`,
552 `fi`,
553 `if ! grep -qF ${pathLit} /etc/fstab; then`,
554 ` echo ${JSON.stringify(`${device} ${path} ${fsType} defaults,nofail 0 2`)} >> /etc/fstab`,
555 `fi`,
556 ].join("\n"),
557 );
558 }
559 });
560
561 const deployUnit = Effect.fn(function* (input: {
562 ssh: SshClient;
563 unitName: string;
564 archive: Uint8Array<ArrayBufferLike>;
565 env: Record<string, unknown>;
566 entryRel?: string;
567 }) {
568 const appDir = `/opt/${input.unitName}`;
569 yield* waitForSsh(input.ssh);
570 yield* input.ssh.exec(
571 [
572 `set -uo pipefail`,
573 `export HOME=/root`,
574 `mkdir -p ${JSON.stringify(appDir)}`,
575 // A fresh server's cloud-init bootstrap (`ALCHEMY_BOOTSTRAP` in
576 // `Server.ts`) installs these same packages and holds the dpkg lock
577 // while it runs; let it finish instead of racing it.
578 `if command -v cloud-init >/dev/null 2>&1; then cloud-init status --wait >/dev/null 2>&1 || true; fi`,
579 `if ! command -v curl >/dev/null 2>&1 || ! command -v unzip >/dev/null 2>&1 || [ ! -f /etc/ssl/certs/ca-certificates.crt ]; then`,
580 ` apt-get -o DPkg::Lock::Timeout=300 update`,
581 ` DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=300 install -y curl unzip ca-certificates`,
582 `fi`,
583 `for tool in curl unzip; do`,
584 ` if ! command -v "$tool" >/dev/null 2>&1; then`,
585 ` echo "$tool install failed" >&2`,
586 ` exit 1`,
587 ` fi`,
588 `done`,
589 `if ! command -v node >/dev/null 2>&1 || ! node -e 'process.exit(Number(process.versions.node.split(".")[0])<26)'; then`,
590 ` for attempt in 1 2 3 4 5; do`,
591 ` curl -fsSL https://deb.nodesource.com/setup_26.x | bash - && DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=300 install -y nodejs && break`,
592 ` sleep 5`,
593 ` done`,
594 `fi`,
595 `if ! command -v node >/dev/null 2>&1; then`,
596 ` echo "node install failed" >&2`,
597 ` exit 1`,
598 `fi`,
599 ].join("\n"),
600 );
601 yield* input.ssh.scp(input.archive, `${appDir}/bundle.zip`);
602 yield* input.ssh.scp(
603 new TextEncoder().encode(renderEnvFile(input.env)),
604 `${appDir}/env`,
605 );
606 yield* input.ssh.scp(
607 new TextEncoder().encode(
608 renderUnit(input.unitName, appDir, input.entryRel ?? "index.mjs"),
609 ),
610 `/etc/systemd/system/${input.unitName}.service`,
611 );
612 yield* input.ssh.exec(
613 [
614 `set -euo pipefail`,
615 `export HOME=/root`,
616 `if ! command -v node >/dev/null 2>&1; then echo "node missing at start" >&2; exit 1; fi`,
617 `unzip -o ${JSON.stringify(`${appDir}/bundle.zip`)} -d ${JSON.stringify(appDir)}`,
618 `if [ -f ${JSON.stringify(`${appDir}/package.json`)} ]; then`,
619 ` (cd ${JSON.stringify(appDir)} && npm install --omit=dev --no-fund --no-audit)`,
620 `fi`,
621 `systemctl daemon-reload`,
622 `systemctl enable --now ${input.unitName}.service`,
623 `systemctl restart ${input.unitName}.service`,
624 ].join("\n"),
625 );
626 const port =
627 typeof input.env.PORT === "string" ? input.env.PORT : undefined;
628 const health =
629 port !== undefined
630 ? `curl -sf -o /dev/null http://127.0.0.1:${port}/health`
631 : "true";
632 yield* input.ssh.exec(
633 [
634 `set -uo pipefail`,
635 `for attempt in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do`,
636 ` if systemctl is-active --quiet ${input.unitName}.service && ${health}; then`,
637 ` exit 0`,
638 ` fi`,
639 ` sleep 2`,
640 `done`,
641 `echo "unit ${input.unitName} not ready" >&2`,
642 `systemctl status ${input.unitName}.service --no-pager || true`,
643 `journalctl -u ${input.unitName}.service -n 80 --no-pager || true`,
644 `ls -la ${JSON.stringify(appDir)} >&2 || true`,
645 `exit 1`,
646 ].join("\n"),
647 );
648 });
649
650 const removeUnit = Effect.fn(function* (input: {
651 ssh: SshClient;
652 unitName: string;
653 }) {
654 const appDir = `/opt/${input.unitName}`;
655 yield* input.ssh
656 .exec(
657 [
658 `systemctl disable --now ${input.unitName}.service || true`,
659 `rm -f /etc/systemd/system/${input.unitName}.service`,
660 `rm -rf ${JSON.stringify(appDir)}`,
661 `systemctl daemon-reload || true`,
662 ].join("\n"),
663 )
664 .pipe(Effect.catchTag("Hetzner.SshError", () => Effect.void));
665 });
666
667 return {
668 alchemyEnv,
669 bundleProgram,
670 attachAndMount,
671 deployUnit,
672 removeUnit,
673 waitForSsh,
674 openSshClient,
675 };
676};