1#!/usr/bin/env python3
2"""Create Jujutsu workspaces and restore dependencies using host-side ZFS reflinks.
3
4Run behind a forced SSH command with a fixed project root and an unprivileged
5account. The guest supplies one JSON request on stdin, never a shell command.
6"""
7
8import argparse
9from contextlib import ExitStack
10import fcntl
11import hashlib
12import json
13import os
14from pathlib import Path
15import re
16import shutil
17import stat
18import subprocess
19import sys
20import tempfile
21
22sys.path.insert(0, str(Path(__file__).with_name("project-tools")))
23import client
24
25
26def parts(value):
27 if not isinstance(value, str) or not value or value.startswith("/"):
28 raise ValueError("Use a relative path inside the project directory.")
29 names = value.split("/")
30 if any(n in ("", ".", "..", ".jj", ".git") or "\\" in n
31 or any(ord(c) < 32 or ord(c) == 127 for c in n) for n in names):
32 raise ValueError("Paths cannot contain traversal or repository metadata.")
33 return names
34
35
36def directory(stack, root, names):
37 for name in names:
38 root = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root)
39 stack.callback(os.close, root)
40 return root
41
42
43def clone_tree(source, destination, *, workspace=False):
44 initial = os.fstat(source)
45 for name in os.listdir(source):
46 if (workspace and name in (".jj", ".git")) or name.startswith("._") or name == ".DS_Store":
47 continue
48 before = os.stat(name, dir_fd=source, follow_symlinks=False)
49 with ExitStack() as stack:
50 if stat.S_ISDIR(before.st_mode):
51 src = directory(stack, source, [name])
52 os.mkdir(name, mode=0o700, dir_fd=destination)
53 dst = directory(stack, destination, [name])
54 clone_tree(src, dst)
55 elif stat.S_ISREG(before.st_mode):
56 src = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=source)
57 stack.callback(os.close, src)
58 if not stat.S_ISREG(os.fstat(src).st_mode):
59 raise ValueError("Target changed during cloning. Retry with an idle donor.")
60 dst = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
61 0o600, dir_fd=destination)
62 stack.callback(os.close, dst)
63 # Linux FICLONE must succeed; a byte-copy fallback defeats warm workspace creation.
64 fcntl.ioctl(dst, 0x40049409, src)
65 elif stat.S_ISLNK(before.st_mode):
66 os.symlink(os.readlink(name, dir_fd=source), name, dir_fd=destination)
67 continue
68 else:
69 raise ValueError(f"Workspace contains a special file: {name}. Remove it before cloning.")
70 current = os.fstat(src)
71 if (before.st_ino, before.st_size, before.st_mtime_ns) != (
72 current.st_ino, current.st_size, current.st_mtime_ns):
73 raise ValueError("Target changed during cloning. Retry with an idle donor.")
74 os.fchmod(dst, stat.S_IMODE(current.st_mode) & 0o777)
75 os.utime(dst, ns=(current.st_atime_ns, current.st_mtime_ns))
76 if os.fstat(source).st_mtime_ns != initial.st_mtime_ns:
77 raise ValueError("Workspace changed during cloning. Retry with an idle donor.")
78
79
80def seed(root, request):
81 if not isinstance(request, dict):
82 raise ValueError("Invalid project-store request.")
83 if request.get("command") == "jj":
84 return run_jj(root, request)
85 if request.get("command") == "read":
86 if set(request) != {"command", "cwd", "tool", "args"} or request["tool"] not in ("rg", "sha256sum"):
87 raise ValueError("Use rg or sha256sum for read-only storage commands.")
88 cwd = shared_directory(root, request["cwd"])
89 args = request["args"]
90 if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args):
91 raise ValueError("Invalid read-only command arguments.")
92 if request["tool"] == "rg":
93 if any(arg in ("--pre", "--hostname-bin") or arg.startswith(("--pre=", "--hostname-bin=")) for arg in args):
94 raise ValueError("Search preprocessors are unavailable on the storage host.")
95 args = ["--no-require-git", *args]
96 else:
97 has_file = False
98 options = True
99 for arg in args:
100 if options and arg == "--":
101 options = False
102 elif options and arg in ("-b", "--binary", "-t", "--text", "--tag", "-z", "--zero"):
103 continue
104 elif arg == "-" or (options and arg.startswith("-")):
105 raise ValueError("Hash named files; stdin and checksum verification are unavailable.")
106 else:
107 path = (cwd / arg).resolve(strict=True)
108 if not path.is_relative_to(root) or not path.is_file():
109 raise ValueError("Hash files inside the project share.")
110 has_file = True
111 if not has_file:
112 raise ValueError("Select at least one file to hash.")
113 result = subprocess.run([request["tool"], *args], cwd=cwd, stdin=subprocess.DEVNULL)
114 return result.returncode if result.returncode >= 0 else 128 - result.returncode
115 if request.get("command") == "pnpm":
116 if (set(request) - {"command", "cwd", "engine", "args", "prepare", "node_env"}
117 or not isinstance(request.get("prepare", False), bool)
118 or request.get("node_env") not in (None, "production")):
119 raise ValueError("Invalid pnpm request.")
120 cwd = shared_directory(root, request["cwd"])
121 args = request["args"]
122 if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args):
123 raise ValueError("Invalid pnpm arguments.")
124 if not args or all(arg.startswith("--") for arg in args):
125 args = ["install", *args]
126 try:
127 parsed = client.pnpm_arguments(args, cwd)
128 except StopIteration:
129 raise ValueError("A dependency command option is missing its value.") from None
130 if parsed[1] is None:
131 raise ValueError("Unsupported dependency command.")
132 cwd, args, _ = parsed
133 cwd = shared_directory(root, cwd)
134 relative = cwd.relative_to(root)
135 if len(relative.parts) < 2:
136 return {"installed": False, "reason": "run the install from the workspace root."}
137 workspace = root.joinpath(*relative.parts[:2])
138 return install(workspace, relative.parts[0], request["engine"], args, cwd=cwd,
139 prepare=request.get("prepare", False), node_env=request.get("node_env"))
140 if set(request) == {"project", "destination", "revision", "engine", "install"}:
141 return checkout(Path(root), request)
142 if set(request) != {"project", "source", "destination", "revision", "targets"}:
143 raise ValueError("Invalid project-store request.")
144 project, source, destination = (parts(request[k]) for k in ("project", "source", "destination"))
145 if (any(len(p) != 1 for p in (project, source, destination)) or source == destination
146 or destination[0] in ("main", "default")):
147 raise ValueError("Select distinct workspaces in one project.")
148 if not isinstance(request["revision"], str) or not request["revision"]:
149 raise ValueError("Select a parent revision.")
150 targets = [parts(p) for p in request["targets"]]
151 if not targets or len({tuple(p) for p in targets}) != len(targets):
152 raise ValueError("Select each target directory once.")
153 root = Path(root).absolute()
154 with ExitStack() as stack:
155 root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
156 stack.callback(os.close, root_fd)
157 project_fd = directory(stack, root_fd, project)
158 src = directory(stack, project_fd, source)
159 directory(stack, project_fd, ["main", ".jj"])
160 for path in targets:
161 directory(stack, src, path)
162 destination_path = root / project[0] / destination[0]
163 if os.path.lexists(destination_path):
164 raise ValueError("Destination exists. Choose a new workspace name.")
165 primary = root / project[0] / "main"
166 jj = ["jj", "--config", "snapshot.auto-track=" + json.dumps('~glob:"**/._*" & ~glob:"**/.DS_Store"')]
167 base = [*jj, "-R", str(primary)]
168 revision = subprocess.run(
169 [*base, "log", "--no-graph", "-r", request["revision"], "-T", 'commit_id ++ "\n"'],
170 text=True, capture_output=True, check=True).stdout.splitlines()
171 if len(revision) != 1:
172 raise ValueError("Select exactly one parent revision.")
173 subprocess.run([*base, "workspace", "add", "--sparse-patterns", "empty",
174 "--name", destination[0], "-r", revision[0], "--", str(destination_path)],
175 stdout=sys.stderr, check=True)
176 try:
177 dst = directory(stack, project_fd, destination)
178 clone_tree(src, dst, workspace=True)
179 local = [*jj, "-R", str(destination_path)]
180 subprocess.run([*local, "sparse", "set", "--add", "."], stdout=sys.stderr, check=True)
181 # Discard donor edits while retaining matching files and ignored build outputs.
182 subprocess.run([*local, "restore", "--from", revision[0]], stdout=sys.stderr, check=True)
183 except BaseException:
184 subprocess.run([*base, "workspace", "forget", destination[0]],
185 stdout=sys.stderr, check=True)
186 shutil.rmtree(destination_path)
187 raise
188 return {"workspace": destination[0]}
189
190
191def dependency_inputs(workspace, revision=None):
192 base = ["jj", "--ignore-working-copy", "-R", str(workspace), "file"]
193 revision_args = ["-r", revision] if revision else []
194 names = subprocess.check_output([*base, "list", *revision_args], cwd=workspace, text=True).splitlines()
195 if revision is None:
196 names = set(names) | {path.name for path in workspace.iterdir() if path.is_file()}
197 result = {}
198 for name in names:
199 path = Path(name)
200 if (path.name in ("package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc")
201 or path.suffix in (".patch", ".tgz") or "pnpmfile" in path.name):
202 target = workspace / path
203 if not revision and target.resolve() != target.absolute():
204 raise ValueError("Dependency input follows a symlink.")
205 if not revision and not target.exists():
206 continue
207 content = (subprocess.check_output([*base, "show", json.dumps(name), *revision_args], cwd=workspace)
208 if revision else target.read_bytes())
209 result[name] = hashlib.sha256(content).hexdigest()
210 return result
211
212
213def checkout(root, request):
214 project, name = (parts(request[k]) for k in ("project", "destination"))
215 if len(project) != 1 or len(name) != 1 or name[0] in ("main", "default"):
216 raise ValueError("Select a new workspace name.")
217 if not isinstance(request["revision"], str) or not request["revision"]:
218 raise ValueError("Select a parent revision.")
219 with ExitStack() as stack:
220 fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
221 stack.callback(os.close, fd)
222 directory(stack, fd, [*project, "main", ".jj"])
223 workspace = root / project[0] / name[0]
224 if os.path.lexists(workspace):
225 raise ValueError("Destination exists. Choose a new workspace name.")
226 subprocess.run(["jj", "-R", str(root / project[0] / "main"), "workspace", "add",
227 "--name", name[0], "-r", request["revision"], "--", str(workspace)],
228 stdout=sys.stderr, check=True)
229 manifest = workspace / "package.json"
230 package = json.loads(manifest.read_text()) if manifest.is_file() else {}
231 result = {"workspace": name[0], "installed": False,
232 "pnpm": package.get("packageManager", "").startswith("pnpm@")}
233 if request["install"] is True:
234 result["installed"] = install(workspace, project[0], request["engine"], [])["installed"]
235 return result
236
237
238def install(workspace, project, engine, args, *, cwd=None, prepare=False, node_env=None):
239 if not args:
240 args = ["install"]
241 cwd = cwd or workspace
242 environment = dict(os.environ, NODE_ENV=node_env) if node_env else None
243 manifest = workspace / "package.json"
244 package = json.loads(manifest.read_text()) if manifest.is_file() else {}
245 manager = re.fullmatch(r"pnpm@(\d+)\.(\d+)\.(\d+)(?:\+.*)?", package.get("packageManager", ""))
246 if not manager:
247 return {"installed": False, "reason": "the workspace does not pin a supported pnpm version."}
248 version = ".".join(manager.groups())
249 major = manager.group(1)
250 executable = Path(__file__).parent / ("pnpm" + major) / {
251 "10": "pnpm.cjs", "11": "pnpm.mjs", "12": "pnpm"}.get(major, "unsupported")
252 if not executable.is_file():
253 return {"installed": False, "reason": f"pnpm {version} is not installed on the storage host."}
254 pnpm = (["node"] if major in ("10", "11") else []) + [str(executable)]
255 if subprocess.check_output([*pnpm, "--version"], text=True, env=environment).strip() != version:
256 return {"installed": False, "reason": f"pnpm {version} is not installed on the storage host."}
257 target = re.fullmatch(r"darwin;arm64;node(\d+(?:\.\d+\.\d+)?)", engine)
258 if not target:
259 return {"installed": False, "reason": "the storage install targets Darwin ARM64."}
260 node_version = target.group(1)
261 profiles = Path(__file__).with_name("profiles.json")
262 profile = json.loads(profiles.read_text()).get(project) if profiles.exists() else None
263 guest_store = ("/Volumes/Code/cache/pnpm12" if major == "12" else
264 "/Users/admin/Library/Caches/project/" + project if major == "11" else
265 "/Volumes/Code/cache/project-pnpm/" + project)
266 if profile and Path(profile["pnpm"]).parent.name == "pnpm" + major:
267 guest_store = str(Path(profile["guest_store"]).parent)
268 workspace_config = workspace / "pnpm-workspace.yaml"
269 config_text = workspace_config.read_text().lower() if workspace_config.is_file() else ""
270 npmrc = workspace / ".npmrc"
271 if npmrc.is_file():
272 config_text += npmrc.read_text().lower()
273 # Knife's reviewed hook behaves identically on Linux and Darwin; edits require review.
274 if ("pnpmfile" in config_text or "configdependencies" in config_text or any(
275 path.name != ".pnpmfile.cjs" or hashlib.sha256(path.read_bytes()).hexdigest() !=
276 "6334e7e025aa23897f0b39365407691b3b65f64bf64fe167b7f1b0d6627b2eed"
277 for path in workspace.glob(".pnpmfile.*"))):
278 return {"installed": False, "store": guest_store, "reason": "workspace pnpmfile hooks require Mac execution."}
279 if prepare:
280 verify = subprocess.check_output([*pnpm, "config", "get", "verify-deps-before-run"],
281 cwd=cwd, text=True, env=environment).strip()
282 if verify == "undefined":
283 verify = "false" if major == "10" else "install"
284 if verify == "false":
285 return {"ready": True}
286 if verify != "install":
287 return {"installed": False, "reason": "the workspace selects its own dependency verification policy."}
288 cached = (not node_env and profile and Path(profile["pnpm"]).parent.name == "pnpm" + major
289 and "darwin;arm64;node" + node_version.split(".")[0] == profile["engine"]
290 and args[0] in ("install", "i")
291 and set(args[1:]) <= {"--frozen-lockfile", "--offline", "--prefer-offline", "--force", "--ignore-scripts"}
292 and dependency_inputs(workspace) == profile["inputs"])
293 if cached:
294 packages = json.loads(subprocess.check_output(
295 [*pnpm, "list", "--recursive", "--depth=-1", "--json"],
296 cwd=workspace, text=True, env=environment))
297 cached = all(str((Path(package["path"]) / "package.json").relative_to(workspace)) in profile["inputs"]
298 for package in packages)
299 Path(guest_store).mkdir(parents=True, exist_ok=True)
300 common = ["--config.store-dir=" + guest_store,
301 "--config.cache-dir=" + str(workspace.parent / ".pnpm-cache"),
302 "--config.enable-global-virtual-store=false",
303 "--config.prefer-symlinked-executables=true",
304 "--reporter=append-only"]
305 if "." in node_version:
306 common.append("--config.node-version=" + node_version)
307 if major == "12":
308 common.extend(('--config.supported-architectures=["darwin-arm64"]',
309 "--config.side-effects-cache=false", "--config.side-effects-cache-readonly=false",
310 "--config.store-index-format=files", "--config.virtual-store-type=local"))
311 else:
312 common.extend(("--config.side-effects-cache-read=false", "--config.cpu=arm64", "--config.os=darwin"))
313 modules = workspace / "node_modules/.modules.yaml"
314 if prepare and modules.is_file():
315 check = subprocess.run([*pnpm, *common, "--config.verify-deps-before-run=error", "exec", "true"],
316 cwd=cwd, text=True, capture_output=True, env=environment)
317 if check.returncode == 0:
318 return {"ready": True}
319 if "VERIFY_DEPS_BEFORE_RUN" not in check.stdout + check.stderr:
320 sys.stderr.write(check.stdout + check.stderr)
321 raise subprocess.CalledProcessError(check.returncode, pnpm)
322 included = json.loads(modules.read_text())["included"]
323 args = ["install"]
324 if included["dependencies"] and not included["devDependencies"]:
325 args.append("--prod")
326 elif included["devDependencies"] and not included["dependencies"]:
327 args.append("--dev")
328 if not included["optionalDependencies"]:
329 args.append("--no-optional")
330 with ExitStack() as stack:
331 lock = os.open(guest_store + "/.project-install.lock", os.O_CREAT | os.O_RDWR, 0o600)
332 stack.callback(os.close, lock)
333 try:
334 fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
335 except BlockingIOError:
336 print("pnpm: Waiting for another install using this package store…", file=sys.stderr, flush=True)
337 fcntl.flock(lock, fcntl.LOCK_EX)
338 subprocess.run([*pnpm, *(arg for arg in args if arg != "--ignore-scripts"),
339 "--config.ignore-scripts=true", "--config.package-import-method=clone", *common], cwd=cwd,
340 stdout=sys.stderr, check=True, env=environment)
341 if cached and "--ignore-scripts" not in args:
342 subprocess.run(["node", profile["restore"], "-r", "rebuild", "--pending", *common],
343 cwd=workspace, stdout=sys.stderr, check=True, env=environment)
344 result = {"installed": True, "workspace": str(workspace)}
345 if "--lockfile-only" not in args:
346 result["modules_hash"] = hashlib.sha256(modules.read_bytes()).hexdigest()
347 return result
348
349
350def shared_directory(root, value):
351 path = Path(value).resolve(strict=True)
352 if not path.is_relative_to(root) or not path.is_dir():
353 raise ValueError("Path is outside the project share.")
354 return path
355
356
357def run_jj(root, request):
358 if set(request) != {"command", "cwd", "args", "config", "version", "ignore"}:
359 raise ValueError("Invalid jj request.")
360 cwd = shared_directory(root, request["cwd"])
361 args = request["args"]
362 if not isinstance(args, list) or not all(isinstance(arg, str) and "\0" not in arg for arg in args):
363 raise ValueError("Invalid jj arguments.")
364 words, options, _ = client.jj_arguments(args)
365 paths = client.jj_paths(words, options)
366 paths.extend(value for name, value in options if name in ("-R", "--repository", "--config-file"))
367 for value in paths:
368 if not (cwd / value).resolve().is_relative_to(root):
369 raise ValueError("Path is outside the project share.")
370 version = subprocess.check_output(["jj-real", "--version"], text=True).strip()
371 if version != request["version"]:
372 raise ValueError("Guest and storage Jujutsu versions differ. Update the project tools together.")
373 with tempfile.TemporaryDirectory(prefix="project-jj-") as temporary:
374 config = Path(temporary) / "jj.toml"
375 config.write_text(request["config"])
376 ignore = Path(temporary) / "git/ignore"
377 ignore.parent.mkdir()
378 ignore.write_text(request["ignore"])
379 # The resolved guest config carries its identity, aliases, and environment overrides.
380 # Global flags must precede `--`, whose remaining arguments are paths or messages.
381 result = subprocess.run(["jj", "--config-file", str(config), "--no-pager", *args], cwd=cwd,
382 env=dict(os.environ, JJ_CONFIG=str(config), XDG_CONFIG_HOME=temporary),
383 stdin=subprocess.DEVNULL)
384 return result.returncode if result.returncode >= 0 else 128 - result.returncode
385
386
387def main():
388 parser = argparse.ArgumentParser(description=__doc__)
389 parser.add_argument("root", help="Fixed host-side project directory")
390 args = parser.parse_args()
391 if os.geteuid() == 0:
392 parser.error("Run project-store as the workspace owner, without root privileges.")
393 request = sys.stdin.buffer.readline(4 * 1024 * 1024 + 1)
394 if len(request) > 4 * 1024 * 1024:
395 parser.error("Project request exceeds 4 MiB.")
396 try:
397 result = seed(Path(args.root).resolve(), json.loads(request))
398 if isinstance(result, int):
399 return result
400 print(json.dumps(result))
401 except subprocess.CalledProcessError as error:
402 return error.returncode if error.returncode >= 0 else 128 - error.returncode
403 except (OSError, ValueError, TypeError, KeyError) as error:
404 print(f"project-store: {error}", file=sys.stderr)
405 return 1
406 return 0
407
408
409if __name__ == "__main__":
410 sys.exit(main())