From 83bb016ae253ba9ec424a98efd19351f9531e6b4 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Fri, 16 Oct 2026 02:35:44 -0700 Subject: [PATCH] Pin external service builds to reviewed source revisions Export pinned build contexts from immutable Jujutsu revisions and retain working-source exports for unpinned manifests. Pin Clover DB to the exact healthy deployed source so a dashboard rollout cannot import unrelated database migrations. Assisted-by: gpt-6.1-sol --- .../clover-source-of-truth/build-source.json | 1 + tools/copyparty.md | 12 ++ tools/deploy-test.py | 127 ++++++++++++++++++ tools/deploy.py | 73 ++++++---- 4 files changed, 190 insertions(+), 23 deletions(-) diff --git a/service/clover-source-of-truth/build-source.json b/service/clover-source-of-truth/build-source.json index 540b73656c2d2e88c0ce3161a10ea0477d7789e3..44d3470b8aa47ecf73920b2a354d58d3944e9397 100644 --- a/service/clover-source-of-truth/build-source.json +++ b/service/clover-source-of-truth/build-source.json @@ -1,5 +1,6 @@ { "source": "../sitegen", + "revision": "13f64d71038f5e72ab34dc1f55805e2fee05f368", "dockerfile": "src/source-of-truth.dockerfile", "include": [ "package.json", diff --git a/tools/copyparty.md b/tools/copyparty.md index a87d8bb153ac36f9b520941c4dc4cff9b00289c0..0276b98fdda5273ae6f276a8a00ad322c857681d 100644 --- a/tools/copyparty.md +++ b/tools/copyparty.md @@ -86,3 +86,15 @@ outside its scroller. Anchoring that hidden text and bounding the list during refresh errors kept the document at 720 × 320 in a 720 × 320 viewport, with only the table scrolling. Its headers and last-row keyboard actions stayed reachable; the hidden live status remained in the accessibility tree. + +Production account creation completed in 0.60 seconds, rotation in 0.47 +seconds, and deletion in 0.15 seconds under the effective host restrictions. +The disposable account was removed; shares and grants remained unchanged. + +The first rollout, `a8982234e99ab23c`, also exported newer sibling Clover DB +sources. That image failed before database startup because its nonroot user +could not create its launcher cache. Clover DB was restored through the +previous immutable release's service deployment. Its newer source also carries +database migrations outside this change's scope. The build manifest now pins +sitegen revision `13f64d71038f5e72ab34dc1f55805e2fee05f368`; all 183 exported +files, file types, and executable bits match the previous healthy context. diff --git a/tools/deploy-test.py b/tools/deploy-test.py index bd4156b6c87c4a851ac988c850109a2710982d4e..5fd561735473ea8beb51f84c32c3d727c123b1bd 100644 --- a/tools/deploy-test.py +++ b/tools/deploy-test.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +from contextlib import ExitStack import json import os from pathlib import Path @@ -75,5 +76,131 @@ class MainExport(unittest.TestCase): deploy.upload(main=True) +class BuildSourceExport(unittest.TestCase): + def setUp(self): + stack = ExitStack() + self.addCleanup(stack.close) + root = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="studio-build-source-"))).resolve() + self.repo = root / "repo" + self.source = root / "source" + self.remote = root / "remote" + for repo in (self.repo, self.source): + subprocess.run(["jj", "git", "init", str(repo)], check=True, capture_output=True) + for name in release.SOURCES: + path = self.repo / name + if "." in name: + path.write_text("fixture") + else: + path.mkdir() + (path / "fixture").write_text("fixture") + (self.repo / "config/excluded-services.json").write_text("[]\n") + (self.source / "lib").mkdir() + (self.source / "lib/committed").write_text("committed library") + (self.source / "src").mkdir() + (self.source / "src/app.dockerfile").write_text("FROM committed\n") + (self.source / "run.sh").write_text("#!/bin/sh\necho committed\n") + (self.source / "run.sh").chmod(0o755) + subprocess.run(["jj", "describe", "-m", "Committed build source"], cwd=self.source, check=True, capture_output=True) + self.commit = subprocess.run(["jj", "--ignore-working-copy", "log", "-r", "@", "--no-graph", "-T", "commit_id"], cwd=self.source, check=True, capture_output=True, text=True).stdout + subprocess.run(["jj", "new"], cwd=self.source, check=True, capture_output=True) + (self.source / "run.sh").write_text("unfinished runner") + (self.source / "run.sh").chmod(0o644) + (self.source / "src/app.dockerfile").write_text("FROM unfinished\n") + (self.source / "lib/committed").unlink() + (self.source / "lib/new").write_text("uncommitted library") + self.manifest = self.repo / "service/app/build-source.json" + self.manifest.parent.mkdir() + self.spec = {"source": "../source", "revision": self.commit, + "dockerfile": "src/app.dockerfile", "include": ["run.sh", "lib"]} + self.manifest.write_text(json.dumps(self.spec)) + for args in [["describe", "-m", "Pinned build source"], ["bookmark", "set", "main"]]: + subprocess.run(["jj", *args], cwd=self.repo, check=True, capture_output=True) + original = subprocess.run + + def transport(argv, **kwargs): + if argv[0] == "ssh": + exists = argv[-1].startswith("test -d ") and Path(argv[-1][8:]).is_dir() + return subprocess.CompletedProcess(argv, 0 if exists else 1) + if argv[0] == "rsync" and "-e" in argv: + destination = Path(argv[-1].partition(":")[2]) + destination.mkdir(parents=True) + for name in argv[4:-1]: + source = Path(name) + if source.is_dir(): + shutil.copytree(source, destination / source.name) + else: + shutil.copy2(source, destination / source.name) + return subprocess.CompletedProcess(argv, 0) + return original(argv, **kwargs) + + stack.enter_context(patch.object(deploy, "REPO", self.repo)) + stack.enter_context(patch.object(deploy, "REMOTE", self.remote)) + stack.enter_context(patch.object(deploy, "ssh")) + stack.enter_context(patch.object(deploy.subprocess, "run", side_effect=transport)) + + def test_pinned_source_ignores_working_files_for_main_and_stage(self): + for main in (True, False): + with self.subTest(main=main): + context = self.remote / "releases" / deploy.upload(main=main) / "service/app/build" + self.assertEqual((context / "run.sh").read_text(), "#!/bin/sh\necho committed\n") + self.assertTrue(os.access(context / "run.sh", os.X_OK)) + self.assertEqual((context / "lib/committed").read_text(), "committed library") + self.assertFalse((context / "lib/new").exists()) + self.assertEqual((context / "Dockerfile").read_text(), "FROM committed\n") + self.assertFalse((context / "src/app.dockerfile").exists()) + + def test_unpinned_source_uses_working_files(self): + del self.spec["revision"] + self.manifest.write_text(json.dumps(self.spec)) + context = self.remote / "releases" / deploy.upload() / "service/app/build" + self.assertEqual((context / "run.sh").read_text(), "unfinished runner") + self.assertFalse(os.access(context / "run.sh", os.X_OK)) + self.assertFalse((context / "lib/committed").exists()) + self.assertEqual((context / "lib/new").read_text(), "uncommitted library") + self.assertEqual((context / "Dockerfile").read_text(), "FROM unfinished\n") + + def test_dockerfile_also_selected_by_include_is_retained(self): + self.manifest.write_text(json.dumps({**self.spec, "include": ["src"]})) + context = self.remote / "releases" / deploy.upload() / "service/app/build" + self.assertEqual((context / "src/app.dockerfile").read_text(), "FROM committed\n") + self.assertEqual((context / "Dockerfile").read_text(), "FROM committed\n") + + def test_invalid_revision_and_paths_are_rejected(self): + for revision in ("main", "f" * 39, "g" * 40, None): + with self.subTest(revision=revision): + self.manifest.write_text(json.dumps({**self.spec, "revision": revision})) + with self.assertRaisesRegex(ValueError, "invalid build source revision"): + deploy.upload() + for name in ("../outside", "/outside", "", "missing", "glob:*"): + with self.subTest(path=name): + self.manifest.write_text(json.dumps({**self.spec, "include": [name]})) + with self.assertRaisesRegex(ValueError, "build source path"): + deploy.upload() + self.manifest.write_text(json.dumps({**self.spec, "source": "../.."})) + with self.assertRaisesRegex(ValueError, "invalid build source"): + deploy.upload() + + def test_selected_symlink_is_rejected(self): + (self.source / "link").symlink_to("run.sh") + subprocess.run(["jj", "describe", "-m", "Source with symlink"], cwd=self.source, check=True, capture_output=True) + commit = subprocess.run(["jj", "--ignore-working-copy", "log", "-r", "@", "--no-graph", "-T", "commit_id"], cwd=self.source, check=True, capture_output=True, text=True).stdout + self.manifest.write_text(json.dumps({**self.spec, "revision": commit, "include": ["link"]})) + with self.assertRaisesRegex(ValueError, "unsupported revision file: link"): + deploy.upload() + + def test_selected_conflict_is_rejected(self): + commits = [] + for content in ("left", "right"): + subprocess.run(["jj", "new", self.commit], cwd=self.source, check=True, capture_output=True) + (self.source / "run.sh").write_text(content) + subprocess.run(["jj", "describe", "-m", content], cwd=self.source, check=True, capture_output=True) + commits.append(subprocess.run(["jj", "--ignore-working-copy", "log", "-r", "@", "--no-graph", "-T", "commit_id"], cwd=self.source, check=True, capture_output=True, text=True).stdout) + subprocess.run(["jj", "new", *commits], cwd=self.source, check=True, capture_output=True) + commit = subprocess.run(["jj", "--ignore-working-copy", "log", "-r", "@", "--no-graph", "-T", "commit_id"], cwd=self.source, check=True, capture_output=True, text=True).stdout + self.manifest.write_text(json.dumps({**self.spec, "revision": commit})) + with self.assertRaisesRegex(ValueError, "unsupported revision file: run.sh"): + deploy.upload() + + if __name__ == "__main__": unittest.main() diff --git a/tools/deploy.py b/tools/deploy.py index 54cb7d4d747d06e361136be172ebe6a8422196ab..a9df7cea0675617fb905ad6d7d9ba01a763e7ee7 100644 --- a/tools/deploy.py +++ b/tools/deploy.py @@ -43,6 +43,32 @@ def sync_manager(release): ssh(f"cp {source}/release.py {source}/data.py {REMOTE}/") +def export_revision(source, commit, paths, destination, excluded=frozenset()): + entries = subprocess.run( + ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T", + '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"', + *("cwd:" + json.dumps(path) for path in paths)], + cwd=source, check=True, capture_output=True, text=True, + ).stdout.splitlines() + for entry in entries: + name, kind, executable = json.loads(entry) + relative = Path(name) + if relative.parts[:1] == ("service",) and ( + set(relative.parts[1:]) & excluded or + relative.suffix == ".pkl" and relative.stem in excluded + ): + continue + if kind != "file" or relative.is_absolute() or ".." in relative.parts: + raise ValueError(f"unsupported revision file: {name}") + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(subprocess.run( + ["jj", "--ignore-working-copy", "file", "show", "-r", commit, name], + cwd=source, check=True, capture_output=True, + ).stdout) + target.chmod(0o755 if executable else 0o644) + + def upload(main=False): excluded = set() if main else excluded_services(REPO) with tempfile.TemporaryDirectory() as temporary: @@ -65,28 +91,7 @@ def upload(main=False): "config/excluded-services.json"], cwd=REPO, check=True, capture_output=True, text=True, ).stdout)) - entries = subprocess.run( - ["jj", "--ignore-working-copy", "file", "list", "-r", commit, "-T", - '\"[\" ++ json(path) ++ \",\" ++ json(file_type) ++ \",\" ++ json(executable) ++ \"]\\n\"', - *SOURCES], cwd=REPO, check=True, capture_output=True, text=True, - ).stdout.splitlines() - for entry in entries: - name, kind, executable = json.loads(entry) - relative = Path(name) - if relative.parts[:1] == ("service",) and ( - set(relative.parts[1:]) & excluded or - relative.suffix == ".pkl" and relative.stem in excluded - ): - continue - if kind != "file" or relative.is_absolute() or ".." in relative.parts: - raise ValueError(f"unsupported main release file: {name}") - destination = snapshot / relative - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(subprocess.run( - ["jj", "--ignore-working-copy", "file", "show", "-r", commit, name], - cwd=REPO, check=True, capture_output=True, - ).stdout) - destination.chmod(0o755 if executable else 0o644) + export_revision(REPO, commit, SOURCES, snapshot, excluded) else: subprocess.run( ["rsync", "-a", "--exclude=.DS_Store", "--exclude=__pycache__", "--exclude=*.pyc", @@ -108,10 +113,32 @@ def upload(main=False): if context.exists(): raise ValueError(f"build context already exists: {manifest.parent.name}") context.mkdir() - for name in [*spec["include"], spec["dockerfile"]]: + paths = [*spec["include"], spec["dockerfile"]] + for name in paths: relative = Path(name) if relative.is_absolute() or ".." in relative.parts or not relative.parts: raise ValueError(f"invalid build source path: {name}") + if "revision" in spec: + commit = spec["revision"] + if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError(f"invalid build source revision: {manifest.parent.name}") + export_revision(source, commit, paths, context) + for name in paths: + if not (context / name).exists(): + raise ValueError(f"missing build source path at revision: {name}") + dockerfile = context / spec["dockerfile"] + if not dockerfile.is_file(): + raise ValueError(f"invalid build source Dockerfile: {spec['dockerfile']}") + if dockerfile != context / "Dockerfile": + if (context / "Dockerfile").exists(): + raise ValueError("build source already includes Dockerfile") + if any(dockerfile.is_relative_to(context / name) for name in spec["include"]): + shutil.copy2(dockerfile, context / "Dockerfile") + else: + dockerfile.rename(context / "Dockerfile") + continue + for name in paths: + relative = Path(name) origin = source / relative if not origin.resolve(strict=True).is_relative_to(source): raise ValueError(f"build source escapes context: {name}") -- 2.54.1