authoravatarclover caruso<git@paperclover.net>2026-10-23 18:18:08-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
logd250b064f061df930c6febd22702fcb9250c9a2b
tree6fa0ec8e6e10d27788a7944a6178c25af982cacc
parent69ba225c6c6854475eff976b11d59f97cea15983
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Let services opt their datasets out of backup snapshots

Services set snapshots = false for disposable managed data; provisioning records it as the snowglobe:snapshots dataset property and backups skip those datasets. Local AI opts out for its context cache. Assisted-by: claude-opus-5.5

6 files changed, 115 insertions(+), 1 deletions(-)

config/Service.pkl+3
...@@ -111,6 +111,8 @@ enabled: Boolean = true...@@ -111,6 +111,8 @@ enabled: Boolean = true
111rollout: String = "simple"111rollout: String = "simple"
112/// Fresh previews create empty storage and new secrets instead of forking production.112/// Fresh previews create empty storage and new secrets instead of forking production.
113stageIsolation: "clone"|"fresh" = "clone"113stageIsolation: "clone"|"fresh" = "clone"
114/// Include the managed dataset in deployment backup snapshots.
115snapshots: Boolean = true
114/// Time allowed for a new allocation to pass its service checks.116/// Time allowed for a new allocation to pass its service checks.
115healthyDeadline: String?117healthyDeadline: String?
116/// Delay before persistent health-check failures may restart a running allocation.118/// Delay before persistent health-check failures may restart a running allocation.
...@@ -165,6 +167,7 @@ output {...@@ -165,6 +167,7 @@ output {
165 access = module.access167 access = module.access
166 rollout = module.rollout168 rollout = module.rollout
167 stageIsolation = module.stageIsolation169 stageIsolation = module.stageIsolation
170 snapshots = module.snapshots
168 healthyDeadline = vmStartupGrace ?? module.healthyDeadline171 healthyDeadline = vmStartupGrace ?? module.healthyDeadline
169 healthRestartGrace = vmStartupGrace ?? module.healthRestartGrace172 healthRestartGrace = vmStartupGrace ?? module.healthRestartGrace
170 hostRoot = "\(site.root)/prod/\(module.id)"173 hostRoot = "\(site.root)/prod/\(module.id)"
service/local-ai/service.pkl+1
...@@ -12,6 +12,7 @@ launcher = false...@@ -12,6 +12,7 @@ launcher = false
12access = "admin"12access = "admin"
13enabled = (read?("env:SNOWGLOBE_LOCAL_AI") ?? "false") == "true"13enabled = (read?("env:SNOWGLOBE_LOCAL_AI") ?? "false") == "true"
14stageIsolation = "fresh"14stageIsolation = "fresh"
15snapshots = false
15prepare = "prepare.py"16prepare = "prepare.py"
16healthyDeadline = "30m"17healthyDeadline = "30m"
17healthRestartGrace = "30m"18healthRestartGrace = "30m"
tools/data-restore.md+5
...@@ -7,6 +7,11 @@ clone produces service-owned database dumps and is destroyed before the...@@ -7,6 +7,11 @@ clone produces service-owned database dumps and is destroyed before the
7backup is published. A failed deployment can leave a completed backup without7backup is published. A failed deployment can leave a completed backup without
8a successful release-history entry.8a successful release-history entry.
99
10Services can set `snapshots = false` in Pkl for disposable managed data, as Local
11AI does for its context cache. Pool provisioning writes the policy to the
12dataset's `snowglobe:snapshots` property. Database dumps and their shared
13PostgreSQL snapshot remain included.
14
10`python3 tools/deploy.py backups` lists backup IDs and services. Each backup's15`python3 tools/deploy.py backups` lists backup IDs and services. Each backup's
11private manifest records the service snapshots, database dumps and checksums,16private manifest records the service snapshots, database dumps and checksums,
12and a private `nomad.snap`. The Nomad snapshot includes jobs, ACLs, and secret17and a private `nomad.snap`. The Nomad snapshot includes jobs, ACLs, and secret
tools/data-test.py created+103
...@@ -0,0 +1,103 @@
1#!/usr/bin/env python3
2from contextlib import ExitStack, nullcontext
3import json
4import os
5from pathlib import Path
6import subprocess
7import tempfile
8import unittest
9from unittest.mock import patch
10
11import data
12import snowglobe
13
14
15class SnapshotPolicy(unittest.TestCase):
16 def test_pkl_policy_reaches_existing_and_new_datasets(self):
17 with tempfile.TemporaryDirectory(prefix="snowglobe-snapshot-policy-") as temporary:
18 root = Path(temporary)
19 fixture = root / "service.pkl"
20 for declaration, expected in (("", True), ("snapshots = false", False)):
21 fixture.write_text(f'''amends {json.dumps((snowglobe.REPO / "config/Service.pkl").as_uri())}
22name = "Fixture"
23container {{ image = "fixture"; volumes {{ ["/cache"] {{}} }} }}
24{declaration}
25''')
26 rendered = json.loads(subprocess.check_output([
27 os.environ.get("PKL", "pkl"), "eval", "-p", "serviceId=fixture", "-p", "uid=3132",
28 "-p", f"root={root}", "-p", "pool=fixture", str(fixture),
29 ], text=True))
30 self.assertIs(rendered["snapshots"], expected)
31 for exists in (True, False):
32 with self.subTest(declaration=declaration, exists=exists), patch.object(
33 snowglobe, "check_pool",
34 ), patch.object(snowglobe, "STATE", root), patch.object(
35 snowglobe.subprocess, "run", return_value=subprocess.CompletedProcess([], 0 if exists else 1),
36 ), patch.object(snowglobe, "command") as command:
37 snowglobe.ensure_pool([rendered])
38 self.assertEqual(command.call_args.args, (
39 "zfs", "set", f"snowglobe:snapshots={str(expected).lower()}", "fixture/prod/fixture",
40 ))
41 self.assertEqual(command.call_count, 1 if exists else 3)
42 rendered["hasManagedVolumes"] = False
43 with patch.object(snowglobe, "check_pool"), patch.object(
44 snowglobe, "STATE", root,
45 ), patch.object(snowglobe, "command") as command:
46 snowglobe.ensure_pool([rendered])
47 command.assert_not_called()
48
49 def test_backup_excludes_cache_and_preserves_database_recovery(self):
50 for policy, database in (("-", False), ("true", False), ("false", False), ("false", True)):
51 with self.subTest(policy=policy, database=database), tempfile.TemporaryDirectory(
52 prefix="snowglobe-backup-policy-",
53 ) as temporary, ExitStack() as stack:
54 root = Path(temporary)
55 (root / "managed-jobs.json").write_text('["cache", "files", "postgres"]')
56 snapshots = []
57
58 def run(*args, **kwargs):
59 if args[:2] == ("zfs", "get"):
60 value = "true" if args[-1].endswith("/files") else policy
61 return subprocess.CompletedProcess(args, 0, value + "\n")
62 if args[:2] == ("zfs", "snapshot"):
63 snapshots.extend(args[2:])
64 elif args[:4] == ("nomad", "operator", "snapshot", "save"):
65 Path(args[4]).write_bytes(b"synthetic Nomad state")
66 elif args[-2:] == ("-c", "SELECT extname FROM pg_extension WHERE extname <> 'plpgsql' ORDER BY extname"):
67 return subprocess.CompletedProcess(args, 0, "")
68 else:
69 self.fail(f"Unexpected command: {args}")
70 return subprocess.CompletedProcess(args, 0)
71
72 for name, value in {
73 "STATE": root, "BACKUPS": root / "backups", "run": run,
74 "current_release": lambda: root / ("a" * 16),
75 "dataset_for": lambda name: "fixture/prod/" + name,
76 "database_inputs": lambda name: {"db": ("fixture", "svc_fixture")} if database and name == "cache" else {},
77 "cloned_postgres": lambda *args: nullcontext("fixture-postgres"),
78 "database_dump": lambda container, name, destination, extensions: destination.write_bytes(b"synthetic database"),
79 }.items():
80 stack.enter_context(patch.object(data, name, value))
81 backup_id = data.backup("a" * 16, "b" * 16)
82 manifest = json.loads((data.BACKUPS / backup_id / "manifest.json").read_text())
83 expected = {"fixture/prod/files"}
84 if policy != "false":
85 expected |= {"fixture/prod/cache", "fixture/prod/postgres"}
86 elif database:
87 expected.add("fixture/prod/postgres")
88 self.assertEqual({item.split("@", 1)[0] for item in snapshots}, expected)
89 services = {"files"}
90 if policy != "false":
91 services |= {"cache", "postgres"}
92 elif database:
93 services.add("cache")
94 self.assertEqual(set(manifest["services"]), services)
95 if database:
96 entry = manifest["services"]["cache"]
97 self.assertNotIn("dataset", entry)
98 dump = entry["databases"]["db"]
99 self.assertEqual(dump["sha256"], data.checksum(data.BACKUPS / backup_id / dump["dump"]))
100
101
102if __name__ == "__main__":
103 unittest.main()
tools/data.py+2-1
...@@ -181,7 +181,8 @@ def backup(from_release, to_release):...@@ -181,7 +181,8 @@ def backup(from_release, to_release):
181 for service in sorted(names):181 for service in sorted(names):
182 entry = {}182 entry = {}
183 dataset = dataset_for(service)183 dataset = dataset_for(service)
184 if dataset:184 if dataset and run("zfs", "get", "-H", "-o", "value", "snowglobe:snapshots", dataset,
185 capture=True, text=True).stdout.strip() != "false":
185 datasets.add(dataset)186 datasets.add(dataset)
186 entry["dataset"] = dataset187 entry["dataset"] = dataset
187 entry["snapshot"] = snapshot188 entry["snapshot"] = snapshot
tools/snowglobe.py+1
...@@ -929,6 +929,7 @@ def ensure_pool(data):...@@ -929,6 +929,7 @@ def ensure_pool(data):
929 raise ValueError(f"refusing to mount over {parent}")929 raise ValueError(f"refusing to mount over {parent}")
930 command("zfs", "create", "-o", "mountpoint=" + parent, pool + "/prod")930 command("zfs", "create", "-o", "mountpoint=" + parent, pool + "/prod")
931 command("zfs", "create", "-o", "mountpoint=" + str(root), dataset)931 command("zfs", "create", "-o", "mountpoint=" + str(root), dataset)
932 command("zfs", "set", "snowglobe:snapshots=" + str(item["snapshots"]).lower(), dataset)
932933
933934
934def main():935def main():