diff --git a/config/Service.pkl b/config/Service.pkl index 86bbab7343c15881c914746f261244acb9d0bd08..170580f8e5f3999353a6c4a64cf4790b74766fc1 100644 --- a/config/Service.pkl +++ b/config/Service.pkl @@ -111,6 +111,8 @@ enabled: Boolean = true rollout: String = "simple" /// Fresh previews create empty storage and new secrets instead of forking production. stageIsolation: "clone"|"fresh" = "clone" +/// Include the managed dataset in deployment backup snapshots. +snapshots: Boolean = true /// Time allowed for a new allocation to pass its service checks. healthyDeadline: String? /// Delay before persistent health-check failures may restart a running allocation. @@ -165,6 +167,7 @@ output { access = module.access rollout = module.rollout stageIsolation = module.stageIsolation + snapshots = module.snapshots healthyDeadline = vmStartupGrace ?? module.healthyDeadline healthRestartGrace = vmStartupGrace ?? module.healthRestartGrace hostRoot = "\(site.root)/prod/\(module.id)" diff --git a/service/local-ai/service.pkl b/service/local-ai/service.pkl index 95296647cc9d3e9a7f24df819788c21453b245f2..5aa848f356b73138e43b8bbc5794e819c0858554 100644 --- a/service/local-ai/service.pkl +++ b/service/local-ai/service.pkl @@ -12,6 +12,7 @@ launcher = false access = "admin" enabled = (read?("env:SNOWGLOBE_LOCAL_AI") ?? "false") == "true" stageIsolation = "fresh" +snapshots = false prepare = "prepare.py" healthyDeadline = "30m" healthRestartGrace = "30m" diff --git a/tools/data-restore.md b/tools/data-restore.md index 5a14dc9b40d15f601db16e30a002e785dea78745..0c28dcb2cee570f6a0a099255dc6b152b2a1e7f0 100644 --- a/tools/data-restore.md +++ b/tools/data-restore.md @@ -7,6 +7,11 @@ clone produces service-owned database dumps and is destroyed before the backup is published. A failed deployment can leave a completed backup without a successful release-history entry. +Services can set `snapshots = false` in Pkl for disposable managed data, as Local +AI does for its context cache. Pool provisioning writes the policy to the +dataset's `snowglobe:snapshots` property. Database dumps and their shared +PostgreSQL snapshot remain included. + `python3 tools/deploy.py backups` lists backup IDs and services. Each backup's private manifest records the service snapshots, database dumps and checksums, and a private `nomad.snap`. The Nomad snapshot includes jobs, ACLs, and secret diff --git a/tools/data-test.py b/tools/data-test.py new file mode 100644 index 0000000000000000000000000000000000000000..96ce06aa2af8a98ee45cf4cf7e4f892e004c1b86 --- /dev/null +++ b/tools/data-test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +from contextlib import ExitStack, nullcontext +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import data +import snowglobe + + +class SnapshotPolicy(unittest.TestCase): + def test_pkl_policy_reaches_existing_and_new_datasets(self): + with tempfile.TemporaryDirectory(prefix="snowglobe-snapshot-policy-") as temporary: + root = Path(temporary) + fixture = root / "service.pkl" + for declaration, expected in (("", True), ("snapshots = false", False)): + fixture.write_text(f'''amends {json.dumps((snowglobe.REPO / "config/Service.pkl").as_uri())} +name = "Fixture" +container {{ image = "fixture"; volumes {{ ["/cache"] {{}} }} }} +{declaration} +''') + rendered = json.loads(subprocess.check_output([ + os.environ.get("PKL", "pkl"), "eval", "-p", "serviceId=fixture", "-p", "uid=3132", + "-p", f"root={root}", "-p", "pool=fixture", str(fixture), + ], text=True)) + self.assertIs(rendered["snapshots"], expected) + for exists in (True, False): + with self.subTest(declaration=declaration, exists=exists), patch.object( + snowglobe, "check_pool", + ), patch.object(snowglobe, "STATE", root), patch.object( + snowglobe.subprocess, "run", return_value=subprocess.CompletedProcess([], 0 if exists else 1), + ), patch.object(snowglobe, "command") as command: + snowglobe.ensure_pool([rendered]) + self.assertEqual(command.call_args.args, ( + "zfs", "set", f"snowglobe:snapshots={str(expected).lower()}", "fixture/prod/fixture", + )) + self.assertEqual(command.call_count, 1 if exists else 3) + rendered["hasManagedVolumes"] = False + with patch.object(snowglobe, "check_pool"), patch.object( + snowglobe, "STATE", root, + ), patch.object(snowglobe, "command") as command: + snowglobe.ensure_pool([rendered]) + command.assert_not_called() + + def test_backup_excludes_cache_and_preserves_database_recovery(self): + for policy, database in (("-", False), ("true", False), ("false", False), ("false", True)): + with self.subTest(policy=policy, database=database), tempfile.TemporaryDirectory( + prefix="snowglobe-backup-policy-", + ) as temporary, ExitStack() as stack: + root = Path(temporary) + (root / "managed-jobs.json").write_text('["cache", "files", "postgres"]') + snapshots = [] + + def run(*args, **kwargs): + if args[:2] == ("zfs", "get"): + value = "true" if args[-1].endswith("/files") else policy + return subprocess.CompletedProcess(args, 0, value + "\n") + if args[:2] == ("zfs", "snapshot"): + snapshots.extend(args[2:]) + elif args[:4] == ("nomad", "operator", "snapshot", "save"): + Path(args[4]).write_bytes(b"synthetic Nomad state") + elif args[-2:] == ("-c", "SELECT extname FROM pg_extension WHERE extname <> 'plpgsql' ORDER BY extname"): + return subprocess.CompletedProcess(args, 0, "") + else: + self.fail(f"Unexpected command: {args}") + return subprocess.CompletedProcess(args, 0) + + for name, value in { + "STATE": root, "BACKUPS": root / "backups", "run": run, + "current_release": lambda: root / ("a" * 16), + "dataset_for": lambda name: "fixture/prod/" + name, + "database_inputs": lambda name: {"db": ("fixture", "svc_fixture")} if database and name == "cache" else {}, + "cloned_postgres": lambda *args: nullcontext("fixture-postgres"), + "database_dump": lambda container, name, destination, extensions: destination.write_bytes(b"synthetic database"), + }.items(): + stack.enter_context(patch.object(data, name, value)) + backup_id = data.backup("a" * 16, "b" * 16) + manifest = json.loads((data.BACKUPS / backup_id / "manifest.json").read_text()) + expected = {"fixture/prod/files"} + if policy != "false": + expected |= {"fixture/prod/cache", "fixture/prod/postgres"} + elif database: + expected.add("fixture/prod/postgres") + self.assertEqual({item.split("@", 1)[0] for item in snapshots}, expected) + services = {"files"} + if policy != "false": + services |= {"cache", "postgres"} + elif database: + services.add("cache") + self.assertEqual(set(manifest["services"]), services) + if database: + entry = manifest["services"]["cache"] + self.assertNotIn("dataset", entry) + dump = entry["databases"]["db"] + self.assertEqual(dump["sha256"], data.checksum(data.BACKUPS / backup_id / dump["dump"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/data.py b/tools/data.py index 0fb47bb73822af74c2a3ae1c9b51232488308528..4771177cfa91902c66e5724d1a40ce446f0f3ae8 100644 --- a/tools/data.py +++ b/tools/data.py @@ -181,7 +181,8 @@ def backup(from_release, to_release): for service in sorted(names): entry = {} dataset = dataset_for(service) - if dataset: + if dataset and run("zfs", "get", "-H", "-o", "value", "snowglobe:snapshots", dataset, + capture=True, text=True).stdout.strip() != "false": datasets.add(dataset) entry["dataset"] = dataset entry["snapshot"] = snapshot diff --git a/tools/snowglobe.py b/tools/snowglobe.py index 1224a8a5e431c0159303848694e417d10dc6b3d3..154b757764ae43f5a0469438c95f8ca9a496eec2 100644 --- a/tools/snowglobe.py +++ b/tools/snowglobe.py @@ -929,6 +929,7 @@ def ensure_pool(data): raise ValueError(f"refusing to mount over {parent}") command("zfs", "create", "-o", "mountpoint=" + parent, pool + "/prod") command("zfs", "create", "-o", "mountpoint=" + str(root), dataset) + command("zfs", "set", "snowglobe:snapshots=" + str(item["snapshots"]).lower(), dataset) def main():