1{ pkgs }:
2pkgs.testers.runNixOSTest {
3 name = "zfs-read-cache";
4 globalTimeout = 600;
5 nodes.server = { lib, ... }: {
6 imports = [ ../zfs-read-cache.nix ];
7 virtualisation = { memorySize = 2048; cores = 2; emptyDiskImages = [ 4096 ]; };
8 networking.hostId = "c0a1c0a1";
9 boot.supportedFilesystems = [ "zfs" ];
10 boot.zfs.forceImportRoot = false;
11 boot.extraModprobeConfig = "options zfs zfs_arc_max=268435456 l2arc_write_max=33554432";
12 environment.variables.SNOWGLOBE_POOL = "cache-test";
13 environment.systemPackages = [ pkgs.python3 pkgs.zfs pkgs.fio ];
14 systemd.services.cache-test-pool = {
15 before = [ "snowglobe-zfs-read-cache.service" ];
16 serviceConfig = { Type = "oneshot"; RemainAfterExit = true; };
17 script = "${pkgs.zfs}/bin/zpool create -o cachefile=none -O mountpoint=/data cache-test /dev/vdb";
18 };
19 systemd.services.snowglobe-zfs-read-cache = {
20 requires = [ "cache-test-pool.service" ];
21 after = [ "cache-test-pool.service" ];
22 serviceConfig.ExecStart = lib.mkForce "${pkgs.python3}/bin/python3 ${../../tools/zfs-read-cache.py} cache-test /var/cache/snowglobe/zfs/l2arc 536870912 33554432";
23 };
24 };
25 testScript = ''
26 import json
27 start_all()
28 server.wait_for_unit("snowglobe-zfs-read-cache.service")
29 image = "/var/cache/snowglobe/zfs/l2arc"
30 helper = "python3 ${../../tools/zfs-read-cache.py}"
31
32 def cache():
33 status = json.loads(server.succeed("zpool status -jP --json-flat-vdevs --json-int cache-test"))
34 return next(v for v in status["pools"]["cache-test"]["vdevs"].values() if v.get("path") == image)
35
36 def counters():
37 return {row.split()[0]: int(row.split()[2]) for row in server.succeed("cat /proc/spl/kstat/zfs/arcstats").splitlines()[2:]}
38
39 with subtest("bounded private cache and idempotent service"):
40 original = cache()
41 assert original["class"] == "l2cache" and original["state"] == "ONLINE", original
42 info = server.succeed(f"stat -c '%s %b %a' {image}").split()
43 assert int(info[0]) == 536870912 and int(info[1]) * 512 >= 536870912 and info[2] == "600", info
44 server.succeed("systemctl restart snowglobe-zfs-read-cache.service")
45 assert cache()["guid"] == original["guid"]
46 server.fail(f"{helper} cache-test {image} 1073741824 0")
47 assert cache()["guid"] == original["guid"]
48
49 with subtest("warm read hits and bounded cache recycling"):
50 server.succeed("fio --name=fill --filename=/data/workload --size=768m --rw=write --bs=1m --refill_buffers=1 --end_fsync=1 --output=/tmp/fill.json --output-format=json")
51 before = counters()
52 server.succeed("fio --name=warm --filename=/data/workload --size=768m --rw=randread --bs=128k --runtime=45 --time_based --output=/tmp/warm.json --output-format=json", timeout=60)
53 after = counters()
54 assert after["l2_hits"] > before["l2_hits"], (before["l2_hits"], after["l2_hits"])
55 assert after["l2_write_bytes"] > 536870912, after["l2_write_bytes"]
56 assert int(server.succeed(f"stat -c %s {image}")) == 536870912
57 print(json.dumps({"cache_hits": after["l2_hits"] - before["l2_hits"], "cache_written_bytes": after["l2_write_bytes"]}))
58
59 with subtest("persistent import and missing-cache fallback"):
60 checksum = server.succeed("sha256sum /data/workload").split()[0]
61 server.succeed("zpool export cache-test; zpool import -d /dev -d /var/cache/snowglobe/zfs -o cachefile=none cache-test")
62 assert cache()["state"] == "ONLINE"
63 assert cache()["guid"] == original["guid"]
64 server.succeed(f"zpool export cache-test; mv {image} {image}.absent; zpool import -d /dev -o cachefile=none cache-test")
65 assert server.succeed("zpool list -H -o health cache-test").strip() == "ONLINE"
66 assert server.succeed("sha256sum /data/workload").split()[0] == checksum
67 server.fail("systemctl restart snowglobe-zfs-read-cache.service")
68 server.succeed("printf writable > /data/after-cache-loss")
69 assert server.succeed("cat /data/after-cache-loss") == "writable"
70 server.succeed(f"zpool remove cache-test {image}; rm {image}.absent; systemctl restart snowglobe-zfs-read-cache.service")
71 assert cache()["state"] == "ONLINE"
72
73 with subtest("no overwrite, symlink escape, or root-disk exhaustion"):
74 server.succeed("printf retained > /var/cache/snowglobe/zfs/occupied")
75 server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/occupied 67108864 0")
76 assert server.succeed("cat /var/cache/snowglobe/zfs/occupied") == "retained"
77 server.succeed("ln -s occupied /var/cache/snowglobe/zfs/link")
78 server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/link 67108864 0")
79 server.fail(f"{helper} cache-test /var/cache/snowglobe/zfs/too-large 161061273600 34359738368")
80 server.succeed("test ! -e /var/cache/snowglobe/zfs/too-large")
81 assert server.succeed("sha256sum /data/workload").split()[0] == checksum
82 '';
83}