Commit e0c36a5d090be3ff5ac6e7c4fdab752ad63f03d6 | home-infra

author	clover caruso <git@paperclover.net> 2026-05-23 12:55:56-07:00
committer	clover caruso <git@paperclover.net> 2026-06-11 01:38:21-07:00
log	e0c36a5d090be3ff5ac6e7c4fdab752ad63f03d6
tree	dd18489d81db6fdfbfd88d1ed71ced5156aea623
parent	6e686c1140e99b56860514fbface0655b333dba3
signature	Signed by SSH key SHA256:xbd+BjjhyBfwk7GVoURf9Yx0gzDerHbvYv7SddNWmAs

backup nonsense

6 files changed, 237 insertions(+), 0 deletions(-)

backup/README.md created+67

...	...	@@ -0,0 +1,67 @@
	1	# sandwich backup
	2
	3	Plain rsync mirror of the Mac Studio (`sandwich`) onto the NAS, versioned with ZFS
	4	snapshots. Replaces the old encrypted Duplicacy repo — the backup here is just
	5	real files you can browse and copy out, with cheap copy-on-write history.
	6
	7	## Pieces
	8
	9	\| Where \| What \| Lives in \|
	10	\|-------\|------\|----------\|
	11	\| sandwich \| hourly rsync push + snapshot trigger \| `~/config` → `users/clover/sandwich/home.nix` (`launchd.agents.sandwich-backup`) \|
	12	\| NAS (zenith) \| `snapshot-prune.sh` — snapshot + tiered prune \| this dir, deployed via `../sync.sh` \|
	13
	14	Flow: every hour sandwich rsyncs `Asset`, `Project`, `Documents` (minus `LMStudio`)
	15	and `~/Desktop` into `storage1/backup/sandwich/`, then — only on a clean rsync —
	16	ssh's to the NAS and runs `snapshot-prune.sh`, so every snapshot is a complete mirror.
	17
	18	## Retention (`snapshot-prune.sh`)
	19
	20	Only `@auto-*` snapshots are created/destroyed; manual ones are never touched.
	21
	22	\| Tier \| Kept \|
	23	\|--------\|------\|
	24	\| hourly (every run) \| 24 h \|
	25	\| daily (newest per UTC day) \| 14 d \|
	26	\| weekly (newest per ISO week) \| 8 wk \|
	27
	28	Dry-run the prune without destroying anything:
	29
	30	```sh
	31	ssh git.paperclover.net /mnt/storage1/apps/home-infra/backup/snapshot-prune.sh storage1/backup/sandwich --dry-run
	32	```
	33
	34	## Observability
	35
	36	```sh
	37	./backup/status.sh
	38	```
	39
	40	Shows the launchd agent state + which volume is currently syncing on sandwich, and
	41	the landed data + snapshots on the NAS. Notes:
	42	- The `lastexit` in `launchctl list` is the previous run's code; it reads `0` after a
	43	clean run. While a sync is in progress the agent shows its live pid instead.
	44	- `~/Library/Logs/sandwich-backup.log` only gets an rsync `--stats` block when each
	45	folder finishes (rsync buffers stats to the end), so during a big initial sync the
	46	most current signal is the NAS `USED` growing, not the log.
	47	- Manually trigger a run: `launchctl kickstart -k gui/$(id -u)/org.nix-community.home.sandwich-backup` (on sandwich).
	48
	49	## Restore
	50
	51	Snapshots are browsable on the NAS under the hidden `.zfs` dir — no rollback needed:
	52
	53	```sh
	54	ls /mnt/storage1/backup/sandwich/.zfs/snapshot/
	55	cp -a /mnt/storage1/backup/sandwich/.zfs/snapshot/auto-20260610-051120/Desktop/notes.txt ~/restored/
	56	```
	57
	58	Or just grab current files straight from `/mnt/storage1/backup/sandwich/`.
	59
	60	## One-time setup
	61
	62	1. Authorize sandwich's SSH pubkey for `clo@zenith` (so push + snapshot trigger work passwordless).
	63	2. On the NAS: `zfs create -o compression=zstd -o atime=off -o recordsize=1M storage1/backup/sandwich`,
	64	`chown clo:chloe` its mountpoint, and delegate snapshot rights:
	65	`zfs allow clo snapshot,destroy,mount,hold,release storage1/backup/sandwich`.
	66	3. Deploy this dir: `../sync.sh` (lands at `/mnt/storage1/apps/home-infra/backup/`), `chmod +x snapshot-prune.sh`.
	67	4. Apply sandwich's Nix: `./switch` in `~/config`.

backup/snapshot-prune.sh created+79

...	...	@@ -0,0 +1,79 @@
	1	#!/usr/bin/env bash
	2	#
	3	# snapshot-prune.sh DATASET [--dry-run]
	4	#
	5	# Takes a timestamped ZFS snapshot of DATASET, then prunes auto-snapshots with a
	6	# tiered (grandfather-father-son) retention policy:
	7	#
	8	# hourly : keep everything younger than 24h
	9	# daily : keep the newest snapshot per UTC day, out to 14 days
	10	# weekly : keep the newest snapshot per ISO week, out to 8 weeks
	11	# older : destroyed
	12	#
	13	# Only snapshots named DATASET@auto-* are ever created or destroyed; any manual
	14	# or replication snapshots are left untouched.
	15	#
	16	# Intended to be invoked over ssh by sandwich right after a successful rsync push:
	17	# ssh nas /mnt/storage1/apps/home-infra/backup/snapshot-prune.sh storage1/backup/sandwich
	18	#
	19	# Snapshot create/destroy is expected to run without sudo via delegated perms:
	20	# zfs allow clo snapshot,destroy,mount,hold,release storage1/backup/sandwich
	21	set -euo pipefail
	22
	23	DS=${1:-}
	24	DRYRUN=0
	25	[ "${2:-}" = "--dry-run" ] && DRYRUN=1
	26	[ "${1:-}" = "--dry-run" ] && { DRYRUN=1; DS=${2:-}; }
	27
	28	if [ -z "$DS" ]; then
	29	echo "usage: $0 DATASET [--dry-run]" >&2
	30	exit 2
	31	fi
	32
	33	# Non-interactive ssh sessions have a minimal PATH; find zfs explicitly.
	34	ZFS=$(command -v zfs \|\| true)
	35	[ -z "$ZFS" ] && for p in /usr/sbin/zfs /sbin/zfs /usr/local/sbin/zfs; do
	36	[ -x "$p" ] && ZFS=$p && break
	37	done
	38	[ -z "$ZFS" ] && { echo "zfs binary not found" >&2; exit 1; }
	39
	40	run() { if [ "$DRYRUN" = 1 ]; then echo "DRY: $*"; else "$@"; fi; }
	41
	42	# Retention windows, in seconds.
	43	DAY=86400
	44	HOURLY_KEEP=$(( 1 * DAY )) # 24h: keep all
	45	DAILY_KEEP=$(( 14 * DAY )) # newest-per-day out to 14d
	46	WEEKLY_KEEP=$(( 56 * DAY )) # newest-per-week out to 8wk
	47
	48	# 1. Take this run's snapshot.
	49	SNAP="$DS@auto-$(date -u +%Y%m%d-%H%M%S)"
	50	run "$ZFS" snapshot "$SNAP"
	51
	52	# 2. Prune. Walk auto-snapshots newest-first so the first one seen in any
	53	# day/week bucket is the newest and is the one we keep.
	54	now=$(date -u +%s)
	55	declare -A seen_day seen_week
	56
	57	while IFS=$'\t' read -r name creation; do
	58	[ -n "$name" ] \|\| continue
	59	age=$(( now - creation ))
	60	keep=0
	61	if (( age < HOURLY_KEEP )); then
	62	keep=1
	63	elif (( age < DAILY_KEEP )); then
	64	key=$(date -u -d "@$creation" +%Y%j) # year + day-of-year
	65	[ -z "${seen_day[$key]:-}" ] && { keep=1; seen_day[$key]=1; }
	66	elif (( age < WEEKLY_KEEP )); then
	67	key=$(date -u -d "@$creation" +%G%V) # ISO year + week number
	68	[ -z "${seen_week[$key]:-}" ] && { keep=1; seen_week[$key]=1; }
	69	fi
	70
	71	if (( keep )); then
	72	echo "keep $name"
	73	else
	74	echo "destroy $name"
	75	run "$ZFS" destroy "$name"
	76	fi
	77	done < <("$ZFS" list -Hp -o name,creation -t snapshot -r "$DS" \
	78	\| awk -F'\t' 'index($1, "@auto-")' \
	79	\| sort -t"$(printf '\t')" -k2,2nr)

backup/status.sh created+33

...	...	@@ -0,0 +1,33 @@
	1	#!/bin/sh
	2	# Observability for the sandwich backup. Run from this Mac: ./backup/status.sh
	3	# Shows the launchd agent state + live rsync target on sandwich, and the
	4	# landed data + snapshots on the NAS.
	5	NAS=git.paperclover.net
	6	SANDWICH=sandwich.local
	7	DS=storage1/backup/sandwich
	8
	9	echo "===== sandwich (agent) ====="
	10	ssh -o BatchMode=yes "$SANDWICH" '
	11	echo "agent (pid / lastexit / label):"
	12	launchctl list 2>/dev/null \| grep sandwich-backup \|\| echo " not loaded"
	13	if [ -d /tmp/sandwich-backup.lock ]; then
	14	echo "state: RUNNING (pid $(cat /tmp/sandwich-backup.lock/pid 2>/dev/null))"
	15	pgrep -fl rsync \| grep -o "/Volumes/[A-Za-z]*" \| head -1 \| sed "s/^/ currently syncing: /"
	16	else
	17	echo "state: idle (waiting for hourly tick or next kickstart)"
	18	fi
	19	echo "--- last log lines ---"
	20	tail -5 "$HOME/Library/Logs/sandwich-backup.log" 2>/dev/null \|\| echo " (no log yet)"
	21	'
	22	echo
	23	echo "===== NAS (storage1/backup/sandwich) ====="
	24	ssh -o BatchMode=yes "$NAS" "
	25	/usr/sbin/zfs list -o name,used,refer $DS
	26	echo
	27	for d in Asset Project Documents Desktop; do
	28	printf ' %-11s%s\n' \"\$d\" \"\$(du -sh /mnt/$DS/\$d 2>/dev/null \| cut -f1)\"
	29	done
	30	echo
	31	echo 'snapshots (newest last):'
	32	/usr/sbin/zfs list -t snapshot -r $DS -o name,used 2>/dev/null \| tail -6
	33	"

compose.yaml+48

...	...	@@ -398,6 +398,13 @@ services:
398	398	net.paperclover.list.domain: jkt
399	399	net.paperclover.list.priority: 40
400	400	net.paperclover.list.access: media-manage
	401	flaresolverr: # port 8191
	402	container_name: flaresolverr
	403	image: ghcr.io/flaresolverr/flaresolverr:latest
	404	restart: unless-stopped
	405	environment:
	406	- "LOG_LEVEL=info"
	407	- "TZ=America/Los_Angelas"
401	408	sonarr: # port 8989
402	409	container_name: sonarr
403	410	environment:
...	...	@@ -731,6 +738,20 @@ services:
731	738	net.paperclover.list.domain: dns
732	739	net.paperclover.list.priority: 9
733	740	net.paperclover.list.access: personal
	741	ddns-updater: # port 8000 — keeps *.${HOME_DOMAIN} pointed at the home public IP
	742	image: qmcgaw/ddns-updater
	743	container_name: ddns-updater
	744	user: "$USER_ID:$GROUP_ID"
	745	volumes:
	746	- "${APP_ROOT}/ddns-updater:/updater/data"
	747	environment:
	748	PERIOD: "5m"
	749	# config is assembled here so the token/zone stay in .env (config.json can't interpolate)
	750	CONFIG: '{"settings":[{"provider":"cloudflare","zone_identifier":"${CLOUDFLARE_ZONE_ID}","domain":"*.${HOME_DOMAIN}","ttl":1,"proxied":false,"ip_version":"ipv4","token":"${CLOUDFLARE_API_TOKEN}"}]}'
	751	restart: unless-stopped
	752	labels:
	753	net.paperclover.list.name: DDNS
	754	net.paperclover.list.access: personal
734	755	# evil inc temporary infrastructure
735	756	evil-forgejo: # port 3000
736	757	container_name: evil-forgejo
...	...	@@ -804,6 +825,33 @@ services:
804	825	PGPASSWORD: "${EVIL_SEARCH_PGPASSWORD}"
805	826	user: "$USER_ID:$GROUP_ID"
806	827	restart: unless-stopped
	828	evil-hedgedoc:
	829	image: quay.io/hedgedoc/hedgedoc:1.10.6
	830	environment:
	831	CMD_DB_URL: "postgres://evil-hedgedoc:${POSTGRES_PASSWORD_EVIL_HEDGEDOC:-}@postgres:5432/evil-hedgedoc"
	832	CMD_DOMAIN: "md.evil.inc"
	833	CMD_PROTOCOL_USESSL: "true"
	834	CMD_EMAIL: "false"
	835	CMD_OAUTH2_PROVIDERNAME: "git.evil.inc"
	836	CMD_OAUTH2_CLIENT_ID: "${EVIL_HEDGEDOC_CLIENT_ID:?}"
	837	CMD_OAUTH2_CLIENT_SECRET: "${EVIL_HEDGEDOC_CLIENT_SECRET:?}"
	838	CMD_OAUTH2_BASEURL: "https://git.evil.inc/login/oauth"
	839	CMD_OAUTH2_USER_PROFILE_URL: "https://git.evil.inc/api/v1/user"
	840	CMD_OAUTH2_AUTHORIZATION_URL: "https://git.evil.inc/login/oauth/authorize"
	841	CMD_OAUTH2_TOKEN_URL: "https://git.evil.inc/login/oauth/access_token"
	842	CMD_OAUTH2_USER_PROFILE_ID_ATTR: "id"
	843	CMD_OAUTH2_USER_PROFILE_USERNAME_ATTR: "username"
	844	CMD_OAUTH2_USER_PROFILE_DISPLAY_NAME_ATTR: "full_name"
	845	CMD_OAUTH2_USER_PROFILE_EMAIL_ATTR: "email"
	846	CMD_ALLOW_ANONYMOUS_EDITS: "false"
	847	CMD_URL_ADDPORT: "false"
	848	CMD_ALLOW_ANONYMOUS: "false"
	849	volumes:
	850	- "${APP_ROOT:?}/evil-infra/hedgedoc:/hedgedoc/public/uploads"
	851	restart: unless-stopped
	852	depends_on:
	853	postgres:
	854	condition: service_healthy
807	855
808	856	secrets:
809	857	openvpn-credentials:

config/Caddyfile+6

...	...	@@ -250,6 +250,12 @@ search.evil.inc {
250	250	}
251	251	reverse_proxy "http://evil-search"
252	252	}
	253	md.evil.inc {
	254	tls {
	255	on_demand
	256	}
	257	reverse_proxy "http://evil-hedgedoc:3000"
	258	}
253	259
254	260	# redirections
255	261	paperclover.dev {

generate-env.sh+4

...	...	@@ -46,6 +46,10 @@ template() {
46	46	add "POSTGRES_PASSWORD_DAWARICH" "$(secret 32)"
47	47	add "POSTGRES_PASSWORD_EVIL_FORGEJO" "$(secret 32)"
48	48
	49	section "cloudflare ddns"
	50	add "CLOUDFLARE_ZONE_ID" ""
	51	add "CLOUDFLARE_API_TOKEN" ""
	52
49	53	section "misc keys"
50	54	add "ANUBIS_PRIVATE_KEY" "$(secret 32)"
51	55	add "FORWARD_AUTH_KEY" "$(secret 32)"