authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-05-23 12:55:56-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-06-11 01:38:21-07:00
loge0c36a5d090be3ff5ac6e7c4fdab752ad63f03d6
treedd18489d81db6fdfbfd88d1ed71ced5156aea623
parent6e686c1140e99b56860514fbface0655b333dba3
signaturebadge-check Signed by SSH key SHA256:xbd+BjjhyBfwk7GVoURf9Yx0gzDerHbvYv7SddNWmAs

backup nonsense

6 files changed, 237 insertions(+), 0 deletions(-)

backup/README.md created+67
...@@ -0,0 +1,67 @@
1# sandwich backup
2
3Plain rsync mirror of the Mac Studio (`sandwich`) onto the NAS, versioned with ZFS
4snapshots. Replaces the old encrypted Duplicacy repo — the backup here is just
5**real files** you can browse and copy out, with cheap copy-on-write history.
6
7## Pieces
8
9| Where | What | Lives in |
10|-------|------|----------|
11| sandwich | hourly rsync push + snapshot trigger | `~/config` → `users/clover/sandwich/home.nix` (`launchd.agents.sandwich-backup`) |
12| NAS (zenith) | `snapshot-prune.sh` — snapshot + tiered prune | this dir, deployed via `../sync.sh` |
13
14Flow: every hour sandwich rsyncs `Asset`, `Project`, `Documents` (minus `LMStudio`)
15and `~/Desktop` into `storage1/backup/sandwich/`, then — only on a clean rsync —
16ssh's to the NAS and runs `snapshot-prune.sh`, so every snapshot is a *complete* mirror.
17
18## Retention (`snapshot-prune.sh`)
19
20Only `@auto-*` snapshots are created/destroyed; manual ones are never touched.
21
22| Tier | Kept |
23|--------|------|
24| hourly (every run) | 24 h |
25| daily (newest per UTC day) | 14 d |
26| weekly (newest per ISO week) | 8 wk |
27
28Dry-run the prune without destroying anything:
29
30```sh
31ssh git.paperclover.net /mnt/storage1/apps/home-infra/backup/snapshot-prune.sh storage1/backup/sandwich --dry-run
32```
33
34## Observability
35
36```sh
37./backup/status.sh
38```
39
40Shows the launchd agent state + which volume is currently syncing on sandwich, and
41the landed data + snapshots on the NAS. Notes:
42- The `lastexit` in `launchctl list` is the *previous* run's code; it reads `0` after a
43 clean run. While a sync is in progress the agent shows its live pid instead.
44- `~/Library/Logs/sandwich-backup.log` only gets an rsync `--stats` block when each
45 folder *finishes* (rsync buffers stats to the end), so during a big initial sync the
46 most current signal is the NAS `USED` growing, not the log.
47- Manually trigger a run: `launchctl kickstart -k gui/$(id -u)/org.nix-community.home.sandwich-backup` (on sandwich).
48
49## Restore
50
51Snapshots are browsable on the NAS under the hidden `.zfs` dir — no rollback needed:
52
53```sh
54ls /mnt/storage1/backup/sandwich/.zfs/snapshot/
55cp -a /mnt/storage1/backup/sandwich/.zfs/snapshot/auto-20260610-051120/Desktop/notes.txt ~/restored/
56```
57
58Or just grab current files straight from `/mnt/storage1/backup/sandwich/`.
59
60## One-time setup
61
621. Authorize sandwich's SSH pubkey for `clo@zenith` (so push + snapshot trigger work passwordless).
632. On the NAS: `zfs create -o compression=zstd -o atime=off -o recordsize=1M storage1/backup/sandwich`,
64 `chown clo:chloe` its mountpoint, and delegate snapshot rights:
65 `zfs allow clo snapshot,destroy,mount,hold,release storage1/backup/sandwich`.
663. Deploy this dir: `../sync.sh` (lands at `/mnt/storage1/apps/home-infra/backup/`), `chmod +x snapshot-prune.sh`.
674. Apply sandwich's Nix: `./switch` in `~/config`.
backup/snapshot-prune.sh created+79
...@@ -0,0 +1,79 @@
1#!/usr/bin/env bash
2#
3# snapshot-prune.sh DATASET [--dry-run]
4#
5# Takes a timestamped ZFS snapshot of DATASET, then prunes auto-snapshots with a
6# tiered (grandfather-father-son) retention policy:
7#
8# hourly : keep everything younger than 24h
9# daily : keep the newest snapshot per UTC day, out to 14 days
10# weekly : keep the newest snapshot per ISO week, out to 8 weeks
11# older : destroyed
12#
13# Only snapshots named DATASET@auto-* are ever created or destroyed; any manual
14# or replication snapshots are left untouched.
15#
16# Intended to be invoked over ssh by sandwich right after a successful rsync push:
17# ssh nas /mnt/storage1/apps/home-infra/backup/snapshot-prune.sh storage1/backup/sandwich
18#
19# Snapshot create/destroy is expected to run without sudo via delegated perms:
20# zfs allow clo snapshot,destroy,mount,hold,release storage1/backup/sandwich
21set -euo pipefail
22
23DS=${1:-}
24DRYRUN=0
25[ "${2:-}" = "--dry-run" ] && DRYRUN=1
26[ "${1:-}" = "--dry-run" ] && { DRYRUN=1; DS=${2:-}; }
27
28if [ -z "$DS" ]; then
29 echo "usage: $0 DATASET [--dry-run]" >&2
30 exit 2
31fi
32
33# Non-interactive ssh sessions have a minimal PATH; find zfs explicitly.
34ZFS=$(command -v zfs || true)
35[ -z "$ZFS" ] && for p in /usr/sbin/zfs /sbin/zfs /usr/local/sbin/zfs; do
36 [ -x "$p" ] && ZFS=$p && break
37done
38[ -z "$ZFS" ] && { echo "zfs binary not found" >&2; exit 1; }
39
40run() { if [ "$DRYRUN" = 1 ]; then echo "DRY: $*"; else "$@"; fi; }
41
42# Retention windows, in seconds.
43DAY=86400
44HOURLY_KEEP=$(( 1 * DAY )) # 24h: keep all
45DAILY_KEEP=$(( 14 * DAY )) # newest-per-day out to 14d
46WEEKLY_KEEP=$(( 56 * DAY )) # newest-per-week out to 8wk
47
48# 1. Take this run's snapshot.
49SNAP="$DS@auto-$(date -u +%Y%m%d-%H%M%S)"
50run "$ZFS" snapshot "$SNAP"
51
52# 2. Prune. Walk auto-snapshots newest-first so the first one seen in any
53# day/week bucket is the newest and is the one we keep.
54now=$(date -u +%s)
55declare -A seen_day seen_week
56
57while IFS=$'\t' read -r name creation; do
58 [ -n "$name" ] || continue
59 age=$(( now - creation ))
60 keep=0
61 if (( age < HOURLY_KEEP )); then
62 keep=1
63 elif (( age < DAILY_KEEP )); then
64 key=$(date -u -d "@$creation" +%Y%j) # year + day-of-year
65 [ -z "${seen_day[$key]:-}" ] && { keep=1; seen_day[$key]=1; }
66 elif (( age < WEEKLY_KEEP )); then
67 key=$(date -u -d "@$creation" +%G%V) # ISO year + week number
68 [ -z "${seen_week[$key]:-}" ] && { keep=1; seen_week[$key]=1; }
69 fi
70
71 if (( keep )); then
72 echo "keep $name"
73 else
74 echo "destroy $name"
75 run "$ZFS" destroy "$name"
76 fi
77done < <("$ZFS" list -Hp -o name,creation -t snapshot -r "$DS" \
78 | awk -F'\t' 'index($1, "@auto-")' \
79 | sort -t"$(printf '\t')" -k2,2nr)
backup/status.sh created+33
...@@ -0,0 +1,33 @@
1#!/bin/sh
2# Observability for the sandwich backup. Run from this Mac: ./backup/status.sh
3# Shows the launchd agent state + live rsync target on sandwich, and the
4# landed data + snapshots on the NAS.
5NAS=git.paperclover.net
6SANDWICH=sandwich.local
7DS=storage1/backup/sandwich
8
9echo "===== sandwich (agent) ====="
10ssh -o BatchMode=yes "$SANDWICH" '
11 echo "agent (pid / lastexit / label):"
12 launchctl list 2>/dev/null | grep sandwich-backup || echo " not loaded"
13 if [ -d /tmp/sandwich-backup.lock ]; then
14 echo "state: RUNNING (pid $(cat /tmp/sandwich-backup.lock/pid 2>/dev/null))"
15 pgrep -fl rsync | grep -o "/Volumes/[A-Za-z]*" | head -1 | sed "s/^/ currently syncing: /"
16 else
17 echo "state: idle (waiting for hourly tick or next kickstart)"
18 fi
19 echo "--- last log lines ---"
20 tail -5 "$HOME/Library/Logs/sandwich-backup.log" 2>/dev/null || echo " (no log yet)"
21'
22echo
23echo "===== NAS (storage1/backup/sandwich) ====="
24ssh -o BatchMode=yes "$NAS" "
25 /usr/sbin/zfs list -o name,used,refer $DS
26 echo
27 for d in Asset Project Documents Desktop; do
28 printf ' %-11s%s\n' \"\$d\" \"\$(du -sh /mnt/$DS/\$d 2>/dev/null | cut -f1)\"
29 done
30 echo
31 echo 'snapshots (newest last):'
32 /usr/sbin/zfs list -t snapshot -r $DS -o name,used 2>/dev/null | tail -6
33"
compose.yaml+48
...@@ -398,6 +398,13 @@ services:...@@ -398,6 +398,13 @@ services:
398 net.paperclover.list.domain: jkt398 net.paperclover.list.domain: jkt
399 net.paperclover.list.priority: 40399 net.paperclover.list.priority: 40
400 net.paperclover.list.access: media-manage400 net.paperclover.list.access: media-manage
401 flaresolverr: # port 8191
402 container_name: flaresolverr
403 image: ghcr.io/flaresolverr/flaresolverr:latest
404 restart: unless-stopped
405 environment:
406 - "LOG_LEVEL=info"
407 - "TZ=America/Los_Angelas"
401 sonarr: # port 8989408 sonarr: # port 8989
402 container_name: sonarr409 container_name: sonarr
403 environment:410 environment:
...@@ -731,6 +738,20 @@ services:...@@ -731,6 +738,20 @@ services:
731 net.paperclover.list.domain: dns738 net.paperclover.list.domain: dns
732 net.paperclover.list.priority: 9739 net.paperclover.list.priority: 9
733 net.paperclover.list.access: personal740 net.paperclover.list.access: personal
741 ddns-updater: # port 8000 — keeps *.${HOME_DOMAIN} pointed at the home public IP
742 image: qmcgaw/ddns-updater
743 container_name: ddns-updater
744 user: "$USER_ID:$GROUP_ID"
745 volumes:
746 - "${APP_ROOT}/ddns-updater:/updater/data"
747 environment:
748 PERIOD: "5m"
749 # config is assembled here so the token/zone stay in .env (config.json can't interpolate)
750 CONFIG: '{"settings":[{"provider":"cloudflare","zone_identifier":"${CLOUDFLARE_ZONE_ID}","domain":"*.${HOME_DOMAIN}","ttl":1,"proxied":false,"ip_version":"ipv4","token":"${CLOUDFLARE_API_TOKEN}"}]}'
751 restart: unless-stopped
752 labels:
753 net.paperclover.list.name: DDNS
754 net.paperclover.list.access: personal
734 # evil inc temporary infrastructure755 # evil inc temporary infrastructure
735 evil-forgejo: # port 3000756 evil-forgejo: # port 3000
736 container_name: evil-forgejo757 container_name: evil-forgejo
...@@ -804,6 +825,33 @@ services:...@@ -804,6 +825,33 @@ services:
804 PGPASSWORD: "${EVIL_SEARCH_PGPASSWORD}"825 PGPASSWORD: "${EVIL_SEARCH_PGPASSWORD}"
805 user: "$USER_ID:$GROUP_ID"826 user: "$USER_ID:$GROUP_ID"
806 restart: unless-stopped827 restart: unless-stopped
828 evil-hedgedoc:
829 image: quay.io/hedgedoc/hedgedoc:1.10.6
830 environment:
831 CMD_DB_URL: "postgres://evil-hedgedoc:${POSTGRES_PASSWORD_EVIL_HEDGEDOC:-}@postgres:5432/evil-hedgedoc"
832 CMD_DOMAIN: "md.evil.inc"
833 CMD_PROTOCOL_USESSL: "true"
834 CMD_EMAIL: "false"
835 CMD_OAUTH2_PROVIDERNAME: "git.evil.inc"
836 CMD_OAUTH2_CLIENT_ID: "${EVIL_HEDGEDOC_CLIENT_ID:?}"
837 CMD_OAUTH2_CLIENT_SECRET: "${EVIL_HEDGEDOC_CLIENT_SECRET:?}"
838 CMD_OAUTH2_BASEURL: "https://git.evil.inc/login/oauth"
839 CMD_OAUTH2_USER_PROFILE_URL: "https://git.evil.inc/api/v1/user"
840 CMD_OAUTH2_AUTHORIZATION_URL: "https://git.evil.inc/login/oauth/authorize"
841 CMD_OAUTH2_TOKEN_URL: "https://git.evil.inc/login/oauth/access_token"
842 CMD_OAUTH2_USER_PROFILE_ID_ATTR: "id"
843 CMD_OAUTH2_USER_PROFILE_USERNAME_ATTR: "username"
844 CMD_OAUTH2_USER_PROFILE_DISPLAY_NAME_ATTR: "full_name"
845 CMD_OAUTH2_USER_PROFILE_EMAIL_ATTR: "email"
846 CMD_ALLOW_ANONYMOUS_EDITS: "false"
847 CMD_URL_ADDPORT: "false"
848 CMD_ALLOW_ANONYMOUS: "false"
849 volumes:
850 - "${APP_ROOT:?}/evil-infra/hedgedoc:/hedgedoc/public/uploads"
851 restart: unless-stopped
852 depends_on:
853 postgres:
854 condition: service_healthy
807855
808secrets:856secrets:
809 openvpn-credentials:857 openvpn-credentials:
config/Caddyfile+6
...@@ -250,6 +250,12 @@ search.evil.inc {...@@ -250,6 +250,12 @@ search.evil.inc {
250 }250 }
251 reverse_proxy "http://evil-search"251 reverse_proxy "http://evil-search"
252}252}
253md.evil.inc {
254 tls {
255 on_demand
256 }
257 reverse_proxy "http://evil-hedgedoc:3000"
258}
253259
254# redirections260# redirections
255paperclover.dev {261paperclover.dev {
generate-env.sh+4
...@@ -46,6 +46,10 @@ template() {...@@ -46,6 +46,10 @@ template() {
46 add "POSTGRES_PASSWORD_DAWARICH" "$(secret 32)"46 add "POSTGRES_PASSWORD_DAWARICH" "$(secret 32)"
47 add "POSTGRES_PASSWORD_EVIL_FORGEJO" "$(secret 32)"47 add "POSTGRES_PASSWORD_EVIL_FORGEJO" "$(secret 32)"
4848
49 section "cloudflare ddns"
50 add "CLOUDFLARE_ZONE_ID" ""
51 add "CLOUDFLARE_API_TOKEN" ""
52
49 section "misc keys"53 section "misc keys"
50 add "ANUBIS_PRIVATE_KEY" "$(secret 32)"54 add "ANUBIS_PRIVATE_KEY" "$(secret 32)"
51 add "FORWARD_AUTH_KEY" "$(secret 32)"55 add "FORWARD_AUTH_KEY" "$(secret 32)"