From e0c36a5d090be3ff5ac6e7c4fdab752ad63f03d6 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sat, 23 May 2026 12:55:56 -0700 Subject: [PATCH] backup nonsense --- backup/README.md | 67 ++++++++++++++++++++++++++++++++++ backup/snapshot-prune.sh | 79 ++++++++++++++++++++++++++++++++++++++++ backup/status.sh | 33 +++++++++++++++++ compose.yaml | 48 ++++++++++++++++++++++++ config/Caddyfile | 6 +++ generate-env.sh | 4 ++ 6 files changed, 237 insertions(+) create mode 100644 backup/README.md create mode 100755 backup/snapshot-prune.sh create mode 100755 backup/status.sh diff --git a/backup/README.md b/backup/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ecbcbd9f504a2b4a9b3fdc32622a3b052128377 --- /dev/null +++ b/backup/README.md @@ -0,0 +1,67 @@ +# sandwich backup + +Plain rsync mirror of the Mac Studio (`sandwich`) onto the NAS, versioned with ZFS +snapshots. Replaces the old encrypted Duplicacy repo — the backup here is just +**real files** you can browse and copy out, with cheap copy-on-write history. + +## Pieces + +| Where | What | Lives in | +|-------|------|----------| +| sandwich | hourly rsync push + snapshot trigger | `~/config` → `users/clover/sandwich/home.nix` (`launchd.agents.sandwich-backup`) | +| NAS (zenith) | `snapshot-prune.sh` — snapshot + tiered prune | this dir, deployed via `../sync.sh` | + +Flow: every hour sandwich rsyncs `Asset`, `Project`, `Documents` (minus `LMStudio`) +and `~/Desktop` into `storage1/backup/sandwich/`, then — only on a clean rsync — +ssh's to the NAS and runs `snapshot-prune.sh`, so every snapshot is a *complete* mirror. + +## Retention (`snapshot-prune.sh`) + +Only `@auto-*` snapshots are created/destroyed; manual ones are never touched. + +| Tier | Kept | +|--------|------| +| hourly (every run) | 24 h | +| daily (newest per UTC day) | 14 d | +| weekly (newest per ISO week) | 8 wk | + +Dry-run the prune without destroying anything: + +```sh +ssh git.paperclover.net /mnt/storage1/apps/home-infra/backup/snapshot-prune.sh storage1/backup/sandwich --dry-run +``` + +## Observability + +```sh +./backup/status.sh +``` + +Shows the launchd agent state + which volume is currently syncing on sandwich, and +the landed data + snapshots on the NAS. Notes: +- The `lastexit` in `launchctl list` is the *previous* run's code; it reads `0` after a + clean run. While a sync is in progress the agent shows its live pid instead. +- `~/Library/Logs/sandwich-backup.log` only gets an rsync `--stats` block when each + folder *finishes* (rsync buffers stats to the end), so during a big initial sync the + most current signal is the NAS `USED` growing, not the log. +- Manually trigger a run: `launchctl kickstart -k gui/$(id -u)/org.nix-community.home.sandwich-backup` (on sandwich). + +## Restore + +Snapshots are browsable on the NAS under the hidden `.zfs` dir — no rollback needed: + +```sh +ls /mnt/storage1/backup/sandwich/.zfs/snapshot/ +cp -a /mnt/storage1/backup/sandwich/.zfs/snapshot/auto-20260610-051120/Desktop/notes.txt ~/restored/ +``` + +Or just grab current files straight from `/mnt/storage1/backup/sandwich/`. + +## One-time setup + +1. Authorize sandwich's SSH pubkey for `clo@zenith` (so push + snapshot trigger work passwordless). +2. On the NAS: `zfs create -o compression=zstd -o atime=off -o recordsize=1M storage1/backup/sandwich`, + `chown clo:chloe` its mountpoint, and delegate snapshot rights: + `zfs allow clo snapshot,destroy,mount,hold,release storage1/backup/sandwich`. +3. Deploy this dir: `../sync.sh` (lands at `/mnt/storage1/apps/home-infra/backup/`), `chmod +x snapshot-prune.sh`. +4. Apply sandwich's Nix: `./switch` in `~/config`. diff --git a/backup/snapshot-prune.sh b/backup/snapshot-prune.sh new file mode 100755 index 0000000000000000000000000000000000000000..76046ff61f2eb139e69d3465638a65a86ae134aa --- /dev/null +++ b/backup/snapshot-prune.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +# snapshot-prune.sh DATASET [--dry-run] +# +# Takes a timestamped ZFS snapshot of DATASET, then prunes auto-snapshots with a +# tiered (grandfather-father-son) retention policy: +# +# hourly : keep everything younger than 24h +# daily : keep the newest snapshot per UTC day, out to 14 days +# weekly : keep the newest snapshot per ISO week, out to 8 weeks +# older : destroyed +# +# Only snapshots named DATASET@auto-* are ever created or destroyed; any manual +# or replication snapshots are left untouched. +# +# Intended to be invoked over ssh by sandwich right after a successful rsync push: +# ssh nas /mnt/storage1/apps/home-infra/backup/snapshot-prune.sh storage1/backup/sandwich +# +# Snapshot create/destroy is expected to run without sudo via delegated perms: +# zfs allow clo snapshot,destroy,mount,hold,release storage1/backup/sandwich +set -euo pipefail + +DS=${1:-} +DRYRUN=0 +[ "${2:-}" = "--dry-run" ] && DRYRUN=1 +[ "${1:-}" = "--dry-run" ] && { DRYRUN=1; DS=${2:-}; } + +if [ -z "$DS" ]; then + echo "usage: $0 DATASET [--dry-run]" >&2 + exit 2 +fi + +# Non-interactive ssh sessions have a minimal PATH; find zfs explicitly. +ZFS=$(command -v zfs || true) +[ -z "$ZFS" ] && for p in /usr/sbin/zfs /sbin/zfs /usr/local/sbin/zfs; do + [ -x "$p" ] && ZFS=$p && break +done +[ -z "$ZFS" ] && { echo "zfs binary not found" >&2; exit 1; } + +run() { if [ "$DRYRUN" = 1 ]; then echo "DRY: $*"; else "$@"; fi; } + +# Retention windows, in seconds. +DAY=86400 +HOURLY_KEEP=$(( 1 * DAY )) # 24h: keep all +DAILY_KEEP=$(( 14 * DAY )) # newest-per-day out to 14d +WEEKLY_KEEP=$(( 56 * DAY )) # newest-per-week out to 8wk + +# 1. Take this run's snapshot. +SNAP="$DS@auto-$(date -u +%Y%m%d-%H%M%S)" +run "$ZFS" snapshot "$SNAP" + +# 2. Prune. Walk auto-snapshots newest-first so the first one seen in any +# day/week bucket is the newest and is the one we keep. +now=$(date -u +%s) +declare -A seen_day seen_week + +while IFS=$'\t' read -r name creation; do + [ -n "$name" ] || continue + age=$(( now - creation )) + keep=0 + if (( age < HOURLY_KEEP )); then + keep=1 + elif (( age < DAILY_KEEP )); then + key=$(date -u -d "@$creation" +%Y%j) # year + day-of-year + [ -z "${seen_day[$key]:-}" ] && { keep=1; seen_day[$key]=1; } + elif (( age < WEEKLY_KEEP )); then + key=$(date -u -d "@$creation" +%G%V) # ISO year + week number + [ -z "${seen_week[$key]:-}" ] && { keep=1; seen_week[$key]=1; } + fi + + if (( keep )); then + echo "keep $name" + else + echo "destroy $name" + run "$ZFS" destroy "$name" + fi +done < <("$ZFS" list -Hp -o name,creation -t snapshot -r "$DS" \ + | awk -F'\t' 'index($1, "@auto-")' \ + | sort -t"$(printf '\t')" -k2,2nr) diff --git a/backup/status.sh b/backup/status.sh new file mode 100755 index 0000000000000000000000000000000000000000..5467e0d96dae01772c2441a6e3ef28c59e641532 --- /dev/null +++ b/backup/status.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# Observability for the sandwich backup. Run from this Mac: ./backup/status.sh +# Shows the launchd agent state + live rsync target on sandwich, and the +# landed data + snapshots on the NAS. +NAS=git.paperclover.net +SANDWICH=sandwich.local +DS=storage1/backup/sandwich + +echo "===== sandwich (agent) =====" +ssh -o BatchMode=yes "$SANDWICH" ' + echo "agent (pid / lastexit / label):" + launchctl list 2>/dev/null | grep sandwich-backup || echo " not loaded" + if [ -d /tmp/sandwich-backup.lock ]; then + echo "state: RUNNING (pid $(cat /tmp/sandwich-backup.lock/pid 2>/dev/null))" + pgrep -fl rsync | grep -o "/Volumes/[A-Za-z]*" | head -1 | sed "s/^/ currently syncing: /" + else + echo "state: idle (waiting for hourly tick or next kickstart)" + fi + echo "--- last log lines ---" + tail -5 "$HOME/Library/Logs/sandwich-backup.log" 2>/dev/null || echo " (no log yet)" +' +echo +echo "===== NAS (storage1/backup/sandwich) =====" +ssh -o BatchMode=yes "$NAS" " + /usr/sbin/zfs list -o name,used,refer $DS + echo + for d in Asset Project Documents Desktop; do + printf ' %-11s%s\n' \"\$d\" \"\$(du -sh /mnt/$DS/\$d 2>/dev/null | cut -f1)\" + done + echo + echo 'snapshots (newest last):' + /usr/sbin/zfs list -t snapshot -r $DS -o name,used 2>/dev/null | tail -6 +" diff --git a/compose.yaml b/compose.yaml index 36c8cef04a223b0e9fd0c61411405726eba4fbd2..e6d33af4c9f0789f80ecf9e36eeadb7f1e641d76 100755 --- a/compose.yaml +++ b/compose.yaml @@ -398,6 +398,13 @@ services: net.paperclover.list.domain: jkt net.paperclover.list.priority: 40 net.paperclover.list.access: media-manage + flaresolverr: # port 8191 + container_name: flaresolverr + image: ghcr.io/flaresolverr/flaresolverr:latest + restart: unless-stopped + environment: + - "LOG_LEVEL=info" + - "TZ=America/Los_Angelas" sonarr: # port 8989 container_name: sonarr environment: @@ -731,6 +738,20 @@ services: net.paperclover.list.domain: dns net.paperclover.list.priority: 9 net.paperclover.list.access: personal + ddns-updater: # port 8000 — keeps *.${HOME_DOMAIN} pointed at the home public IP + image: qmcgaw/ddns-updater + container_name: ddns-updater + user: "$USER_ID:$GROUP_ID" + volumes: + - "${APP_ROOT}/ddns-updater:/updater/data" + environment: + PERIOD: "5m" + # config is assembled here so the token/zone stay in .env (config.json can't interpolate) + CONFIG: '{"settings":[{"provider":"cloudflare","zone_identifier":"${CLOUDFLARE_ZONE_ID}","domain":"*.${HOME_DOMAIN}","ttl":1,"proxied":false,"ip_version":"ipv4","token":"${CLOUDFLARE_API_TOKEN}"}]}' + restart: unless-stopped + labels: + net.paperclover.list.name: DDNS + net.paperclover.list.access: personal # evil inc temporary infrastructure evil-forgejo: # port 3000 container_name: evil-forgejo @@ -804,6 +825,33 @@ services: PGPASSWORD: "${EVIL_SEARCH_PGPASSWORD}" user: "$USER_ID:$GROUP_ID" restart: unless-stopped + evil-hedgedoc: + image: quay.io/hedgedoc/hedgedoc:1.10.6 + environment: + CMD_DB_URL: "postgres://evil-hedgedoc:${POSTGRES_PASSWORD_EVIL_HEDGEDOC:-}@postgres:5432/evil-hedgedoc" + CMD_DOMAIN: "md.evil.inc" + CMD_PROTOCOL_USESSL: "true" + CMD_EMAIL: "false" + CMD_OAUTH2_PROVIDERNAME: "git.evil.inc" + CMD_OAUTH2_CLIENT_ID: "${EVIL_HEDGEDOC_CLIENT_ID:?}" + CMD_OAUTH2_CLIENT_SECRET: "${EVIL_HEDGEDOC_CLIENT_SECRET:?}" + CMD_OAUTH2_BASEURL: "https://git.evil.inc/login/oauth" + CMD_OAUTH2_USER_PROFILE_URL: "https://git.evil.inc/api/v1/user" + CMD_OAUTH2_AUTHORIZATION_URL: "https://git.evil.inc/login/oauth/authorize" + CMD_OAUTH2_TOKEN_URL: "https://git.evil.inc/login/oauth/access_token" + CMD_OAUTH2_USER_PROFILE_ID_ATTR: "id" + CMD_OAUTH2_USER_PROFILE_USERNAME_ATTR: "username" + CMD_OAUTH2_USER_PROFILE_DISPLAY_NAME_ATTR: "full_name" + CMD_OAUTH2_USER_PROFILE_EMAIL_ATTR: "email" + CMD_ALLOW_ANONYMOUS_EDITS: "false" + CMD_URL_ADDPORT: "false" + CMD_ALLOW_ANONYMOUS: "false" + volumes: + - "${APP_ROOT:?}/evil-infra/hedgedoc:/hedgedoc/public/uploads" + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy secrets: openvpn-credentials: diff --git a/config/Caddyfile b/config/Caddyfile index 1f05b8f3154a87118505e153e3357d2ab18308d6..2fe1eef8ea47b5f1ea160fd938646510c2f2c2f2 100644 --- a/config/Caddyfile +++ b/config/Caddyfile @@ -250,6 +250,12 @@ search.evil.inc { } reverse_proxy "http://evil-search" } +md.evil.inc { + tls { + on_demand + } + reverse_proxy "http://evil-hedgedoc:3000" +} # redirections paperclover.dev { diff --git a/generate-env.sh b/generate-env.sh index 625eee94da2ad64ae63af189b72352e28ded8a79..f2c8850399d518647023ad540c64c16512b76ae3 100644 --- a/generate-env.sh +++ b/generate-env.sh @@ -46,6 +46,10 @@ template() { add "POSTGRES_PASSWORD_DAWARICH" "$(secret 32)" add "POSTGRES_PASSWORD_EVIL_FORGEJO" "$(secret 32)" + section "cloudflare ddns" + add "CLOUDFLARE_ZONE_ID" "" + add "CLOUDFLARE_API_TOKEN" "" + section "misc keys" add "ANUBIS_PRIVATE_KEY" "$(secret 32)" add "FORWARD_AUTH_KEY" "$(secret 32)" -- 2.54.0