| 1 | #!/usr/bin/env bash |
| 2 | set -euo pipefail |
| 3 | |
| 4 | instance=${1:-ytdl} |
| 5 | [[ $instance == ytdl || $instance =~ ^ytdl-preview-[0-9a-f]{8}$ ]] || { |
| 6 | echo 'Expected ytdl or its preview ID' >&2 |
| 7 | exit 1 |
| 8 | } |
| 9 | |
| 10 | source_host=${STUDIO_MIGRATION_SOURCE:-zenith} |
| 11 | target_host=${STUDIO_DEPLOY_HOST:-root@127.0.0.1} |
| 12 | target_port=${STUDIO_DEPLOY_PORT:-2222} |
| 13 | remote=(ssh -p "$target_port" "$target_host") |
| 14 | source_copy_host=$source_host |
| 15 | source_rsh=ssh |
| 16 | offline=false |
| 17 | if [[ -n ${STUDIO_LEGACY_HANDOFF:-} ]]; then |
| 18 | [[ $instance == ytdl && -n ${STUDIO_DEPLOY_HOST:-} ]] || { |
| 19 | echo 'Offline handoff requires production YouTube Triage and a target' >&2; exit 1; |
| 20 | } |
| 21 | sh "$(dirname "$0")/check-legacy-handoff.sh" "$STUDIO_LEGACY_HANDOFF" "$target_host" "$target_port" |
| 22 | source_copy_host=$target_host |
| 23 | source_rsh="ssh -p $target_port" |
| 24 | offline=true |
| 25 | fi |
| 26 | if [[ $instance == ytdl ]]; then |
| 27 | root=/srv/prod/ytdl |
| 28 | "${remote[@]}" 'test "$(findmnt -n -o FSTYPE --mountpoint /srv/clover/Media)" = zfs; test -d "/srv/clover/Media/Intake - Music"; test -d "/srv/clover/Media/Indie Shows"; test -d /srv/clover/Media/Videos/Independent' || { |
| 29 | echo 'Mount Media with Intake - Music, Indie Shows, and Videos/Independent before importing YouTube Triage' >&2 |
| 30 | exit 1 |
| 31 | } |
| 32 | if [[ $offline == false ]]; then |
| 33 | source_running=$(ssh "$source_host" "sudo -n docker inspect -f '{{.State.Running}}' yt-feed") |
| 34 | [[ $source_running == false ]] || { echo 'Stop Zenith yt-feed before importing production data' >&2; exit 1; } |
| 35 | fi |
| 36 | response=$("${remote[@]}" "curl -s -w '\n%{http_code}' -H \"X-Nomad-Token: \$(cat /var/lib/studio/nomad.token)\" http://127.0.0.1:4646/v1/job/ytdl") |
| 37 | status=${response##*$'\n'} |
| 38 | if [[ $status == 200 ]]; then |
| 39 | stopped=$(python3 -c 'import json,sys; print(str(json.load(sys.stdin)["Stop"]).lower())' <<<"${response%$'\n'*}") |
| 40 | [[ $stopped == true ]] || { echo 'Stop production YTDL before importing' >&2; exit 1; } |
| 41 | elif [[ $status != 404 ]]; then |
| 42 | echo "Could not verify production YTDL job state: $status" >&2 |
| 43 | exit 1 |
| 44 | fi |
| 45 | else |
| 46 | root=/srv/staging/$instance |
| 47 | source_id=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/stages/$instance.json\"))[\"sourceId\"])'") |
| 48 | [[ $source_id == ytdl ]] || { echo 'Preview belongs to another service' >&2; exit 1; } |
| 49 | fi |
| 50 | dataset=$("${remote[@]}" "findmnt -n -o SOURCE --mountpoint $root") |
| 51 | [[ $dataset == */prod/ytdl || $dataset == */staging/"$instance" ]] || { |
| 52 | echo "Expected a mounted YTDL dataset at $root" >&2 |
| 53 | exit 1 |
| 54 | } |
| 55 | |
| 56 | scratch=$(mktemp -d) |
| 57 | trap 'rm -rf "$scratch"' EXIT |
| 58 | rsync -a -e "$source_rsh" "$source_copy_host:/mnt/storage1/apps/yt-feed/" "$scratch/" |
| 59 | python3 - "$scratch" <<'PY' |
| 60 | import json |
| 61 | from pathlib import Path |
| 62 | import sys |
| 63 | |
| 64 | root = Path(sys.argv[1]) |
| 65 | files = list(root.glob("*.json")) |
| 66 | if not files: |
| 67 | raise ValueError("no YouTube Triage JSON state was copied") |
| 68 | for file in files: |
| 69 | json.loads(file.read_text()) |
| 70 | print(f"Copied {len(files)} valid JSON state files") |
| 71 | PY |
| 72 | drift=$(rsync -rnc --delete --out-format='%n' -e "$source_rsh" "$source_copy_host:/mnt/storage1/apps/yt-feed/" "$scratch/") |
| 73 | [[ -z $drift ]] || { echo 'Source changed during copy; retry the import' >&2; exit 1; } |
| 74 | |
| 75 | if [[ $instance != ytdl ]]; then |
| 76 | "${remote[@]}" "NOMAD_TOKEN=\$(cat /var/lib/studio/nomad.token) nomad job stop -yes $instance" |
| 77 | else |
| 78 | "${remote[@]}" 'systemctl stop studio-dashboard' |
| 79 | fi |
| 80 | for _ in {1..30}; do |
| 81 | running=$("${remote[@]}" "curl -s -H \"X-Nomad-Token: \$(cat /var/lib/studio/nomad.token)\" http://127.0.0.1:4646/v1/job/$instance/allocations" | |
| 82 | python3 -c 'import json,sys; print(sum(x["ClientStatus"] == "running" for x in json.load(sys.stdin)))') |
| 83 | [[ $running == 0 ]] && break |
| 84 | sleep 2 |
| 85 | done |
| 86 | [[ $running == 0 ]] || { echo "$instance still has a running allocation" >&2; exit 1; } |
| 87 | |
| 88 | snapshot="$dataset@before-yt-feed-import-$(date +%s)-$$" |
| 89 | "${remote[@]}" "zfs snapshot $snapshot" |
| 90 | rsync -a --delete -e "ssh -p $target_port" "$scratch/" "$target_host:$root/data/" |
| 91 | uid=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/identities.json\"))[\"ytdl\"])'") |
| 92 | "${remote[@]}" "chown -R $uid:$uid $root/data" |
| 93 | drift=$(rsync -rnc --delete --out-format='%n' -e "ssh -p $target_port" "$scratch/" "$target_host:$root/data/") |
| 94 | [[ -z $drift ]] || { echo "Copied state differs from source; restore $snapshot" >&2; exit 1; } |
| 95 | |
| 96 | if [[ $instance == ytdl ]]; then |
| 97 | if [[ $offline == false ]]; then |
| 98 | [[ $(ssh "$source_host" "sudo -n docker inspect -f '{{.State.Running}}' yt-feed") == false ]] || { |
| 99 | echo 'Zenith yt-feed restarted during import; leave the home server stopped' >&2 |
| 100 | exit 1 |
| 101 | } |
| 102 | fi |
| 103 | "${remote[@]}" 'systemctl start studio-dashboard' |
| 104 | echo "Imported production YTDL from Zenith. Previous dataset state: $snapshot" |
| 105 | else |
| 106 | python3 "$(dirname "$0")/deploy.py" stage ytdl |
| 107 | "${remote[@]}" "zfs destroy $snapshot" |
| 108 | echo "Imported and tested $instance" |
| 109 | fi |