1#!/usr/bin/env bash
2set -euo pipefail
3
4instance=${1:?usage: tools/import-shale.sh shale|shale-preview-XXXXXXXX}
5[[ $instance == shale || $instance =~ ^shale-preview-[0-9a-f]{8}$ ]] || { echo 'Expected Shale or its preview ID' >&2; exit 1; }
6source_host=${STUDIO_MIGRATION_SOURCE:-zenith}
7target_host=${STUDIO_DEPLOY_HOST:-root@127.0.0.1}
8target_port=${STUDIO_DEPLOY_PORT:-2222}
9remote=(ssh -p "$target_port" "$target_host")
10job_status=200
11handoff=${STUDIO_LEGACY_HANDOFF:-}
12if [[ -n $handoff ]]; then
13 [[ $instance == shale && -n ${STUDIO_DEPLOY_HOST:-} ]] || {
14 echo 'Offline handoff requires production Shale and a target' >&2
15 exit 1
16 }
17 sh "$(dirname "$0")/check-legacy-handoff.sh" "$handoff" "$target_host" "$target_port"
18 source_copy_host=$target_host
19 source_rsh="ssh -p $target_port"
20else
21 source_copy_host=$source_host
22 source_rsh=ssh
23fi
24
25if [[ $instance == shale ]]; then
26 [[ -n ${STUDIO_DEPLOY_HOST:-} ]] || { echo 'Set STUDIO_DEPLOY_HOST to the production target' >&2; exit 1; }
27 if [[ -z $handoff ]]; then
28 [[ $(ssh "$source_host" "sudo -n docker inspect -f '{{.State.Running}}' shale") == false ]] || { echo 'Stop Zenith Shale before importing production data' >&2; exit 1; }
29 fi
30 root=/srv/prod/shale
31 source_root=/mnt/storage1/apps/shale
32 job_response=$("${remote[@]}" 'curl -s -w "\n%{http_code}" -H "X-Nomad-Token: $(cat /var/lib/studio/nomad.token)" http://127.0.0.1:4646/v1/job/shale')
33 job_status=${job_response##*$'\n'}
34 if [[ $job_status == 200 ]]; then
35 stopped=$(python3 -c 'import json,sys; print(str(json.load(sys.stdin)["Stop"]).lower())' <<<"${job_response%$'\n'*}")
36 [[ $stopped == true ]] || { echo 'Stop Shale on the home server before importing production data' >&2; exit 1; }
37 elif [[ $job_status != 404 ]]; then
38 echo "Could not verify production Shale job: $job_status" >&2
39 exit 1
40 fi
41else
42 root=/srv/staging/$instance
43 source_id=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/stages/$instance.json\"))[\"sourceId\"])'")
44 [[ $source_id == shale ]] || { echo 'Preview belongs to another service' >&2; exit 1; }
45 source_snapshot=$(ssh "$source_host" '/usr/sbin/zfs list -H -t snapshot -o name -s creation -r storage1/apps | tail -n 1')
46 [[ $source_snapshot == storage1/apps@* ]] || { echo 'No Zenith app snapshot is available' >&2; exit 1; }
47 source_root=/mnt/storage1/apps/.zfs/snapshot/${source_snapshot#*@}/shale
48fi
49
50dataset=$("${remote[@]}" "findmnt -n -o SOURCE --mountpoint $root")
51if [[ $instance == shale ]]; then
52 [[ $dataset == */prod/shale ]] || { echo 'Production Shale dataset is not mounted' >&2; exit 1; }
53else
54 [[ $dataset == */staging/$instance ]] || { echo 'Preview Shale dataset is not mounted' >&2; exit 1; }
55fi
56
57scratch=$(mktemp -d)
58trap 'rm -rf "$scratch"' EXIT
59for name in data repositories_owned repositories_mirrors; do
60 rsync -aH -e "$source_rsh" "$source_copy_host:$source_root/$name/" "$scratch/$name/"
61 drift=$(rsync -rlnc --delete --out-format='%n' -e "$source_rsh" "$source_copy_host:$source_root/$name/" "$scratch/$name/")
62 [[ -z $drift ]] || { echo "Zenith Shale $name changed during copy" >&2; exit 1; }
63done
64python3 - "$scratch/data/astheno.shale.db" <<'PY'
65import sqlite3
66import sys
67
68db = sqlite3.connect(f"file:{sys.argv[1]}?mode=ro", uri=True)
69assert db.execute("PRAGMA integrity_check").fetchone()[0] == "ok"
70PY
71
72if [[ $instance != shale ]]; then
73 "${remote[@]}" "NOMAD_TOKEN=\$(cat /var/lib/studio/nomad.token) nomad job stop -yes $instance"
74fi
75if [[ $job_status == 200 || $instance != shale ]]; then
76 for _ in {1..30}; do
77 running=$("${remote[@]}" "NOMAD_TOKEN=\$(cat /var/lib/studio/nomad.token) nomad job allocs -json $instance" |
78 python3 -c 'import json,sys; print(sum(a["ClientStatus"] == "running" for a in json.load(sys.stdin)))')
79 [[ $running == 0 ]] && break
80 sleep 2
81 done
82 [[ $running == 0 ]] || { echo 'Shale allocation did not stop' >&2; exit 1; }
83fi
84
85snapshot=$dataset@before-shale-import-$(date +%s)-$$
86"${remote[@]}" "zfs snapshot $snapshot"
87for name in data repositories_owned repositories_mirrors; do
88 "${remote[@]}" "mkdir -p $root/$name"
89 rsync -aH --delete -e "ssh -p $target_port" "$scratch/$name/" "$target_host:$root/$name/"
90 drift=$(rsync -rlnc --delete --out-format='%n' -e "ssh -p $target_port" "$scratch/$name/" "$target_host:$root/$name/")
91 [[ -z $drift ]] || { echo "Shale $name on the home server differs from the source copy; restore $snapshot" >&2; exit 1; }
92done
93uid=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/identities.json\"))[\"shale\"])'")
94"${remote[@]}" "chown -R $uid:$uid $root/data $root/repositories_owned $root/repositories_mirrors"
95"${remote[@]}" "python3 -c 'import sqlite3; db=sqlite3.connect(\"file:$root/data/astheno.shale.db?mode=ro\", uri=True); assert db.execute(\"PRAGMA integrity_check\").fetchone()[0] == \"ok\"'"
96
97if [[ $instance == shale ]]; then
98 if [[ -z $handoff ]]; then
99 [[ $(ssh "$source_host" "sudo -n docker inspect -f '{{.State.Running}}' shale") == false ]] || { echo 'Zenith Shale restarted during import; leave the home server stopped' >&2; exit 1; }
100 fi
101 echo "Imported production Shale. Previous dataset state: $snapshot"
102else
103 python3 "$(dirname "$0")/deploy.py" stage shale
104 "${remote[@]}" "zfs destroy $snapshot"
105 echo "Imported and tested $instance from $source_snapshot"
106fi