1#!/usr/bin/env python3
2"""Builds Snowbound for the web (crates/snowbound for wasm32, with crates/snowbound/web) as a
3static folder and deploys it to the VPS, which serves it at https://snowbound.paperclover.net.
4
5 python3 tools/release_web.py --out /tmp/snowbound-web/dist # build only
6 python3 tools/release_web.py --deploy # build, then replace the site
7
8The module and its JavaScript go in a folder named by their contents, b/HASH/, which
9index.html names, so a page never pairs one build's module with another's JavaScript. The
10server is snowbound-site (crates/relay), which pm2's `snowbound-site` runs from the VPS's
11snowbound-web/ on 127.0.0.1:23593, serving site/ and the Live Share codes' pages: it has
12index.html checked on every load and keeps a build's folder for good. Deploying builds it
13for the VPS too, and restarts it only when its binary changed.
14"""
15import argparse
16import gzip
17import hashlib
18import itertools
19import json
20import os
21import re
22from pathlib import Path
23import shutil
24import subprocess
25import sys
26import tempfile
27import time
28import urllib.request
29
30ROOT = Path(__file__).resolve().parents[1]
31# The one target folder wasm builds share, apart from the native builds in target/.
32TARGET = ROOT / 'target/wasm'
33WEB = ROOT / 'crates/snowbound/web'
34FONTS = ROOT / 'crates/canvas/assets/fonts'
35# Where the loading shell's icons are looked up by name, in order.
36ICONS = [ROOT / 'crates/snowbound/assets/icons', ROOT / 'crates/canvas/assets/tags', ROOT / 'crates/ui/assets']
37# The VPS's folder: snowbound-site, and the site/ it serves on 127.0.0.1:23593 as pm2's
38# `snowbound-site`, behind https://snowbound.paperclover.net.
39HOST = 'clo@paperclover.net'
40DEPLOYED = 'snowbound-web'
41SERVER = 'snowbound-site'
42URL = 'https://snowbound.paperclover.net'
43# Builds the server keeps beside the newest, for pages still running one.
44KEPT = 3
45EMOJI_URL = 'https://github.com/googlefonts/noto-emoji/raw/v2.051/fonts/Noto-COLRv1.ttf'
46EMOJI_SHA256 = '0ae57fe58645638523ba35f388d93739d292539a9acb84df5700c81b1e1a28d2'
47# As `EMOJI` in src/web.rs names it.
48EMOJI_FONT = 'Noto-COLRv1.ttf.gz'
49# Spelling dictionaries: the name the page asks for, the nixpkgs hunspellDicts package, its
50# files' stem, and the SPDX licences it comes under, each checked to allow redistribution
51# (the LGPL's text with the GPL's, which it amends). Add a row to offer another language.
52DICTIONARIES = [
53 ('en_US', 'en_US', 'en_US', ['BSD-3-Clause']),
54 ('en_GB', 'en_GB-ise', 'en_GB', ['BSD-3-Clause']),
55 ('de_DE', 'de_DE', 'de_DE', ['GPL-2.0-only', 'GPL-3.0-only']),
56 ('fr_FR', 'fr-moderne', 'fr-moderne', ['MPL-2.0']),
57 ('es_ES', 'es_ES', 'es_ES', ['GPL-3.0-only', 'LGPL-3.0-only', 'MPL-1.1']),
58 ('pt_BR', 'pt_BR', 'pt_BR', ['LGPL-3.0-only', 'GPL-3.0-only']),
59 ('pt_PT', 'pt_PT', 'pt_PT', ['GPL-2.0-only', 'LGPL-2.1-only', 'MPL-1.1']),
60 ('it_IT', 'it_IT', 'it_IT', ['GPL-3.0-only']),
61 ('nl_NL', 'nl_NL', 'nl_NL', ['BSD-3-Clause', 'CC-BY-3.0']),
62 ('sv_SE', 'sv_SE', 'sv_SE', ['LGPL-3.0-only', 'GPL-3.0-only']),
63 ('ru_RU', 'ru_RU', 'ru_RU', ['MPL-2.0', 'LGPL-3.0-only', 'GPL-3.0-only']),
64]
65# Size over speed where it costs little: the module is most of the first load.
66PROFILE = ['--config', 'profile.release.opt-level="s"']
67
68
69def run(command, **kwargs):
70 print('+', ' '.join(map(str, command)), flush=True)
71 subprocess.run(command, cwd=ROOT, check=True, **kwargs)
72
73
74def tool(name, nix=None, wasm=False, store=False):
75 """`name` from PATH or Cargo's bin folder, or else from nixpkgs package `nix`; with `wasm`,
76 from nixpkgs only, as the system's clang may not build for wasm32; with `store`, the
77 package's folder."""
78 found = None if wasm or store else shutil.which(name) or shutil.which(name, path=str(Path.home() / '.cargo/bin'))
79 if found:
80 return found
81 if nix and shutil.which('nix'):
82 built = subprocess.check_output(['nix', 'build', f'nixpkgs#{nix}', '--no-link', '--print-out-paths'],
83 text=True).split()[-1]
84 return built if store else str(Path(built) / 'bin' / name)
85 sys.exit(f'{name} is missing: cargo install wasm-bindgen-cli at the version Cargo.lock pins, '
86 'and binaryen for wasm-opt')
87
88
89def environment():
90 """Cargo's environment for wasm32: the wasm target folder, and a clang and llvm-ar that
91 build for wasm32, which SQLite (through sqlite-wasm-rs) needs and Apple's clang lacks."""
92 tools = {}
93 for variable, package, name in (('CC_wasm32_unknown_unknown', 'llvmPackages.clang-unwrapped', 'clang'),
94 ('AR_wasm32_unknown_unknown', 'llvmPackages.llvm', 'llvm-ar')):
95 tools[variable] = os.environ.get(variable) or tool(name, package, wasm=True)
96 return {**os.environ, 'CARGO_TARGET_DIR': str(TARGET), **tools}
97
98
99def build(out):
100 """Writes index.html, the module, its JavaScript and the fonts to `out`, the module knowing
101 when it was built."""
102 built = str(int(time.time()))
103 run(['cargo', 'build', '--locked', '-p', 'snowbound', '--release', '--target', 'wasm32-unknown-unknown', '--no-default-features', '--features', 'wgpu,live',
104 *PROFILE], env={**environment(), 'SNOWBOUND_WEB_BUILD': built})
105 if out.exists():
106 shutil.rmtree(out)
107 out.mkdir(parents=True)
108 module = TARGET / 'wasm32-unknown-unknown/release/snowbound.wasm'
109 code = out / 'b/next'
110 run([tool('wasm-bindgen'), '--target', 'web', '--no-typescript', '--out-name', 'snowbound_web',
111 '--out-dir', code, module])
112 bound = code / 'snowbound_web_bg.wasm'
113 run([tool('wasm-opt', 'binaryen'), '-Oz', '--strip-debug', '--strip-producers', bound, '-o', bound])
114 digest = hashlib.sha256()
115 for path in sorted(code.rglob('*')):
116 if path.is_file():
117 digest.update(str(path.relative_to(code)).encode() + b'\0' + path.read_bytes())
118 code = code.rename(out / 'b' / digest.hexdigest()[:16])
119 dictionaries(out / 'dictionaries')
120 (out / 'fonts').mkdir()
121 for font in sorted(FONTS.glob('*')):
122 if font.suffix in ('.ttf', '.txt'):
123 shutil.copy(font, out / 'fonts')
124 fallbacks(out / 'fonts')
125 page(out, code.relative_to(out).as_posix())
126 for path in sorted(out.rglob('*')):
127 if path.is_file():
128 print(f'{path.stat().st_size:>12,} {path.relative_to(out)}')
129
130
131def page(out, code):
132 """Writes index.html with its loading shell's icons inlined, the folder `code` it takes
133 the module and its JavaScript from, and the sizes of the files it fetches before the app
134 starts, for its progress bar."""
135 count = itertools.count()
136
137 def inline(match):
138 name = match[2]
139 svg = next(folder / f'{name}.svg' for folder in ICONS if (folder / f'{name}.svg').exists()).read_text()
140 # Each copy's gradients keep ids of their own in the one document.
141 prefix = f'i{next(count)}-'
142 svg = re.sub(r'id="([^"]+)"', rf'id="{prefix}\1"', svg)
143 svg = re.sub(r'url\(#([^)]+)\)', rf'url(#{prefix}\1)', svg)
144 return f'<i{match[1]}>{svg.strip()}</i>'
145 html = re.sub(r'<i([^>]*) icon="([^"]+)"></i>', inline, (WEB / 'index.html').read_text())
146 sizes = {path.relative_to(out).as_posix(): path.stat().st_size
147 for path in [out / code / 'snowbound_web_bg.wasm', *sorted((out / 'fonts').glob('*.ttf'))]}
148 for marker, written in [('const SIZES = {};', f'const SIZES = {json.dumps(sizes)};'),
149 ('"./snowbound_web.js"', f'"./{code}/snowbound_web.js"'),
150 ('const MODULE = "snowbound_web_bg.wasm";', f'const MODULE = "{code}/snowbound_web_bg.wasm";')]:
151 if marker not in html:
152 sys.exit(f'index.html lacks `{marker}`')
153 html = html.replace(marker, written)
154 (out / 'index.html').write_text(html)
155
156
157def dictionaries(folder):
158 """The Hunspell dictionaries in DICTIONARIES as UTF-8, gzipped for the glue to inflate,
159 each beside its readme, the texts of the licences they come under in licenses/, and their
160 names in index.json for the page to pick from (`pick` in canvas/src/spelling.rs)."""
161 (folder / 'licenses').mkdir(parents=True)
162 texts = Path(tool('spdx', 'spdx-license-list-data.text', store=True)) / 'text'
163 for name, package, stem, licenses in DICTIONARIES:
164 store = Path(tool('hunspell', f'hunspellDicts.{package}', store=True))
165 affix = (store / f'share/hunspell/{stem}.aff').read_bytes()
166 # Hunspell names the files' encoding in the affix file's SET line.
167 encoding = next((line.split()[1] for line in affix.decode('latin-1').splitlines()
168 if line.startswith('SET ')), 'UTF-8')
169 for kind in ('aff', 'dic'):
170 text = (store / f'share/hunspell/{stem}.{kind}').read_bytes().decode(encoding)
171 # One encoding and one line ending for spellbook, which reads only UTF-8. An
172 # 8-bit file's flags are its characters by default, as FLAG UTF-8 keeps them.
173 utf8 = 'SET UTF-8' if encoding == 'UTF-8' or 'FLAG ' in text else 'SET UTF-8\nFLAG UTF-8'
174 text = ''.join((utf8 if line.startswith('SET ') else line) + '\n'
175 for line in text.splitlines())
176 (folder / f'{name}.{kind}.gz').write_bytes(gzip.compress(text.encode(), 9, mtime=0))
177 # Where a package has no readme, its affix file's header comment names the licences.
178 readme = list((store / 'share/doc').glob('*.txt'))
179 notice = readme[0].read_bytes() if readme else \
180 ''.join(line + '\n' for line in affix.decode(encoding).splitlines() if line.startswith('#')).encode()
181 (folder / f'{name}.txt').write_bytes(notice + f'\nLicences: {", ".join(licenses)} (licenses/)\n'.encode())
182 for license in licenses:
183 shutil.copyfile(texts / f'{license}.txt', folder / 'licenses' / f'{license}.txt')
184 (folder / 'index.json').write_text(json.dumps([name for name, *_ in DICTIONARIES]))
185
186
187def fallbacks(fonts):
188 """Noto's faces for scripts the bundled ones lack, which the page fetches as it needs them
189 (`FALLBACKS` in src/web.rs), under the SIL Open Font License."""
190 noto = Path(tool('noto', 'noto-fonts', store=True)) / 'share/fonts/noto'
191 for name in ('NotoSansArabic.ttf', 'NotoSansHebrew.ttf', 'NotoSansDevanagari.ttf', 'NotoSansThai.ttf',
192 'NotoSansSymbols2-Regular.otf'):
193 shutil.copy(noto / name, fonts)
194 # Cutting the CJK face takes a minute or two, so the cut is kept between builds.
195 cjk = TARGET / 'NotoSansCJK.otf'
196 if not cjk.exists():
197 collection = Path(tool('noto', 'noto-fonts-cjk-sans', store=True)) / \
198 'share/fonts/opentype/noto-cjk/NotoSansCJK-VF.otf.ttc'
199 run(['uv', 'run', '--no-project', '--with', 'fonttools', 'python', ROOT / 'tools/web/subset_cjk.py',
200 collection, cjk])
201 shutil.copy(cjk, fonts)
202 # Noto Color Emoji as COLRv1 outlines (4.8 MB), which src/render/colr.rs in draw paints,
203 # gzipped to 2.7 MB for the glue to inflate; nixpkgs has only its 10 MB of bitmaps.
204 emoji = TARGET / EMOJI_FONT
205 if not emoji.exists():
206 data = urllib.request.urlopen(EMOJI_URL).read()
207 if hashlib.sha256(data).hexdigest() != EMOJI_SHA256:
208 sys.exit(f'{EMOJI_URL} is not the font this build pins')
209 emoji.write_bytes(gzip.compress(data, 9, mtime=0))
210 shutil.copy(emoji, fonts)
211 shutil.copy(ROOT / 'crates/draw/assets/NotoColorEmoji-OFL.txt', fonts)
212 (fonts / 'Noto-OFL.txt').write_text('Noto fonts: SIL Open Font License 1.1, https://openfontlicense.org\n')
213
214
215def deploy(built):
216 """Makes the VPS's site `built`, each file taking its place once all have arrived, and
217 leaves the last builds' folders for pages still running them; replaces its server, and
218 restarts it, where the server built now differs."""
219 sync = ['rsync', '--recursive', '--links', '--times', '--compress', '--itemize-changes']
220 arch = subprocess.check_output(['ssh', HOST, 'uname -m'], text=True).strip()
221 with tempfile.TemporaryDirectory() as scratch:
222 run([sys.executable, ROOT / 'tools/release_relay.py', '--architectures', arch, '--output', scratch])
223 server = subprocess.run([*sync, '--checksum', Path(scratch) / f'{SERVER}-linux-{arch}',
224 f'{HOST}:{DEPLOYED}/{SERVER}'],
225 check=True, capture_output=True, text=True).stdout
226 run([*sync, '--delete', '--delay-updates', '--filter=P b/*', f'{built}/', f'{HOST}:{DEPLOYED}/site/'])
227 run(['ssh', HOST, f'cd {DEPLOYED}/site/b && ls -t | tail -n +{KEPT + 2} | xargs -r rm -rf --'])
228 if server.strip():
229 run(['ssh', HOST, f'pm2 restart {SERVER}'])
230 print(f'Deployed {URL}')
231
232
233def main():
234 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
235 parser.add_argument('--out', type=Path, help='Build into this folder (default: a temporary one)')
236 parser.add_argument('--deploy', action='store_true', help=f'Copy the build to {DEPLOYED}')
237 args = parser.parse_args()
238 if not args.out and not args.deploy:
239 parser.error('give --out, --deploy or both')
240 with tempfile.TemporaryDirectory() as scratch:
241 out = args.out or Path(scratch) / 'web'
242 build(out)
243 if args.deploy:
244 deploy(out)
245
246
247if __name__ == '__main__':
248 main()