1#!/usr/bin/env python3
2"""Serve the HTML diagnostic editor on a new notebook copy."""
3import argparse
4import hashlib
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json
7import mimetypes
8import os
9from pathlib import Path
10import re
11import shutil
12import subprocess
13from threading import Lock
14import time
15from urllib.parse import parse_qs, urlsplit
16import uuid
17
18from document_model import BRIDGE, walk
19from notebook_report import generate
20
21ROOT = Path(__file__).resolve().parent.parent
22
23
24def bridge(mode, source, destination, edit=None):
25 try:
26 result = subprocess.run([BRIDGE, mode, source, destination],
27 input=json.dumps(edit) if edit is not None else None,
28 capture_output=True, text=True, timeout=120)
29 if result.returncode:
30 raise RuntimeError(result.stderr or f'Diagnostic process exited {result.returncode}')
31 return json.loads(result.stdout)
32 except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error:
33 return {'ok': False, 'state': 'Unknown' if mode == 'commit' else 'NotCommitted',
34 'kind': 'Process', 'error': str(error)}
35
36
37class Session:
38 def __init__(self, source, output):
39 source = source.resolve(strict=True)
40 self.output = output.resolve()
41 if self.output.is_relative_to(source):
42 raise ValueError('Choose a session directory outside the source notebook.')
43 self.output.mkdir(parents=True, exist_ok=False)
44 self.lock = Lock()
45 hashes = {p.relative_to(source).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
46 for p in source.rglob('*') if p.is_file()}
47 shutil.copytree(source, self.output / 'notebook')
48 for name, digest in hashes.items():
49 copied = self.output / 'notebook' / name
50 if hashlib.sha256(copied.read_bytes()).hexdigest() != digest or hashlib.sha256((source / name).read_bytes()).hexdigest() != digest:
51 raise ValueError('The source changed during copying; start a fresh session.')
52 copied.chmod(copied.stat().st_mode | 0o600)
53 (self.output / 'source.json').write_text(json.dumps({'root': str(source), 'sha256': hashes}, indent=2))
54 (self.output / 'g').mkdir()
55 (self.output / 'objects').mkdir()
56 self.snapshot()
57
58 def snapshot(self):
59 number = max((int(p.name) for p in (self.output / 'g').iterdir() if p.name.isdecimal()), default=-1) + 1
60 pending = self.output / 'g' / (str(number) + '-' + uuid.uuid4().hex + '.building')
61 source = pending / 'snapshot'
62 source.mkdir(parents=True)
63 for path in sorted((self.output / 'notebook').rglob('*')):
64 if path.suffix.lower() not in ('.one', '.onetoc2', '.onebin'): continue
65 saved = source / path.relative_to(self.output / 'notebook')
66 saved.parent.mkdir(parents=True, exist_ok=True)
67 result = bridge('snapshot', path, saved)
68 if not result['ok']: raise RuntimeError(result['error'])
69 digest = hashlib.sha256(saved.read_bytes()).hexdigest()
70 blob = self.output / 'objects' / digest
71 if blob.exists(): saved.unlink()
72 else:
73 saved.rename(blob)
74 blob.chmod(0o444)
75 os.link(blob, saved)
76 previous = self.output / 'g' / str(number - 1) / 'report' if number else None
77 generate(source, pending / 'report', editable=True, previous=previous)
78 pending.rename(self.output / 'g' / str(number))
79 return number
80
81 def page(self, generation, page):
82 if type(generation) is not int or generation < 0:
83 raise ValueError('Choose a page from this report.')
84 folder = self.output / 'g' / str(generation)
85 pages = json.loads((folder / 'report/pages.json').read_text())
86 row = next(p for p in pages if p['report'] == page)
87 if row['category'] != 'Page':
88 raise ValueError('Choose an active page. Conflicts, templates, deleted pages and history are read-only here.')
89 sources = json.loads((folder / 'report/source.json').read_text())
90 index = next(i for i, source in enumerate(sources) if source['path'] == row['section'])
91 model = folder / 'report/model' / str(index)
92 document = json.loads((model / 'document.json').read_text())
93 revision = document['spaces'][row['space']]['revisions'][row['revision']]
94 return row, folder / 'snapshot' / row['section'], revision, model
95
96 def selection(self, generation, page, oid, run):
97 if type(run) is not int or run < 0:
98 raise ValueError('Choose a text run from this report.')
99 row, source, revision, model = self.page(generation, page)
100 node = next(node for key, node in walk(revision, row['object']) if key == oid)
101 if node['kind']['type'] != 'RichText': raise ValueError('Choose a text run.')
102 selected = node['kind']['runs'][run]
103 text = json.loads((model / 'text.json').read_text())[row['space']][row['revision']][oid][run]['text']
104 request = {'space': row['space'], 'object': oid,
105 'action': {'type': 'Text', 'start': selected['start'], 'end': selected['end'], 'replacement': text}}
106 return row, source, request
107
108 def location(self, generation, row=None):
109 page = 'index.html'
110 if row:
111 pages = json.loads((self.output / 'g' / str(generation) / 'report/pages.json').read_text())
112 page = next((p['report'] for p in pages if (p['section'], p['space'], p['object'], p['context']) ==
113 (row['section'], row['space'], row['object'], row['context'])), page)
114 return f'/g/{generation}/report/{page}'
115
116 def save(self, data):
117 fields = {'text': {'run', 'replacement'}, 'format': {'run', 'start', 'end', 'attributes'},
118 'paragraph': {'before', 'text', 'author'}, 'outline': {'x', 'y', 'text', 'author'}}
119 action = data.get('action')
120 if action not in fields or set(data) != {'generation', 'page', 'object', 'action'} | fields[action]:
121 raise ValueError('Choose text, formatting, a paragraph or an outline to save.')
122 if action in ('text', 'format'):
123 row, source, edit = self.selection(data['generation'], data['page'], data['object'], data['run'])
124 selected = edit['action']
125 if action == 'text':
126 if not isinstance(data['replacement'], str): raise ValueError('Enter replacement text.')
127 selected['replacement'] = data['replacement']
128 else:
129 if (type(data['start']) is not int or type(data['end']) is not int
130 or not 0 <= data['start'] <= data['end'] <= selected['end'] - selected['start']):
131 raise ValueError('Select text within this run.')
132 edit['action'] = {'type': 'Format', 'start': selected['start'] + data['start'],
133 'end': selected['start'] + data['end'], 'attributes': data['attributes']}
134 else:
135 row, source, revision, _ = self.page(data['generation'], data['page'])
136 if data['object'] not in {oid for oid, _ in walk(revision, row['object'])}:
137 raise ValueError('Choose a container on this page.')
138 edit = {'space': row['space'], 'object': data['object'],
139 'action': {'type': action.title(), **{key: data[key] for key in fields[action]}}}
140 operation = uuid.uuid4().hex
141 result = {'ok': False, 'state': 'NotCommitted'}
142 try:
143 with (self.output / 'operations.jsonl').open('a') as log:
144 intent = {'event': 'intent', 'operation': operation, 'started_ms': time.time_ns() // 1000000,
145 'selection': {key: data[key] for key in ('generation', 'page', 'object', 'action')},
146 'edit': edit, 'section': row['section']}
147 log.write(json.dumps(intent, ensure_ascii=True) + '\n')
148 log.flush(); os.fsync(log.fileno())
149 result = {'ok': False, 'state': 'Unknown'}
150 result = bridge('commit', self.output / 'notebook' / row['section'], source, edit)
151 log.write(json.dumps({'event': 'outcome', 'operation': operation, 'finished_ms': time.time_ns() // 1000000, **result}) + '\n')
152 log.flush(); os.fsync(log.fileno())
153 except Exception as error:
154 result = {**result, 'ok': False, 'error': str(error)}
155 try:
156 generation = self.snapshot()
157 result['location'] = self.location(generation, row)
158 except Exception as error:
159 result = {**result, 'ok': False, 'report_error': str(error)}
160 return result
161
162
163class Handler(BaseHTTPRequestHandler):
164 def redirect(self, location):
165 self.send_response(302)
166 self.send_header('Location', location)
167 self.send_header('Cache-Control', 'no-store')
168 self.send_header('Content-Length', '0')
169 self.end_headers()
170
171 def reply(self, status, value):
172 content = json.dumps(value, ensure_ascii=True).encode()
173 self.send_response(status)
174 self.send_header('Content-Type', 'application/json')
175 self.send_header('Content-Length', str(len(content)))
176 self.send_header('Cache-Control', 'no-store')
177 self.end_headers()
178 self.wfile.write(content)
179
180 def do_GET(self):
181 session = self.server.session
182 url = urlsplit(self.path)
183 try:
184 if url.path == '/':
185 latest = max(int(p.name) for p in (session.output / 'g').iterdir() if p.name.isdecimal())
186 self.redirect(session.location(latest))
187 return
188 if url.path in ('/api/run', '/api/page', '/latest'):
189 query = parse_qs(url.query, strict_parsing=True)
190 if url.path == '/latest':
191 row = session.page(int(query['generation'][0]), query['page'][0])[0] if query else None
192 with session.lock:
193 self.redirect(session.location(session.snapshot(), row))
194 return
195 if url.path == '/api/page':
196 row, _, revision, _ = session.page(int(query['generation'][0]), query['page'][0])
197 def label(oid):
198 node = revision['nodes'][oid]
199 text = ' '.join(n['kind']['text'] for _, n in walk(revision, oid) if n['kind']['type'] == 'RichText')
200 return node['kind']['type'] + (' · ' + text[:80] if text else '')
201 targets = []
202 pending = [row['object']]
203 while pending:
204 oid = pending.pop()
205 node = revision['nodes'][oid]
206 if node['kind']['type'] == 'Title': continue
207 if node['kind']['type'] in ('Outline', 'Paragraph', 'OutlineGroup', 'Cell'):
208 targets.append({'object': oid, 'label': label(oid),
209 'children': [{'object': child, 'label': label(child)} for child in node['children']]})
210 pending.extend(reversed(node['structure'] + node['content'] + node['children']))
211 self.reply(200, {'ok': True, 'object': row['object'], 'targets': targets})
212 return
213 row, source, edit = session.selection(int(query['generation'][0]), query['page'][0], query['object'][0], int(query['run'][0]))
214 result = bridge('check', source, '-', edit)
215 self.reply(200 if result['ok'] else 422, {**result, 'text': edit['action']['replacement']})
216 return
217 if url.path in ('/editor.js', '/editor.css'):
218 path = ROOT / 'tools/diagnostic' / url.path[1:]
219 else:
220 match = re.fullmatch(r'/g/(\d+)/report/(.+)', url.path)
221 if not match: raise FileNotFoundError()
222 root = session.output / 'g' / match[1] / 'report'
223 path = (root / match[2]).resolve(strict=True)
224 if not path.is_relative_to(root): raise FileNotFoundError()
225 with path.open('rb') as file:
226 self.send_response(200)
227 self.send_header('Content-Type', mimetypes.guess_type(path)[0] or 'application/octet-stream')
228 self.send_header('Content-Length', str(os.fstat(file.fileno()).st_size))
229 self.send_header('Cache-Control', 'no-store')
230 self.end_headers()
231 shutil.copyfileobj(file, self.wfile)
232 except (ValueError, KeyError, IndexError, StopIteration) as error:
233 self.reply(422, {'ok': False, 'error': str(error) or 'The selected text is unavailable.'})
234 except OSError:
235 self.reply(404, {'ok': False, 'error': 'Report unavailable. Open the notebook index.'})
236 except Exception as error:
237 self.reply(503, {'ok': False, 'error': str(error)})
238
239 def do_POST(self):
240 session = self.server.session
241 host = self.headers.get('Host')
242 allowed = {f'127.0.0.1:{self.server.server_port}', f'localhost:{self.server.server_port}'}
243 if host not in allowed or self.headers.get('Origin', 'http://' + host) != 'http://' + host or self.headers.get('X-OneNote-Diagnostic') != '1':
244 self.reply(403, {'ok': False, 'error': 'Open this editor on its local address.'})
245 return
246 if self.path != '/api/save':
247 self.reply(404, {'ok': False, 'error': 'Unknown diagnostic action.'})
248 return
249 try:
250 length = int(self.headers.get('Content-Length', '0'))
251 if not 0 < length <= 65536 or self.headers.get('Content-Type') != 'application/json':
252 raise ValueError('Send a JSON edit smaller than 64 KiB.')
253 data = json.loads(self.rfile.read(length))
254 with session.lock:
255 result = session.save(data)
256 status = 200 if result['ok'] else 409 if result.get('kind') == 'ResourceBusy' else 422 if result.get('kind') in ('InvalidData', 'Input') else 503
257 self.reply(status, result)
258 except (ValueError, KeyError, IndexError, StopIteration, TypeError) as error:
259 self.reply(422, {'ok': False, 'state': 'NotCommitted', 'error': str(error) or 'The selected text is unavailable.'})
260 except Exception as error:
261 self.reply(503, {'ok': False, 'state': 'Unknown' if self.path == '/api/save' else 'NotCommitted', 'error': str(error)})
262
263
264if __name__ == '__main__':
265 parser = argparse.ArgumentParser(description=__doc__)
266 parser.add_argument('source', type=Path)
267 parser.add_argument('session', type=Path)
268 parser.add_argument('--port', type=int, default=8782)
269 args = parser.parse_args()
270 server = ThreadingHTTPServer(('127.0.0.1', args.port), Handler)
271 server.session = Session(args.source, args.session)
272 print(f'Diagnostic editor: http://127.0.0.1:{server.server_port}/', flush=True)
273 print(f'Editable notebook copy: {server.session.output / "notebook"}', flush=True)
274 try:
275 server.serve_forever()
276 except KeyboardInterrupt:
277 pass
278 finally:
279 server.server_close()