1// Static and dynamic assets are built alongside the server binary.
2// This module implements decoding and serving of the asset blobs,
3// but also implements patching of dynamic assets. The `Manifest`
4// is generated by `incremental.ts`
5const console = log.scoped("assets");
6const root = import.meta.dirname;
7let current: Loaded | null = null;
8
9// TODO: rename all these types
10type DynamicId = string;
11export type { Key };
12export type Manifest =
13 & {
14 [K in Key]: StaticAsset | DynamicAsset;
15 }
16 & {
17 [string: string]: StaticAsset | DynamicAsset;
18 };
19export interface StaticAsset extends AssetBase {
20 type: 0;
21}
22interface AssetBase {
23 headers: Record<string, string> & { etag: string };
24 raw: BufferView;
25 gzip: BufferView;
26 zstd: BufferView;
27}
28export interface DynamicAsset {
29 type: 1;
30 id: DynamicId;
31}
32interface Loaded {
33 map: Manifest;
34 static: Buffer;
35 dynamic: Map<DynamicId, DynamicEntry>;
36}
37export interface DynamicEntry extends AssetBase {
38 buffer: Buffer;
39}
40type CacheMode = "auto" | "long-term" | "temporary" | "no-cache";
41
42export async function reload() {
43 const map = await fs.readJson<Manifest>(path.join(root, "asset.json"));
44 const statics = await fs.readFile(path.join(root, "asset.blob"));
45 const dynamic = new Map(
46 await Promise.all(
47 Object.entries(map)
48 .filter((entry): entry is [string, DynamicAsset] => entry[1].type === 1)
49 .map(async ([k, v]) =>
50 [
51 v.id,
52 await fs.readFile(path.join(root, "dynamic", v.id))
53 .then(loadRegenerative),
54 ] as const
55 ),
56 ),
57 );
58 return (current = { map, static: statics, dynamic });
59}
60
61export async function middleware(c: Context, next: Next) {
62 if (!current) current = await reload();
63 const asset = current.map[c.req.path];
64 if (asset) return assetInner(c, c.req.path, asset, 200);
65 return next();
66}
67
68export async function notFound(c: Context) {
69 if (!current) current = await reload();
70 let pathname = c.req.path;
71 do {
72 const asset = current.map[pathname + "/404"];
73 if (asset) return assetInner(c, pathname + "/404", asset, 404);
74 pathname = pathname.slice(0, pathname.lastIndexOf("/"));
75 } while (pathname);
76 const asset = current.map["/404"];
77 if (asset) return assetInner(c, "/404", asset, 404);
78 return c.text("the 'Not Found' page was not found", 404);
79}
80
81export async function serveAsset(
82 c: Context,
83 id: Key,
84 status: StatusCode = 200,
85 cache: CacheMode = "auto",
86) {
87 return assetInner(
88 c,
89 id,
90 (current ?? (await reload())).map[id],
91 status,
92 cache,
93 );
94}
95
96export function etagMatches(etag: string, ifNoneMatch: string) {
97 return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
98}
99
100function assetInner(
101 c: Context,
102 key: string,
103 asset: Manifest[Key],
104 status: StatusCode,
105 cache: CacheMode = "auto",
106) {
107 ASSERT(current);
108 if (asset.type === 0) {
109 return respondWithBufferAndViews(
110 c,
111 key,
112 current.static,
113 asset,
114 status,
115 cache,
116 );
117 } else {
118 const entry = UNWRAP(current.dynamic.get(asset.id));
119 return respondWithBufferAndViews(
120 c,
121 key,
122 entry.buffer,
123 entry,
124 status,
125 cache,
126 );
127 }
128}
129
130function respondWithBufferAndViews(
131 c: Context,
132 key: string,
133 buffer: Buffer,
134 asset: AssetBase,
135 status: StatusCode,
136 cache: CacheMode,
137) {
138 const ifNoneMatch = c.req.header("If-None-Match");
139 if (ifNoneMatch) {
140 const etag = asset.headers.etag;
141 if (etagMatches(etag, ifNoneMatch)) {
142 return (c.res = new Response(null, {
143 status: 304,
144 statusText: "Not Modified",
145 headers: {
146 ETag: etag,
147 },
148 }));
149 }
150 }
151 const acceptEncoding = c.req.header("Accept-Encoding") ?? "";
152 let body;
153 let headers = asset.headers;
154 if (acceptEncoding.includes("zstd")) {
155 body = buffer.subarray(...asset.zstd);
156 headers = {
157 ...asset.headers,
158 "Content-Encoding": "zstd",
159 };
160 } else if (acceptEncoding.includes("gzip")) {
161 body = buffer.subarray(...asset.gzip);
162 headers = {
163 ...asset.headers,
164 "Content-Encoding": "gzip",
165 };
166 } else {
167 body = buffer.subarray(...asset.raw);
168 }
169 console.info(
170 `${key} encoding=${headers["Content-Encoding"] ?? "raw"} status=${status}`,
171 );
172 if (!Object.keys(headers).some((x) => x.toLowerCase() === "cache-control")) {
173 if (cache === "auto") {
174 if (status < 200 || status >= 300) {
175 cache = "no-cache";
176 } else if (headers["content-type"]?.includes("text/html")) {
177 cache = "temporary";
178 } else {
179 cache = "long-term";
180 }
181 }
182 if (cache === "no-cache") {
183 headers["Cache-Control"] = "no-store";
184 } else if (cache === "long-term") {
185 headers["Cache-Control"] = "public, max-age=7200, stale-while-revalidate=7200, immutable";
186 } else if (cache === "temporary") {
187 headers["Cache-Control"] = "public, max-age=7200, stale-while-revalidate=7200";
188 }
189 }
190 return (c.res = new Response(body, { headers, status }));
191}
192
193export function packDynamicBuffer(
194 raw: Buffer,
195 gzip: Buffer,
196 zstd: Buffer,
197 headers: Record<string, string>,
198) {
199 const headersBuffer = Buffer.from(
200 Object.entries(headers)
201 .map((entry) => entry.join(":"))
202 .join("\n"),
203 "utf-8",
204 );
205 const header = new Uint32Array(3);
206 header[0] = headersBuffer.byteLength + header.byteLength;
207 header[1] = header[0] + raw.byteLength;
208 header[2] = header[1] + gzip.byteLength;
209 return Buffer.concat([
210 Buffer.from(header.buffer),
211 headersBuffer,
212 raw,
213 gzip,
214 zstd,
215 ]);
216}
217
218function loadRegenerative(buffer: Buffer): DynamicEntry {
219 const headersEnd = buffer.readUInt32LE(0);
220 const headers = Object.fromEntries(
221 buffer
222 .subarray(3 * 4, headersEnd)
223 .toString("utf-8")
224 .split("\n")
225 .map((line) => {
226 const i = line.indexOf(":");
227 return [line.slice(0, i), line.slice(i + 1)];
228 }),
229 );
230 const raw = buffer.readUInt32LE(4);
231 const gzip = buffer.readUInt32LE(8);
232 const hasEtag = (v: object): v is typeof v & { etag: string } => "etag" in v && typeof v.etag === "string";
233 ASSERT(hasEtag(headers));
234 return {
235 headers,
236 buffer,
237 raw: [headersEnd, raw],
238 gzip: [raw, gzip],
239 zstd: [gzip, buffer.byteLength],
240 };
241}
242
243const gzip = util.promisify(zlib.gzip);
244const zstdCompress = util.promisify(zlib.zstdCompress);
245
246export async function overwriteDynamic(
247 key: Key,
248 value: string | Buffer,
249 headers: Record<string, string>,
250) {
251 if (!current) current = await reload();
252 const asset = UNWRAP(current.map[key]);
253 ASSERT(asset.type === 1);
254 UNWRAP(current.dynamic.has(asset.id));
255 const buffer = Buffer.from(value);
256 const etag = JSON.stringify(
257 crypto.createHash("sha1").update(buffer).digest("hex"),
258 );
259 const [gzipBuffer, zstdBuffer] = await Promise.all([
260 gzip(buffer),
261 zstdCompress(buffer),
262 ]);
263 const packed = packDynamicBuffer(buffer, gzipBuffer, zstdBuffer, {
264 ...headers,
265 etag,
266 });
267 current.dynamic.set(asset.id, loadRegenerative(packed));
268 await fs.writeFile(path.join(root, "dynamic", asset.id), packed);
269}
270
271process.on("message", (msg: any) => {
272 if (msg?.type === "clover.assets.reload") reload();
273});
274
275import * as fs from "#sitegen/fs";
276import { ASSERT, UNWRAP } from "@clo/lib/assert";
277import * as log from "@clo/lib/log";
278import type { Context, Next } from "hono";
279import type { StatusCode } from "hono/utils/http-status";
280import { Buffer } from "node:buffer";
281import * as crypto from "node:crypto";
282import * as path from "node:path";
283import * as util from "node:util";
284import * as zlib from "node:zlib";
285import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts";
286import type { BufferView } from "../incremental.ts";