| 1 | /** |
| 2 | * Object store unit tests (src/Git/Store/ObjectStore.ts, Jobs/Compact.ts) |
| 3 | * over the bun:sqlite + in-memory blob harness. Every emitted pack is |
| 4 | * re-parsed by the real pack parser, so the bytes — not just the counts — |
| 5 | * are verified. |
| 6 | */ |
| 7 | import { |
| 8 | encodeTypeSize, |
| 9 | makeSha1, |
| 10 | hashObject, |
| 11 | type Oid, |
| 12 | type ObjectType, |
| 13 | } from "@/Git/Protocol/ObjectCodec.ts"; |
| 14 | import { bufferRandomAccess, ingestPack } from "@/Git/Protocol/PackParser.ts"; |
| 15 | import { packHeader } from "@/Git/Protocol/PackWriter.ts"; |
| 16 | import type { ManifestEntry } from "@/Git/Protocol/Store.ts"; |
| 17 | import * as Zlib from "@/Git/Protocol/Zlib.ts"; |
| 18 | import { |
| 19 | runCompactJob, |
| 20 | runGeometricMergeJob, |
| 21 | shouldCompact, |
| 22 | BLOB_TYPE, |
| 23 | } from "@/Git/Jobs/Compact.ts"; |
| 24 | import { packKey, packKeyOf, wirePackId } from "@/Git/Store/Keys.ts"; |
| 25 | import { |
| 26 | makeObjectStore, |
| 27 | STAGE_INSERT_ROWS, |
| 28 | WINDOW_BYTES, |
| 29 | WINDOW_CACHE_BYTES, |
| 30 | } from "@/Git/Store/ObjectStore.ts"; |
| 31 | import { describe, expect, test } from "alchemy-test"; |
| 32 | import * as Effect from "effect/Effect"; |
| 33 | import { RuntimeContext } from "@/RuntimeContext.ts"; |
| 34 | import * as Stream from "effect/Stream"; |
| 35 | import { concat, verifyPack } from "./harness/pack.ts"; |
| 36 | import { makeMemoryBlobStore, makeTestSqlClient } from "./harness/store.ts"; |
| 37 | |
| 38 | const REPO = "01TESTREPO0000000000000000"; |
| 39 | |
| 40 | /** Runs an effect, surfacing typed failures with their fields (not just `message`). */ |
| 41 | const run = <A>(effect: Effect.Effect<A, unknown, RuntimeContext>) => |
| 42 | Effect.runPromise( |
| 43 | effect.pipe( |
| 44 | Effect.provide(RuntimeContext.phantom), |
| 45 | Effect.mapError((e) => |
| 46 | e instanceof Error && !("reason" in e) |
| 47 | ? e |
| 48 | : new Error(JSON.stringify(e)), |
| 49 | ), |
| 50 | ), |
| 51 | ); |
| 52 | |
| 53 | /** Incompressible bytes; `seed` only guarantees distinct content per call. */ |
| 54 | const bytes = (n: number, seed: number) => { |
| 55 | const out = new Uint8Array(n); |
| 56 | for (let at = 0; at < n; at += 65536) { |
| 57 | crypto.getRandomValues(out.subarray(at, Math.min(at + 65536, n))); |
| 58 | } |
| 59 | out[0] = seed & 0xff; |
| 60 | return out; |
| 61 | }; |
| 62 | |
| 63 | interface Fixture { |
| 64 | readonly oid: Oid; |
| 65 | readonly type: ObjectType; |
| 66 | readonly content: Uint8Array; |
| 67 | readonly zdata: Uint8Array; |
| 68 | } |
| 69 | |
| 70 | const makeFixture = (type: ObjectType, content: Uint8Array) => |
| 71 | Effect.gen(function* () { |
| 72 | const oid = yield* hashObject(type, content); |
| 73 | const zdata = yield* Zlib.deflate(content); |
| 74 | return { oid, type, content, zdata } satisfies Fixture; |
| 75 | }); |
| 76 | |
| 77 | /** A repo with `blobs` incompressible blobs of `blobSize` and `trees` small trees. */ |
| 78 | const seedRepo = (options: { |
| 79 | blobs: number; |
| 80 | blobSize: number; |
| 81 | trees: number; |
| 82 | }) => |
| 83 | Effect.gen(function* () { |
| 84 | const sql = makeTestSqlClient(); |
| 85 | const blobs = makeMemoryBlobStore(); |
| 86 | const store = makeObjectStore({ sql, blobs, repoId: REPO }); |
| 87 | const fixtures: Array<Fixture> = []; |
| 88 | for (let i = 0; i < options.blobs; i++) { |
| 89 | fixtures.push(yield* makeFixture(3, bytes(options.blobSize, i + 1))); |
| 90 | } |
| 91 | for (let i = 0; i < options.trees; i++) { |
| 92 | // Not a real tree encoding; the store never parses tree bytes here. |
| 93 | fixtures.push( |
| 94 | yield* makeFixture( |
| 95 | 2, |
| 96 | new TextEncoder().encode(`tree ${i} ${"x".repeat(40)}`), |
| 97 | ), |
| 98 | ); |
| 99 | } |
| 100 | yield* sql.transactionSync((raw) => { |
| 101 | for (const f of fixtures) { |
| 102 | raw.exec( |
| 103 | `INSERT INTO objects (oid, type, size, zsize, location, zdata) VALUES (?, ?, ?, ?, 'row', ?)`, |
| 104 | f.oid, |
| 105 | f.type, |
| 106 | f.content.length, |
| 107 | f.zdata.length, |
| 108 | f.zdata.buffer.slice( |
| 109 | f.zdata.byteOffset, |
| 110 | f.zdata.byteOffset + f.zdata.byteLength, |
| 111 | ), |
| 112 | ); |
| 113 | } |
| 114 | }); |
| 115 | return { sql, blobs, store, fixtures }; |
| 116 | }); |
| 117 | |
| 118 | const manifestOf = ( |
| 119 | fixtures: ReadonlyArray<Fixture>, |
| 120 | location: ManifestEntry["location"], |
| 121 | ) => |
| 122 | fixtures.map((f): ManifestEntry => ({ |
| 123 | oid: f.oid, |
| 124 | type: f.type, |
| 125 | size: f.content.length, |
| 126 | zsize: f.zdata.length, |
| 127 | location, |
| 128 | })); |
| 129 | |
| 130 | /** Emits a full pack (header + entries + trailer) through `packEntries`. */ |
| 131 | const buildPack = ( |
| 132 | store: ReturnType<typeof makeObjectStore>, |
| 133 | entries: ReadonlyArray<ManifestEntry>, |
| 134 | ) => |
| 135 | Effect.gen(function* () { |
| 136 | const body = Array.from( |
| 137 | yield* Stream.runCollect(store.packEntries(entries)), |
| 138 | ); |
| 139 | const head = packHeader(entries.length); |
| 140 | const sha = makeSha1(); |
| 141 | sha.update(head); |
| 142 | for (const b of body) sha.update(b); |
| 143 | return concat([head, ...body, sha.digest()]); |
| 144 | }); |
| 145 | |
| 146 | /** Parses a pack with the real parser and returns the oids it yields. */ |
| 147 | const parsePack = ( |
| 148 | pack: Uint8Array, |
| 149 | store: ReturnType<typeof makeObjectStore>, |
| 150 | ) => |
| 151 | Effect.gen(function* () { |
| 152 | const seen = new Map<string, number>(); |
| 153 | const summary = yield* ingestPack({ |
| 154 | source: bufferRandomAccess(pack), |
| 155 | store, |
| 156 | sink: (entry) => |
| 157 | Effect.sync(() => { |
| 158 | seen.set(entry.oid, entry.size); |
| 159 | }), |
| 160 | }); |
| 161 | return { count: summary.count, seen }; |
| 162 | }); |
| 163 | |
| 164 | const compactAll = ( |
| 165 | sql: ReturnType<typeof makeTestSqlClient>, |
| 166 | blobs: ReturnType<typeof makeMemoryBlobStore>, |
| 167 | maxObjects = 100, |
| 168 | ) => |
| 169 | Effect.gen(function* () { |
| 170 | const packIds: Array<string> = []; |
| 171 | for (;;) { |
| 172 | const out = yield* runCompactJob({ |
| 173 | repoId: REPO, |
| 174 | sql, |
| 175 | blobs, |
| 176 | maxObjects, |
| 177 | maxBytes: 64 * 1024 * 1024, |
| 178 | }); |
| 179 | if (out.packId !== undefined) packIds.push(out.packId); |
| 180 | if (!out.more) break; |
| 181 | } |
| 182 | return packIds; |
| 183 | }); |
| 184 | |
| 185 | describe("compaction is blob-only", { tags: ["unit", "local"] }, () => { |
| 186 | test("shouldCompact ignores commits/trees; runCompactJob leaves them as rows", async () => { |
| 187 | await run( |
| 188 | Effect.gen(function* () { |
| 189 | const { sql, blobs, fixtures } = yield* seedRepo({ |
| 190 | blobs: 5, |
| 191 | blobSize: 100, |
| 192 | trees: 50, |
| 193 | }); |
| 194 | expect(BLOB_TYPE).toBe(3); |
| 195 | // 55 rows total, but only 5 blobs: below a count threshold of 10. |
| 196 | expect( |
| 197 | yield* shouldCompact(sql, { |
| 198 | countThreshold: 10, |
| 199 | bytesThreshold: 1 << 30, |
| 200 | }), |
| 201 | ).toBe(false); |
| 202 | expect( |
| 203 | yield* shouldCompact(sql, { |
| 204 | countThreshold: 5, |
| 205 | bytesThreshold: 1 << 30, |
| 206 | }), |
| 207 | ).toBe(true); |
| 208 | const packs = yield* compactAll(sql, blobs); |
| 209 | expect(packs.length).toBe(1); |
| 210 | const rows = yield* sql.all<{ |
| 211 | location: string; |
| 212 | type: number; |
| 213 | n: number; |
| 214 | }>( |
| 215 | `SELECT location, type, COUNT(*) AS n FROM objects GROUP BY location, type ORDER BY location, type`, |
| 216 | ); |
| 217 | expect(rows).toEqual([ |
| 218 | { location: "pack", type: 3, n: 5 }, |
| 219 | { location: "row", type: 2, n: 50 }, |
| 220 | ]); |
| 221 | expect( |
| 222 | yield* runCompactJob({ repoId: REPO, sql, blobs }), |
| 223 | ).toMatchObject({ moved: 0, more: false }); |
| 224 | expect(fixtures.length).toBe(55); |
| 225 | }), |
| 226 | ); |
| 227 | }); |
| 228 | }); |
| 229 | |
| 230 | describe("packEntries", { tags: ["unit", "local"] }, () => { |
| 231 | test( |
| 232 | "rows + multi-window pack (with straddling objects) emit a pack the parser accepts", |
| 233 | async () => { |
| 234 | await run( |
| 235 | Effect.gen(function* () { |
| 236 | // 300 × 30 KiB incompressible ≈ 9 MiB → 3 windows, with objects |
| 237 | // straddling both window edges. |
| 238 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 239 | blobs: 300, |
| 240 | blobSize: 30 * 1024, |
| 241 | trees: 40, |
| 242 | }); |
| 243 | const packs = yield* compactAll(sql, blobs, 1000); // one pack |
| 244 | expect(packs.length).toBe(1); |
| 245 | const packBytes = blobs.objects.get(packKey(REPO, packs[0]!))!; |
| 246 | expect(packBytes.length).toBeGreaterThan(2 * WINDOW_BYTES); // 3 windows |
| 247 | const coords = yield* sql.all<{ pack_id: string; n: number }>( |
| 248 | `SELECT pack_id, COUNT(*) AS n FROM objects WHERE location='pack' GROUP BY pack_id`, |
| 249 | ); |
| 250 | expect(coords.map((c) => c.n)).toEqual([300]); |
| 251 | |
| 252 | // Manifest in the order a closure would produce (trees first). |
| 253 | const entries = [ |
| 254 | ...manifestOf( |
| 255 | fixtures.filter((f) => f.type === 2), |
| 256 | "row", |
| 257 | ), |
| 258 | ...manifestOf( |
| 259 | fixtures.filter((f) => f.type === 3), |
| 260 | "pack", |
| 261 | ), |
| 262 | ]; |
| 263 | blobs.gets.length = 0; |
| 264 | const pack = yield* buildPack(store, entries); |
| 265 | expect(verifyPack(pack).error).toBeUndefined(); |
| 266 | const parsed = yield* parsePack(pack, store); |
| 267 | expect(parsed.count).toBe(340); |
| 268 | expect(parsed.seen.size).toBe(340); |
| 269 | for (const f of fixtures) |
| 270 | expect(parsed.seen.get(f.oid)).toBe(f.content.length); |
| 271 | |
| 272 | // One ranged GET per window touched, not per object. |
| 273 | const windowGets = blobs.gets.filter( |
| 274 | (g) => g.length === WINDOW_BYTES, |
| 275 | ); |
| 276 | const straddleGets = blobs.gets.filter( |
| 277 | (g) => g.length !== WINDOW_BYTES, |
| 278 | ); |
| 279 | expect(windowGets.length).toBe(3); |
| 280 | expect(straddleGets.length).toBeLessThanOrEqual(2); // ≤ one per window edge |
| 281 | }), |
| 282 | ); |
| 283 | }, |
| 284 | { timeout: 60_000 }, |
| 285 | ); |
| 286 | |
| 287 | test("a manifest entry that compaction moved after the closure is still emitted (re-dispatch)", async () => { |
| 288 | await run( |
| 289 | Effect.gen(function* () { |
| 290 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 291 | blobs: 20, |
| 292 | blobSize: 2048, |
| 293 | trees: 3, |
| 294 | }); |
| 295 | // The closure saw everything as rows… |
| 296 | const entries = manifestOf(fixtures, "row"); |
| 297 | // …then compaction ran before emission. |
| 298 | yield* compactAll(sql, blobs); |
| 299 | const pack = yield* buildPack(store, entries); |
| 300 | expect(verifyPack(pack).error).toBeUndefined(); |
| 301 | const parsed = yield* parsePack(pack, store); |
| 302 | expect(parsed.seen.size).toBe(23); |
| 303 | }), |
| 304 | ); |
| 305 | }); |
| 306 | |
| 307 | test("a truly absent object fails the stream instead of emitting a short pack", async () => { |
| 308 | await run( |
| 309 | Effect.gen(function* () { |
| 310 | const { store, fixtures } = yield* seedRepo({ |
| 311 | blobs: 2, |
| 312 | blobSize: 100, |
| 313 | trees: 0, |
| 314 | }); |
| 315 | const ghost = { |
| 316 | ...manifestOf(fixtures, "row")[0]!, |
| 317 | oid: "0".repeat(40) as Oid, |
| 318 | }; |
| 319 | const result = yield* Effect.result( |
| 320 | Stream.runCollect(store.packEntries([ghost])), |
| 321 | ); |
| 322 | expect(result._tag).toBe("Failure"); |
| 323 | }), |
| 324 | ); |
| 325 | }); |
| 326 | }); |
| 327 | |
| 328 | describe("readContentBatch", { tags: ["unit", "local"] }, () => { |
| 329 | test("returns inflated content for rows and packed objects alike", async () => { |
| 330 | await run( |
| 331 | Effect.gen(function* () { |
| 332 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 333 | blobs: 30, |
| 334 | blobSize: 512, |
| 335 | trees: 10, |
| 336 | }); |
| 337 | yield* compactAll(sql, blobs); |
| 338 | const out = yield* store.readContentBatch(fixtures.map((f) => f.oid)); |
| 339 | expect(out.size).toBe(40); |
| 340 | for (const f of fixtures) { |
| 341 | expect(Array.from(out.get(f.oid)!)).toEqual(Array.from(f.content)); |
| 342 | } |
| 343 | }), |
| 344 | ); |
| 345 | }); |
| 346 | }); |
| 347 | |
| 348 | describe("window cache", { tags: ["unit", "local"] }, () => { |
| 349 | test( |
| 350 | "is bounded by WINDOW_CACHE_BYTES: re-reading the first of nine windows refetches", |
| 351 | async () => { |
| 352 | await run( |
| 353 | Effect.gen(function* () { |
| 354 | // 36 × 1 MiB incompressible → one 36 MiB pack = 9 windows > 8 cached. |
| 355 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 356 | blobs: 36, |
| 357 | blobSize: 1024 * 1024, |
| 358 | trees: 0, |
| 359 | }); |
| 360 | const out = yield* runCompactJob({ |
| 361 | repoId: REPO, |
| 362 | sql, |
| 363 | blobs, |
| 364 | maxObjects: 100, |
| 365 | maxBytes: 64 * 1024 * 1024, |
| 366 | }); |
| 367 | expect(out.moved).toBe(36); |
| 368 | expect(WINDOW_CACHE_BYTES / WINDOW_BYTES).toBe(8); |
| 369 | const sorted = [...fixtures]; |
| 370 | const order = yield* sql.all<{ |
| 371 | oid: string; |
| 372 | pack_offset: number; |
| 373 | zsize: number; |
| 374 | }>( |
| 375 | `SELECT oid, pack_offset, zsize FROM objects ORDER BY pack_offset`, |
| 376 | ); |
| 377 | const byOid = new Map(sorted.map((f) => [f.oid, f])); |
| 378 | const first = byOid.get(order[0]!.oid as Oid)!; |
| 379 | // An object lying entirely inside window 7 (objects that straddle a |
| 380 | // window edge take a ranged read that bypasses the cache by design). |
| 381 | const inside7 = order.find( |
| 382 | (r) => |
| 383 | Math.floor(r.pack_offset / WINDOW_BYTES) === 7 && |
| 384 | r.pack_offset + r.zsize <= 8 * WINDOW_BYTES, |
| 385 | )!; |
| 386 | const last = byOid.get(inside7.oid as Oid)!; |
| 387 | blobs.gets.length = 0; |
| 388 | yield* store.readContentBatch([first.oid]); |
| 389 | const afterFirst = blobs.gets.length; |
| 390 | expect(afterFirst).toBeGreaterThanOrEqual(1); |
| 391 | // Touch every window. |
| 392 | yield* store.readContentBatch(order.map((r) => r.oid as Oid)); |
| 393 | blobs.gets.length = 0; |
| 394 | // The first window has been evicted (9 > 8): reading it fetches again. |
| 395 | yield* store.readContentBatch([first.oid]); |
| 396 | expect(blobs.gets.length).toBeGreaterThanOrEqual(1); |
| 397 | // The last window is still hot: no fetch. |
| 398 | blobs.gets.length = 0; |
| 399 | yield* store.readContentBatch([last.oid]); |
| 400 | expect(blobs.gets.length).toBe(0); |
| 401 | }), |
| 402 | ); |
| 403 | }, |
| 404 | { timeout: 60_000 }, |
| 405 | ); |
| 406 | }); |
| 407 | |
| 408 | describe("insertStagedBatch", { tags: ["unit", "local"] }, () => { |
| 409 | test("stages a batch with multi-row inserts; staged rows are invisible to live reads", async () => { |
| 410 | await run( |
| 411 | Effect.gen(function* () { |
| 412 | const sql = makeTestSqlClient(); |
| 413 | const store = makeObjectStore({ |
| 414 | sql, |
| 415 | blobs: makeMemoryBlobStore(), |
| 416 | repoId: REPO, |
| 417 | }); |
| 418 | const objects = []; |
| 419 | for (let i = 0; i < STAGE_INSERT_ROWS * 2 + 7; i++) { |
| 420 | const f = yield* makeFixture(3, bytes(64, i + 1)); |
| 421 | objects.push({ |
| 422 | oid: f.oid, |
| 423 | type: f.type, |
| 424 | size: f.content.length, |
| 425 | zdata: f.zdata, |
| 426 | }); |
| 427 | } |
| 428 | yield* store.insertStagedBatch("push-A", objects); |
| 429 | const staged = yield* sql.first<{ n: number }>( |
| 430 | `SELECT COUNT(*) AS n FROM objects WHERE staged_push = 'push-A'`, |
| 431 | ); |
| 432 | expect(staged?.n).toBe(objects.length); |
| 433 | expect(yield* store.has(objects[0]!.oid)).toBe(false); |
| 434 | }), |
| 435 | ); |
| 436 | }); |
| 437 | |
| 438 | test("a committed push's rows are live before the flip, cannot be adopted, and GC flips them (DESIGN §22.14)", async () => { |
| 439 | await run( |
| 440 | Effect.gen(function* () { |
| 441 | const sql = makeTestSqlClient(); |
| 442 | const store = makeObjectStore({ |
| 443 | sql, |
| 444 | blobs: makeMemoryBlobStore(), |
| 445 | repoId: REPO, |
| 446 | }); |
| 447 | const objects = []; |
| 448 | for (let i = 0; i < 6; i++) { |
| 449 | const f = yield* makeFixture(3, bytes(64, i + 300)); |
| 450 | objects.push({ |
| 451 | oid: f.oid, |
| 452 | type: f.type, |
| 453 | size: f.content.length, |
| 454 | zdata: f.zdata, |
| 455 | }); |
| 456 | } |
| 457 | yield* sql.run( |
| 458 | `INSERT INTO pushes (push_id, started_at, state) VALUES ('push-live', ?, 'committed')`, |
| 459 | Date.now(), |
| 460 | ); |
| 461 | yield* store.insertStagedBatch("push-live", objects); |
| 462 | // Live for readers although `staged_push` is still set. |
| 463 | expect(yield* store.has(objects[0]!.oid)).toBe(true); |
| 464 | const missing = yield* store.missingObjects( |
| 465 | objects.map((o) => o.oid), |
| 466 | "push-other", |
| 467 | ); |
| 468 | expect(missing).toEqual([]); |
| 469 | // A later push re-staging the same oids must not steal them. |
| 470 | yield* store.insertStagedBatch("push-other", objects.slice(0, 3)); |
| 471 | const still = yield* sql.first<{ n: number }>( |
| 472 | `SELECT COUNT(*) AS n FROM objects WHERE staged_push = 'push-live'`, |
| 473 | ); |
| 474 | expect(still?.n).toBe(6); |
| 475 | // The deferred flip (here: what GC does) nulls the column. |
| 476 | yield* sql.run( |
| 477 | `UPDATE objects SET staged_push = NULL WHERE staged_push IN (SELECT push_id FROM pushes WHERE state = 'committed')`, |
| 478 | ); |
| 479 | const flipped = yield* sql.first<{ n: number }>( |
| 480 | `SELECT COUNT(*) AS n FROM objects WHERE staged_push IS NULL`, |
| 481 | ); |
| 482 | expect(flipped?.n).toBe(6); |
| 483 | expect(yield* store.has(objects[5]!.oid)).toBe(true); |
| 484 | }), |
| 485 | ); |
| 486 | }); |
| 487 | |
| 488 | test("rows left by a crashed push are adopted; live objects are left alone", async () => { |
| 489 | await run( |
| 490 | Effect.gen(function* () { |
| 491 | const sql = makeTestSqlClient(); |
| 492 | const store = makeObjectStore({ |
| 493 | sql, |
| 494 | blobs: makeMemoryBlobStore(), |
| 495 | repoId: REPO, |
| 496 | }); |
| 497 | const objects = []; |
| 498 | for (let i = 0; i < 10; i++) { |
| 499 | const f = yield* makeFixture(3, bytes(64, i + 100)); |
| 500 | objects.push({ |
| 501 | oid: f.oid, |
| 502 | type: f.type, |
| 503 | size: f.content.length, |
| 504 | zdata: f.zdata, |
| 505 | }); |
| 506 | } |
| 507 | yield* store.insertStagedBatch("push-crashed", objects.slice(0, 4)); |
| 508 | yield* store.insertStagedBatch("push-B", objects); |
| 509 | const byPush = yield* sql.all<{ staged_push: string; n: number }>( |
| 510 | `SELECT staged_push, COUNT(*) AS n FROM objects GROUP BY staged_push`, |
| 511 | ); |
| 512 | expect(byPush).toEqual([{ staged_push: "push-B", n: 10 }]); |
| 513 | yield* sql.run( |
| 514 | `UPDATE objects SET staged_push = NULL WHERE oid = ?`, |
| 515 | objects[0]!.oid, |
| 516 | ); |
| 517 | yield* store.insertStagedBatch("push-C", objects.slice(0, 2)); |
| 518 | const live = yield* sql.first<{ staged_push: string | null }>( |
| 519 | `SELECT staged_push FROM objects WHERE oid = ?`, |
| 520 | objects[0]!.oid, |
| 521 | ); |
| 522 | expect(live?.staged_push).toBeNull(); |
| 523 | const other = yield* sql.first<{ staged_push: string | null }>( |
| 524 | `SELECT staged_push FROM objects WHERE oid = ?`, |
| 525 | objects[1]!.oid, |
| 526 | ); |
| 527 | expect(other?.staged_push).toBe("push-C"); |
| 528 | }), |
| 529 | ); |
| 530 | }); |
| 531 | }); |
| 532 | |
| 533 | describe( |
| 534 | "promoted wire packs (DESIGN §22.5)", |
| 535 | { tags: ["unit", "local"] }, |
| 536 | () => { |
| 537 | /** Lays fixtures out the way a wire pack does: typeSize header + zdata per entry. */ |
| 538 | const layout = (fixtures: ReadonlyArray<Fixture>, base: number) => { |
| 539 | const pieces: Array<Uint8Array> = [new Uint8Array(base)]; |
| 540 | const offsets = new Map<Oid, number>(); |
| 541 | let at = base; |
| 542 | for (const f of fixtures) { |
| 543 | const head = encodeTypeSize(f.type as 3, f.content.length); |
| 544 | pieces.push(head, f.zdata); |
| 545 | offsets.set(f.oid, at + head.length); |
| 546 | at += head.length + f.zdata.length; |
| 547 | } |
| 548 | return { bytes: concat(pieces), offsets }; |
| 549 | }; |
| 550 | |
| 551 | test("rows staged as pack references read back through the wire key on every path", async () => { |
| 552 | await run( |
| 553 | Effect.gen(function* () { |
| 554 | const sql = makeTestSqlClient(); |
| 555 | const blobs = makeMemoryBlobStore(); |
| 556 | const store = makeObjectStore({ sql, blobs, repoId: REPO }); |
| 557 | const fixtures: Array<Fixture> = []; |
| 558 | for (let i = 0; i < 40; i++) |
| 559 | fixtures.push(yield* makeFixture(3, bytes(3000, i + 1))); |
| 560 | const packId = wirePackId("01RECEIVE00000000000000000"); |
| 561 | const { bytes: wire, offsets } = layout(fixtures, 137); // 137 bytes of pkt-line commands first |
| 562 | yield* blobs.put(packKeyOf(REPO, packId), wire); |
| 563 | expect(packKeyOf(REPO, packId)).toContain("/incoming/"); |
| 564 | yield* store.insertStagedBatch( |
| 565 | "push-P", |
| 566 | fixtures.map((f) => ({ |
| 567 | oid: f.oid, |
| 568 | type: f.type, |
| 569 | size: f.content.length, |
| 570 | zdata: f.zdata, |
| 571 | pack: { packId, offset: offsets.get(f.oid)! }, |
| 572 | })), |
| 573 | ); |
| 574 | const staged = yield* sql.all<{ |
| 575 | location: string; |
| 576 | n: number; |
| 577 | withBlob: number; |
| 578 | }>( |
| 579 | `SELECT location, COUNT(*) AS n, SUM(zdata IS NOT NULL) AS withBlob FROM objects GROUP BY location`, |
| 580 | ); |
| 581 | expect(staged).toEqual([{ location: "pack", n: 40, withBlob: 0 }]); |
| 582 | yield* sql.run( |
| 583 | `UPDATE objects SET staged_push = NULL WHERE staged_push = 'push-P'`, |
| 584 | ); |
| 585 | // Single reads, batched reads, and pack emission all resolve the wire key. |
| 586 | const one = yield* store.readContent(fixtures[7]!.oid); |
| 587 | expect(Array.from(one)).toEqual(Array.from(fixtures[7]!.content)); |
| 588 | const many = yield* store.readContentBatch( |
| 589 | fixtures.map((f) => f.oid), |
| 590 | ); |
| 591 | expect(many.size).toBe(40); |
| 592 | const pack = yield* buildPack(store, manifestOf(fixtures, "pack")); |
| 593 | expect(verifyPack(pack).error).toBeUndefined(); |
| 594 | const parsed = yield* parsePack( |
| 595 | pack, |
| 596 | makeObjectStore({ |
| 597 | sql: makeTestSqlClient(), |
| 598 | blobs: makeMemoryBlobStore(), |
| 599 | repoId: "X", |
| 600 | }), |
| 601 | ); |
| 602 | expect(parsed.seen.size).toBe(40); |
| 603 | }), |
| 604 | ); |
| 605 | }); |
| 606 | |
| 607 | test("compaction leaves promoted rows alone; geometric merge REWRITES a wire pack into the merged pack", async () => { |
| 608 | await run( |
| 609 | Effect.gen(function* () { |
| 610 | const { sql, blobs, store, fixtures } = yield* seedRepo({ |
| 611 | blobs: 30, |
| 612 | blobSize: 2000, |
| 613 | trees: 3, |
| 614 | }); |
| 615 | const promotedFixtures: Array<Fixture> = []; |
| 616 | for (let i = 0; i < 12; i++) |
| 617 | promotedFixtures.push(yield* makeFixture(3, bytes(2000, i + 500))); |
| 618 | const packId = wirePackId("01RECEIVE00000000000000001"); |
| 619 | // The wire object carries junk around the referenced spans (a real |
| 620 | // one carries trees, commits and delta entries): only spans move. |
| 621 | const { bytes: wire, offsets } = layout(promotedFixtures, 333); |
| 622 | const withJunk = concat([wire, bytes(5000, 9)]); |
| 623 | yield* blobs.put(packKeyOf(REPO, packId), withJunk); |
| 624 | yield* store.insertStagedBatch( |
| 625 | "push-Q", |
| 626 | promotedFixtures.map((f) => ({ |
| 627 | oid: f.oid, |
| 628 | type: f.type, |
| 629 | size: f.content.length, |
| 630 | zdata: f.zdata, |
| 631 | pack: { packId, offset: offsets.get(f.oid)! }, |
| 632 | })), |
| 633 | ); |
| 634 | yield* sql.run(`UPDATE objects SET staged_push = NULL`); |
| 635 | const p1 = yield* runCompactJob({ |
| 636 | repoId: REPO, |
| 637 | sql, |
| 638 | blobs, |
| 639 | maxObjects: 15, |
| 640 | maxBytes: 1 << 30, |
| 641 | }); |
| 642 | const p2 = yield* runCompactJob({ |
| 643 | repoId: REPO, |
| 644 | sql, |
| 645 | blobs, |
| 646 | maxObjects: 15, |
| 647 | maxBytes: 1 << 30, |
| 648 | }); |
| 649 | expect(p1.moved + p2.moved).toBe(30); |
| 650 | const merge = yield* runGeometricMergeJob({ |
| 651 | repoId: REPO, |
| 652 | sql, |
| 653 | blobs, |
| 654 | }); |
| 655 | expect(merge.packs).toBe(3); |
| 656 | expect(merge.pendingDelete).toContain(packKeyOf(REPO, packId)); |
| 657 | const packs = yield* sql.all<{ pack_id: string; n: number }>( |
| 658 | `SELECT pack_id, COUNT(*) AS n FROM objects WHERE location = 'pack' GROUP BY pack_id`, |
| 659 | ); |
| 660 | expect(packs).toEqual([{ pack_id: merge.packId, n: 42 }]); |
| 661 | // The merged pack is a clean packfile the parser accepts, and every |
| 662 | // object — promoted ones included — reads back through it. |
| 663 | const merged = blobs.objects.get(packKey(REPO, merge.packId!))!; |
| 664 | expect(verifyPack(merged)).toEqual({ objects: 42 }); |
| 665 | const all = yield* store.readContentBatch( |
| 666 | [...fixtures, ...promotedFixtures].map((f) => f.oid), |
| 667 | ); |
| 668 | expect(all.size).toBe(45); |
| 669 | for (const f of promotedFixtures) |
| 670 | expect(Array.from(all.get(f.oid)!)).toEqual(Array.from(f.content)); |
| 671 | }), |
| 672 | ); |
| 673 | }); |
| 674 | }, |
| 675 | ); |
| 676 | |
| 677 | describe("prepared object view", { tags: ["unit", "local"] }, () => { |
| 678 | test("reads only its own staged objects and live objects, with a size cap", () => |
| 679 | run( |
| 680 | Effect.gen(function* () { |
| 681 | const sql = makeTestSqlClient(); |
| 682 | const store = makeObjectStore({ |
| 683 | sql, |
| 684 | blobs: makeMemoryBlobStore(), |
| 685 | repoId: REPO, |
| 686 | }); |
| 687 | const own = yield* makeFixture(3, new TextEncoder().encode("own")); |
| 688 | const other = yield* makeFixture(3, new TextEncoder().encode("other")); |
| 689 | yield* store.insertStaged("mine", { ...own, size: own.content.length }); |
| 690 | yield* store.insertStaged("theirs", { |
| 691 | ...other, |
| 692 | size: other.content.length, |
| 693 | }); |
| 694 | expect(yield* store.getMeta(own.oid)).toBeUndefined(); |
| 695 | expect( |
| 696 | (yield* store.readPrepared("mine", own.oid, 3))?.content, |
| 697 | ).toEqual(own.content); |
| 698 | expect( |
| 699 | yield* store.readPrepared("mine", other.oid, 100), |
| 700 | ).toBeUndefined(); |
| 701 | const tooBig = yield* Effect.result( |
| 702 | store.readPrepared("mine", own.oid, 2), |
| 703 | ); |
| 704 | expect(tooBig._tag).toBe("Failure"); |
| 705 | }), |
| 706 | )); |
| 707 | }); |