1import { assert, describe, it } from "@effect/vitest";
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import { ProviderDriverKind } from "@t3tools/contracts";
4import * as Effect from "effect/Effect";
5import * as FileSystem from "effect/FileSystem";
6import * as Layer from "effect/Layer";
7import * as Path from "effect/Path";
8import * as TestClock from "effect/testing/TestClock";
9import { HttpClient, HttpClientResponse } from "effect/http";
10
11import * as ServerConfig from "../config.ts";
12import * as ServerSettings from "../serverSettings.ts";
13import * as ModelManifest from "./ModelManifest.ts";
14
15/**
16 * Test policy: this file covers manifest machinery, not manifest contents.
17 * Do not add assertions for real model slugs, names, status, aliases, or
18 * profiles when editing model-manifest.json. Add tests only when fetch/cache
19 * behavior or the provider-neutral resolver semantics change, and use
20 * synthetic models for resolver coverage.
21 */
22
23describe("resolveProviderCatalog", () => {
24 it("resolves generic model presentation through a reusable profile", () => {
25 const manifest: ModelManifest.ModelManifestData = {
26 version: 1,
27 currentModels: {},
28 providers: {
29 synthetic: {
30 defaults: { chat: "model-next" },
31 profiles: {
32 standard: {
33 capabilities: {
34 optionDescriptors: [
35 {
36 id: "mode",
37 label: "Mode",
38 type: "select",
39 options: [{ id: "fast", label: "Fast", isDefault: true }],
40 },
41 ],
42 },
43 adapter: { opaque: true },
44 },
45 },
46 models: [
47 {
48 slug: "model-next",
49 name: "Model Next",
50 aliases: ["next"],
51 status: "current",
52 badge: "new",
53 profile: "standard",
54 },
55 ],
56 },
57 },
58 };
59
60 const catalog = ModelManifest.resolveProviderCatalog(
61 manifest,
62 ProviderDriverKind.make("synthetic"),
63 );
64 assert.deepStrictEqual(catalog?.models[0], {
65 slug: "model-next",
66 name: "Model Next",
67 aliases: ["next"],
68 badge: "new",
69 status: "current",
70 capabilities: manifest.providers!.synthetic!.profiles.standard!.capabilities!,
71 adapter: undefined,
72 profileAdapter: { opaque: true },
73 });
74 assert.strictEqual(catalog?.defaultChatModel, "model-next");
75 });
76
77 it("rejects invalid catalog references", () => {
78 const invalidCatalog = (input: {
79 readonly models: NonNullable<ModelManifest.ModelManifestData["providers"]>[string]["models"];
80 readonly defaultChat?: string;
81 }): ModelManifest.ModelManifestData => ({
82 version: 1,
83 currentModels: {},
84 providers: {
85 synthetic: {
86 ...(input.defaultChat ? { defaults: { chat: input.defaultChat } } : {}),
87 profiles: {},
88 models: input.models,
89 },
90 },
91 });
92
93 for (const invalid of [
94 invalidCatalog({
95 models: [
96 { slug: "duplicate", name: "First", status: "current" },
97 { slug: "duplicate", name: "Second", status: "current" },
98 ],
99 }),
100 invalidCatalog({
101 models: [
102 {
103 slug: "missing-profile",
104 name: "Missing Profile",
105 status: "current",
106 profile: "missing",
107 },
108 ],
109 }),
110 invalidCatalog({
111 models: [{ slug: "present", name: "Present", status: "current" }],
112 defaultChat: "absent",
113 }),
114 ]) {
115 assert.isNull(
116 ModelManifest.resolveProviderCatalog(invalid, ProviderDriverKind.make("synthetic")),
117 );
118 }
119 });
120});
121
122// Remote fixtures date after the bundle so a fetch still outranks it.
123const REMOTE_UPDATED_AT = "2099-01-01T00:00:00Z";
124
125const REMOTE_MANIFEST: ModelManifest.ModelManifestData = {
126 version: 1,
127 updatedAt: REMOTE_UPDATED_AT,
128 currentModels: {
129 codex: ["remote-model"],
130 claudeAgent: ["remote-agent-model"],
131 },
132};
133
134const REMOTE_CLAUDE_MANIFEST: ModelManifest.ModelManifestData = {
135 version: 1,
136 updatedAt: REMOTE_UPDATED_AT,
137 currentModels: {},
138 providers: {
139 claudeAgent: {
140 profiles: {
141 synthetic: {
142 adapter: { claudeCode: { effortMap: { extreme: "high" } } },
143 },
144 },
145 models: [
146 {
147 slug: "remote-only-model",
148 name: "Remote Only Model",
149 status: "current",
150 profile: "synthetic",
151 },
152 ],
153 },
154 },
155};
156
157const remoteClaudeManifestWithCompatibility = (
158 compatibility: unknown,
159): ModelManifest.ModelManifestData => ({
160 ...REMOTE_CLAUDE_MANIFEST,
161 providers: {
162 claudeAgent: {
163 profiles: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.profiles,
164 models: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models.map((model) => ({
165 ...model,
166 adapter: { claudeCode: compatibility },
167 })),
168 },
169 },
170});
171
172const INVALID_REMOTE_MANIFESTS: ReadonlyArray<ModelManifest.ModelManifestData> = [
173 {
174 ...REMOTE_CLAUDE_MANIFEST,
175 providers: {
176 claudeAgent: {
177 profiles: {
178 synthetic: {
179 adapter: { claudeCode: { effortMap: { extreme: 123 } } },
180 },
181 },
182 models: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models,
183 },
184 },
185 },
186 {
187 ...REMOTE_CLAUDE_MANIFEST,
188 providers: {
189 claudeAgent: {
190 profiles: {},
191 models: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models,
192 },
193 },
194 },
195 {
196 ...REMOTE_CLAUDE_MANIFEST,
197 providers: {
198 claudeAgent: {
199 profiles: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.profiles,
200 models: [
201 ...REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models,
202 {
203 slug: "remote-only-model",
204 name: "Duplicate Remote Model",
205 status: "current",
206 profile: "synthetic",
207 },
208 ],
209 },
210 },
211 },
212 {
213 ...REMOTE_CLAUDE_MANIFEST,
214 providers: {
215 claudeAgent: {
216 defaults: { chat: "absent-model" },
217 profiles: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.profiles,
218 models: REMOTE_CLAUDE_MANIFEST.providers!.claudeAgent!.models,
219 },
220 },
221 },
222 remoteClaudeManifestWithCompatibility({ minVersion: "2.x" }),
223 remoteClaudeManifestWithCompatibility({ maxVersionExclusive: "2.x" }),
224 remoteClaudeManifestWithCompatibility({
225 minVersion: "2.2",
226 maxVersionExclusive: "2.1",
227 }),
228];
229
230const layerHttpClient = (handler: () => Response) =>
231 Layer.succeed(
232 HttpClient.HttpClient,
233 HttpClient.make((request) => Effect.succeed(HttpClientResponse.fromWeb(request, handler()))),
234 );
235
236const layerService = (input: {
237 readonly prefix: string;
238 readonly response: () => Response;
239 readonly settings?: Parameters<typeof ServerSettings.layerTest>[0];
240}) =>
241 ServerConfig.layerTest(process.cwd(), { prefix: input.prefix }).pipe(
242 Layer.provideMerge(NodeServices.layer),
243 Layer.provideMerge(ServerSettings.layerTest(input.settings ?? {})),
244 Layer.provideMerge(layerHttpClient(input.response)),
245 );
246
247describe("ModelManifest service", () => {
248 it.live("explicit refresh bypasses fresh memory and disk caches", () => {
249 let fetchCount = 0;
250 const updated: ModelManifest.ModelManifestData = {
251 ...REMOTE_MANIFEST,
252 currentModels: { codex: ["gpt-reloaded"] },
253 };
254 return Effect.gen(function* () {
255 const service = yield* ModelManifest.make;
256 assert.deepStrictEqual(yield* service.refresh, REMOTE_MANIFEST);
257 assert.deepStrictEqual(yield* service.refresh, REMOTE_MANIFEST);
258 assert.strictEqual(fetchCount, 1);
259
260 const rebooted = yield* ModelManifest.make;
261 assert.deepStrictEqual(yield* rebooted.refresh, REMOTE_MANIFEST);
262 assert.strictEqual(fetchCount, 1);
263 assert.deepStrictEqual(yield* rebooted.forceRefresh, updated);
264 assert.strictEqual(fetchCount, 2);
265 assert.deepStrictEqual(yield* rebooted.current, updated);
266 assert.deepStrictEqual(yield* (yield* ModelManifest.make).current, updated);
267 }).pipe(
268 Effect.scoped,
269 Effect.provide(
270 layerService({
271 prefix: "model-manifest-force-refresh-test",
272 response: () => Response.json(fetchCount++ === 0 ? REMOTE_MANIFEST : updated),
273 }),
274 ),
275 );
276 });
277
278 it.live("explicit refresh retries immediately after failure and preserves last-good data", () => {
279 let fetchCount = 0;
280 return Effect.gen(function* () {
281 const service = yield* ModelManifest.make;
282 assert.deepStrictEqual(yield* service.refresh, REMOTE_MANIFEST);
283 assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST);
284 assert.deepStrictEqual(yield* service.current, REMOTE_MANIFEST);
285 assert.deepStrictEqual(yield* (yield* ModelManifest.make).current, REMOTE_MANIFEST);
286 assert.strictEqual(fetchCount, 2);
287 assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST);
288 assert.strictEqual(fetchCount, 3);
289 }).pipe(
290 Effect.scoped,
291 Effect.provide(
292 layerService({
293 prefix: "model-manifest-force-retry-test",
294 response: () =>
295 fetchCount++ === 1
296 ? new Response(null, { status: 503 })
297 : Response.json(REMOTE_MANIFEST),
298 }),
299 ),
300 );
301 });
302
303 it.live("explicit refresh bypasses the retry delay after an initial failure", () => {
304 let fetchCount = 0;
305 return Effect.gen(function* () {
306 const service = yield* ModelManifest.make;
307 assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
308 assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
309 assert.strictEqual(fetchCount, 1);
310 assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST);
311 assert.strictEqual(fetchCount, 2);
312 }).pipe(
313 Effect.scoped,
314 Effect.provide(
315 layerService({
316 prefix: "model-manifest-force-initial-retry-test",
317 response: () =>
318 fetchCount++ === 0
319 ? new Response(null, { status: 503 })
320 : Response.json(REMOTE_MANIFEST),
321 }),
322 ),
323 );
324 });
325
326 it.live("prefers a fetched manifest over the bundle and caches it to disk", () =>
327 Effect.gen(function* () {
328 const service = yield* ModelManifest.make;
329 const refreshed = yield* service.refresh;
330 assert.deepStrictEqual(refreshed, REMOTE_MANIFEST);
331
332 // A fresh service instance sees the disk cache without another fetch:
333 // its HTTP layer is still stubbed, but `current` never fetches at all.
334 const rebooted = yield* ModelManifest.make;
335 assert.deepStrictEqual(yield* rebooted.current, REMOTE_MANIFEST);
336 }).pipe(
337 Effect.scoped,
338 Effect.provide(
339 layerService({
340 prefix: "model-manifest-fetch-test",
341 response: () => Response.json(REMOTE_MANIFEST),
342 }),
343 ),
344 ),
345 );
346
347 it.live("ignores older remote edits without replacing the current manifest or disk cache", () => {
348 let remote = { ...REMOTE_MANIFEST, updatedAt: "2000-01-01T00:00:00Z" };
349 return Effect.gen(function* () {
350 const service = yield* ModelManifest.make;
351 assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
352 assert.deepStrictEqual(yield* service.current, ModelManifest.BUNDLED_MODEL_MANIFEST);
353
354 remote = { ...REMOTE_MANIFEST, updatedAt: REMOTE_UPDATED_AT };
355 assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST);
356 remote = { ...REMOTE_MANIFEST, updatedAt: ModelManifest.BUNDLED_MODEL_MANIFEST.updatedAt! };
357 assert.deepStrictEqual(yield* service.forceRefresh, REMOTE_MANIFEST);
358 const rebooted = yield* ModelManifest.make;
359 assert.deepStrictEqual(yield* rebooted.current, REMOTE_MANIFEST);
360 }).pipe(
361 Effect.scoped,
362 Effect.provide(
363 layerService({
364 prefix: "model-manifest-stale-fetch-test",
365 response: () => Response.json(remote),
366 }),
367 ),
368 );
369 });
370
371 it.live("keeps the bundled manifest when the remote payload is malformed", () =>
372 Effect.gen(function* () {
373 const service = yield* ModelManifest.make;
374 assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
375 }).pipe(
376 Effect.scoped,
377 Effect.provide(
378 layerService({
379 prefix: "model-manifest-malformed-test",
380 response: () => Response.json({ version: 999, nonsense: true }),
381 }),
382 ),
383 ),
384 );
385
386 it.effect("preserves the last-good remote cache when later payloads are invalid", () => {
387 let responseIndex = 0;
388 const responses = [REMOTE_CLAUDE_MANIFEST, ...INVALID_REMOTE_MANIFESTS];
389
390 return Effect.gen(function* () {
391 const service = yield* ModelManifest.make;
392 assert.deepStrictEqual(yield* service.refresh, REMOTE_CLAUDE_MANIFEST);
393
394 for (const _invalid of INVALID_REMOTE_MANIFESTS) {
395 yield* TestClock.adjust("1 hour");
396 responseIndex += 1;
397 assert.deepStrictEqual(yield* service.refresh, REMOTE_CLAUDE_MANIFEST);
398 }
399
400 const rebooted = yield* ModelManifest.make;
401 assert.deepStrictEqual(yield* rebooted.current, REMOTE_CLAUDE_MANIFEST);
402 }).pipe(
403 Effect.scoped,
404 Effect.provide(
405 layerService({
406 prefix: "model-manifest-last-good-test",
407 response: () => Response.json(responses[responseIndex]),
408 }),
409 ),
410 );
411 });
412
413 it.live("drops a disk cache of a manifest older than the bundled one", () =>
414 Effect.gen(function* () {
415 const fs = yield* FileSystem.FileSystem;
416 const path = yield* Path.Path;
417 const config = yield* ServerConfig.ServerConfig;
418 const cachePath = path.join(config.stateDir, "model-manifest.json");
419 // A cache of the manifest as it was before the release edited it. The
420 // fetch time is irrelevant: the remote may be unreachable now, so
421 // `current` must already prefer the bundle.
422 const { updatedAt: _undated, ...undatedManifest } = REMOTE_MANIFEST;
423 for (const stale of [
424 undatedManifest,
425 { ...REMOTE_MANIFEST, updatedAt: "2000-01-01T00:00:00Z" },
426 ]) {
427 yield* fs.writeFileString(
428 cachePath,
429 yield* ModelManifest.encodeManifestCache({ fetchedAtMs: 0, manifest: stale }),
430 );
431 const service = yield* ModelManifest.make;
432 assert.deepStrictEqual(yield* service.current, ModelManifest.BUNDLED_MODEL_MANIFEST);
433 }
434
435 // A cache of a newer edit still outranks the bundle.
436 yield* fs.writeFileString(
437 cachePath,
438 yield* ModelManifest.encodeManifestCache({ fetchedAtMs: 0, manifest: REMOTE_MANIFEST }),
439 );
440 const later = yield* ModelManifest.make;
441 assert.deepStrictEqual(yield* later.current, REMOTE_MANIFEST);
442 }).pipe(
443 Effect.scoped,
444 Effect.provide(
445 layerService({
446 prefix: "model-manifest-newer-bundle-test",
447 response: () => Response.json(REMOTE_MANIFEST),
448 }),
449 ),
450 ),
451 );
452
453 it.live("does not fetch when provider update checks are disabled", () =>
454 Effect.gen(function* () {
455 let fetchCount = 0;
456 const service = yield* ModelManifest.make.pipe(
457 Effect.provide(
458 layerHttpClient(() => {
459 fetchCount += 1;
460 return Response.json(REMOTE_MANIFEST);
461 }),
462 ),
463 );
464 assert.deepStrictEqual(yield* service.refresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
465 assert.deepStrictEqual(yield* service.forceRefresh, ModelManifest.BUNDLED_MODEL_MANIFEST);
466 assert.strictEqual(fetchCount, 0);
467 }).pipe(
468 Effect.scoped,
469 Effect.provide(
470 layerService({
471 prefix: "model-manifest-optout-test",
472 response: () => Response.json(REMOTE_MANIFEST),
473 settings: { enableProviderUpdateChecks: false },
474 }),
475 ),
476 ),
477 );
478});
479
480it.effect("caches valid compatibility policies and keeps them after a malformed refresh", () => {
481 const remote: ModelManifest.ModelManifestData = {
482 ...REMOTE_MANIFEST,
483 compatibility: [
484 {
485 driver: "codex",
486 t3CodeRange: ">=0.0.42",
487 recommendedVersion: "2.0.0",
488 ranges: [{ range: "=2.0.0", status: "supported" }],
489 },
490 ],
491 };
492 let invalid = false;
493 return Effect.gen(function* () {
494 const service = yield* ModelManifest.make;
495 assert.deepStrictEqual((yield* service.refresh).compatibility, remote.compatibility);
496 invalid = true;
497 yield* TestClock.adjust("1 hour");
498 assert.deepStrictEqual((yield* service.refresh).compatibility, remote.compatibility);
499 const rebooted = yield* ModelManifest.make;
500 assert.deepStrictEqual((yield* rebooted.current).compatibility, remote.compatibility);
501 }).pipe(
502 Effect.scoped,
503 Effect.provide(
504 layerService({
505 prefix: "model-manifest-compatibility-test",
506 response: () =>
507 Response.json(
508 invalid
509 ? {
510 ...remote,
511 compatibility: [{ ...remote.compatibility![0], recommendedVersion: "3.0.0" }],
512 }
513 : remote,
514 ),
515 }),
516 ),
517 );
518});