1import { Generated, OpenRouterClient, OpenRouterLanguageModel } from "@effect/ai-openrouter"
2import { assert, describe, it } from "@effect/vitest"
3import { deepStrictEqual, strictEqual } from "@effect/vitest/utils"
4import { Array, Context, Effect, Layer, Redacted, Ref, Schema, Stream } from "effect"
5import { LanguageModel, Prompt, Tool, Toolkit } from "effect/ai"
6import { HttpClient, type HttpClientError, type HttpClientRequest, HttpClientResponse } from "effect/http"
7
8describe("OpenRouterLanguageModel", () => {
9 describe("strictJsonSchema", () => {
10 it.effect("omits false from requests while preserving response strictness", () =>
11 Effect.gen(function*() {
12 yield* LanguageModel.generateObject({
13 prompt: "Give me a name",
14 schema: Schema.Struct({ name: Schema.String })
15 }).pipe(Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini", { strictJsonSchema: false })))
16
17 const requests = yield* MockHttpClient.requests
18 const body = yield* getRequestBody(requests[0])
19 strictEqual(body.response_format.json_schema.strict, false)
20 assert.notProperty(body, "strictJsonSchema")
21 }).pipe(Effect.provide(makeTestLayer({
22 body: {
23 choices: [{
24 finish_reason: "stop",
25 index: 0,
26 message: { role: "assistant", content: JSON.stringify({ name: "Alice" }) }
27 }]
28 }
29 }))))
30
31 it.effect("omits true from streaming requests while preserving tool strictness", () =>
32 Effect.gen(function*() {
33 const tool = Tool.make("FlexibleTool", { parameters: Schema.Struct({ query: Schema.String }) })
34 .annotate(Tool.Strict, false)
35 yield* LanguageModel.streamText({
36 prompt: "Use a tool",
37 toolkit: Toolkit.make(tool),
38 disableToolCallResolution: true
39 }).pipe(
40 Stream.runDrain,
41 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini", { strictJsonSchema: true }))
42 )
43
44 const requests = yield* MockHttpClient.requests
45 const body = yield* getRequestBody(requests[0])
46 strictEqual(body.stream, true)
47 strictEqual(body.tools[0].function.strict, false)
48 assert.notProperty(body, "strictJsonSchema")
49 }).pipe(Effect.provide(makeStreamTestLayer([]))))
50 })
51
52 describe("generateText", () => {
53 describe("message preparation", () => {
54 describe("audio file parts", () => {
55 it.effect("converts audio bytes to input_audio", () =>
56 Effect.gen(function*() {
57 const audioData = new Uint8Array([0x49, 0x44, 0x33, 0x04]) // ID3v2 magic bytes
58
59 yield* LanguageModel.generateText({
60 prompt: Prompt.make([{
61 role: "user",
62 content: [
63 Prompt.filePart({
64 mediaType: "audio/mpeg",
65 data: audioData
66 })
67 ]
68 }])
69 }).pipe(Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")))
70
71 const requests = yield* MockHttpClient.requests
72 const body = yield* getRequestBody(requests[0])
73
74 const userMessage = body.messages.find((message: any) => message.role === "user")
75 deepStrictEqual(userMessage.content, [{
76 type: "input_audio",
77 input_audio: {
78 data: "SUQzBA==",
79 format: "mp3"
80 }
81 }])
82 }).pipe(Effect.provide(makeTestLayer())))
83
84 it.effect("converts base64 data url audio to input_audio", () =>
85 Effect.gen(function*() {
86 yield* LanguageModel.generateText({
87 prompt: Prompt.make([{
88 role: "user",
89 content: [
90 Prompt.filePart({
91 mediaType: "audio/wav",
92 data: "data:audio/wav;base64,UklGRg=="
93 })
94 ]
95 }])
96 }).pipe(Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")))
97
98 const requests = yield* MockHttpClient.requests
99 const body = yield* getRequestBody(requests[0])
100
101 const userMessage = body.messages.find((message: any) => message.role === "user")
102 deepStrictEqual(userMessage.content, [{
103 type: "input_audio",
104 input_audio: {
105 data: "UklGRg==",
106 format: "wav"
107 }
108 }])
109 }).pipe(Effect.provide(makeTestLayer())))
110
111 it.effect("maps audio media types to OpenRouter audio formats", () =>
112 Effect.gen(function*() {
113 const mediaTypes: ReadonlyArray<readonly [mediaType: string, format: string]> = [
114 ["audio/aac", "aac"],
115 ["audio/x-aiff", "aiff"],
116 ["audio/flac", "flac"],
117 ["audio/L16", "pcm16"],
118 ["audio/mp4", "m4a"],
119 ["audio/mp3", "mp3"],
120 ["audio/ogg", "ogg"],
121 ["audio/x-wav", "wav"]
122 ]
123
124 yield* LanguageModel.generateText({
125 prompt: Prompt.make([{
126 role: "user",
127 content: mediaTypes.map(([mediaType]) =>
128 Prompt.filePart({
129 mediaType,
130 data: new Uint8Array([0x00])
131 })
132 )
133 }])
134 }).pipe(Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")))
135
136 const requests = yield* MockHttpClient.requests
137 const body = yield* getRequestBody(requests[0])
138
139 const userMessage = body.messages.find((message: any) => message.role === "user")
140 deepStrictEqual(
141 userMessage.content.map((item: any) => item.input_audio.format),
142 mediaTypes.map(([, format]) => format)
143 )
144 }).pipe(Effect.provide(makeTestLayer())))
145
146 it.effect("fails on unsupported audio media types", () =>
147 Effect.gen(function*() {
148 const error = yield* LanguageModel.generateText({
149 prompt: Prompt.make([{
150 role: "user",
151 content: [
152 Prompt.filePart({
153 mediaType: "audio/webm",
154 data: new Uint8Array([0x00])
155 })
156 ]
157 }])
158 }).pipe(
159 Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")),
160 Effect.flip
161 )
162
163 strictEqual(error.reason._tag, "InvalidUserInputError")
164 assert.include(error.message, "audio/webm")
165 }).pipe(Effect.provide(makeTestLayer())))
166
167 it.effect("fails on audio URLs", () =>
168 Effect.gen(function*() {
169 const error = yield* LanguageModel.generateText({
170 prompt: Prompt.make([{
171 role: "user",
172 content: [
173 Prompt.filePart({
174 mediaType: "audio/mpeg",
175 data: new URL("https://example.com/audio.mp3")
176 })
177 ]
178 }])
179 }).pipe(
180 Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")),
181 Effect.flip
182 )
183
184 strictEqual(error.reason._tag, "InvalidUserInputError")
185 }).pipe(Effect.provide(makeTestLayer())))
186
187 it.effect("converts non-audio files to file blocks", () =>
188 Effect.gen(function*() {
189 const pdfData = new Uint8Array([0x25, 0x50, 0x44, 0x46]) // %PDF
190
191 yield* LanguageModel.generateText({
192 prompt: Prompt.make([{
193 role: "user",
194 content: [
195 Prompt.filePart({
196 mediaType: "application/pdf",
197 fileName: "document.pdf",
198 data: pdfData
199 })
200 ]
201 }])
202 }).pipe(Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")))
203
204 const requests = yield* MockHttpClient.requests
205 const body = yield* getRequestBody(requests[0])
206
207 const userMessage = body.messages.find((message: any) => message.role === "user")
208 deepStrictEqual(userMessage.content, [{
209 type: "file",
210 file: {
211 filename: "document.pdf",
212 file_data: "data:application/pdf;base64,JVBERg=="
213 }
214 }])
215 }).pipe(Effect.provide(makeTestLayer())))
216 })
217
218 it.effect("preserves string tool results", () =>
219 Effect.gen(function*() {
220 yield* LanguageModel.generateText({
221 prompt: Prompt.make([
222 { role: "user", content: "Use the tool" },
223 {
224 role: "assistant",
225 content: [Prompt.toolCallPart({
226 id: "call_text",
227 name: "text_tool",
228 params: {},
229 providerExecuted: false
230 })]
231 },
232 {
233 role: "tool",
234 content: [Prompt.toolResultPart({
235 id: "call_text",
236 name: "text_tool",
237 result: "PLAIN_TEXT_SENTINEL\n",
238 isFailure: false,
239 providerExecuted: false
240 })]
241 }
242 ]),
243 disableToolCallResolution: true
244 }).pipe(Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")))
245
246 const requests = yield* MockHttpClient.requests
247 const body = yield* getRequestBody(requests[0])
248 const toolResult = body.messages.find((message: any) => message.role === "tool")
249
250 assert.isDefined(toolResult)
251 strictEqual(toolResult.content, "PLAIN_TEXT_SENTINEL\n")
252 }).pipe(Effect.provide(makeTestLayer())))
253 })
254
255 describe("tool preparation", () => {
256 it.effect("passes raw JSON schema for dynamic tools", () =>
257 Effect.gen(function*() {
258 const inputSchema = {
259 type: "object",
260 properties: {
261 query: { type: "string" },
262 limit: { type: "number" }
263 },
264 required: ["query"],
265 additionalProperties: false
266 } as const
267
268 const DynamicTool = Tool.dynamic("DynamicTool", {
269 description: "A dynamic tool",
270 parameters: inputSchema
271 })
272
273 yield* LanguageModel.generateText({
274 prompt: "Use the dynamic tool",
275 toolkit: Toolkit.make(DynamicTool),
276 disableToolCallResolution: true
277 }).pipe(Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")))
278
279 const requests = yield* MockHttpClient.requests
280 const body = yield* getRequestBody(requests[0])
281
282 const tool = body.tools?.find((entry: any) =>
283 entry.type === "function" && entry.function.name === "DynamicTool"
284 )
285 assert.isDefined(tool)
286 strictEqual(tool.function.description, "A dynamic tool")
287 deepStrictEqual(tool.function.parameters, inputSchema)
288 }).pipe(Effect.provide(makeTestLayer())))
289 })
290
291 describe("usage", () => {
292 it.effect("derives text and uncached tokens when details are subsets of their totals", () =>
293 Effect.gen(function*() {
294 const result = yield* LanguageModel.generateText({ prompt: "Hello" }).pipe(
295 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini"))
296 )
297
298 deepStrictEqual(
299 result.usage.inputTokens,
300 { uncached: 70, total: 100, cacheRead: 30, cacheWrite: 0 },
301 "subset input usage"
302 )
303 deepStrictEqual(result.usage.outputTokens, { total: 50, text: 30, reasoning: 20 }, "subset output usage")
304 }).pipe(Effect.provide(makeTestLayer({
305 body: {
306 usage: {
307 prompt_tokens: 100,
308 prompt_tokens_details: { cached_tokens: 30 },
309 completion_tokens: 50,
310 completion_tokens_details: { reasoning_tokens: 20 },
311 total_tokens: 150
312 }
313 }
314 }))))
315
316 it.effect("preserves totals when detail counts equal their parent counts", () =>
317 Effect.gen(function*() {
318 const result = yield* LanguageModel.generateText({ prompt: "Hello" }).pipe(
319 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini"))
320 )
321
322 deepStrictEqual(
323 result.usage.inputTokens,
324 { uncached: 0, total: 100, cacheRead: 100, cacheWrite: 0 },
325 "input usage at equality"
326 )
327 deepStrictEqual(result.usage.outputTokens, { total: 20, text: 0, reasoning: 20 }, "output usage at equality")
328 }).pipe(Effect.provide(makeTestLayer({
329 body: {
330 usage: {
331 prompt_tokens: 100,
332 prompt_tokens_details: { cached_tokens: 100 },
333 completion_tokens: 20,
334 completion_tokens_details: { reasoning_tokens: 20 },
335 total_tokens: 120
336 }
337 }
338 }))))
339
340 it.effect("treats reasoning tokens as disjoint when they exceed completion tokens", () =>
341 Effect.gen(function*() {
342 const result = yield* LanguageModel.generateText({ prompt: "Hello" }).pipe(
343 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini"))
344 )
345
346 deepStrictEqual(result.usage.outputTokens, { total: 30, text: 10, reasoning: 20 }, "disjoint reasoning usage")
347 }).pipe(Effect.provide(makeTestLayer({
348 body: {
349 usage: {
350 prompt_tokens: 100,
351 completion_tokens: 10,
352 completion_tokens_details: { reasoning_tokens: 20 },
353 total_tokens: 110
354 }
355 }
356 }))))
357
358 it.effect("treats cached tokens as disjoint when they exceed prompt tokens", () =>
359 Effect.gen(function*() {
360 const result = yield* LanguageModel.generateText({ prompt: "Hello" }).pipe(
361 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini"))
362 )
363
364 deepStrictEqual(
365 result.usage.inputTokens,
366 { uncached: 100, total: 400, cacheRead: 300, cacheWrite: 0 },
367 "disjoint cached usage"
368 )
369 }).pipe(Effect.provide(makeTestLayer({
370 body: {
371 usage: {
372 prompt_tokens: 100,
373 prompt_tokens_details: { cached_tokens: 300 },
374 completion_tokens: 10,
375 total_tokens: 110
376 }
377 }
378 }))))
379 })
380 })
381
382 describe("streamText", () => {
383 describe("usage", () => {
384 it.effect("treats streamed reasoning tokens as disjoint when they exceed completion tokens", () =>
385 Effect.gen(function*() {
386 const parts = yield* LanguageModel.streamText({ prompt: "Hello" }).pipe(
387 Stream.runCollect,
388 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini")),
389 Effect.provide(makeStreamTestLayer([{
390 id: "response-1",
391 object: "chat.completion.chunk",
392 model: "openai/gpt-4o-mini",
393 created: 1,
394 choices: [],
395 usage: {
396 prompt_tokens: 100,
397 completion_tokens: 10,
398 completion_tokens_details: { reasoning_tokens: 20 },
399 total_tokens: 110
400 }
401 }]))
402 )
403
404 const finishPart = parts.find((part) => part.type === "finish")
405 deepStrictEqual(
406 finishPart?.usage.outputTokens,
407 { total: 30, text: 10, reasoning: 20 },
408 "streamed disjoint reasoning usage"
409 )
410 }))
411 })
412
413 it.effect("preserves streamed citation start and end indexes", () =>
414 Effect.gen(function*() {
415 const parts = yield* LanguageModel.streamText({ prompt: "cite a source" }).pipe(
416 Stream.runCollect,
417 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini")),
418 Effect.provide(makeStreamTestLayer([{
419 id: "response-1",
420 object: "chat.completion.chunk",
421 model: "openai/gpt-4o-mini",
422 created: 1,
423 choices: [{
424 index: 0,
425 delta: {
426 annotations: [{
427 type: "url_citation",
428 url_citation: {
429 url: "https://example.com/source",
430 title: "source",
431 start_index: 2,
432 end_index: 9
433 }
434 }]
435 }
436 }]
437 }]))
438 )
439
440 const source = globalThis.Array.from(parts).find((part) => part.type === "source")
441 assert.isDefined(source)
442 if (source?.type === "source") {
443 assert.deepStrictEqual(source.metadata, {
444 openrouter: { startIndex: 2, endIndex: 9 }
445 })
446 }
447 }))
448
449 it.effect("uses lowercase openrouter reasoning-end metadata", () =>
450 Effect.gen(function*() {
451 const reasoningDetails = [{
452 type: "reasoning.text",
453 text: "thinking",
454 signature: "signature-final",
455 format: "unknown"
456 }] as const
457 const parts = yield* LanguageModel.streamText({ prompt: "reason then answer" }).pipe(
458 Stream.runCollect,
459 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini")),
460 Effect.provide(makeStreamTestLayer([
461 {
462 id: "response-1",
463 object: "chat.completion.chunk",
464 model: "openai/gpt-4o-mini",
465 created: 1,
466 choices: [{ index: 0, delta: { reasoning_details: reasoningDetails } }]
467 },
468 {
469 id: "response-1",
470 object: "chat.completion.chunk",
471 model: "openai/gpt-4o-mini",
472 created: 1,
473 choices: [{ index: 0, finish_reason: "stop", delta: { content: "answer" } }]
474 }
475 ]))
476 )
477
478 const reasoningEnd = parts.find((part) => part.type === "reasoning-end")
479 deepStrictEqual(reasoningEnd?.metadata, { openrouter: { reasoningDetails } })
480 }))
481
482 it.effect("emits incremental tool parameter fragments", () =>
483 Effect.gen(function*() {
484 const ProbeTool = Tool.make("ProbeTool", {
485 parameters: Schema.Struct({ a: Schema.Number }),
486 success: Schema.String
487 })
488 const toolkit = Toolkit.make(ProbeTool)
489 const parts = yield* LanguageModel.streamText({
490 prompt: "call the tool",
491 toolkit,
492 disableToolCallResolution: true
493 }).pipe(
494 Stream.runCollect,
495 Effect.provide(OpenRouterLanguageModel.model("openai/gpt-4o-mini")),
496 Effect.provide(toolkit.toLayer({ ProbeTool: () => Effect.succeed("ok") })),
497 Effect.provide(makeStreamTestLayer([
498 {
499 id: "response-1",
500 object: "chat.completion.chunk",
501 model: "openai/gpt-4o-mini",
502 created: 1,
503 choices: [{
504 index: 0,
505 delta: {
506 tool_calls: [{
507 index: 0,
508 id: "call-1",
509 type: "function",
510 function: { name: "ProbeTool", arguments: "{\"a\":" }
511 }]
512 }
513 }]
514 },
515 {
516 id: "response-1",
517 object: "chat.completion.chunk",
518 model: "openai/gpt-4o-mini",
519 created: 1,
520 choices: [{
521 index: 0,
522 delta: { tool_calls: [{ index: 0 }] }
523 }]
524 },
525 {
526 id: "response-1",
527 object: "chat.completion.chunk",
528 model: "openai/gpt-4o-mini",
529 created: 1,
530 choices: [{
531 index: 0,
532 finish_reason: "tool_calls",
533 delta: { tool_calls: [{ index: 0, function: { arguments: "1}" } }] }
534 }]
535 }
536 ]))
537 )
538
539 deepStrictEqual(
540 globalThis.Array.from(parts)
541 .filter((part) => part.type === "tool-params-delta")
542 .map((part) => part.delta),
543 ["{\"a\":", "1}"]
544 )
545 }))
546 })
547})
548
549// =============================================================================
550// Test Infrastructure
551// =============================================================================
552
553class MockOpenRouterResponse extends Context.Service<MockOpenRouterResponse, {
554 readonly status: number
555 readonly body: typeof Generated.SendChatCompletionRequest200.Type
556 readonly headers?: Record<string, string> | undefined
557}>()("MockOpenRouterResponse") {}
558
559class MockHttpClient extends Context.Service<MockHttpClient, {
560 readonly requests: Effect.Effect<ReadonlyArray<HttpClientRequest.HttpClientRequest>>
561}>()("MockHttpClient") {
562 static requests = Effect.service(MockHttpClient).pipe(
563 Effect.flatMap((client) => client.requests)
564 )
565}
566
567const encodeResponse = Schema.encodeEffect(Generated.SendChatCompletionRequest200)
568
569const makeHttpClient = Effect.gen(function*() {
570 const capturedRequests = yield* Ref.make<ReadonlyArray<HttpClientRequest.HttpClientRequest>>([])
571 const response = yield* MockOpenRouterResponse
572 const body = yield* Effect.orDie(encodeResponse(response.body))
573
574 const httpClient = HttpClient.makeWith(
575 Effect.fnUntraced(function*(requestEffect) {
576 const request = yield* requestEffect
577 yield* Ref.update(capturedRequests, Array.append(request))
578 return HttpClientResponse.fromWeb(
579 request,
580 new Response(JSON.stringify(body), {
581 headers: response.headers ?? {},
582 status: response.status
583 })
584 )
585 }),
586 Effect.succeed as HttpClient.HttpClient.Preprocess<HttpClientError.HttpClientError, never>
587 )
588
589 return Context.make(HttpClient.HttpClient, httpClient).pipe(
590 Context.add(MockHttpClient, MockHttpClient.of({ requests: Ref.get(capturedRequests) }))
591 )
592})
593
594const HttpClientLayer = Layer.effectContext(makeHttpClient)
595
596const makeDefaultResponse = (
597 overrides: Partial<typeof Generated.SendChatCompletionRequest200.Type> = {}
598): typeof Generated.SendChatCompletionRequest200.Type => ({
599 id: "gen-test123",
600 choices: [{
601 finish_reason: "stop",
602 index: 0,
603 message: {
604 role: "assistant",
605 content: "Hello!"
606 }
607 }],
608 created: 1234567890,
609 model: "google/gemini-2.5-flash",
610 object: "chat.completion",
611 system_fingerprint: null,
612 ...overrides
613})
614
615const makeTestLayer = (options: {
616 readonly body?: Partial<typeof Generated.SendChatCompletionRequest200.Type>
617 readonly status?: number
618 readonly headers?: Record<string, string>
619} = {}) =>
620 OpenRouterClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe(
621 Layer.provideMerge(HttpClientLayer),
622 Layer.provide(Layer.succeed(MockOpenRouterResponse, {
623 body: makeDefaultResponse(options.body),
624 status: options.status ?? 200,
625 headers: options.headers ?? {}
626 }))
627 )
628
629const getRequestBody = (request: HttpClientRequest.HttpClientRequest) =>
630 Effect.gen(function*() {
631 const body = request.body
632 if (body._tag === "Uint8Array") {
633 const text = new TextDecoder().decode(body.body)
634 return JSON.parse(text)
635 }
636 return yield* Effect.die(new Error("Expected Uint8Array body"))
637 })
638
639const makeStreamTestLayer = (events: ReadonlyArray<typeof Generated.ChatStreamChunk.Encoded>) => {
640 const body = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("") + "data: [DONE]\n\n"
641 const httpClientLayer = Layer.effectContext(Effect.gen(function*() {
642 const capturedRequests = yield* Ref.make<ReadonlyArray<HttpClientRequest.HttpClientRequest>>([])
643 const httpClient = HttpClient.makeWith(
644 Effect.fnUntraced(function*(requestEffect) {
645 const request = yield* requestEffect
646 yield* Ref.update(capturedRequests, Array.append(request))
647 return HttpClientResponse.fromWeb(
648 request,
649 new Response(body, {
650 status: 200,
651 headers: { "content-type": "text/event-stream" }
652 })
653 )
654 }),
655 Effect.succeed as HttpClient.HttpClient.Preprocess<HttpClientError.HttpClientError, never>
656 )
657 return Context.make(HttpClient.HttpClient, httpClient).pipe(
658 Context.add(MockHttpClient, MockHttpClient.of({ requests: Ref.get(capturedRequests) }))
659 )
660 }))
661 return OpenRouterClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe(
662 Layer.provideMerge(httpClientLayer)
663 )
664}