1import { OpenRouterClient, OpenRouterConfig, OpenRouterDecisionModel } from "@effect/ai-openrouter"
2import { assert, describe, it } from "@effect/vitest"
3import { Effect, Layer, Redacted, Result, Schema } from "effect"
4import { Decision, DecisionModel, Model } from "effect/ai"
5import { HttpClient, type HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http"
6
7const Ticket = Schema.Struct({
8 message: Schema.String,
9 orderId: Schema.String
10})
11
12const TicketTriage = Decision.make({
13 input: Ticket,
14 decisions: {
15 department: Decision.classify({
16 instructions: "Which team should handle this",
17 criteria: {
18 billing: "Payment or subscription issues",
19 technical: "Bugs or integration problems",
20 sales: "Pricing or account questions"
21 }
22 }),
23 frustration: Decision.rate({
24 instructions: "How frustrated the customer appears",
25 criteria: ["Calm, just stating facts", "Frustrated but civil", "Very angry, strong language"]
26 }),
27 urgent: Decision.probability({
28 instructions: "The message conveys urgency or time-sensitivity",
29 criteria: {
30 false: "No time pressure",
31 true: "Needs action now"
32 }
33 })
34 }
35})
36
37const ticket = {
38 message: "Hi, I've been trying to connect my Stripe account for 3 days and it keeps failing. Please help ASAP.",
39 orderId: "A-104"
40}
41
42const decisionsResponse = {
43 model: "test/decision-model",
44 answers: {
45 department: {
46 type: "choice",
47 choice: "billing",
48 probabilities: { billing: 0.84, technical: 0.159, sales: 0.001 },
49 confidence: 0.596
50 },
51 frustration: {
52 type: "score",
53 score: 1.6,
54 legend: {
55 "0": "Calm, just stating facts",
56 "1": "Frustrated but civil",
57 "2": "Very angry, strong language"
58 },
59 probabilities: { "0": 0.05, "1": 0.3, "2": 0.65 },
60 confidence: 0.78
61 },
62 urgent: {
63 type: "noul",
64 noul: 0.999
65 }
66 },
67 usage: { input_tokens: 312, output_tokens: 48 }
68}
69
70describe("OpenRouterDecisionModel", () => {
71 it.effect("accepts rounded choice probabilities that total 0.99", () =>
72 Effect.gen(function*() {
73 const definition = Decision.make({
74 input: Schema.String,
75 decisions: {
76 neutral3: Decision.classify({
77 instructions: "Choose a sentiment",
78 criteria: { negative: "Negative", neutral: "Neutral", positive: "Positive" }
79 })
80 }
81 })
82 const { answers } = yield* DecisionModel.decide(definition, { input: "It is fine" }).pipe(
83 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
84 Effect.provide(makeClientLayer((request) =>
85 Effect.succeed(jsonResponse(request, {
86 model: "test/decision-model",
87 answers: {
88 neutral3: {
89 type: "choice",
90 choice: "neutral",
91 probabilities: { negative: 0.02, neutral: 0.93, positive: 0.04 },
92 confidence: 0.93
93 }
94 },
95 usage: { input_tokens: 10, output_tokens: 5 }
96 }))
97 ))
98 )
99
100 assert.strictEqual(answers.neutral3.label, "neutral")
101 assert.closeTo(Object.values(answers.neutral3.probabilities).reduce((sum, value) => sum + value, 0), 1, 1e-6)
102 }))
103
104 for (
105 const [name, probabilities, succeeds] of [
106 ["extra provider key", { a: 0.5, b: 0.5, extra: 0.01 }, true],
107 ["drift outside two-decimal rounding", { a: 0.49, b: 0.49 }, false],
108 ["out-of-range probability", { a: 1.01, b: -0.02 }, false],
109 ["all-zero distribution", { a: 0, b: 0 }, false],
110 ["eight rounded labels", { a: 0.12, b: 0.12, c: 0.12, d: 0.12, e: 0.12, f: 0.12, g: 0.12, h: 0.13 }, true]
111 ] as const
112 ) {
113 it.effect(name, () =>
114 Effect.gen(function*() {
115 const definition = Decision.make({
116 input: Schema.String,
117 decisions: {
118 choice: Decision.classify({
119 instructions: "Pick a label",
120 criteria: Object.fromEntries(
121 Object.keys(probabilities).filter((key) => key !== "extra").map((key) => [key, key])
122 )
123 })
124 }
125 })
126 const result = yield* Effect.result(
127 DecisionModel.decide(definition, { input: "test" }).pipe(
128 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
129 Effect.provide(makeClientLayer((request) =>
130 Effect.succeed(jsonResponse(request, {
131 model: "test/decision-model",
132 answers: { choice: { type: "choice", choice: "a", probabilities, confidence: 0.5 } },
133 usage: { input_tokens: 1, output_tokens: 1 }
134 }))
135 ))
136 )
137 )
138 if (succeeds) {
139 assert.isTrue(Result.isSuccess(result))
140 if (Result.isSuccess(result)) {
141 assert.closeTo(
142 Object.values(result.success.answers.choice.probabilities).reduce((sum, value) => sum + value, 0),
143 1,
144 1e-6
145 )
146 if (name === "extra provider key") {
147 assert.deepStrictEqual(result.success.answers.choice.probabilities, { a: 0.5, b: 0.5 })
148 }
149 }
150 } else {
151 assert.isTrue(Result.isFailure(result))
152 if (Result.isFailure(result)) assert.strictEqual(result.failure.reason._tag, "InvalidOutputError")
153 }
154 }))
155 }
156
157 it.effect("normalizes rounded score probabilities", () =>
158 Effect.gen(function*() {
159 const definition = Decision.make({
160 input: Schema.String,
161 decisions: { score: Decision.rate({ instructions: "Rate", criteria: ["low", "mid", "high"] }) }
162 })
163 const { answers } = yield* DecisionModel.decide(definition, { input: "test" }).pipe(
164 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
165 Effect.provide(makeClientLayer((request) =>
166 Effect.succeed(jsonResponse(request, {
167 model: "test/decision-model",
168 answers: {
169 score: { type: "score", score: 1, probabilities: { "0": 0.1, "1": 0.7, "2": 0.19 }, confidence: 0.7 }
170 },
171 usage: { input_tokens: 1, output_tokens: 1 }
172 }))
173 ))
174 )
175 assert.closeTo(Object.values(answers.score.probabilities).reduce((sum, value) => sum + value, 0), 1, 1e-6)
176 }))
177
178 it.effect("omits absent probability criteria from the encoded question", () =>
179 Effect.gen(function*() {
180 const definition = Decision.make({
181 input: Schema.String,
182 decisions: {
183 urgent: Decision.probability({ instructions: "Needs action now" })
184 }
185 })
186 const { answers } = yield* DecisionModel.decide(definition, { input: "Help ASAP" }).pipe(
187 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
188 Effect.provide(makeClientLayer((request) =>
189 Effect.gen(function*() {
190 const { questions } = yield* getRequestBody(request)
191 assert.deepStrictEqual(questions.urgent, {
192 type: "noul",
193 instructions: "Needs action now"
194 })
195
196 return jsonResponse(request, {
197 model: "test/decision-model",
198 answers: { urgent: { type: "noul", noul: 0.75 } },
199 usage: { input_tokens: 20, output_tokens: 4 }
200 })
201 })
202 ))
203 )
204
205 assert.deepStrictEqual(answers.urgent, { probability: 0.75 })
206 }))
207
208 for (const key of ["__proto__", "constructor", "toString"]) {
209 it.effect("round trips a classification label named " + key, () =>
210 Effect.gen(function*() {
211 const definition = Decision.make({
212 input: Schema.String,
213 decisions: {
214 category: Decision.classify({
215 instructions: "Choose a category",
216 criteria: { [key]: "Special category", ordinary: "Ordinary category" }
217 })
218 }
219 })
220 const { answers } = yield* DecisionModel.decide(definition, { input: "Help" }).pipe(
221 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
222 Effect.provide(makeClientLayer((request) =>
223 Effect.gen(function*() {
224 const { questions } = yield* getRequestBody(request)
225 const criteria = questions.category.criteria
226
227 assert.isTrue(Object.hasOwn(criteria, key))
228 assert.isTrue(Object.getOwnPropertyDescriptor(criteria, key)?.enumerable)
229 assert.deepStrictEqual(Object.keys(criteria), [key, "ordinary"])
230 assert.strictEqual(criteria[key], "Special category")
231 assert.strictEqual(Object.getPrototypeOf(criteria), Object.prototype)
232
233 return jsonResponse(request, {
234 model: "test/decision-model",
235 answers: {
236 category: {
237 type: "choice",
238 choice: key,
239 probabilities: { [key]: 0.75, ordinary: 0.25 },
240 confidence: 0.5
241 }
242 },
243 usage: { input_tokens: 10, output_tokens: 5 }
244 })
245 })
246 ))
247 )
248 const { label, probabilities } = answers.category
249
250 assert.isTrue(Object.hasOwn(probabilities, key))
251 assert.isTrue(Object.getOwnPropertyDescriptor(probabilities, key)?.enumerable)
252 assert.deepStrictEqual(Object.keys(probabilities), [key, "ordinary"])
253 assert.deepStrictEqual(probabilities, { [key]: 0.75, ordinary: 0.25 })
254 assert.strictEqual(label, key)
255 assert.strictEqual(Object.getPrototypeOf(probabilities), null)
256 }))
257
258 it.effect("maps a score distribution to a rate level named " + key, () =>
259 Effect.gen(function*() {
260 const definition = Decision.make({
261 input: Schema.String,
262 decisions: {
263 intensity: Decision.rate({ instructions: "How intense", criteria: ["low", key, "high"] })
264 }
265 })
266 const { answers } = yield* DecisionModel.decide(definition, { input: "Help" }).pipe(
267 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
268 Effect.provide(makeClientLayer((request) =>
269 Effect.gen(function*() {
270 const { questions } = yield* getRequestBody(request)
271 assert.deepStrictEqual(questions.intensity.criteria, ["low", key, "high"])
272
273 return jsonResponse(request, {
274 model: "test/decision-model",
275 answers: {
276 intensity: {
277 type: "score",
278 score: 1.1,
279 legend: { "0": "low", "1": key, "2": "high" },
280 probabilities: { "0": 0.1, "1": 0.7, "2": 0.2 },
281 confidence: 0.5
282 }
283 },
284 usage: { input_tokens: 10, output_tokens: 5 }
285 })
286 })
287 ))
288 )
289 const { label, probabilities, rating } = answers.intensity
290
291 assert.isTrue(Object.hasOwn(probabilities, key))
292 assert.isTrue(Object.getOwnPropertyDescriptor(probabilities, key)?.enumerable)
293 assert.deepStrictEqual(Object.keys(probabilities), ["low", key, "high"])
294 assert.deepStrictEqual(probabilities, { low: 0.1, [key]: 0.7, high: 0.2 })
295 assert.strictEqual(label, key)
296 assert.strictEqual(rating, 1.1)
297 assert.strictEqual(Object.getPrototypeOf(probabilities), null)
298 }))
299
300 it.effect("sends an own enumerable question named " + key, () =>
301 Effect.gen(function*() {
302 const definition = Decision.make({
303 input: Schema.String,
304 decisions: {
305 [key]: Decision.probability({
306 instructions: "Needs action now",
307 criteria: { false: "No", true: "Yes" }
308 })
309 }
310 })
311 const { answers } = yield* DecisionModel.decide(definition, { input: "Help" }).pipe(
312 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
313 Effect.provide(makeClientLayer((request) =>
314 Effect.gen(function*() {
315 const { questions } = yield* getRequestBody(request)
316
317 assert.strictEqual(Object.getPrototypeOf(questions), Object.prototype)
318 assert.isTrue(Object.hasOwn(questions, key))
319 assert.isTrue(Object.getOwnPropertyDescriptor(questions, key)?.enumerable)
320 assert.deepStrictEqual(Object.keys(questions), [key])
321 assert.deepStrictEqual(questions[key], {
322 type: "noul",
323 instructions: "Needs action now",
324 criteria: { false: "No", true: "Yes" }
325 })
326
327 return jsonResponse(request, {
328 model: "test/decision-model",
329 answers: { [key]: { type: "noul", noul: 0.75 } },
330 usage: { input_tokens: 20, output_tokens: 4 }
331 })
332 })
333 ))
334 )
335
336 assert.strictEqual(Object.getPrototypeOf(answers), null)
337 assert.isTrue(Object.hasOwn(answers, key))
338 assert.isTrue(Object.getOwnPropertyDescriptor(answers, key)?.enumerable)
339 assert.deepStrictEqual(Object.keys(answers), [key])
340 assert.deepStrictEqual(answers[key], { probability: 0.75 })
341 }))
342 }
343
344 for (
345 const apiUrl of [
346 "https://proxy.test/openrouter/v1",
347 "https://proxy.test/openrouter/v1/",
348 "https://proxy.test/openrouter",
349 "https://proxy.test/openrouter/"
350 ]
351 ) {
352 it.effect("resolves the alpha endpoint for " + apiUrl, () =>
353 Effect.gen(function*() {
354 const requests: Array<HttpClientRequest.HttpClientRequest> = []
355 yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
356 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
357 Effect.provide(makeClientLayer((request) => {
358 requests.push(request)
359 return Effect.succeed(jsonResponse(request, decisionsResponse))
360 }, apiUrl))
361 )
362 assert.strictEqual(requests.length, 1)
363 assert.strictEqual(requests[0].url, "https://proxy.test/openrouter/alpha/decisions")
364 }))
365 }
366
367 it.effect("applies a scoped client transform only to the enclosed decisions request", () =>
368 Effect.gen(function*() {
369 const requests: Array<HttpClientRequest.HttpClientRequest> = []
370 yield* Effect.gen(function*() {
371 yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
372 OpenRouterConfig.withClientTransform(
373 HttpClient.mapRequest(HttpClientRequest.setHeader("x-decision-scope", "scoped"))
374 )
375 )
376 yield* DecisionModel.decide(TicketTriage, { input: ticket })
377 }).pipe(
378 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
379 Effect.provide(makeClientLayer((request) => {
380 requests.push(request)
381 return Effect.succeed(jsonResponse(request, decisionsResponse))
382 }))
383 )
384 assert.strictEqual(requests.length, 2)
385 assert.strictEqual(requests[0].headers["x-decision-scope"], "scoped")
386 assert.isUndefined(requests[1].headers["x-decision-scope"])
387 }))
388
389 it.effect("rejects numeric state before making an HTTP request", () =>
390 Effect.gen(function*() {
391 const definition = Decision.make({ input: Schema.Number, decisions: TicketTriage.decisions })
392 const error = yield* DecisionModel.decide(definition, { input: 42 }).pipe(
393 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
394 Effect.provide(makeClientLayer(() => Effect.die(new Error("Unexpected HTTP request")))),
395 Effect.flip
396 )
397 assert.strictEqual(error._tag, "AiError")
398 assert.strictEqual(error.reason._tag, "InvalidUserInputError")
399 }))
400
401 it.effect("maps a malformed successful response to a client InvalidOutputError", () =>
402 Effect.gen(function*() {
403 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
404 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
405 Effect.provide(makeClientLayer((request) =>
406 Effect.succeed(jsonResponse(request, {
407 ...decisionsResponse,
408 usage: { input_tokens: "not a number", output_tokens: 48 }
409 }))
410 )),
411 Effect.flip
412 )
413 assert.strictEqual(error._tag, "AiError")
414 assert.strictEqual(error.module, "OpenRouterClient")
415 assert.strictEqual(error.method, "createDecisions")
416 assert.strictEqual(error.reason._tag, "InvalidOutputError")
417 }))
418
419 it.effect("make constructs a DecisionModel service", () =>
420 Effect.gen(function*() {
421 const service = yield* OpenRouterDecisionModel.make({ model: "test/decision-model" })
422 assert.isFunction(service.decide)
423 }).pipe(Effect.provide(makeClientLayer(() => Effect.die(new Error("Unexpected HTTP request"))))))
424
425 it.effect("forwards request config and scoped overrides", () =>
426 Effect.gen(function*() {
427 const requests: Array<HttpClientRequest.HttpClientRequest> = []
428 yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
429 Effect.provideService(OpenRouterDecisionModel.Config, { user: "scoped-user", trace: { trace_id: "scoped" } }),
430 Effect.provide(OpenRouterDecisionModel.layer({
431 model: "test/decision-model",
432 config: {
433 provider: { order: ["test-provider"] },
434 session_id: "session-1",
435 user: "default-user",
436 trace: { trace_id: "default" }
437 }
438 })),
439 Effect.provide(makeClientLayer((request) => {
440 requests.push(request)
441 return Effect.succeed(jsonResponse(request, decisionsResponse))
442 }))
443 )
444 const body = yield* getRequestBody(requests[0])
445 assert.deepStrictEqual(body.provider, { order: ["test-provider"] })
446 assert.strictEqual(body.session_id, "session-1")
447 assert.strictEqual(body.user, "scoped-user")
448 assert.deepStrictEqual(body.trace, { trace_id: "scoped" })
449 }))
450
451 it.effect("accepts absent confidence, cost, id and provider without inventing values", () =>
452 Effect.gen(function*() {
453 const { confidence: _choiceConfidence, ...department } = decisionsResponse.answers.department
454 const { confidence: _scoreConfidence, legend: _legend, ...frustration } = decisionsResponse.answers.frustration
455 const { answers, usage } = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
456 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
457 Effect.provide(makeClientLayer((request) =>
458 Effect.succeed(jsonResponse(request, {
459 ...decisionsResponse,
460 answers: { ...decisionsResponse.answers, department, frustration }
461 }))
462 ))
463 )
464 assert.isUndefined(answers.department.confidence)
465 assert.isUndefined(answers.frustration.confidence)
466 assert.strictEqual(answers.frustration.label, "Very angry, strong language")
467 assert.strictEqual(answers.urgent.probability, 0.999)
468 assert.deepStrictEqual({ ...usage }, { inputTokens: 312, outputTokens: 48 })
469 }))
470
471 it.effect("drops provider-specific cost, id and provider metadata", () =>
472 Effect.gen(function*() {
473 const result = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
474 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
475 Effect.provide(makeClientLayer((request) =>
476 Effect.succeed(jsonResponse(request, {
477 ...decisionsResponse,
478 id: "decision-1",
479 provider: "test-provider",
480 usage: { ...decisionsResponse.usage, cost: 0.001 }
481 }))
482 ))
483 )
484 assert.deepStrictEqual({ ...result.usage }, { inputTokens: 312, outputTokens: 48 })
485 assert.isFalse("id" in result)
486 assert.isFalse("provider" in result)
487 assert.isFalse("cost" in result.answers.department)
488 }))
489
490 it.effect("fails with InvalidOutputError when a choice answer omits probabilities", () =>
491 Effect.gen(function*() {
492 const { probabilities: _probabilities, ...department } = decisionsResponse.answers.department
493 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
494 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
495 Effect.provide(makeClientLayer((request) =>
496 Effect.succeed(jsonResponse(request, {
497 ...decisionsResponse,
498 answers: { ...decisionsResponse.answers, department }
499 }))
500 )),
501 Effect.flip
502 )
503 assert.strictEqual(error._tag, "AiError")
504 assert.strictEqual(error.reason._tag, "InvalidOutputError")
505 }))
506
507 it.effect("model provides DecisionModel with openrouter provider metadata", () =>
508 Effect.gen(function*() {
509 const provider = yield* Model.ProviderName
510 const modelName = yield* Model.ModelName
511 const model = yield* DecisionModel.DecisionModel
512
513 assert.strictEqual(provider, "openrouter")
514 assert.strictEqual(modelName, "test/decision-model")
515 assert.isDefined(model.decide)
516 }).pipe(
517 Effect.provide(OpenRouterDecisionModel.model("test/decision-model")),
518 Effect.provide(makeClientLayer(() => Effect.die(new Error("Unexpected HTTP request"))))
519 ))
520
521 it.effect("sends the encoded input as state and every decision as a question in one request", () =>
522 Effect.gen(function*() {
523 const requests: Array<HttpClientRequest.HttpClientRequest> = []
524 const clientLayer = makeClientLayer((request) => {
525 requests.push(request)
526 return Effect.succeed(jsonResponse(request, decisionsResponse))
527 })
528
529 yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
530 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
531 Effect.provide(clientLayer)
532 )
533
534 assert.strictEqual(requests.length, 1)
535 assert.strictEqual(requests[0].method, "POST")
536 assert.strictEqual(requests[0].url, "https://openrouter.ai/api/alpha/decisions")
537 assert.strictEqual(requests[0].headers["authorization"], "Bearer or-test-key")
538
539 assert.strictEqual(requests[0].headers["http-referer"], "https://example.com")
540 assert.strictEqual(requests[0].headers["x-title"], "Decision contract tests")
541
542 const body = yield* getRequestBody(requests[0])
543 assert.deepStrictEqual(body, {
544 state: {
545 message: ticket.message,
546 orderId: "A-104"
547 },
548 model: "test/decision-model",
549 questions: {
550 department: {
551 type: "choice",
552 instructions: "Which team should handle this",
553 criteria: {
554 billing: "Payment or subscription issues",
555 technical: "Bugs or integration problems",
556 sales: "Pricing or account questions"
557 }
558 },
559 frustration: {
560 type: "score",
561 instructions: "How frustrated the customer appears",
562 criteria: ["Calm, just stating facts", "Frustrated but civil", "Very angry, strong language"]
563 },
564 urgent: {
565 type: "noul",
566 instructions: "The message conveys urgency or time-sensitivity",
567 criteria: {
568 false: "No time pressure",
569 true: "Needs action now"
570 }
571 }
572 }
573 })
574 }))
575
576 it.effect("maps choice, score, and noul answers onto classify, rate, and probability answers", () =>
577 Effect.gen(function*() {
578 const { answers, usage } = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
579 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
580 Effect.provide(makeClientLayer((request) => Effect.succeed(jsonResponse(request, decisionsResponse))))
581 )
582
583 assert.strictEqual(answers.department.label, "billing")
584 assert.deepStrictEqual(answers.department.probabilities, { billing: 0.84, technical: 0.159, sales: 0.001 })
585 assert.strictEqual(answers.department.confidence, 0.596)
586
587 assert.strictEqual(answers.frustration.rating, 1.6)
588 assert.strictEqual(answers.frustration.label, "Very angry, strong language")
589 assert.deepStrictEqual(answers.frustration.probabilities, {
590 "Calm, just stating facts": 0.05,
591 "Frustrated but civil": 0.3,
592 "Very angry, strong language": 0.65
593 })
594 assert.strictEqual(answers.frustration.confidence, 0.78)
595
596 assert.strictEqual(answers.urgent.probability, 0.999)
597 assert.isFalse("confidence" in answers.urgent)
598
599 assert.strictEqual(usage.inputTokens, 312)
600 assert.strictEqual(usage.outputTokens, 48)
601 }))
602
603 it.effect("encodes an explicitly undefined optional input field as JSON null", () =>
604 Effect.gen(function*() {
605 const requests: Array<HttpClientRequest.HttpClientRequest> = []
606 const definition = Decision.make({
607 input: Schema.Struct({ message: Schema.String, orderId: Schema.optional(Schema.String) }),
608 decisions: TicketTriage.decisions
609 })
610 const { answers } = yield* DecisionModel.decide(definition, {
611 input: { message: ticket.message, orderId: undefined }
612 }).pipe(
613 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
614 Effect.provide(makeClientLayer((request) => {
615 requests.push(request)
616 return Effect.succeed(jsonResponse(request, decisionsResponse))
617 }))
618 )
619
620 assert.strictEqual(requests.length, 1)
621 const body = yield* getRequestBody(requests[0])
622 assert.deepStrictEqual(body.state, { message: ticket.message, orderId: null })
623 assert.strictEqual(answers.urgent.probability, 0.999)
624 }))
625
626 it.effect("sends a pinned model id through unchanged", () =>
627 Effect.gen(function*() {
628 const requests: Array<HttpClientRequest.HttpClientRequest> = []
629
630 const modelName = yield* Effect.gen(function*() {
631 yield* DecisionModel.decide(TicketTriage, { input: ticket })
632 return yield* Model.ModelName
633 }).pipe(
634 Effect.provide(OpenRouterDecisionModel.model("test/decision-model-pinned")),
635 Effect.provide(makeClientLayer((request) => {
636 requests.push(request)
637 return Effect.succeed(jsonResponse(request, { ...decisionsResponse, model: "test/decision-model-pinned" }))
638 }))
639 )
640
641 const body = yield* getRequestBody(requests[0])
642 assert.strictEqual(modelName, "test/decision-model-pinned")
643 assert.strictEqual(body.model, "test/decision-model-pinned")
644 }))
645
646 it.effect("passes a string input through as a string state", () =>
647 Effect.gen(function*() {
648 const requests: Array<HttpClientRequest.HttpClientRequest> = []
649 const Sentiment = Decision.make({
650 input: Schema.String,
651 decisions: {
652 urgent: Decision.probability({
653 instructions: "The message conveys urgency",
654 criteria: { false: "No time pressure", true: "Needs action now" }
655 })
656 }
657 })
658
659 yield* DecisionModel.decide(Sentiment, { input: "My card was charged twice." }).pipe(
660 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
661 Effect.provide(makeClientLayer((request) => {
662 requests.push(request)
663 return Effect.succeed(jsonResponse(request, {
664 model: "test/decision-model",
665 answers: { urgent: { type: "noul", noul: 0.93 } },
666 usage: { input_tokens: 20, output_tokens: 4 }
667 }))
668 }))
669 )
670
671 const body = yield* getRequestBody(requests[0])
672 assert.strictEqual(body.state, "My card was charged twice.")
673 }))
674
675 it.effect("passes an array input through as an array state", () =>
676 Effect.gen(function*() {
677 const requests: Array<HttpClientRequest.HttpClientRequest> = []
678 const Conversation = Decision.make({
679 input: Schema.Array(Schema.String),
680 decisions: {
681 urgent: Decision.probability({
682 instructions: "The message conveys urgency",
683 criteria: { false: "No time pressure", true: "Needs action now" }
684 })
685 }
686 })
687
688 yield* DecisionModel.decide(Conversation, {
689 input: ["Hi", "My customer number is TS1337.", "My card was charged twice."]
690 }).pipe(
691 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
692 Effect.provide(makeClientLayer((request) => {
693 requests.push(request)
694 return Effect.succeed(jsonResponse(request, {
695 model: "test/decision-model",
696 answers: { urgent: { type: "noul", noul: 0.93 } },
697 usage: { input_tokens: 20, output_tokens: 4 }
698 }))
699 }))
700 )
701
702 const body = yield* getRequestBody(requests[0])
703 assert.deepStrictEqual(body.state, ["Hi", "My customer number is TS1337.", "My card was charged twice."])
704 }))
705
706 it.effect("fails with InvalidOutputError when an answer type does not match the decision", () =>
707 Effect.gen(function*() {
708 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
709 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
710 Effect.provide(makeClientLayer((request) =>
711 Effect.succeed(jsonResponse(request, {
712 ...decisionsResponse,
713 answers: {
714 ...decisionsResponse.answers,
715 urgent: {
716 type: "choice",
717 choice: "true",
718 probabilities: { true: 0.9, false: 0.1 },
719 confidence: 0.8
720 }
721 }
722 }))
723 )),
724 Effect.flip
725 )
726
727 assert.strictEqual(error._tag, "AiError")
728 assert.strictEqual(error.reason._tag, "InvalidOutputError")
729 }))
730
731 it.effect("fails with InvalidOutputError when an answer is missing", () =>
732 Effect.gen(function*() {
733 const { urgent: _urgent, ...answers } = decisionsResponse.answers
734 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
735 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
736 Effect.provide(
737 makeClientLayer((request) => Effect.succeed(jsonResponse(request, { ...decisionsResponse, answers })))
738 ),
739 Effect.flip
740 )
741
742 assert.strictEqual(error._tag, "AiError")
743 assert.strictEqual(error.reason._tag, "InvalidOutputError")
744 }))
745
746 it.effect("fails with InvalidOutputError when a score answer omits probabilities", () =>
747 Effect.gen(function*() {
748 const { probabilities: _probabilities, ...frustration } = decisionsResponse.answers.frustration
749 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
750 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
751 Effect.provide(makeClientLayer((request) =>
752 Effect.succeed(jsonResponse(request, {
753 ...decisionsResponse,
754 answers: { ...decisionsResponse.answers, frustration }
755 }))
756 )),
757 Effect.flip
758 )
759
760 assert.strictEqual(error._tag, "AiError")
761 assert.strictEqual(error.reason._tag, "InvalidOutputError")
762 }))
763
764 it.effect("fails with InvalidOutputError when a score distribution skips a level index", () =>
765 Effect.gen(function*() {
766 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
767 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
768 Effect.provide(makeClientLayer((request) =>
769 Effect.succeed(jsonResponse(request, {
770 ...decisionsResponse,
771 answers: {
772 ...decisionsResponse.answers,
773 frustration: {
774 ...decisionsResponse.answers.frustration,
775 probabilities: { "0": 0.35, "2": 0.65 }
776 }
777 }
778 }))
779 )),
780 Effect.flip
781 )
782
783 assert.strictEqual(error._tag, "AiError")
784 assert.strictEqual(error.reason._tag, "InvalidOutputError")
785 }))
786
787 for (
788 const [status, reason] of [
789 [400, "InvalidRequestError"],
790 [401, "AuthenticationError"],
791 [402, "UnknownError"],
792 [403, "AuthenticationError"],
793 [404, "InvalidRequestError"],
794 [413, "UnknownError"],
795 [429, "RateLimitError"],
796 [500, "InternalProviderError"],
797 [502, "InternalProviderError"],
798 [503, "InternalProviderError"],
799 [524, "InternalProviderError"],
800 [529, "InternalProviderError"]
801 ] as const
802 ) {
803 it.effect("maps HTTP " + status + " using the existing OpenRouter error mapping", () =>
804 Effect.gen(function*() {
805 const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe(
806 Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })),
807 Effect.provide(makeClientLayer((request) =>
808 Effect.succeed(HttpClientResponse.fromWeb(
809 request,
810 new Response(
811 JSON.stringify({ error: { message: "Decision request failed", code: status } }),
812 { status, headers: { "content-type": "application/json", "retry-after": "3" } }
813 )
814 ))
815 )),
816 Effect.flip
817 )
818 assert.strictEqual(error._tag, "AiError")
819 assert.strictEqual(error.reason._tag, reason)
820 }))
821 }
822})
823
824const makeClientLayer = (
825 handler: (
826 request: HttpClientRequest.HttpClientRequest
827 ) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>,
828 apiUrl?: string
829) =>
830 OpenRouterClient.layer({
831 apiUrl,
832 apiKey: Redacted.make("or-test-key"),
833 siteReferrer: "https://example.com",
834 siteTitle: "Decision contract tests"
835 }).pipe(
836 Layer.provide(Layer.succeed(HttpClient.HttpClient, makeHttpClient(handler)))
837 )
838
839const makeHttpClient = (
840 handler: (
841 request: HttpClientRequest.HttpClientRequest
842 ) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>
843) =>
844 HttpClient.makeWith(
845 Effect.fnUntraced(function*(requestEffect) {
846 const request = yield* requestEffect
847 return yield* handler(request)
848 }),
849 Effect.succeed as HttpClient.HttpClient.Preprocess<HttpClientError.HttpClientError, never>
850 )
851
852const jsonResponse = (
853 request: HttpClientRequest.HttpClientRequest,
854 body: unknown
855): HttpClientResponse.HttpClientResponse =>
856 HttpClientResponse.fromWeb(
857 request,
858 new Response(JSON.stringify(body), {
859 status: 200,
860 headers: {
861 "content-type": "application/json"
862 }
863 })
864 )
865
866const getRequestBody = (request: HttpClientRequest.HttpClientRequest) =>
867 Effect.gen(function*() {
868 const body = request.body
869 if (body._tag === "Uint8Array") {
870 const text = new TextDecoder().decode(body.body)
871 return JSON.parse(text)
872 }
873 return yield* Effect.die(new Error("Expected Uint8Array body"))
874 })