1import * as Cloudflare from "@/Cloudflare";
2import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment";
3import * as Provider from "@/Provider";
4import * as Test from "@/Test/Alchemy";
5import * as aiGateway from "@distilled.cloud/cloudflare/ai-gateway";
6import { expect } from "alchemy-test";
7import * as Data from "effect/Data";
8import * as Effect from "effect/Effect";
9import { MinimumLogLevel } from "effect/References";
10import * as Schedule from "effect/Schedule";
11
12const { test } = Test.make({ providers: Cloudflare.providers() });
13
14const logLevel = Effect.provideService(
15 MinimumLogLevel,
16 process.env.DEBUG ? "Debug" : "Info",
17);
18
19const GATEWAY_ID = "alchemy-test-aigw-eval";
20
21class EvaluationStillExists extends Data.TaggedError("EvaluationStillExists") {}
22
23// A deleted evaluation surfaces as `EvaluationNotFound` (Cloudflare error
24// code 7002, which also covers a deleted parent gateway) — that's the
25// success condition here.
26const expectGone = (
27 accountId: string,
28 gatewayId: string,
29 evaluationId: string,
30) =>
31 aiGateway.getEvaluation({ accountId, gatewayId, id: evaluationId }).pipe(
32 Effect.flatMap(() => Effect.fail(new EvaluationStillExists())),
33 Effect.retry({
34 while: (e): e is EvaluationStillExists =>
35 e instanceof EvaluationStillExists,
36 schedule: Schedule.max([
37 Schedule.exponential("250 millis"),
38 Schedule.recurs(10),
39 ]),
40 }),
41 Effect.catchTag("EvaluationNotFound", () => Effect.void),
42 );
43
44test.provider(
45 "create, noop, replace, delete an evaluation",
46 (stack) =>
47 Effect.gen(function* () {
48 const { accountId } = yield* yield* CloudflareEnvironment;
49
50 yield* stack.destroy();
51
52 // The mandatory evaluation types (speed, cost) are account-global
53 // constants — discover their ids out-of-band.
54 const types = yield* Cloudflare.AI.listEvaluationTypes(accountId);
55 const typeIds = types
56 .filter((t) => t.mandatory)
57 .map((t) => t.id)
58 .sort();
59 expect(typeIds.length).toBeGreaterThan(0);
60
61 const program = (name: string) =>
62 Effect.gen(function* () {
63 const gateway = yield* Cloudflare.AI.Gateway("EvalGateway", {
64 id: GATEWAY_ID,
65 });
66 const dataset = yield* Cloudflare.AI.Dataset("EvalDataset", {
67 gatewayId: gateway.gatewayId,
68 name: "alchemy-test-eval-dataset",
69 filters: [{ key: "success", operator: "eq", value: [true] }],
70 });
71 const evaluation = yield* Cloudflare.AI.Evaluation("Evaluation", {
72 gatewayId: gateway.gatewayId,
73 name,
74 datasetIds: [dataset.datasetId],
75 evaluationTypeIds: typeIds,
76 });
77 return { dataset, evaluation };
78 });
79
80 const initial = yield* stack.deploy(program("alchemy-test-eval"));
81
82 expect(initial.evaluation.evaluationId).toBeDefined();
83 expect(initial.evaluation.accountId).toEqual(accountId);
84 expect(initial.evaluation.gatewayId).toEqual(GATEWAY_ID);
85 expect(initial.evaluation.name).toEqual("alchemy-test-eval");
86 expect(initial.evaluation.datasetIds).toEqual([
87 initial.dataset.datasetId,
88 ]);
89 expect([...initial.evaluation.evaluationTypeIds].sort()).toEqual(typeIds);
90 expect(initial.evaluation.totalLogs).toEqual(0);
91
92 // Verify out-of-band via the API.
93 const live = yield* aiGateway.getEvaluation({
94 accountId,
95 gatewayId: GATEWAY_ID,
96 id: initial.evaluation.evaluationId,
97 });
98 expect(live.name).toEqual("alchemy-test-eval");
99 // The API returns an empty `datasets` list for a fresh evaluation, so
100 // there is nothing to assert on `live.datasets`.
101
102 // Redeploying identical props is a no-op (still the same evaluation).
103 const noop = yield* stack.deploy(program("alchemy-test-eval"));
104 expect(noop.evaluation.evaluationId).toEqual(
105 initial.evaluation.evaluationId,
106 );
107 // The observed evaluation lists no datasets; state keeps the requested ids.
108 expect(noop.evaluation.datasetIds).toEqual([initial.dataset.datasetId]);
109
110 // Evaluations are create-only — renaming is a replacement.
111 const renamed = yield* stack.deploy(program("alchemy-test-eval-v2"));
112 expect(renamed.evaluation.name).toEqual("alchemy-test-eval-v2");
113 expect(renamed.evaluation.evaluationId).not.toEqual(
114 initial.evaluation.evaluationId,
115 );
116
117 // The replaced evaluation is gone.
118 yield* expectGone(accountId, GATEWAY_ID, initial.evaluation.evaluationId);
119
120 yield* stack.destroy();
121
122 yield* expectGone(accountId, GATEWAY_ID, renamed.evaluation.evaluationId);
123 }).pipe(logLevel),
124 { tags: ["provider:cloudflare", "provider:cloudflare:ai", "live"] },
125);
126
127test.provider(
128 "list enumerates the deployed evaluation",
129 (stack) =>
130 Effect.gen(function* () {
131 const { accountId } = yield* yield* CloudflareEnvironment;
132
133 yield* stack.destroy();
134
135 const types = yield* Cloudflare.AI.listEvaluationTypes(accountId);
136 const typeIds = types
137 .filter((t) => t.mandatory)
138 .map((t) => t.id)
139 .sort();
140 expect(typeIds.length).toBeGreaterThan(0);
141
142 const deployed = yield* stack.deploy(
143 Effect.gen(function* () {
144 const gateway = yield* Cloudflare.AI.Gateway("EvalGateway", {
145 id: GATEWAY_ID,
146 });
147 const dataset = yield* Cloudflare.AI.Dataset("EvalDataset", {
148 gatewayId: gateway.gatewayId,
149 name: "alchemy-test-eval-list-dataset",
150 filters: [{ key: "success", operator: "eq", value: [true] }],
151 });
152 const evaluation = yield* Cloudflare.AI.Evaluation("Evaluation", {
153 gatewayId: gateway.gatewayId,
154 name: "alchemy-test-eval-list",
155 datasetIds: [dataset.datasetId],
156 evaluationTypeIds: typeIds,
157 });
158 return { evaluation };
159 }),
160 );
161
162 const provider = yield* Provider.findProvider(Cloudflare.AI.Evaluation);
163 const all = yield* provider.list();
164
165 const found = all.find(
166 (e) => e.evaluationId === deployed.evaluation.evaluationId,
167 );
168 expect(found).toBeDefined();
169 expect(found?.accountId).toEqual(accountId);
170 expect(found?.gatewayId).toEqual(GATEWAY_ID);
171 expect(found?.name).toEqual("alchemy-test-eval-list");
172
173 yield* stack.destroy();
174 }).pipe(logLevel),
175 { tags: ["provider:cloudflare", "provider:cloudflare:ai", "live"] },
176);