1import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { Unowned } from "../../AdoptPolicy.ts";
7import { isResolved } from "../../Diff.ts";
8import * as Provider from "../../Provider.ts";
9import { Resource } from "../../Resource.ts";
10import { tagRecord } from "../../Tags.ts";
11import { GcpEnvironment } from "../Environment.ts";
12import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
13import type { Providers } from "../Providers.ts";
14import { waitForOperation } from "./operations.ts";
15import {
16 hasAlchemyLabelMap,
17 normalizeLocation,
18 parseResourceName,
19 toPhysicalRfc1035,
20 userLabels,
21} from "./helpers.ts";
22
23const MAX_NAME_LENGTH = 63;
24
25export type EvaluationRunProps = {
26 /**
27 * Evaluation run id (the `{evaluation_run}` segment). Assigned by
28 * Vertex on create. Provide to target an existing run.
29 */
30 evaluationRunId?: string;
31 /**
32 * Region. Immutable — changing it replaces the run.
33 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
34 */
35 location?: string;
36 /**
37 * Display name. Defaults to a generated id when omitted. Immutable.
38 */
39 displayName?: string;
40 /**
41 * User labels. Alchemy ownership labels are merged in automatically.
42 */
43 labels?: Record<string, string>;
44 /**
45 * Arbitrary caller metadata.
46 */
47 metadata?: unknown;
48 /**
49 * Data source (an Evaluation Set name or a BigQuery request set).
50 * Immutable.
51 */
52 dataSource: aiplatform.GoogleCloudAiplatformV1EvaluationRunDataSource;
53 /**
54 * Candidate-to-inference config map. Immutable.
55 */
56 inferenceConfigs?: Record<
57 string,
58 aiplatform.GoogleCloudAiplatformV1EvaluationRunInferenceConfig
59 >;
60 /**
61 * Metrics, rubric, and output configuration. Immutable.
62 */
63 evaluationConfig?: aiplatform.GoogleCloudAiplatformV1EvaluationRunEvaluationConfig;
64};
65
66export type EvaluationRun = Resource<
67 "GCP.AIPlatform.EvaluationRun",
68 EvaluationRunProps,
69 {
70 /** Full resource name. */
71 name: string;
72 /** Evaluation run id (last path segment). */
73 evaluationRunId: string;
74 /** Project id. */
75 project: string;
76 /** Location id. */
77 location: string;
78 /** Display name. */
79 displayName: string | undefined;
80 /** Run state. */
81 state:
82 | aiplatform.GoogleCloudAiplatformV1EvaluationRunStateEnum
83 | (string & {})
84 | undefined;
85 /** Data source. */
86 dataSource:
87 | aiplatform.GoogleCloudAiplatformV1EvaluationRunDataSource
88 | undefined;
89 /** Inference configs. */
90 inferenceConfigs:
91 | aiplatform.GoogleCloudAiplatformV1EvaluationRunInferenceConfigMap
92 | undefined;
93 /** Evaluation config. */
94 evaluationConfig:
95 | aiplatform.GoogleCloudAiplatformV1EvaluationRunEvaluationConfig
96 | undefined;
97 /** Snapshot evaluation set. */
98 evaluationSetSnapshot: string | undefined;
99 /** Results when the run succeeded. */
100 evaluationResults:
101 | aiplatform.GoogleCloudAiplatformV1EvaluationResults
102 | undefined;
103 /** Caller metadata. */
104 metadata: unknown;
105 /** User labels (Alchemy ownership labels stripped). */
106 labels: Record<string, string>;
107 /** Error when FAILED or CANCELLED. */
108 error: aiplatform.GoogleRpcStatus | undefined;
109 /** RFC3339 creation timestamp. */
110 createTime: string | undefined;
111 /** RFC3339 completion timestamp. */
112 completionTime: string | undefined;
113 },
114 never,
115 Providers
116>;
117
118/**
119 * A Vertex AI Evaluation Run — one execution of metrics over a set of
120 * evaluation items or a BigQuery request set.
121 *
122 * Vertex assigns the resource id. Creating a run starts evaluation. There
123 * is no update API: changing location, display name, data source, or
124 * config replaces the run.
125 *
126 * ### Creating an Evaluation Run
127 * **Example:** Run metrics over an evaluation set
128 * ```typescript
129 * const run = yield* GCP.AIPlatform.EvaluationRun("Quality", {
130 * dataSource: { evaluationSet: evaluationSet.name },
131 * evaluationConfig: {
132 * metrics: [
133 * {
134 * metric: "instruction_following_v1",
135 * predefinedMetricSpec: { metricSpecName: "instruction_following_v1" },
136 * },
137 * ],
138 * },
139 * labels: { env: "test" },
140 * });
141 * ```
142 *
143 * @resource
144 * @category AIPlatform
145 */
146export const EvaluationRun = Resource<EvaluationRun>(
147 "GCP.AIPlatform.EvaluationRun",
148);
149
150export class EvaluationRunNotResolved extends Data.TaggedError(
151 "GCP.AIPlatform.EvaluationRunNotResolved",
152)<{
153 name: string;
154}> {}
155
156export class EvaluationRunStillExists extends Data.TaggedError(
157 "GCP.AIPlatform.EvaluationRunStillExists",
158)<{
159 name: string;
160}> {}
161
162const resourceName = (project: string, location: string, runId: string) =>
163 `projects/${project}/locations/${location}/evaluationRuns/${runId}`;
164
165const toAttrs = (
166 run: aiplatform.GoogleCloudAiplatformV1EvaluationRun,
167 project: string,
168) => {
169 const name = run.name ?? "";
170 const parsed = parseResourceName(name, "evaluationRuns");
171 return {
172 name,
173 evaluationRunId: parsed.id,
174 project: parsed.project || project,
175 location: parsed.location,
176 displayName: run.displayName,
177 state: run.state,
178 dataSource: run.dataSource,
179 inferenceConfigs: run.inferenceConfigs,
180 evaluationConfig: run.evaluationConfig,
181 evaluationSetSnapshot: run.evaluationSetSnapshot,
182 evaluationResults: run.evaluationResults,
183 metadata: run.metadata,
184 labels: userLabels(run.labels),
185 error: run.error,
186 createTime: run.createTime,
187 completionTime: run.completionTime,
188 };
189};
190
191const getByName = (name: string) =>
192 aiplatform
193 .getProjectsLocationsEvaluationRuns({ name })
194 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
195
196const listOwned = (project: string, location = "-") =>
197 aiplatform.listProjectsLocationsEvaluationRuns
198 .pages({
199 parent: `projects/${project}/locations/${location}`,
200 pageSize: 100,
201 })
202 .pipe(
203 Stream.flatMap((page) => Stream.fromIterable(page.evaluationRuns ?? [])),
204 Stream.filter((run) => hasAlchemyLabelMap(run.labels)),
205 Stream.runCollect,
206 Effect.map((chunk) => Array.from(chunk)),
207 Effect.catchTag("NotFound", () => Effect.succeed([])),
208 );
209
210const findOwned = (id: string, project: string, location?: string) =>
211 Effect.gen(function* () {
212 const runs = yield* listOwned(project, location);
213 for (const run of runs) {
214 if (yield* hasAlchemyLabels(id, tagRecord(run.labels))) {
215 return run;
216 }
217 }
218 return undefined;
219 });
220
221const waitUntilGone = (name: string) =>
222 getByName(name).pipe(
223 Effect.flatMap((run) =>
224 run === undefined
225 ? Effect.void
226 : Effect.fail(new EvaluationRunStillExists({ name })),
227 ),
228 Effect.retry({
229 while: (error) =>
230 error._tag === "GCP.AIPlatform.EvaluationRunStillExists",
231 times: 10,
232 schedule: Schedule.spaced("2 seconds"),
233 }),
234 );
235
236export const EvaluationRunProvider = () =>
237 Provider.succeed(EvaluationRun, {
238 stables: ["name", "evaluationRunId", "project", "location", "createTime"],
239
240 diff: Effect.fn(function* ({ news, olds, output }) {
241 const env = yield* GcpEnvironment.current;
242 if (!isResolved(news)) return undefined;
243 const previousId = olds?.evaluationRunId ?? output?.evaluationRunId;
244 const nextId = news.evaluationRunId ?? previousId;
245 const previousLocation = normalizeLocation(
246 olds?.location ?? output?.location,
247 env.region,
248 );
249 const nextLocation = normalizeLocation(
250 news.location ?? output?.location,
251 env.region,
252 );
253 const previousDisplay = olds?.displayName ?? output?.displayName ?? "";
254 const nextDisplay = news.displayName ?? previousDisplay;
255 const replace =
256 (previousId !== undefined &&
257 nextId !== undefined &&
258 nextId !== previousId) ||
259 previousLocation !== nextLocation ||
260 (news.displayName !== undefined && nextDisplay !== previousDisplay);
261 if (!replace) return undefined;
262 return { action: "replace" as const, deleteFirst: false };
263 }),
264
265 read: Effect.fn(function* ({ id, olds, output }) {
266 const env = yield* GcpEnvironment.current;
267 const location = normalizeLocation(
268 olds?.location ?? output?.location,
269 env.region,
270 );
271 const runId = olds?.evaluationRunId ?? output?.evaluationRunId;
272 const name =
273 output?.name ??
274 (runId ? resourceName(env.project, location, runId) : undefined);
275 const existing = name
276 ? yield* getByName(name)
277 : yield* findOwned(id, env.project, location);
278 if (existing === undefined) return undefined;
279 const attrs = toAttrs(existing, env.project);
280 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels)))
281 ? attrs
282 : Unowned(attrs);
283 }),
284
285 list: () =>
286 Effect.gen(function* () {
287 const env = yield* GcpEnvironment.current;
288 const runs = yield* listOwned(env.project);
289 return runs.map((run) => toAttrs(run, env.project));
290 }),
291
292 reconcile: Effect.fn(function* ({ id, news, output }) {
293 const env = yield* GcpEnvironment.current;
294 const location = normalizeLocation(
295 news.location ?? output?.location,
296 env.region,
297 );
298 const runId = news.evaluationRunId ?? output?.evaluationRunId;
299 const name =
300 output?.name ??
301 (runId ? resourceName(env.project, location, runId) : undefined);
302 const desiredLabels = {
303 ...toLabels(news.labels),
304 ...(yield* createInternalLabels(id)),
305 };
306 const displayName =
307 news.displayName ??
308 (yield* toPhysicalRfc1035(id, undefined, undefined, MAX_NAME_LENGTH));
309
310 let current = name
311 ? yield* getByName(name)
312 : yield* findOwned(id, env.project, location);
313
314 if (current === undefined) {
315 current = yield* aiplatform
316 .createProjectsLocationsEvaluationRuns({
317 parent: `projects/${env.project}/locations/${location}`,
318 body: {
319 displayName,
320 dataSource: news.dataSource,
321 inferenceConfigs: news.inferenceConfigs,
322 evaluationConfig: news.evaluationConfig,
323 metadata: news.metadata,
324 labels: desiredLabels,
325 },
326 })
327 .pipe(
328 Effect.catchTag("Conflict", () =>
329 findOwned(id, env.project, location),
330 ),
331 );
332 }
333
334 if (current === undefined) {
335 return yield* new EvaluationRunNotResolved({
336 name: name ?? displayName,
337 });
338 }
339
340 return toAttrs(current, env.project);
341 }),
342
343 delete: Effect.fn(function* ({ output }) {
344 // A running evaluation blocks the delete with BadRequest "There are
345 // other operations running on the EvaluationRun" until it finishes
346 // (cancel is unimplemented: HTTP 501).
347 const operation = yield* aiplatform
348 .deleteProjectsLocationsEvaluationRuns({ name: output.name })
349 .pipe(
350 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
351 Effect.retry({
352 while: (error) =>
353 error._tag === "Conflict" || error._tag === "BadRequest",
354 times: 24,
355 schedule: Schedule.spaced("5 seconds"),
356 }),
357 );
358 if (operation !== undefined) {
359 yield* waitForOperation(operation, { notFoundOk: true });
360 }
361 yield* waitUntilGone(output.name);
362 }),
363 });