1import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { Unowned } from "../../AdoptPolicy.ts";
7import { isResolved } from "../../Diff.ts";
8import * as Provider from "../../Provider.ts";
9import { Resource } from "../../Resource.ts";
10import { tagRecord } from "../../Tags.ts";
11import { GcpEnvironment } from "../Environment.ts";
12import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
13import type { Providers } from "../Providers.ts";
14import { waitForOperation } from "./operations.ts";
15import {
16 hasAlchemyLabelMap,
17 normalizeLocation,
18 parseResourceName,
19 toPhysicalRfc1035,
20 userLabels,
21} from "./helpers.ts";
22
23const MAX_NAME_LENGTH = 63;
24
25export type EvaluationItemProps = {
26 /**
27 * Evaluation item id (the `{evaluation_item}` segment). Assigned by
28 * Vertex on create. Provide to target an existing item.
29 */
30 evaluationItemId?: string;
31 /**
32 * Region. Immutable — changing it replaces the item.
33 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
34 */
35 location?: string;
36 /**
37 * Display name. Defaults to a generated id when omitted.
38 */
39 displayName?: string;
40 /**
41 * Item type (`REQUEST` or `RESULT`). Immutable.
42 */
43 evaluationItemType:
44 | aiplatform.GoogleCloudAiplatformV1EvaluationItemEvaluationItemTypeEnum
45 | (string & {});
46 /**
47 * Cloud Storage object holding the request or response.
48 */
49 gcsUri?: string;
50 /**
51 * Inlined evaluation request (prompt, golden response, candidates).
52 */
53 evaluationRequest?: aiplatform.GoogleCloudAiplatformV1EvaluationRequest;
54 /**
55 * Arbitrary caller metadata.
56 */
57 metadata?: unknown;
58 /**
59 * User labels. Alchemy ownership labels are merged in automatically.
60 */
61 labels?: Record<string, string>;
62};
63
64export type EvaluationItem = Resource<
65 "GCP.AIPlatform.EvaluationItem",
66 EvaluationItemProps,
67 {
68 /** Full resource name. */
69 name: string;
70 /** Evaluation item id (last path segment). */
71 evaluationItemId: string;
72 /** Project id. */
73 project: string;
74 /** Location id. */
75 location: string;
76 /** Display name. */
77 displayName: string | undefined;
78 /** Item type. */
79 evaluationItemType: string | undefined;
80 /** GCS object URI, if set. */
81 gcsUri: string | undefined;
82 /** Inlined evaluation request, if set. */
83 evaluationRequest:
84 | aiplatform.GoogleCloudAiplatformV1EvaluationRequest
85 | undefined;
86 /** Evaluation result (RESULT items). */
87 evaluationResponse:
88 | aiplatform.GoogleCloudAiplatformV1EvaluationResult
89 | undefined;
90 /** Caller metadata. */
91 metadata: unknown;
92 /** User labels (Alchemy ownership labels stripped). */
93 labels: Record<string, string>;
94 /** Error, if any. */
95 error: aiplatform.GoogleRpcStatus | undefined;
96 /** RFC3339 creation timestamp. */
97 createTime: string | undefined;
98 },
99 never,
100 Providers
101>;
102
103/**
104 * A Vertex AI Evaluation Item — one request or result row used by
105 * Evaluation Sets and Evaluation Runs.
106 *
107 * Vertex assigns the resource id. There is no update API: changing type,
108 * location, display name, request, or GCS URI replaces the item. Labels
109 * are stamped at create time.
110 *
111 * ### Creating an Evaluation Item
112 * **Example:** Request item with an inlined prompt
113 * ```typescript
114 * const item = yield* GCP.AIPlatform.EvaluationItem("Prompt", {
115 * evaluationItemType: "REQUEST",
116 * evaluationRequest: { prompt: { text: "What is 2+2?" } },
117 * labels: { env: "test" },
118 * });
119 * ```
120 *
121 * @resource
122 * @category AIPlatform
123 */
124export const EvaluationItem = Resource<EvaluationItem>(
125 "GCP.AIPlatform.EvaluationItem",
126);
127
128export class EvaluationItemNotResolved extends Data.TaggedError(
129 "GCP.AIPlatform.EvaluationItemNotResolved",
130)<{
131 name: string;
132}> {}
133
134export class EvaluationItemStillExists extends Data.TaggedError(
135 "GCP.AIPlatform.EvaluationItemStillExists",
136)<{
137 name: string;
138}> {}
139
140const resourceName = (project: string, location: string, itemId: string) =>
141 `projects/${project}/locations/${location}/evaluationItems/${itemId}`;
142
143const toAttrs = (
144 item: aiplatform.GoogleCloudAiplatformV1EvaluationItem,
145 project: string,
146) => {
147 const name = item.name ?? "";
148 const parsed = parseResourceName(name, "evaluationItems");
149 return {
150 name,
151 evaluationItemId: parsed.id,
152 project: parsed.project || project,
153 location: parsed.location,
154 displayName: item.displayName,
155 evaluationItemType: item.evaluationItemType,
156 gcsUri: item.gcsUri,
157 evaluationRequest: item.evaluationRequest,
158 evaluationResponse: item.evaluationResponse,
159 metadata: item.metadata,
160 labels: userLabels(item.labels),
161 error: item.error,
162 createTime: item.createTime,
163 };
164};
165
166const getByName = (name: string) =>
167 aiplatform
168 .getProjectsLocationsEvaluationItems({ name })
169 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
170
171const listOwned = (project: string, location = "-") =>
172 aiplatform.listProjectsLocationsEvaluationItems
173 .pages({
174 parent: `projects/${project}/locations/${location}`,
175 pageSize: 100,
176 })
177 .pipe(
178 Stream.flatMap((page) => Stream.fromIterable(page.evaluationItems ?? [])),
179 Stream.filter((item) => hasAlchemyLabelMap(item.labels)),
180 Stream.runCollect,
181 Effect.map((chunk) => Array.from(chunk)),
182 Effect.catchTag("NotFound", () => Effect.succeed([])),
183 );
184
185const findOwned = (id: string, project: string, location?: string) =>
186 Effect.gen(function* () {
187 const items = yield* listOwned(project, location);
188 for (const item of items) {
189 if (yield* hasAlchemyLabels(id, tagRecord(item.labels))) {
190 return item;
191 }
192 }
193 return undefined;
194 });
195
196const waitUntilGone = (name: string) =>
197 getByName(name).pipe(
198 Effect.flatMap((item) =>
199 item === undefined
200 ? Effect.void
201 : Effect.fail(new EvaluationItemStillExists({ name })),
202 ),
203 Effect.retry({
204 while: (error) =>
205 error._tag === "GCP.AIPlatform.EvaluationItemStillExists",
206 times: 10,
207 schedule: Schedule.spaced("2 seconds"),
208 }),
209 );
210
211export const EvaluationItemProvider = () =>
212 Provider.succeed(EvaluationItem, {
213 stables: [
214 "name",
215 "evaluationItemId",
216 "project",
217 "location",
218 "createTime",
219 "evaluationItemType",
220 ],
221
222 diff: Effect.fn(function* ({ news, olds, output }) {
223 const env = yield* GcpEnvironment.current;
224 if (!isResolved(news)) return undefined;
225 const previousId = olds?.evaluationItemId ?? output?.evaluationItemId;
226 const nextId = news.evaluationItemId ?? previousId;
227 const previousLocation = normalizeLocation(
228 olds?.location ?? output?.location,
229 env.region,
230 );
231 const nextLocation = normalizeLocation(
232 news.location ?? output?.location,
233 env.region,
234 );
235 const previousType =
236 olds?.evaluationItemType ?? output?.evaluationItemType ?? "";
237 const nextType = news.evaluationItemType ?? previousType;
238 const previousDisplay = olds?.displayName ?? output?.displayName ?? "";
239 const nextDisplay = news.displayName ?? previousDisplay;
240 const replace =
241 (previousId !== undefined &&
242 nextId !== undefined &&
243 nextId !== previousId) ||
244 previousLocation !== nextLocation ||
245 previousType !== nextType ||
246 (news.displayName !== undefined && nextDisplay !== previousDisplay);
247 if (!replace) return undefined;
248 return { action: "replace" as const, deleteFirst: false };
249 }),
250
251 read: Effect.fn(function* ({ id, olds, output }) {
252 const env = yield* GcpEnvironment.current;
253 const location = normalizeLocation(
254 olds?.location ?? output?.location,
255 env.region,
256 );
257 const itemId = olds?.evaluationItemId ?? output?.evaluationItemId;
258 const name =
259 output?.name ??
260 (itemId ? resourceName(env.project, location, itemId) : undefined);
261 const existing = name
262 ? yield* getByName(name)
263 : yield* findOwned(id, env.project, location);
264 if (existing === undefined) return undefined;
265 const attrs = toAttrs(existing, env.project);
266 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels)))
267 ? attrs
268 : Unowned(attrs);
269 }),
270
271 list: () =>
272 Effect.gen(function* () {
273 const env = yield* GcpEnvironment.current;
274 const items = yield* listOwned(env.project);
275 return items.map((item) => toAttrs(item, env.project));
276 }),
277
278 reconcile: Effect.fn(function* ({ id, news, output }) {
279 const env = yield* GcpEnvironment.current;
280 const location = normalizeLocation(
281 news.location ?? output?.location,
282 env.region,
283 );
284 const itemId = news.evaluationItemId ?? output?.evaluationItemId;
285 const name =
286 output?.name ??
287 (itemId ? resourceName(env.project, location, itemId) : undefined);
288 const desiredLabels = {
289 ...toLabels(news.labels),
290 ...(yield* createInternalLabels(id)),
291 };
292 const displayName =
293 news.displayName ??
294 (yield* toPhysicalRfc1035(id, undefined, undefined, MAX_NAME_LENGTH));
295
296 let current = name
297 ? yield* getByName(name)
298 : yield* findOwned(id, env.project, location);
299
300 if (current === undefined) {
301 current = yield* aiplatform
302 .createProjectsLocationsEvaluationItems({
303 parent: `projects/${env.project}/locations/${location}`,
304 body: {
305 displayName,
306 evaluationItemType: news.evaluationItemType,
307 gcsUri: news.gcsUri,
308 evaluationRequest: news.evaluationRequest,
309 metadata: news.metadata,
310 labels: desiredLabels,
311 },
312 })
313 .pipe(
314 Effect.catchTag("Conflict", () =>
315 findOwned(id, env.project, location),
316 ),
317 );
318 }
319
320 if (current === undefined) {
321 return yield* new EvaluationItemNotResolved({
322 name: name ?? displayName,
323 });
324 }
325
326 return toAttrs(current, env.project);
327 }),
328
329 delete: Effect.fn(function* ({ output }) {
330 const operation = yield* aiplatform
331 .deleteProjectsLocationsEvaluationItems({ name: output.name })
332 .pipe(
333 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
334 Effect.retry({
335 while: (error) => error._tag === "Conflict",
336 times: 8,
337 schedule: Schedule.spaced("5 seconds"),
338 }),
339 );
340 if (operation !== undefined) {
341 yield* waitForOperation(operation, { notFoundOk: true });
342 }
343 yield* waitUntilGone(output.name);
344 }),
345 });