1import * as GCP from "@/GCP";
2import { GcpEnvironment } from "@/GCP/Environment";
3import * as Test from "@/Test/Alchemy";
4import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
5import { expect } from "alchemy-test";
6import * as Effect from "effect/Effect";
7import { MinimumLogLevel } from "effect/References";
8import * as Schedule from "effect/Schedule";
9
10const { test } = Test.make({ providers: GCP.providers() });
11
12const logLevel = Effect.provideService(
13 MinimumLogLevel,
14 process.env.DEBUG ? "Debug" : "Info",
15);
16
17const waitUntilGone = (name: string) =>
18 aiplatform.getProjectsLocationsEvaluationSets({ name }).pipe(
19 Effect.as("found" as const),
20 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
21 Effect.repeat({
22 schedule: Schedule.spaced("2 seconds"),
23 until: (status) => status === "gone",
24 times: 10,
25 }),
26 );
27
28test.provider(
29 "getProjectsLocationsEvaluationSets on a missing set fails with a typed tag",
30 (stack) =>
31 Effect.gen(function* () {
32 const { project } = yield* GcpEnvironment.current;
33 const parent = `projects/${project}/locations/us-central1`;
34 yield* stack.destroy();
35
36 const error = yield* Effect.flip(
37 aiplatform.getProjectsLocationsEvaluationSets({
38 name: `${parent}/evaluationSets/1234567890123456789`,
39 }),
40 );
41 expect(error._tag).toEqual("NotFound");
42 const page = yield* aiplatform.listProjectsLocationsEvaluationSets({
43 parent,
44 pageSize: 10,
45 });
46 expect(
47 (page.evaluationSets ?? []).map((item) => item.name),
48 ).not.toContain(`${parent}/evaluationSets/1234567890123456789`);
49
50 yield* stack.destroy();
51 }).pipe(logLevel),
52 {
53 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
54 timeout: 90_000,
55 },
56);
57
58test.provider(
59 "create, update, and delete a vertex evaluation set",
60 (stack) =>
61 Effect.gen(function* () {
62 yield* stack.destroy();
63
64 const created = yield* stack.deploy(
65 Effect.gen(function* () {
66 const item = yield* GCP.AIPlatform.EvaluationItem("Prompt", {
67 location: "us-central1",
68 displayName: "alchemy-eval-set-item",
69 evaluationItemType: "REQUEST",
70 evaluationRequest: { prompt: { text: "What is 2+2?" } },
71 labels: { env: "test" },
72 });
73 const set = yield* GCP.AIPlatform.EvaluationSet("Prompts", {
74 location: "us-central1",
75 displayName: "alchemy-eval-set",
76 evaluationItems: [item.name],
77 });
78 return { item, set };
79 }),
80 );
81
82 expect(created.set.name).toContain("/evaluationSets/");
83 expect(created.set.location).toEqual("us-central1");
84 expect(created.set.evaluationItems).toContain(created.item.name);
85
86 const fetched = yield* aiplatform.getProjectsLocationsEvaluationSets({
87 name: created.set.name,
88 });
89 expect(fetched.name).toEqual(created.set.name);
90
91 const updated = yield* stack.deploy(
92 Effect.gen(function* () {
93 const item = yield* GCP.AIPlatform.EvaluationItem("Prompt", {
94 evaluationItemId: created.item.evaluationItemId,
95 location: "us-central1",
96 displayName: "alchemy-eval-set-item",
97 evaluationItemType: "REQUEST",
98 evaluationRequest: { prompt: { text: "What is 2+2?" } },
99 labels: { env: "test" },
100 });
101 const second = yield* GCP.AIPlatform.EvaluationItem("Followup", {
102 location: "us-central1",
103 displayName: "alchemy-eval-set-item-2",
104 evaluationItemType: "REQUEST",
105 evaluationRequest: { prompt: { text: "What is 3+3?" } },
106 labels: { env: "test" },
107 });
108 const set = yield* GCP.AIPlatform.EvaluationSet("Prompts", {
109 evaluationSetId: created.set.evaluationSetId,
110 location: "us-central1",
111 displayName: "alchemy-eval-set",
112 evaluationItems: [item.name, second.name],
113 });
114 return { item, second, set };
115 }),
116 );
117
118 expect(updated.set.name).toEqual(created.set.name);
119 expect(updated.set.evaluationItems).toContain(updated.second.name);
120
121 yield* stack.destroy();
122
123 const gone = yield* waitUntilGone(created.set.name);
124 expect(gone).toEqual("gone");
125 }).pipe(logLevel),
126 {
127 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
128 timeout: 180_000,
129 },
130);