1import * as GCP from "@/GCP";
2import { GcpEnvironment } from "@/GCP/Environment";
3import * as Test from "@/Test/Alchemy";
4import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
5import { expect } from "alchemy-test";
6import * as Effect from "effect/Effect";
7import { MinimumLogLevel } from "effect/References";
8import * as Schedule from "effect/Schedule";
9
10const { test } = Test.make({ providers: GCP.providers() });
11
12const logLevel = Effect.provideService(
13 MinimumLogLevel,
14 process.env.DEBUG ? "Debug" : "Info",
15);
16
17// Persistent training clusters take 5-15 minutes to provision and tear down.
18const runLifecycle = !!process.env.GCP_TEST_SLOW && !process.env.FAST;
19
20const waitUntilGone = (name: string) =>
21 aiplatform.getProjectsLocationsPersistentResources({ name }).pipe(
22 Effect.as("found" as const),
23 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
24 Effect.repeat({
25 schedule: Schedule.spaced("2 seconds"),
26 until: (status) => status === "gone",
27 times: 10,
28 }),
29 );
30
31test.provider(
32 "getProjectsLocationsPersistentResources on a missing resource fails with a typed tag",
33 (stack) =>
34 Effect.gen(function* () {
35 const { project } = yield* GcpEnvironment.current;
36 const parent = `projects/${project}/locations/us-central1`;
37 yield* stack.destroy();
38
39 const error = yield* Effect.flip(
40 aiplatform.getProjectsLocationsPersistentResources({
41 name: `${parent}/persistentResources/alchemy-missing`,
42 }),
43 );
44 expect(error._tag).toEqual("NotFound");
45 const page = yield* aiplatform.listProjectsLocationsPersistentResources({
46 parent,
47 pageSize: 10,
48 });
49 expect(
50 (page.persistentResources ?? []).map((item) => item.name),
51 ).not.toContain(`${parent}/persistentResources/alchemy-missing`);
52
53 yield* stack.destroy();
54 }).pipe(logLevel),
55 {
56 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
57 timeout: 90_000,
58 },
59);
60
61test.provider.skipIf(!runLifecycle)(
62 "create, update, and delete a persistent resource",
63 (stack) =>
64 Effect.gen(function* () {
65 yield* stack.destroy();
66
67 const created = yield* stack.deploy(
68 Effect.gen(function* () {
69 return yield* GCP.AIPlatform.PersistentResource("Train", {
70 location: "us-central1",
71 displayName: "alchemy-persistent",
72 resourcePools: [
73 {
74 id: "worker",
75 replicaCount: "1",
76 machineSpec: { machineType: "n1-standard-4" },
77 },
78 ],
79 labels: { env: "test" },
80 });
81 }),
82 );
83
84 expect(created.name).toContain("/persistentResources/");
85 expect(created.location).toEqual("us-central1");
86 expect(created.labels).toMatchObject({ env: "test" });
87
88 const fetched = yield* aiplatform.getProjectsLocationsPersistentResources(
89 {
90 name: created.name,
91 },
92 );
93 expect(fetched.name).toEqual(created.name);
94 expect(fetched.labels?.env).toEqual("test");
95
96 const updated = yield* stack.deploy(
97 Effect.gen(function* () {
98 return yield* GCP.AIPlatform.PersistentResource("Train", {
99 persistentResourceId: created.persistentResourceId,
100 location: "us-central1",
101 displayName: "alchemy-persistent",
102 resourcePools: [
103 {
104 id: "worker",
105 // Non-Ray persistent resources cannot be updated in place,
106 // so a replica change replaces the resource.
107 replicaCount: "2",
108 machineSpec: { machineType: "n1-standard-4" },
109 },
110 ],
111 labels: { env: "test" },
112 });
113 }),
114 );
115
116 // Replaced under the same id.
117 expect(updated.name).toEqual(created.name);
118 const scaled = yield* aiplatform.getProjectsLocationsPersistentResources({
119 name: created.name,
120 });
121 expect(scaled.resourcePools?.[0]?.replicaCount).toEqual("2");
122
123 yield* stack.destroy();
124
125 const gone = yield* waitUntilGone(created.name);
126 expect(gone).toEqual("gone");
127 }).pipe(logLevel),
128 // Provisioning the worker pool takes several minutes; so does teardown.
129 {
130 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
131 timeout: 2_400_000,
132 retry: 0,
133 },
134);