1import * as GCP from "@/GCP";
2import { GcpEnvironment } from "@/GCP/Environment";
3import * as Test from "@/Test/Alchemy";
4import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
5import { expect } from "alchemy-test";
6import * as Effect from "effect/Effect";
7import { MinimumLogLevel } from "effect/References";
8import * as Schedule from "effect/Schedule";
9
10const { test } = Test.make({ providers: GCP.providers() });
11
12const logLevel = Effect.provideService(
13 MinimumLogLevel,
14 process.env.DEBUG ? "Debug" : "Info",
15);
16
17// Monitoring needs an endpoint with a deployed model: against an empty
18// endpoint create fails with BadRequest "Deployed Model ID: `0` does not
19// match existing Deployed Models". Set GCP_TEST_AIPLATFORM_DEPLOYED_ENDPOINT
20// to `{endpoint name}#{deployed model id}` to run the lifecycle.
21const deployedEndpoint = process.env.GCP_TEST_AIPLATFORM_DEPLOYED_ENDPOINT;
22const runLifecycle = !process.env.FAST && !!deployedEndpoint;
23const [endpointName = "", deployedModelId = ""] = (
24 deployedEndpoint ?? ""
25).split("#");
26
27const waitUntilGone = (name: string) =>
28 aiplatform.getProjectsLocationsModelDeploymentMonitoringJobs({ name }).pipe(
29 Effect.as("found" as const),
30 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
31 Effect.repeat({
32 schedule: Schedule.spaced("2 seconds"),
33 until: (status) => status === "gone",
34 times: 10,
35 }),
36 );
37
38test.provider(
39 "getProjectsLocationsModelDeploymentMonitoringJobs on a missing job fails with a typed tag",
40 (stack) =>
41 Effect.gen(function* () {
42 const { project } = yield* GcpEnvironment.current;
43 const parent = `projects/${project}/locations/us-central1`;
44 yield* stack.destroy();
45
46 const error = yield* Effect.flip(
47 aiplatform.getProjectsLocationsModelDeploymentMonitoringJobs({
48 name: `${parent}/modelDeploymentMonitoringJobs/1234567890123456789`,
49 }),
50 );
51 expect(error._tag).toEqual("NotFound");
52 const page =
53 yield* aiplatform.listProjectsLocationsModelDeploymentMonitoringJobs({
54 parent,
55 pageSize: 10,
56 });
57 expect(
58 (page.modelDeploymentMonitoringJobs ?? []).map((item) => item.name),
59 ).not.toContain(
60 `${parent}/modelDeploymentMonitoringJobs/1234567890123456789`,
61 );
62
63 yield* stack.destroy();
64 }).pipe(logLevel),
65 {
66 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
67 timeout: 90_000,
68 },
69);
70
71test.provider.skipIf(!runLifecycle)(
72 "create, update, and delete a model deployment monitoring job",
73 (stack) =>
74 Effect.gen(function* () {
75 yield* stack.destroy();
76
77 const created = yield* stack.deploy(
78 Effect.gen(function* () {
79 const job = yield* GCP.AIPlatform.ModelDeploymentMonitoringJob(
80 "Watch",
81 {
82 location: "us-central1",
83 displayName: "alchemy-mdm",
84 endpoint: endpointName,
85 loggingSamplingStrategy: {
86 randomSampleConfig: { sampleRate: 0.1 },
87 },
88 modelDeploymentMonitoringScheduleConfig: {
89 monitorInterval: "3600s",
90 },
91 modelDeploymentMonitoringObjectiveConfigs: [
92 {
93 deployedModelId,
94 objectiveConfig: {
95 predictionDriftDetectionConfig: {},
96 },
97 },
98 ],
99 labels: { env: "test" },
100 },
101 );
102 return { job };
103 }),
104 );
105
106 expect(created.job.name).toContain("/modelDeploymentMonitoringJobs/");
107 expect(created.job.endpoint).toEqual(endpointName);
108 expect(created.job.labels).toMatchObject({ env: "test" });
109
110 const fetched =
111 yield* aiplatform.getProjectsLocationsModelDeploymentMonitoringJobs({
112 name: created.job.name,
113 });
114 expect(fetched.name).toEqual(created.job.name);
115
116 const updated = yield* stack.deploy(
117 Effect.gen(function* () {
118 const job = yield* GCP.AIPlatform.ModelDeploymentMonitoringJob(
119 "Watch",
120 {
121 location: "us-central1",
122 displayName: "alchemy-mdm-v2",
123 endpoint: endpointName,
124 loggingSamplingStrategy: {
125 randomSampleConfig: { sampleRate: 0.2 },
126 },
127 modelDeploymentMonitoringScheduleConfig: {
128 monitorInterval: "3600s",
129 },
130 modelDeploymentMonitoringObjectiveConfigs: [
131 {
132 deployedModelId,
133 objectiveConfig: {
134 predictionDriftDetectionConfig: {},
135 },
136 },
137 ],
138 labels: { env: "prod" },
139 },
140 );
141 return { job };
142 }),
143 );
144
145 expect(updated.job.name).toEqual(created.job.name);
146 expect(updated.job.displayName).toEqual("alchemy-mdm-v2");
147 expect(updated.job.labels).toMatchObject({ env: "prod" });
148
149 yield* stack.destroy();
150 const gone = yield* waitUntilGone(created.job.name);
151 expect(gone).toEqual("gone");
152 }).pipe(logLevel),
153 {
154 tags: ["provider:gcp", "provider:gcp:aiplatform", "live"],
155 timeout: 180_000,
156 },
157);