1import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { Unowned } from "../../AdoptPolicy.ts";
7import { isResolved } from "../../Diff.ts";
8import * as Provider from "../../Provider.ts";
9import { Resource } from "../../Resource.ts";
10import { tagRecord } from "../../Tags.ts";
11import { GcpEnvironment } from "../Environment.ts";
12import { hasAlchemyLabels } from "../Labels.ts";
13import type { Providers } from "../Providers.ts";
14import {
15 createInternalLabels,
16 hasAlchemyPrefix,
17 labelsDiffer,
18 listLocations,
19 locationOf,
20 lastSegment,
21 projectOf,
22 stableJson,
23 toDisplayName,
24 toLabels,
25 toResourceId,
26 userLabels,
27} from "./names.ts";
28import { waitForOperation } from "./operations.ts";
29
30const DEFAULT_SCHEMA_TITLE = "system.Artifact";
31
32export type MetadataStoresArtifactProps = {
33 /**
34 * Parent MetadataStore resource name
35 * `projects/{project}/locations/{location}/metadataStores/{metadatastore}`.
36 * Immutable — changing it replaces the artifact.
37 */
38 metadataStore: string;
39 /**
40 * Artifact id. If omitted, a unique id is generated. Must be 4-128
41 * characters of `a-z` and `-`. Immutable.
42 */
43 artifactId?: string;
44 /**
45 * Display name (max 128 Unicode characters).
46 */
47 displayName?: string;
48 /**
49 * Human-readable description.
50 */
51 description?: string;
52 /**
53 * User labels. Alchemy ownership labels are merged in automatically.
54 */
55 labels?: Record<string, string>;
56 /**
57 * Uniform resource identifier of the artifact file.
58 */
59 uri?: string;
60 /**
61 * Artifact lifecycle state.
62 */
63 state?: aiplatform.GoogleCloudAiplatformV1ArtifactStateEnum | (string & {});
64 /**
65 * Schema title registered in the MetadataStore. Vertex AI rejects
66 * entries without one.
67 * @default "system.Artifact"
68 */
69 schemaTitle?: string;
70 /**
71 * Schema version registered in the MetadataStore.
72 */
73 schemaVersion?: string;
74 /**
75 * Arbitrary metadata properties.
76 */
77 metadata?: aiplatform.DocumentMap;
78};
79
80export type MetadataStoresArtifact = Resource<
81 "GCP.AIPlatform.MetadataStoresArtifact",
82 MetadataStoresArtifactProps,
83 {
84 /** Full resource name. */
85 name: string;
86 /** Artifact id (last path segment). */
87 artifactId: string;
88 /** Parent MetadataStore resource name. */
89 metadataStore: string;
90 /** Project id. */
91 project: string;
92 /** Location id. */
93 location: string;
94 /** Display name. */
95 displayName: string | undefined;
96 /** User-provided description. */
97 description: string | undefined;
98 /** User labels (Alchemy ownership labels stripped). */
99 labels: Record<string, string>;
100 /** Artifact URI. */
101 uri: string | undefined;
102 /** Artifact state. */
103 state: string | undefined;
104 /** Schema title. */
105 schemaTitle: string | undefined;
106 /** Schema version. */
107 schemaVersion: string | undefined;
108 /** RFC3339 creation timestamp. */
109 createTime: string | undefined;
110 /** RFC3339 last-update timestamp. */
111 updateTime: string | undefined;
112 },
113 never,
114 Providers
115>;
116
117/**
118 * A Vertex ML Metadata Artifact associated with a MetadataStore.
119 *
120 * Changing `metadataStore` or `artifactId` replaces the artifact. Display
121 * name, description, labels, URI, state, schema, and metadata update in
122 * place.
123 *
124 * ### Creating an Artifact
125 * **Example:** Generated id
126 * ```typescript
127 * const artifact = yield* GCP.AIPlatform.MetadataStoresArtifact("Model", {
128 * metadataStore: store.name,
129 * displayName: "trained-model",
130 * uri: "gs://bucket/model",
131 * labels: { env: "prod" },
132 * });
133 * ```
134 *
135 * @resource
136 * @category AIPlatform
137 */
138export const MetadataStoresArtifact = Resource<MetadataStoresArtifact>(
139 "GCP.AIPlatform.MetadataStoresArtifact",
140);
141
142export class MetadataStoresArtifactNotResolved extends Data.TaggedError(
143 "GCP.AIPlatform.MetadataStoresArtifactNotResolved",
144)<{
145 name: string;
146}> {}
147
148export class MetadataStoresArtifactStillExists extends Data.TaggedError(
149 "GCP.AIPlatform.MetadataStoresArtifactStillExists",
150)<{
151 name: string;
152}> {}
153
154const parentOf = (name: string) => {
155 const at = name.lastIndexOf("/artifacts/");
156 return at >= 0 ? name.slice(0, at) : name;
157};
158
159const resourceName = (metadataStore: string, artifactId: string) =>
160 `${metadataStore}/artifacts/${artifactId}`;
161
162const toAttrs = (
163 artifact: aiplatform.GoogleCloudAiplatformV1Artifact,
164 project: string,
165) => {
166 const name = artifact.name ?? "";
167 return {
168 name,
169 artifactId: lastSegment(name),
170 metadataStore: parentOf(name),
171 project: projectOf(name, project),
172 location: locationOf(name),
173 displayName: artifact.displayName,
174 description: artifact.description,
175 labels: userLabels(artifact.labels),
176 uri: artifact.uri,
177 state: artifact.state,
178 schemaTitle: artifact.schemaTitle,
179 schemaVersion: artifact.schemaVersion,
180 createTime: artifact.createTime,
181 updateTime: artifact.updateTime,
182 };
183};
184
185const getByName = (name: string) =>
186 aiplatform
187 .getProjectsLocationsMetadataStoresArtifacts({ name })
188 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
189
190const listArtifacts = (parent: string) =>
191 aiplatform.listProjectsLocationsMetadataStoresArtifacts
192 .pages({ parent, pageSize: 100 })
193 .pipe(
194 Stream.runCollect,
195 Effect.map((pages) =>
196 Array.from(pages).flatMap((page) => page.artifacts ?? []),
197 ),
198 Effect.catchTag("NotFound", () =>
199 Effect.succeed([] as aiplatform.GoogleCloudAiplatformV1Artifact[]),
200 ),
201 );
202
203const listStores = (parent: string) =>
204 aiplatform.listProjectsLocationsMetadataStores
205 .pages({ parent, pageSize: 100 })
206 .pipe(
207 Stream.runCollect,
208 Effect.map((pages) =>
209 Array.from(pages).flatMap((page) => page.metadataStores ?? []),
210 ),
211 Effect.catchTag("NotFound", () =>
212 Effect.succeed([] as aiplatform.GoogleCloudAiplatformV1MetadataStore[]),
213 ),
214 );
215
216const waitUntilGone = (name: string) =>
217 getByName(name).pipe(
218 Effect.filterOrFail(
219 (artifact) => artifact === undefined,
220 () => new MetadataStoresArtifactStillExists({ name }),
221 ),
222 Effect.asVoid,
223 Effect.retry({
224 while: (error) =>
225 error._tag === "GCP.AIPlatform.MetadataStoresArtifactStillExists",
226 times: 10,
227 schedule: Schedule.spaced("1 second"),
228 }),
229 );
230
231export const MetadataStoresArtifactProvider = () =>
232 Provider.succeed(MetadataStoresArtifact, {
233 stables: [
234 "name",
235 "artifactId",
236 "metadataStore",
237 "project",
238 "location",
239 "createTime",
240 ],
241
242 diff: Effect.fn(function* ({ news, olds, output }) {
243 if (!isResolved(news)) return undefined;
244 const previousParent = olds?.metadataStore ?? output?.metadataStore;
245 const nextParent = news.metadataStore ?? previousParent;
246 const previousId = olds?.artifactId ?? output?.artifactId;
247 const nextId = news.artifactId ?? previousId;
248 if (
249 (previousParent !== undefined &&
250 nextParent !== undefined &&
251 nextParent !== previousParent) ||
252 (previousId !== undefined &&
253 nextId !== undefined &&
254 nextId !== previousId)
255 ) {
256 return {
257 action: "replace" as const,
258 deleteFirst:
259 previousParent === nextParent &&
260 previousId !== undefined &&
261 nextId === previousId,
262 };
263 }
264 return undefined;
265 }),
266
267 read: Effect.fn(function* ({ id, olds, output }) {
268 const env = yield* GcpEnvironment.current;
269 const artifactId = yield* toResourceId(
270 id,
271 olds?.artifactId,
272 output?.artifactId,
273 );
274 const metadataStore = olds?.metadataStore ?? output?.metadataStore;
275 const name =
276 output?.name ??
277 (metadataStore !== undefined
278 ? resourceName(metadataStore, artifactId)
279 : "");
280 if (name.length === 0) return undefined;
281 const existing = yield* getByName(name);
282 if (existing === undefined) return undefined;
283 const attrs = toAttrs(existing, env.project);
284 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels)))
285 ? attrs
286 : Unowned(attrs);
287 }),
288
289 list: () =>
290 Effect.gen(function* () {
291 const env = yield* GcpEnvironment.current;
292 const stores = yield* Effect.forEach(
293 listLocations(env.region),
294 (location) =>
295 listStores(`projects/${env.project}/locations/${location}`),
296 { concurrency: 4 },
297 );
298 const artifacts = yield* Effect.forEach(
299 stores.flat().filter((store) => store.name !== undefined),
300 (store) => listArtifacts(store.name!),
301 { concurrency: 4 },
302 );
303 return artifacts
304 .flat()
305 .filter((artifact) => hasAlchemyPrefix(artifact.labels))
306 .map((artifact) => toAttrs(artifact, env.project));
307 }),
308
309 reconcile: Effect.fn(function* ({ id, news, output }) {
310 const env = yield* GcpEnvironment.current;
311 const artifactId = yield* toResourceId(
312 id,
313 news.artifactId,
314 output?.artifactId,
315 );
316 const metadataStore = news.metadataStore;
317 const name = resourceName(metadataStore, artifactId);
318 const displayName = yield* toDisplayName(
319 id,
320 news.displayName,
321 output?.displayName,
322 );
323 const desiredLabels = {
324 ...toLabels(news.labels),
325 ...(yield* createInternalLabels(id)),
326 };
327
328 let current = yield* getByName(output?.name ?? name);
329
330 if (current === undefined) {
331 const created = yield* aiplatform
332 .createProjectsLocationsMetadataStoresArtifacts({
333 parent: metadataStore,
334 artifactId,
335 body: {
336 displayName,
337 description: news.description,
338 labels: desiredLabels,
339 uri: news.uri,
340 state: news.state,
341 schemaTitle: news.schemaTitle ?? DEFAULT_SCHEMA_TITLE,
342 schemaVersion: news.schemaVersion,
343 metadata: news.metadata,
344 },
345 })
346 .pipe(Effect.catchTag("Conflict", () => getByName(name)));
347 current = created ?? undefined;
348 }
349
350 if (current === undefined) {
351 return yield* new MetadataStoresArtifactNotResolved({ name });
352 }
353
354 const descriptionChanged =
355 (current.description ?? "") !== (news.description ?? "");
356 const displayChanged = (current.displayName ?? "") !== displayName;
357 const labelsChanged = labelsDiffer(current.labels, desiredLabels);
358 const uriChanged = (current.uri ?? "") !== (news.uri ?? "");
359 const stateChanged = (current.state ?? "") !== (news.state ?? "");
360 const schemaTitleChanged =
361 (current.schemaTitle ?? "") !==
362 (news.schemaTitle ?? DEFAULT_SCHEMA_TITLE);
363 const schemaVersionChanged =
364 (current.schemaVersion ?? "") !== (news.schemaVersion ?? "");
365 const metadataChanged =
366 news.metadata !== undefined &&
367 stableJson(current.metadata) !== stableJson(news.metadata);
368
369 if (
370 descriptionChanged ||
371 displayChanged ||
372 labelsChanged ||
373 uriChanged ||
374 stateChanged ||
375 schemaTitleChanged ||
376 schemaVersionChanged ||
377 metadataChanged
378 ) {
379 current =
380 yield* aiplatform.patchProjectsLocationsMetadataStoresArtifacts({
381 name,
382 updateMask: [
383 descriptionChanged ? "description" : undefined,
384 displayChanged ? "display_name" : undefined,
385 labelsChanged ? "labels" : undefined,
386 uriChanged ? "uri" : undefined,
387 stateChanged ? "state" : undefined,
388 schemaTitleChanged ? "schema_title" : undefined,
389 schemaVersionChanged ? "schema_version" : undefined,
390 metadataChanged ? "metadata" : undefined,
391 ]
392 .filter((field): field is string => field !== undefined)
393 .join(","),
394 body: {
395 name,
396 displayName,
397 description: news.description,
398 labels: desiredLabels,
399 uri: news.uri,
400 state: news.state,
401 schemaTitle: news.schemaTitle ?? DEFAULT_SCHEMA_TITLE,
402 schemaVersion: news.schemaVersion,
403 metadata: news.metadata,
404 etag: current.etag,
405 },
406 });
407 }
408
409 return toAttrs(current, env.project);
410 }),
411
412 delete: Effect.fn(function* ({ output }) {
413 const operation = yield* aiplatform
414 .deleteProjectsLocationsMetadataStoresArtifacts({
415 name: output.name,
416 })
417 .pipe(
418 Effect.retry({
419 while: (error) => error._tag === "Conflict",
420 times: 8,
421 schedule: Schedule.spaced("2 seconds"),
422 }),
423 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
424 );
425 if (operation !== undefined) {
426 yield* waitForOperation(operation, { notFoundOk: true });
427 }
428 yield* waitUntilGone(output.name);
429 }),
430 });