1import * as ml from "@distilled.cloud/gcp/ml_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { createPhysicalName } from "../../PhysicalName.ts";
7import { tagRecord } from "../../Tags.ts";
8import {
9 alchemyLabelKeys,
10 ALCHEMY_LABEL_PREFIX,
11 createInternalLabels,
12 hasAlchemyLabels,
13 stripInternalLabels,
14} from "../Labels.ts";
15
16export const MAX_NAME_LENGTH = 63;
17export const MAX_DESCRIPTION_LENGTH = 8000;
18
19export class ResourceNotResolved extends Data.TaggedError(
20 "GCP.ML.ResourceNotResolved",
21)<{
22 name: string;
23}> {}
24
25export class ResourceStillExists extends Data.TaggedError(
26 "GCP.ML.ResourceStillExists",
27)<{
28 name: string;
29}> {}
30
31export class VersionNotReady extends Data.TaggedError(
32 "GCP.ML.VersionNotReady",
33)<{
34 name: string;
35 state: string;
36}> {}
37
38export class VersionFailed extends Data.TaggedError("GCP.ML.VersionFailed")<{
39 name: string;
40 message: string;
41}> {}
42
43export const lastSegment = (value: string) => {
44 const trimmed = value.replace(/\/+$/, "");
45 const parts = trimmed.split("/");
46 return parts[parts.length - 1] || trimmed;
47};
48
49export const parentOf = (name: string) => {
50 const parts = name.split("/").filter((part) => part.length > 0);
51 return parts.slice(0, -2).join("/");
52};
53
54export const rfc1035 = (name: string, fallback: string): string => {
55 let next = name
56 .toLowerCase()
57 .replace(/[^a-z0-9-]/g, "-")
58 .replace(/-+/g, "-")
59 .replace(/^-+|-+$/g, "");
60 if (!/^[a-z]/.test(next)) next = `m${next}`;
61 next = next.slice(0, MAX_NAME_LENGTH).replace(/-+$/g, "");
62 if (next.length === 0) return fallback;
63 if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, MAX_NAME_LENGTH - 1)}0`;
64 return next.slice(0, MAX_NAME_LENGTH);
65};
66
67export const toPhysicalId = (
68 id: string,
69 explicit: string | undefined,
70 existing: string | undefined,
71 fallback: string,
72) =>
73 Effect.gen(function* () {
74 if (explicit !== undefined && explicit.length > 0) {
75 return rfc1035(lastSegment(explicit), fallback);
76 }
77 if (existing !== undefined && existing.length > 0) {
78 return lastSegment(existing);
79 }
80 return rfc1035(
81 yield* createPhysicalName({
82 id,
83 maxLength: MAX_NAME_LENGTH,
84 lowercase: true,
85 }),
86 fallback,
87 );
88 });
89
90export const projectParent = (project: string) => `projects/${project}`;
91
92export const modelResourceName = (project: string, modelId: string) =>
93 `${projectParent(project)}/models/${modelId}`;
94
95export const versionResourceName = (modelName: string, versionId: string) =>
96 `${modelName}/versions/${versionId}`;
97
98export const expandModel = (project: string, value: string) => {
99 const next = value.replace(/\/+$/, "");
100 if (next.includes("/models/")) return next;
101 return modelResourceName(project, lastSegment(next));
102};
103
104export const parseModelName = (name: string, fallbackProject = "") => {
105 const parts = name.split("/").filter((part) => part.length > 0);
106 const modelsAt = parts.lastIndexOf("models");
107 const projectsAt = parts.lastIndexOf("projects");
108 return {
109 project:
110 projectsAt >= 0 && parts[projectsAt + 1]
111 ? parts[projectsAt + 1]!
112 : fallbackProject,
113 modelId:
114 modelsAt >= 0 && parts[modelsAt + 1]
115 ? parts[modelsAt + 1]!
116 : lastSegment(name),
117 parent:
118 modelsAt > 0 ? parts.slice(0, modelsAt).join("/") : projectParent(""),
119 };
120};
121
122export const parseVersionName = (name: string, fallbackProject = "") => {
123 const parts = name.split("/").filter((part) => part.length > 0);
124 const versionsAt = parts.lastIndexOf("versions");
125 const parsed = parseModelName(name, fallbackProject);
126 return {
127 ...parsed,
128 versionId:
129 versionsAt >= 0 && parts[versionsAt + 1]
130 ? parts[versionsAt + 1]!
131 : lastSegment(name),
132 model:
133 versionsAt > 0
134 ? parts.slice(0, versionsAt).join("/")
135 : modelResourceName(parsed.project, parsed.modelId),
136 };
137};
138
139export const userLabels = (
140 labels: Record<string, string | undefined> | null | undefined,
141): Record<string, string> => stripInternalLabels(tagRecord(labels));
142
143export const hasAlchemyLabelMap = (
144 labels: Record<string, string | undefined> | null | undefined,
145) =>
146 Object.keys(labels ?? {}).some((key) => key.startsWith(ALCHEMY_LABEL_PREFIX));
147
148export const sameText = (left: string | undefined, right: string | undefined) =>
149 (left ?? "") === (right ?? "");
150
151export const sameStringList = (
152 left: readonly string[] | undefined,
153 right: readonly string[] | undefined,
154) =>
155 JSON.stringify(
156 [...(left ?? [])].map((value) => value.toLowerCase()).sort(),
157 ) ===
158 JSON.stringify([...(right ?? [])].map((value) => value.toLowerCase()).sort());
159
160export const normalizeRegions = (
161 regions: readonly string[] | undefined,
162 fallback: readonly string[],
163): string[] => {
164 const next = [...(regions ?? [])]
165 .map((region) => lastSegment(region).toLowerCase())
166 .filter((region) => region.length > 0);
167 return next.length > 0 ? next : [...fallback];
168};
169
170export const fieldMask = (fields: Array<string | false | undefined>) =>
171 fields
172 .filter((field): field is string => typeof field === "string")
173 .join(",");
174
175const markerOf = (
176 labels: Record<string, string>,
177 stack: string,
178 stage: string,
179 id: string,
180) =>
181 `[alchemy ${alchemyLabelKeys.stack}=${stack} ${alchemyLabelKeys.stage}=${stage} ${alchemyLabelKeys.id}=${id}]`;
182
183const fitMarker = (labels: Record<string, string>, maxLength: number) => {
184 let stack = labels[alchemyLabelKeys.stack] ?? "x";
185 let stage = labels[alchemyLabelKeys.stage] ?? "x";
186 let id = labels[alchemyLabelKeys.id] ?? "x";
187 let marker = markerOf(labels, stack, stage, id);
188 while (
189 marker.length > maxLength &&
190 (stack.length > 1 || stage.length > 1 || id.length > 1)
191 ) {
192 if (stack.length >= stage.length && stack.length >= id.length) {
193 stack = stack.slice(0, -1);
194 } else if (stage.length >= id.length) {
195 stage = stage.slice(0, -1);
196 } else {
197 id = id.slice(0, -1);
198 }
199 marker = markerOf(labels, stack, stage, id);
200 }
201 return marker.length <= maxLength
202 ? marker
203 : `${marker.slice(0, Math.max(0, maxLength - 1))}]`;
204};
205
206export const encodeOwnership = (
207 labels: Record<string, string>,
208 text: string | undefined,
209 maxLength = MAX_DESCRIPTION_LENGTH,
210): string => {
211 const marker = fitMarker(labels, maxLength);
212 const trimmed = text?.trim();
213 if (!trimmed) return marker;
214 return `${marker}\n${trimmed}`.slice(0, maxLength);
215};
216
217export const parseOwnership = (
218 text: string | undefined,
219): {
220 labels: Record<string, string>;
221 text: string | undefined;
222} => {
223 if (!text?.startsWith("[alchemy ")) {
224 return { labels: {}, text };
225 }
226 const end = text.indexOf("]");
227 if (end < 0) return { labels: {}, text };
228 const labels: Record<string, string> = {};
229 for (const part of text.slice("[alchemy ".length, end).split(/\s+/)) {
230 const eq = part.indexOf("=");
231 if (eq > 0) {
232 labels[part.slice(0, eq)] = part.slice(eq + 1);
233 }
234 }
235 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
236 return { labels, text: rest.length > 0 ? rest : undefined };
237};
238
239export const hasOwnershipMarker = (text: string | undefined) =>
240 Object.keys(parseOwnership(text).labels).some((key) =>
241 key.startsWith(ALCHEMY_LABEL_PREFIX),
242 );
243
244const prefixMatch = (expected: string, observed: string) =>
245 expected === observed ||
246 expected.startsWith(observed) ||
247 observed.startsWith(expected);
248
249export const ownedByAlchemy = (id: string, text: string | undefined) =>
250 Effect.gen(function* () {
251 const expected = yield* createInternalLabels(id);
252 const { labels } = parseOwnership(text);
253 if (!hasOwnershipMarker(text)) return false;
254 const exact = yield* hasAlchemyLabels(id, labels);
255 if (exact) return true;
256 return (
257 prefixMatch(
258 expected[alchemyLabelKeys.stack] ?? "",
259 labels[alchemyLabelKeys.stack] ?? "",
260 ) &&
261 prefixMatch(
262 expected[alchemyLabelKeys.stage] ?? "",
263 labels[alchemyLabelKeys.stage] ?? "",
264 ) &&
265 prefixMatch(
266 expected[alchemyLabelKeys.id] ?? "",
267 labels[alchemyLabelKeys.id] ?? "",
268 )
269 );
270 });
271
272export const modelOwnedByAlchemy = (
273 id: string,
274 model: ml.GoogleCloudMlV1__Model,
275) =>
276 Effect.gen(function* () {
277 if (yield* hasAlchemyLabels(id, tagRecord(model.labels))) return true;
278 return yield* ownedByAlchemy(id, model.description);
279 });
280
281export const versionOwnedByAlchemy = (
282 id: string,
283 version: ml.GoogleCloudMlV1__Version,
284) =>
285 Effect.gen(function* () {
286 if (yield* hasAlchemyLabels(id, tagRecord(version.labels))) return true;
287 return yield* ownedByAlchemy(id, version.description);
288 });
289
290export const modelHasOwnership = (model: ml.GoogleCloudMlV1__Model) =>
291 hasAlchemyLabelMap(model.labels) || hasOwnershipMarker(model.description);
292
293export const versionHasOwnership = (version: ml.GoogleCloudMlV1__Version) =>
294 hasAlchemyLabelMap(version.labels) || hasOwnershipMarker(version.description);
295
296export const replaceOnIdentity = (input: {
297 previousId?: string;
298 nextId?: string;
299 previousParent?: string;
300 nextParent?: string;
301 extra?: boolean;
302}) => {
303 const idChanged =
304 input.previousId !== undefined &&
305 input.nextId !== undefined &&
306 input.previousId !== input.nextId;
307 const parentChanged =
308 (input.previousParent ?? "") !== "" &&
309 (input.nextParent ?? "") !== "" &&
310 (input.previousParent ?? "") !== (input.nextParent ?? "");
311 if (!idChanged && !parentChanged && input.extra !== true) {
312 return undefined;
313 }
314 return {
315 action: "replace" as const,
316 deleteFirst:
317 !idChanged &&
318 !parentChanged &&
319 input.extra === true &&
320 input.previousId !== undefined &&
321 input.nextId === input.previousId,
322 };
323};
324
325export const getModel = (name: string) =>
326 name.length === 0
327 ? Effect.succeed(undefined)
328 : ml
329 .getProjectsModels({ name })
330 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
331
332export const getVersion = (name: string) =>
333 name.length === 0
334 ? Effect.succeed(undefined)
335 : ml
336 .getProjectsModelsVersions({ name })
337 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
338
339const emptyList = <A>() => Effect.succeed([] as A[]);
340
341export const listModels = (project: string) =>
342 ml.listProjectsModels
343 .pages({
344 parent: projectParent(project),
345 pageSize: 100,
346 })
347 .pipe(
348 Stream.runCollect,
349 Effect.map((pages) =>
350 Array.from(pages).flatMap((page) => page.models ?? []),
351 ),
352 Effect.catchTag("NotFound", () => emptyList<ml.GoogleCloudMlV1__Model>()),
353 );
354
355export const listVersions = (parent: string) =>
356 parent.length === 0
357 ? emptyList<ml.GoogleCloudMlV1__Version>()
358 : ml.listProjectsModelsVersions
359 .pages({
360 parent,
361 pageSize: 100,
362 })
363 .pipe(
364 Stream.runCollect,
365 Effect.map((pages) =>
366 Array.from(pages).flatMap((page) => page.versions ?? []),
367 ),
368 Effect.catchTag("NotFound", () =>
369 emptyList<ml.GoogleCloudMlV1__Version>(),
370 ),
371 );
372
373export const listOwnedModels = (project: string) =>
374 listModels(project).pipe(
375 Effect.map((models) => models.filter(modelHasOwnership)),
376 );
377
378export const listOwnedVersions = (project: string) =>
379 Effect.gen(function* () {
380 const models = yield* listOwnedModels(project);
381 const pages = yield* Effect.forEach(
382 models,
383 (model) =>
384 model.name
385 ? listVersions(model.name)
386 : emptyList<ml.GoogleCloudMlV1__Version>(),
387 { concurrency: 4 },
388 );
389 return pages.flat().filter((version) => versionHasOwnership(version));
390 });
391
392type Tagged = { readonly _tag: string };
393
394const versionState = (version: ml.GoogleCloudMlV1__Version) =>
395 (version.state ?? "").toUpperCase();
396
397export const waitUntilGone = <A, R>(
398 get: Effect.Effect<A, Tagged, R>,
399 name: string,
400): Effect.Effect<void, Tagged, R> =>
401 get.pipe(
402 Effect.filterOrFail(
403 (value) => value === undefined,
404 () => new ResourceStillExists({ name }),
405 ),
406 Effect.retry({
407 while: (error) => error._tag === "GCP.ML.ResourceStillExists",
408 times: 10,
409 schedule: Schedule.spaced("2 seconds"),
410 }),
411 Effect.asVoid,
412 );
413
414export const waitUntilExists = <A, R>(
415 get: Effect.Effect<A, Tagged, R>,
416 name: string,
417): Effect.Effect<Exclude<A, undefined>, Tagged, R> =>
418 get.pipe(
419 Effect.filterOrFail(
420 (value): value is Exclude<A, undefined> => value !== undefined,
421 () => new ResourceNotResolved({ name }),
422 ),
423 Effect.retry({
424 while: (error) => error._tag === "GCP.ML.ResourceNotResolved",
425 times: 8,
426 schedule: Schedule.spaced("1 second"),
427 }),
428 );
429
430export const waitUntilVersionReady = (
431 name: string,
432): Effect.Effect<
433 ml.GoogleCloudMlV1__Version,
434 | ml.Forbidden
435 | ml.GcpOpError
436 | ResourceNotResolved
437 | VersionFailed
438 | VersionNotReady,
439 ml.GcpOpContext
440> =>
441 getVersion(name).pipe(
442 Effect.filterOrFail(
443 (version): version is ml.GoogleCloudMlV1__Version =>
444 version !== undefined,
445 () => new ResourceNotResolved({ name }),
446 ),
447 Effect.filterOrFail(
448 (version) => {
449 const state = versionState(version);
450 return state === "" || state === "READY" || state === "FAILED";
451 },
452 (version) =>
453 new VersionNotReady({
454 name,
455 state: version.state ?? "",
456 }),
457 ),
458 Effect.flatMap(
459 (version): Effect.Effect<ml.GoogleCloudMlV1__Version, VersionFailed> =>
460 versionState(version) === "FAILED"
461 ? Effect.fail(
462 new VersionFailed({
463 name,
464 message: version.errorMessage ?? "version failed",
465 }),
466 )
467 : Effect.succeed(version),
468 ),
469 Effect.retry({
470 while: (error) =>
471 error._tag === "GCP.ML.ResourceNotResolved" ||
472 error._tag === "GCP.ML.VersionNotReady",
473 times: 10,
474 schedule: Schedule.spaced("5 seconds"),
475 }),
476 );