1import * as documentai from "@distilled.cloud/gcp/documentai_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import * as Stream from "effect/Stream";
6import { createPhysicalName } from "../../PhysicalName.ts";
7import { tagRecord } from "../../Tags.ts";
8import {
9 alchemyLabelKeys,
10 createInternalLabels,
11 hasAlchemyLabels,
12 stripInternalLabels,
13} from "../Labels.ts";
14import { isTransientGcpError } from "../Errors.ts";
15
16// Document AI is only served from the `us` / `eu` multi-region deployments, so the
17// default stays `us` rather than following `GCP.Region`.
18export const DEFAULT_LOCATION = "us";
19/**
20 * The distilled Document AI client talks to `documentai.googleapis.com`
21 * (the US deployment). Listing processors/schemas in `eu` or regional
22 * locations against that host returns HTTP 400 INVALID_ARGUMENT
23 * (`Invalid location: '…' must match the server deployment 'us'`).
24 */
25export const LIST_LOCATIONS = ["us"] as const;
26export const MAX_PROCESSOR_DISPLAY_NAME_LENGTH = 64;
27export const MAX_DISPLAY_NAME_LENGTH = 128;
28export const DEFAULT_PROCESSOR_TYPE = "OCR_PROCESSOR";
29
30export class ResourceNotResolved extends Data.TaggedError(
31 "GCP.DocumentAI.ResourceNotResolved",
32)<{
33 name: string;
34}> {}
35
36export class ResourceStillExists extends Data.TaggedError(
37 "GCP.DocumentAI.ResourceStillExists",
38)<{
39 name: string;
40}> {}
41
42export class ProcessorPending extends Data.TaggedError(
43 "GCP.DocumentAI.ProcessorPending",
44)<{
45 name: string;
46 state: string;
47}> {}
48
49export class ProcessorFailed extends Data.TaggedError(
50 "GCP.DocumentAI.ProcessorFailed",
51)<{
52 name: string;
53 state: string;
54}> {}
55
56export const lastSegment = (value: string) => {
57 const trimmed = value.replace(/\/+$/, "");
58 const parts = trimmed.split("/");
59 return parts[parts.length - 1] || trimmed;
60};
61
62export const locationParent = (project: string, location: string) =>
63 `projects/${project}/locations/${location}`;
64
65export const normalizeLocation = (location: string | undefined) =>
66 lastSegment(location ?? DEFAULT_LOCATION).toLowerCase();
67
68export const parseResourceName = (name: string, collection: string) => {
69 const parts = name.split("/").filter((part) => part.length > 0);
70 const collectionAt = parts.lastIndexOf(collection);
71 const locationsAt = parts.lastIndexOf("locations");
72 const projectsAt = parts.lastIndexOf("projects");
73 return {
74 project:
75 projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
76 location:
77 locationsAt >= 0 && parts[locationsAt + 1]
78 ? parts[locationsAt + 1]!
79 : DEFAULT_LOCATION,
80 id:
81 collectionAt >= 0 && parts[collectionAt + 1]
82 ? parts[collectionAt + 1]!
83 : lastSegment(name),
84 parent:
85 collectionAt > 0
86 ? parts.slice(0, collectionAt).join("/")
87 : parts.slice(0, Math.max(0, parts.length - 1)).join("/"),
88 };
89};
90
91export const expandParent = (
92 value: string,
93 project: string,
94 location: string,
95 collection: string,
96) => {
97 if (value.includes("/")) return value.replace(/\/+$/, "");
98 return `${locationParent(project, location)}/${collection}/${value}`;
99};
100
101export const sameText = (left: string | undefined, right: string | undefined) =>
102 (left ?? "") === (right ?? "");
103
104export const canonical = (value: unknown): unknown => {
105 if (value === undefined || value === null) return undefined;
106 if (typeof value === "boolean" || typeof value === "number") return value;
107 if (typeof value === "string") {
108 return value.length === 0 ? undefined : value;
109 }
110 if (Array.isArray(value)) {
111 const items = value.map(canonical).filter((item) => item !== undefined);
112 return items.length === 0 ? undefined : items;
113 }
114 if (typeof value === "object") {
115 const entries = Object.entries(value as Record<string, unknown>)
116 .map(([key, item]) => [key, canonical(item)] as const)
117 .filter(([, item]) => item !== undefined)
118 .sort(([left], [right]) => left.localeCompare(right));
119 if (entries.length === 0) return undefined;
120 return Object.fromEntries(entries);
121 }
122 return undefined;
123};
124
125export const fingerprint = (value: unknown): string =>
126 JSON.stringify(canonical(value) ?? null);
127
128export const sameJson = (left: unknown, right: unknown) =>
129 fingerprint(left) === fingerprint(right);
130
131export const updateMaskOf = (...fields: Array<string | undefined>) =>
132 fields.filter((field): field is string => field !== undefined).join(",");
133
134export const userLabels = (
135 labels: Record<string, string | undefined> | null | undefined,
136): Record<string, string> => stripInternalLabels(tagRecord(labels));
137
138export const hasAlchemyLabelMap = (
139 labels: Record<string, string | undefined> | null | undefined,
140) => Object.keys(labels ?? {}).some((key) => key.startsWith("alchemy-"));
141
142const compactMarkerOf = (stack: string, stage: string, id: string) =>
143 `[alc ${stack} ${stage} ${id}]`;
144
145const shrinkMarker = (
146 labels: Record<string, string>,
147 maxLength: number,
148 build: (stack: string, stage: string, id: string) => string,
149) => {
150 let stack = labels[alchemyLabelKeys.stack] ?? "x";
151 let stage = labels[alchemyLabelKeys.stage] ?? "x";
152 let id = labels[alchemyLabelKeys.id] ?? "x";
153 let marker = build(stack, stage, id);
154 while (
155 marker.length > maxLength &&
156 (stack.length > 1 || stage.length > 1 || id.length > 1)
157 ) {
158 if (stack.length >= stage.length && stack.length >= id.length) {
159 stack = stack.slice(0, -1);
160 } else if (stage.length >= id.length) {
161 stage = stage.slice(0, -1);
162 } else {
163 id = id.slice(0, -1);
164 }
165 marker = build(stack, stage, id);
166 }
167 return marker.slice(0, maxLength);
168};
169
170export const encodeOwnershipLine = (
171 labels: Record<string, string>,
172 text: string | undefined,
173 maxLength = MAX_DISPLAY_NAME_LENGTH,
174): string => {
175 const trimmed = text?.replace(/[\r\n]+/g, " ").trim();
176 const suffix = trimmed && trimmed.length > 0 ? ` ${trimmed}` : "";
177 const markerMax = Math.min(
178 maxLength,
179 Math.max(12, maxLength - suffix.length),
180 );
181 const marker = shrinkMarker(labels, markerMax, compactMarkerOf);
182 return `${marker}${suffix}`.slice(0, maxLength);
183};
184
185export const parseOwnership = (
186 text: string | undefined,
187): {
188 labels: Record<string, string>;
189 text: string | undefined;
190} => {
191 if (text?.startsWith("[alc ")) {
192 const end = text.indexOf("]");
193 if (end < 0) return { labels: {}, text };
194 const parts = text.slice("[alc ".length, end).trim().split(/\s+/);
195 const labels: Record<string, string> = {};
196 if (parts[0]) labels[alchemyLabelKeys.stack] = parts[0]!;
197 if (parts[1]) labels[alchemyLabelKeys.stage] = parts[1]!;
198 if (parts[2]) labels[alchemyLabelKeys.id] = parts[2]!;
199 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
200 return { labels, text: rest.length > 0 ? rest : undefined };
201 }
202 if (!text?.startsWith("[alchemy ")) {
203 return { labels: {}, text };
204 }
205 const end = text.indexOf("]");
206 if (end < 0) return { labels: {}, text };
207 const labels: Record<string, string> = {};
208 for (const part of text.slice("[alchemy ".length, end).split(/\s+/)) {
209 const eq = part.indexOf("=");
210 if (eq > 0) {
211 labels[part.slice(0, eq)] = part.slice(eq + 1);
212 }
213 }
214 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
215 return { labels, text: rest.length > 0 ? rest : undefined };
216};
217
218export const hasOwnershipMarker = (text: string | undefined) =>
219 Object.keys(parseOwnership(text).labels).some((key) =>
220 key.startsWith("alchemy-"),
221 );
222
223const prefixMatch = (expected: string, observed: string) =>
224 expected === observed ||
225 expected.startsWith(observed) ||
226 observed.startsWith(expected);
227
228export const ownedByAlchemy = (id: string, text: string | undefined) =>
229 Effect.gen(function* () {
230 const expected = yield* createInternalLabels(id);
231 const { labels } = parseOwnership(text);
232 if (!hasOwnershipMarker(text)) return false;
233 const exact = yield* hasAlchemyLabels(id, labels);
234 if (exact) return true;
235 return (
236 prefixMatch(
237 expected[alchemyLabelKeys.stack] ?? "",
238 labels[alchemyLabelKeys.stack] ?? "",
239 ) &&
240 prefixMatch(
241 expected[alchemyLabelKeys.stage] ?? "",
242 labels[alchemyLabelKeys.stage] ?? "",
243 ) &&
244 prefixMatch(
245 expected[alchemyLabelKeys.id] ?? "",
246 labels[alchemyLabelKeys.id] ?? "",
247 )
248 );
249 });
250
251export const toPhysicalId = (
252 id: string,
253 explicit: string | undefined,
254 existing: string | undefined,
255 maxLength = 32,
256) =>
257 Effect.gen(function* () {
258 if (explicit !== undefined) return explicit;
259 if (existing !== undefined) return existing;
260 return yield* createPhysicalName({
261 id,
262 maxLength,
263 lowercase: true,
264 });
265 });
266
267export const replaceOnIdentity = (input: {
268 previousId?: string;
269 nextId?: string;
270 previousLocation?: string;
271 nextLocation?: string;
272 extra?: boolean;
273}) => {
274 const previousLocation = normalizeLocation(input.previousLocation);
275 const nextLocation = normalizeLocation(input.nextLocation);
276 const replace =
277 (input.extra ?? false) ||
278 (input.previousId !== undefined &&
279 input.nextId !== undefined &&
280 input.nextId !== input.previousId) ||
281 (input.previousLocation !== undefined && previousLocation !== nextLocation);
282 if (!replace) return undefined;
283 const samePhysical =
284 previousLocation === nextLocation &&
285 input.previousId !== undefined &&
286 input.nextId === input.previousId;
287 return {
288 action: "replace" as const,
289 deleteFirst: samePhysical,
290 };
291};
292
293export const retryTransient = <A, E extends { readonly _tag: string }, R>(
294 effect: Effect.Effect<A, E, R>,
295) =>
296 effect.pipe(
297 Effect.retry({
298 while: isTransientGcpError,
299 times: 8,
300 schedule: Schedule.exponential("250 millis"),
301 }),
302 );
303
304export const waitUntilGone = <A, E, R>(
305 get: Effect.Effect<A | undefined, E, R>,
306 name: string,
307): Effect.Effect<void, E | ResourceStillExists, R> =>
308 get.pipe(
309 Effect.filterOrFail(
310 (value) => value === undefined,
311 () => new ResourceStillExists({ name }),
312 ),
313 Effect.retry({
314 while: (error) => error instanceof ResourceStillExists,
315 times: 10,
316 schedule: Schedule.spaced("2 seconds"),
317 }),
318 Effect.asVoid,
319 );
320
321export const collectPages = <Page, Item, E, R>(
322 pages: Stream.Stream<Page, E, R>,
323 items: (page: Page) => readonly Item[] | null | undefined,
324) =>
325 pages.pipe(
326 Stream.flatMap((page) => Stream.fromIterable(items(page) ?? [])),
327 Stream.runCollect,
328 Effect.map((chunk) => Array.from(chunk) as Item[]),
329 );
330
331const emptyList = <A>() => Effect.succeed<A[]>([]);
332
333export const listLocationParents = (project: string) =>
334 Effect.succeed(
335 LIST_LOCATIONS.map((location) => locationParent(project, location)),
336 );
337
338export const listProcessorsAt = (parent: string) =>
339 parent.length === 0
340 ? emptyList<documentai.GoogleCloudDocumentaiV1Processor>()
341 : collectPages(
342 documentai.listProjectsLocationsProcessors.pages({
343 parent,
344 pageSize: 100,
345 }),
346 (page) => page.processors,
347 ).pipe(
348 Effect.catchTag("NotFound", () =>
349 emptyList<documentai.GoogleCloudDocumentaiV1Processor>(),
350 ),
351 );
352
353export const listSchemasAt = (parent: string) =>
354 parent.length === 0
355 ? emptyList<documentai.GoogleCloudDocumentaiV1NextSchema>()
356 : collectPages(
357 documentai.listProjectsLocationsSchemas.pages({
358 parent,
359 pageSize: 20,
360 }),
361 (page) => page.schemas,
362 ).pipe(
363 Effect.catchTag("NotFound", () =>
364 emptyList<documentai.GoogleCloudDocumentaiV1NextSchema>(),
365 ),
366 );
367
368export const listSchemaVersionsAt = (parent: string) =>
369 parent.length === 0
370 ? emptyList<documentai.GoogleCloudDocumentaiV1SchemaVersion>()
371 : collectPages(
372 documentai.listProjectsLocationsSchemasSchemaVersions.pages({
373 parent,
374 pageSize: 20,
375 }),
376 (page) => page.schemaVersions,
377 ).pipe(
378 Effect.catchTag("NotFound", () =>
379 emptyList<documentai.GoogleCloudDocumentaiV1SchemaVersion>(),
380 ),
381 );
382
383export const listProjectProcessors = (project: string) =>
384 Effect.gen(function* () {
385 const parents = yield* listLocationParents(project);
386 const groups = yield* Effect.forEach(parents, listProcessorsAt, {
387 concurrency: 2,
388 });
389 const seen = new Set<string>();
390 const processors: documentai.GoogleCloudDocumentaiV1Processor[] = [];
391 for (const processor of groups.flat()) {
392 const name = processor.name ?? "";
393 if (name.length === 0 || seen.has(name)) continue;
394 seen.add(name);
395 processors.push(processor);
396 }
397 return processors;
398 });
399
400export const listProjectSchemas = (project: string) =>
401 Effect.gen(function* () {
402 const parents = yield* listLocationParents(project);
403 const groups = yield* Effect.forEach(parents, listSchemasAt, {
404 concurrency: 2,
405 });
406 const seen = new Set<string>();
407 const schemas: documentai.GoogleCloudDocumentaiV1NextSchema[] = [];
408 for (const schema of groups.flat()) {
409 const name = schema.name ?? "";
410 if (name.length === 0 || seen.has(name)) continue;
411 seen.add(name);
412 schemas.push(schema);
413 }
414 return schemas;
415 });
416
417export const findOwnedByDisplayName = <
418 T extends { name?: string; displayName?: string },
419>(
420 id: string,
421 items: readonly T[],
422) =>
423 Effect.gen(function* () {
424 for (const item of items) {
425 if (yield* ownedByAlchemy(id, item.displayName)) return item;
426 }
427 return undefined as T | undefined;
428 });
429
430export const findOwnedByLabels = <
431 T extends {
432 name?: string;
433 labels?: Record<string, string | undefined> | null;
434 },
435>(
436 id: string,
437 items: readonly T[],
438) =>
439 Effect.gen(function* () {
440 for (const item of items) {
441 if (yield* hasAlchemyLabels(id, tagRecord(item.labels))) return item;
442 }
443 return undefined as T | undefined;
444 });