1import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2import * as Effect from "effect/Effect";
3import * as Schedule from "effect/Schedule";
4import { Unowned } from "../../AdoptPolicy.ts";
5import { isResolved } from "../../Diff.ts";
6import * as Provider from "../../Provider.ts";
7import { Resource } from "../../Resource.ts";
8import { tagRecord } from "../../Tags.ts";
9import { GcpEnvironment } from "../Environment.ts";
10import {
11 createInternalLabels,
12 diffLabels,
13 hasAlchemyLabels,
14 toLabels,
15} from "../Labels.ts";
16import type { Providers } from "../Providers.ts";
17import {
18 DataplexNotResolved,
19 collectPages,
20 fingerprint,
21 hasAlchemyLabelMap,
22 listAtLocation,
23 normalizeLocation,
24 parentOf,
25 parseName,
26 replaceOnIdentity,
27 retryQuota,
28 toPhysicalId,
29 userLabels,
30 waitForOperation,
31 waitUntilExists,
32 waitUntilGone,
33} from "./internal.ts";
34
35export type DataProductAccessGroup =
36 dataplex.GoogleCloudDataplexV1DataProductAccessGroup;
37
38export type DataProductAccessApprovalConfig =
39 dataplex.GoogleCloudDataplexV1DataProductAccessApprovalConfig;
40
41export type DataProductProps = {
42 /**
43 * Data product id. If omitted, a unique name is generated. Must match
44 * `^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$`. Immutable — changing it
45 * replaces the product.
46 */
47 dataProductId?: string;
48 /**
49 * Region (`us-central1`, …). Immutable — changing it replaces the
50 * product.
51 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
52 */
53 location?: string;
54 /**
55 * User-friendly display name. Required.
56 */
57 displayName: string;
58 /**
59 * Description of the data product.
60 */
61 description?: string;
62 /**
63 * User labels. Alchemy ownership labels are merged in automatically.
64 */
65 labels?: Record<string, string>;
66 /**
67 * Emails of the data product owners. Required.
68 */
69 ownerEmails: string[];
70 /**
71 * Base64-encoded icon (max 3 MiB).
72 */
73 icon?: string;
74 /**
75 * Access groups keyed by group id.
76 */
77 accessGroups?: Record<string, DataProductAccessGroup>;
78 /**
79 * Access-approval configuration.
80 */
81 accessApprovalConfig?: DataProductAccessApprovalConfig;
82};
83
84export type DataProduct = Resource<
85 "GCP.Dataplex.DataProduct",
86 DataProductProps,
87 {
88 /** Full resource name. */
89 name: string;
90 /** Data product id (last path segment). */
91 dataProductId: string;
92 /** Project id. */
93 project: string;
94 /** Location id. */
95 location: string;
96 /** Display name. */
97 displayName: string | undefined;
98 /** Description. */
99 description: string | undefined;
100 /** User labels (Alchemy ownership labels stripped). */
101 labels: Record<string, string>;
102 /** Owner emails. */
103 ownerEmails: string[];
104 /** Icon payload. */
105 icon: string | undefined;
106 /** Access groups. */
107 accessGroups:
108 | Record<string, DataProductAccessGroup | undefined>
109 | undefined;
110 /** Access-approval configuration. */
111 accessApprovalConfig: DataProductAccessApprovalConfig | undefined;
112 /** Number of attached data assets. */
113 assetCount: number | undefined;
114 /** Server etag. */
115 etag: string | undefined;
116 /** System uid. */
117 uid: string | undefined;
118 /** RFC3339 creation timestamp. */
119 createTime: string | undefined;
120 /** RFC3339 last-update timestamp. */
121 updateTime: string | undefined;
122 },
123 never,
124 Providers
125>;
126
127/**
128 * A Dataplex Data Product — a curated collection of data assets packaged
129 * for a use case.
130 *
131 * Location and product id are immutable. Display name, description,
132 * labels, owners, icon, access groups, and approval config update in
133 * place.
134 *
135 * ### Creating a Data Product
136 * **Example:** Product with an owner
137 * ```typescript
138 * const product = yield* GCP.Dataplex.DataProduct("Sales", {
139 * displayName: "Sales mart",
140 * ownerEmails: ["owner@example.com"],
141 * labels: { env: "test" },
142 * });
143 * ```
144 *
145 * @resource
146 * @category Dataplex
147 */
148export const DataProduct = Resource<DataProduct>("GCP.Dataplex.DataProduct");
149
150const resourceName = (
151 project: string,
152 location: string,
153 dataProductId: string,
154) => `projects/${project}/locations/${location}/dataProducts/${dataProductId}`;
155
156const toAttrs = (
157 product: dataplex.GoogleCloudDataplexV1DataProduct,
158 project: string,
159) => {
160 const name = product.name ?? "";
161 const parsed = parseName(name, "dataProducts");
162 return {
163 name,
164 dataProductId: parsed.id,
165 project: parsed.project || project,
166 location: parsed.location,
167 displayName: product.displayName,
168 description: product.description,
169 labels: userLabels(product.labels),
170 ownerEmails: [...(product.ownerEmails ?? [])],
171 icon: product.icon,
172 accessGroups: product.accessGroups,
173 accessApprovalConfig: product.accessApprovalConfig,
174 assetCount: product.assetCount,
175 etag: product.etag,
176 uid: product.uid,
177 createTime: product.createTime,
178 updateTime: product.updateTime,
179 };
180};
181
182const getByName = (name: string) =>
183 retryQuota(dataplex.getProjectsLocationsDataProducts({ name })).pipe(
184 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
185 );
186
187const listProducts = (project: string, region: string) => {
188 const collect = (parent: string) =>
189 collectPages(
190 dataplex.listProjectsLocationsDataProducts.pages({
191 parent,
192 pageSize: 1000,
193 }),
194 (page) => page.dataProducts,
195 ).pipe(
196 Effect.map((items) =>
197 items.filter((item) => hasAlchemyLabelMap(item.labels)),
198 ),
199 Effect.catchTag("NotFound", () => Effect.succeed([])),
200 );
201 return listAtLocation(project, region, collect);
202};
203
204export const listAlchemyDataProducts = (project: string, region: string) =>
205 listProducts(project, region);
206
207export const DataProductProvider = () =>
208 Provider.succeed(DataProduct, {
209 stables: [
210 "name",
211 "dataProductId",
212 "project",
213 "location",
214 "uid",
215 "createTime",
216 ],
217
218 diff: Effect.fn(function* ({ news, olds, output }) {
219 if (!isResolved(news)) return undefined;
220 const env = yield* GcpEnvironment.current;
221 return replaceOnIdentity({
222 previousId: olds?.dataProductId ?? output?.dataProductId,
223 nextId:
224 news.dataProductId ?? olds?.dataProductId ?? output?.dataProductId,
225 previousLocation: normalizeLocation(
226 olds?.location ?? output?.location,
227 env.region,
228 ),
229 nextLocation: normalizeLocation(
230 news.location ?? olds?.location ?? output?.location,
231 env.region,
232 ),
233 });
234 }),
235
236 read: Effect.fn(function* ({ id, olds, output }) {
237 const env = yield* GcpEnvironment.current;
238 const dataProductId = yield* toPhysicalId(
239 id,
240 olds?.dataProductId,
241 output?.dataProductId,
242 "dataproduct",
243 );
244 const location = normalizeLocation(
245 olds?.location ?? output?.location,
246 env.region,
247 );
248 const name =
249 output?.name ?? resourceName(env.project, location, dataProductId);
250 const existing = yield* getByName(name);
251 if (existing === undefined) return undefined;
252 const attrs = toAttrs(existing, env.project);
253 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels)))
254 ? attrs
255 : Unowned(attrs);
256 }),
257
258 list: () =>
259 Effect.gen(function* () {
260 const env = yield* GcpEnvironment.current;
261 const items = yield* listProducts(env.project, env.region);
262 return items.map((item) => toAttrs(item, env.project));
263 }),
264
265 reconcile: Effect.fn(function* ({ id, news, output }) {
266 const env = yield* GcpEnvironment.current;
267 const dataProductId = yield* toPhysicalId(
268 id,
269 news.dataProductId,
270 output?.dataProductId,
271 "dataproduct",
272 );
273 const location = normalizeLocation(
274 news.location ?? output?.location,
275 env.region,
276 );
277 const name = resourceName(env.project, location, dataProductId);
278 const desiredLabels = {
279 ...toLabels(news.labels),
280 ...(yield* createInternalLabels(id)),
281 };
282
283 let current = yield* getByName(output?.name ?? name);
284
285 if (current === undefined) {
286 const created = yield* retryQuota(
287 dataplex.createProjectsLocationsDataProducts({
288 parent: parentOf(env.project, location),
289 dataProductId,
290 body: {
291 displayName: news.displayName,
292 description: news.description,
293 labels: desiredLabels,
294 ownerEmails: news.ownerEmails,
295 icon: news.icon,
296 accessGroups: news.accessGroups,
297 accessApprovalConfig: news.accessApprovalConfig,
298 },
299 }),
300 ).pipe(Effect.catchTag("Conflict", () => Effect.succeed(undefined)));
301 if (created !== undefined) {
302 yield* waitForOperation(created);
303 }
304 current = yield* waitUntilExists(getByName(name), name);
305 }
306
307 if (current === undefined) {
308 return yield* new DataplexNotResolved({ name });
309 }
310
311 const observedLabels = tagRecord(current.labels);
312 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
313 const labelsChanged = upsert.length > 0 || removed.length > 0;
314 const descriptionChanged =
315 (current.description ?? "") !== (news.description ?? "");
316 const displayNameChanged =
317 (current.displayName ?? "") !== news.displayName;
318 const ownersChanged =
319 fingerprint([...(current.ownerEmails ?? [])].sort()) !==
320 fingerprint([...(news.ownerEmails ?? [])].sort());
321 const iconChanged = (current.icon ?? "") !== (news.icon ?? "");
322 const groupsChanged =
323 fingerprint(current.accessGroups) !== fingerprint(news.accessGroups);
324 const approvalChanged =
325 fingerprint(current.accessApprovalConfig) !==
326 fingerprint(news.accessApprovalConfig);
327
328 if (
329 labelsChanged ||
330 descriptionChanged ||
331 displayNameChanged ||
332 ownersChanged ||
333 iconChanged ||
334 groupsChanged ||
335 approvalChanged
336 ) {
337 const operation = yield* dataplex.patchProjectsLocationsDataProducts({
338 name: current.name ?? name,
339 updateMask: [
340 labelsChanged ? "labels" : undefined,
341 descriptionChanged ? "description" : undefined,
342 displayNameChanged ? "displayName" : undefined,
343 ownersChanged ? "ownerEmails" : undefined,
344 iconChanged ? "icon" : undefined,
345 groupsChanged ? "accessGroups" : undefined,
346 approvalChanged ? "accessApprovalConfig" : undefined,
347 ]
348 .filter((field): field is string => field !== undefined)
349 .join(","),
350 body: {
351 name: current.name ?? name,
352 etag: current.etag,
353 displayName: news.displayName,
354 description: news.description,
355 labels: desiredLabels,
356 ownerEmails: news.ownerEmails,
357 icon: news.icon,
358 accessGroups: news.accessGroups,
359 accessApprovalConfig: news.accessApprovalConfig,
360 },
361 });
362 yield* waitForOperation(operation);
363 current = yield* waitUntilExists(
364 getByName(current.name ?? name),
365 current.name ?? name,
366 );
367 }
368
369 return toAttrs(current, env.project);
370 }),
371
372 delete: Effect.fn(function* ({ output }) {
373 const operation = yield* dataplex
374 .deleteProjectsLocationsDataProducts({
375 name: output.name,
376 etag: output.etag,
377 })
378 .pipe(
379 Effect.retry({
380 while: (error) => error._tag === "Conflict",
381 times: 8,
382 schedule: Schedule.spaced("2 seconds"),
383 }),
384 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
385 );
386 if (operation !== undefined) {
387 yield* waitForOperation(operation, { notFoundOk: true });
388 }
389 yield* waitUntilGone(getByName(output.name), output.name);
390 }),
391 });