1import * as dlp from "@distilled.cloud/gcp/dlp_v2";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import { Unowned } from "../../AdoptPolicy.ts";
6import { isResolved } from "../../Diff.ts";
7import * as Provider from "../../Provider.ts";
8import { Resource } from "../../Resource.ts";
9import { tagRecord } from "../../Tags.ts";
10import { GcpEnvironment } from "../Environment.ts";
11import { createInternalLabels, stripInternalLabels } from "../Labels.ts";
12import type { Providers } from "../Providers.ts";
13import { createPhysicalName } from "../../PhysicalName.ts";
14import {
15 collectPages,
16 encodeOwnership,
17 hasHybridOwnership,
18 hasOwnershipMarker,
19 hybridLabels,
20 jsonEqual,
21 lastSegment,
22 locationOf,
23 locationParent,
24 normalizeLocation,
25 ownedByAlchemy,
26 parseOwnership,
27} from "./internal.ts";
28
29type InspectJobConfig = dlp.GooglePrivacyDlpV2InspectJobConfig;
30type RiskAnalysisJobConfig = dlp.GooglePrivacyDlpV2RiskAnalysisJobConfig;
31
32const LOCATION = "global";
33const MAX_JOB_ID_LENGTH = 64;
34
35const stripJobPrefix = (jobId: string) => jobId.replace(/^[ir]-/, "");
36
37const prefixedJobId = (jobId: string, risk: boolean) => {
38 const bare = stripJobPrefix(jobId);
39 return `${risk ? "r" : "i"}-${bare}`;
40};
41
42const toJobId = (
43 id: string,
44 requested: string | undefined,
45 existing: string | undefined,
46) =>
47 Effect.gen(function* () {
48 if (requested !== undefined) {
49 return stripJobPrefix(requested).slice(0, MAX_JOB_ID_LENGTH);
50 }
51 if (existing !== undefined) return stripJobPrefix(existing);
52 const generated = yield* createPhysicalName({
53 id,
54 maxLength: MAX_JOB_ID_LENGTH,
55 lowercase: true,
56 });
57 const next = /^[a-zA-Z]/.test(generated) ? generated : `d${generated}`;
58 return next.replace(/[^a-zA-Z0-9_-]/g, "-").slice(0, MAX_JOB_ID_LENGTH);
59 });
60
61const DEFAULT_INSPECT_JOB: InspectJobConfig = {
62 inspectConfig: { infoTypes: [{ name: "EMAIL_ADDRESS" }] },
63 storageConfig: { hybridOptions: {} },
64};
65
66export type LocationsDlpJobProps = {
67 /**
68 * Job id (the `{dlpJob}` segment of
69 * `projects/{project}/locations/{location}/dlpJobs/{dlpJob}`). If
70 * omitted, a unique name is generated. Must match `[a-zA-Z0-9_-]+` and
71 * is at most 64 characters. Immutable — changing it replaces the job.
72 */
73 jobId?: string;
74 /**
75 * Processing location (`global`, `us`, `europe-west1`, …). Immutable —
76 * changing it replaces the job. `us-central1` is not a valid DLP job
77 * location.
78 * @default "global"
79 */
80 location?: string;
81 /**
82 * Inspect-job configuration. Defaults to a hybrid EMAIL_ADDRESS
83 * inspect job. DlpJob has no update API — changing this replaces the
84 * job.
85 */
86 inspectJob?: InspectJobConfig;
87 /**
88 * Risk-analysis job configuration. Mutually exclusive with
89 * `inspectJob`. Immutable after create.
90 */
91 riskJob?: RiskAnalysisJobConfig;
92};
93
94export type LocationsDlpJob = Resource<
95 "GCP.DLP.LocationsDlpJob",
96 LocationsDlpJobProps,
97 {
98 /** Full resource name `projects/{project}/locations/{location}/dlpJobs/{id}`. */
99 name: string;
100 /** Job id (last path segment). */
101 jobId: string;
102 /** Location id. */
103 location: string;
104 /** Project id. */
105 project: string;
106 /** Job type (`INSPECT_JOB` or `RISK_ANALYSIS_JOB`). */
107 type: string | undefined;
108 /** Job state (`PENDING`, `RUNNING`, `ACTIVE`, `DONE`, …). */
109 state: string | undefined;
110 /** Inspect-job configuration, Alchemy labels stripped. */
111 inspectJob: InspectJobConfig | undefined;
112 /** Risk-analysis job configuration. */
113 riskJob: RiskAnalysisJobConfig | undefined;
114 /** Trigger that created this job, if any. */
115 jobTriggerName: string | undefined;
116 /** RFC3339 creation timestamp. */
117 createTime: string | undefined;
118 /** RFC3339 start timestamp. */
119 startTime: string | undefined;
120 /** RFC3339 end timestamp. */
121 endTime: string | undefined;
122 },
123 never,
124 Providers
125>;
126
127/**
128 * A regional Cloud DLP job (`projects.locations.dlpJobs`).
129 *
130 * Creating a DlpJob starts it immediately. Inspect jobs are stored as
131 * `i-{jobId}` and risk jobs as `r-{jobId}`. There is no update API, so
132 * reconcile is observe-ensure (create if missing). Delete cancels a
133 * running/active job, then deletes it. DLP jobs have no labels field —
134 * Alchemy stamps ownership into hybrid inspect `storageConfig.hybridOptions`
135 * (labels + description) so `list` / nuke can find them.
136 *
137 * ### Creating a DLP Job
138 * **Example:** Hybrid inspect job
139 * ```typescript
140 * const job = yield* GCP.DLP.LocationsDlpJob("Scan", {
141 * inspectJob: {
142 * inspectConfig: { infoTypes: [{ name: "EMAIL_ADDRESS" }] },
143 * storageConfig: { hybridOptions: { description: "inbox" } },
144 * },
145 * });
146 * ```
147 *
148 * @resource
149 * @category DLP
150 */
151export const LocationsDlpJob = Resource<LocationsDlpJob>(
152 "GCP.DLP.LocationsDlpJob",
153);
154
155export class LocationsDlpJobNotResolved extends Data.TaggedError(
156 "GCP.DLP.LocationsDlpJobNotResolved",
157)<{
158 name: string;
159}> {}
160
161const resourceName = (
162 project: string,
163 location: string,
164 jobId: string,
165 risk = false,
166) =>
167 `${locationParent(project, location)}/dlpJobs/${prefixedJobId(jobId, risk)}`;
168
169const stampInspectJob = (
170 inspectJob: InspectJobConfig | undefined,
171 ownership: Record<string, string>,
172): InspectJobConfig | undefined => {
173 if (inspectJob === undefined) return undefined;
174 const storage = inspectJob.storageConfig;
175 const isHybrid =
176 storage === undefined ||
177 storage.hybridOptions !== undefined ||
178 (storage.cloudStorageOptions === undefined &&
179 storage.bigQueryOptions === undefined &&
180 storage.datastoreOptions === undefined);
181 if (!isHybrid) return inspectJob;
182 const userHybrid = storage?.hybridOptions ?? {};
183 return {
184 ...inspectJob,
185 storageConfig: {
186 ...storage,
187 hybridOptions: {
188 ...userHybrid,
189 labels: {
190 ...tagRecord(userHybrid.labels),
191 ...hybridLabels(ownership),
192 },
193 description: encodeOwnership(ownership, userHybrid.description),
194 },
195 },
196 };
197};
198
199const userInspectJob = (
200 inspectJob: InspectJobConfig | undefined,
201): InspectJobConfig | undefined => {
202 if (inspectJob === undefined) return undefined;
203 const hybrid = inspectJob.storageConfig?.hybridOptions;
204 if (hybrid === undefined) return inspectJob;
205 const parsed = parseOwnership(hybrid.description);
206 return {
207 ...inspectJob,
208 storageConfig: {
209 ...inspectJob.storageConfig,
210 hybridOptions: {
211 ...hybrid,
212 description: parsed.text,
213 labels: stripInternalLabels(tagRecord(hybrid.labels)),
214 },
215 },
216 };
217};
218
219const hybridOf = (job: dlp.GooglePrivacyDlpV2DlpJob) =>
220 job.inspectDetails?.requestedOptions?.jobConfig?.storageConfig?.hybridOptions;
221
222const jobOwned = (job: dlp.GooglePrivacyDlpV2DlpJob) => {
223 const hybrid = hybridOf(job);
224 return (
225 hasOwnershipMarker(hybrid?.description) ||
226 hasHybridOwnership(hybrid?.labels)
227 );
228};
229
230const toAttrs = (job: dlp.GooglePrivacyDlpV2DlpJob, project: string) => {
231 const name = job.name ?? "";
232 return {
233 name,
234 jobId: lastSegment(name),
235 location: locationOf(name, LOCATION),
236 project,
237 type: job.type,
238 state: job.state,
239 inspectJob: userInspectJob(job.inspectDetails?.requestedOptions?.jobConfig),
240 riskJob: job.riskDetails?.requestedOptions?.jobConfig,
241 jobTriggerName: job.jobTriggerName,
242 createTime: job.createTime,
243 startTime: job.startTime,
244 endTime: job.endTime,
245 };
246};
247
248const getByName = (name: string) =>
249 name.length === 0
250 ? Effect.succeed(undefined)
251 : dlp
252 .getProjectsLocationsDlpJobs({ name })
253 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
254
255const listType = (parent: string, type: "INSPECT_JOB" | "RISK_ANALYSIS_JOB") =>
256 collectPages(
257 dlp.listProjectsLocationsDlpJobs.pages({
258 parent,
259 pageSize: 100,
260 type,
261 }),
262 (page) => page.jobs,
263 ).pipe(
264 Effect.catchTag("NotFound", () =>
265 Effect.succeed([] as dlp.GooglePrivacyDlpV2DlpJob[]),
266 ),
267 );
268
269export const LocationsDlpJobProvider = () =>
270 Provider.succeed(LocationsDlpJob, {
271 stables: ["name", "jobId", "location", "project", "createTime", "type"],
272
273 diff: Effect.fn(function* ({ news, olds, output }) {
274 if (!isResolved(news)) return undefined;
275 const previousId = olds?.jobId ?? output?.jobId;
276 const idChanged =
277 previousId !== undefined &&
278 news.jobId !== undefined &&
279 news.jobId !== previousId;
280 const previousLocation = olds?.location ?? output?.location;
281 const locationChanged =
282 previousLocation !== undefined &&
283 normalizeLocation(news.location, LOCATION) !==
284 normalizeLocation(previousLocation, LOCATION);
285 const inspectChanged =
286 output !== undefined &&
287 news.inspectJob !== undefined &&
288 !jsonEqual(userInspectJob(news.inspectJob), output.inspectJob);
289 const riskChanged =
290 output !== undefined &&
291 news.riskJob !== undefined &&
292 !jsonEqual(news.riskJob, output.riskJob);
293 if (idChanged || locationChanged) {
294 return { action: "replace" as const, deleteFirst: false };
295 }
296 if (inspectChanged || riskChanged) {
297 return { action: "replace" as const, deleteFirst: true };
298 }
299 return undefined;
300 }),
301
302 read: Effect.fn(function* ({ id, olds, output }) {
303 const env = yield* GcpEnvironment.current;
304 const location = normalizeLocation(
305 olds?.location ?? output?.location,
306 LOCATION,
307 );
308 const jobId = yield* toJobId(id, olds?.jobId, output?.jobId);
309 const name =
310 output?.name ??
311 resourceName(
312 env.project,
313 location,
314 jobId,
315 olds?.riskJob !== undefined || output?.type === "RISK_ANALYSIS_JOB",
316 );
317 const existing = yield* getByName(name);
318 if (existing === undefined) return undefined;
319 const attrs = toAttrs(existing, env.project);
320 const hybrid = hybridOf(existing);
321 const owned = yield* ownedByAlchemy(id, hybrid?.description);
322 return owned || hasHybridOwnership(hybrid?.labels)
323 ? attrs
324 : Unowned(attrs);
325 }),
326
327 list: () =>
328 Effect.gen(function* () {
329 const env = yield* GcpEnvironment.current;
330 const parent = locationParent(env.project, LOCATION);
331 const inspect = yield* listType(parent, "INSPECT_JOB");
332 const risk = yield* listType(parent, "RISK_ANALYSIS_JOB");
333 const rows = yield* Effect.forEach(
334 [...inspect, ...risk],
335 (job) =>
336 Effect.gen(function* () {
337 let current = job;
338 if (
339 current.inspectDetails === undefined &&
340 current.riskDetails === undefined &&
341 current.name
342 ) {
343 current = (yield* getByName(current.name)) ?? current;
344 }
345 if (!jobOwned(current)) return undefined;
346 return toAttrs(current, env.project);
347 }),
348 { concurrency: 8 },
349 );
350 return rows.filter((row) => row !== undefined);
351 }),
352
353 reconcile: Effect.fn(function* ({ id, news, output }) {
354 const env = yield* GcpEnvironment.current;
355 const location = normalizeLocation(
356 news.location ?? output?.location,
357 LOCATION,
358 );
359 const jobId = yield* toJobId(id, news.jobId, output?.jobId);
360 const name = resourceName(
361 env.project,
362 location,
363 jobId,
364 news.riskJob !== undefined,
365 );
366 const ownership = yield* createInternalLabels(id);
367 const inspectJob = stampInspectJob(
368 news.riskJob === undefined
369 ? (news.inspectJob ?? DEFAULT_INSPECT_JOB)
370 : news.inspectJob,
371 ownership,
372 );
373
374 let current = yield* getByName(output?.name ?? name);
375
376 if (current === undefined) {
377 const created = yield* dlp
378 .createProjectsLocationsDlpJobs({
379 parent: locationParent(env.project, location),
380 body: {
381 jobId,
382 inspectJob,
383 riskJob: news.riskJob,
384 },
385 })
386 .pipe(Effect.catchTag("Conflict", () => getByName(name)));
387 current = created ?? undefined;
388 }
389
390 if (current === undefined) {
391 return yield* new LocationsDlpJobNotResolved({ name });
392 }
393
394 return toAttrs(current, env.project);
395 }),
396
397 delete: Effect.fn(function* ({ output }) {
398 yield* dlp
399 .cancelProjectsLocationsDlpJobs({ name: output.name, body: {} })
400 .pipe(
401 Effect.catchTag(
402 ["NotFound", "BadRequest", "Conflict"],
403 () => Effect.void,
404 ),
405 );
406 yield* dlp.deleteProjectsLocationsDlpJobs({ name: output.name }).pipe(
407 Effect.catchTag("NotFound", () => Effect.void),
408 Effect.retry({
409 while: (e) => e._tag === "BadRequest" || e._tag === "Conflict",
410 times: 8,
411 schedule: Schedule.exponential("500 millis"),
412 }),
413 );
414 }),
415 });