1import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Stream from "effect/Stream";
5import { Unowned } from "../../AdoptPolicy.ts";
6import { isResolved } from "../../Diff.ts";
7import { createPhysicalName } from "../../PhysicalName.ts";
8import * as Provider from "../../Provider.ts";
9import { Resource } from "../../Resource.ts";
10import { tagRecord } from "../../Tags.ts";
11import { GcpEnvironment } from "../Environment.ts";
12import {
13 createInternalLabels,
14 diffLabels,
15 hasAlchemyLabels,
16 stripInternalLabels,
17 toLabels,
18} from "../Labels.ts";
19import type { Providers } from "../Providers.ts";
20
21const MAX_DATASET_ID_LENGTH = 1024;
22
23export type DatasetAccess = bigquery.DatasetAccessItem;
24
25export type DatasetEncryption = {
26 /**
27 * Cloud KMS key used as the default encryption key for new tables,
28 * as `projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`.
29 */
30 kmsKeyName?: string;
31};
32
33export type DatasetProps = {
34 /**
35 * Dataset id (the `{datasetId}` segment of
36 * `projects/{project}/datasets/{datasetId}`). If omitted, a unique name
37 * is generated from the stack, stage, and logical id. Must contain only
38 * letters, numbers, and underscores; max 1024 characters. Immutable —
39 * changing it replaces the dataset.
40 */
41 datasetId?: string;
42 /**
43 * Geographic location (`US`, `EU`, `US-CENTRAL1`, `us-central1`, …).
44 * Immutable — changing it replaces the dataset.
45 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
46 */
47 location?: string;
48 /**
49 * User labels. Alchemy ownership labels are merged in automatically.
50 */
51 labels?: Record<string, string>;
52 /**
53 * User-friendly description of the dataset.
54 */
55 description?: string;
56 /**
57 * Descriptive name shown in the console.
58 */
59 friendlyName?: string;
60 /**
61 * Default lifetime of newly created tables, in milliseconds. Minimum
62 * `"3600000"` (one hour). Set to `"0"` to clear.
63 */
64 defaultTableExpirationMs?: string;
65 /**
66 * Default partition expiration for new partitioned tables, in
67 * milliseconds. Set to `"0"` to clear.
68 */
69 defaultPartitionExpirationMs?: string;
70 /**
71 * Time travel window in hours (`"48"` to `"168"`). Defaults to `"168"`
72 * when omitted at create.
73 */
74 maxTimeTravelHours?: string;
75 /**
76 * Storage billing model (`LOGICAL` or `PHYSICAL`).
77 */
78 storageBillingModel?: bigquery.DatasetStorageBillingModelEnum | (string & {});
79 /**
80 * When true, dataset and table names are case-insensitive.
81 * @default false
82 */
83 isCaseInsensitive?: boolean;
84 /**
85 * Default collation for new tables (`"und:ci"` or `""`).
86 */
87 defaultCollation?: string;
88 /**
89 * Default rounding mode for new tables
90 * (`ROUND_HALF_AWAY_FROM_ZERO`, `ROUND_HALF_EVEN`).
91 */
92 defaultRoundingMode?: bigquery.DatasetDefaultRoundingModeEnum | (string & {});
93 /**
94 * Default Cloud KMS encryption for newly created tables.
95 */
96 defaultEncryptionConfiguration?: DatasetEncryption;
97 /**
98 * Access control entries. If omitted, BigQuery installs its default
99 * project-owner/reader/writer grants. Providing this list replaces the
100 * entire ACL.
101 */
102 access?: DatasetAccess[];
103 /**
104 * Delete all tables, views, and routines before destroying the dataset.
105 * @default false
106 */
107 forceDestroy?: boolean;
108};
109
110export type Dataset = Resource<
111 "GCP.BigQuery.Dataset",
112 DatasetProps,
113 {
114 /** Resource name `projects/{project}/datasets/{datasetId}`. */
115 name: string;
116 /** Dataset id. */
117 datasetId: string;
118 /** Project id. */
119 project: string;
120 /** Fully-qualified id `projectId:datasetId`. */
121 id: string;
122 /** Geographic location. */
123 location: string;
124 /** User labels (Alchemy ownership labels stripped). */
125 labels: Record<string, string>;
126 /** User-friendly description. */
127 description: string | undefined;
128 /** Descriptive console name. */
129 friendlyName: string | undefined;
130 /** Default table expiration, in milliseconds. */
131 defaultTableExpirationMs: string | undefined;
132 /** Default partition expiration, in milliseconds. */
133 defaultPartitionExpirationMs: string | undefined;
134 /** Time travel window in hours. */
135 maxTimeTravelHours: string | undefined;
136 /** Storage billing model. */
137 storageBillingModel: string | undefined;
138 /** Whether names are case-insensitive. */
139 isCaseInsensitive: boolean;
140 /** Default collation for new tables. */
141 defaultCollation: string | undefined;
142 /** Default rounding mode for new tables. */
143 defaultRoundingMode: string | undefined;
144 /** Default KMS key for new tables, if any. */
145 kmsKeyName: string | undefined;
146 /** Access control entries currently on the dataset. */
147 access: DatasetAccess[];
148 /** Dataset type (`DEFAULT`, `LINKED`, `EXTERNAL`, …). */
149 type: string | undefined;
150 /** Self-link URL. */
151 selfLink: string | undefined;
152 /** Server etag. */
153 etag: string | undefined;
154 /** Creation time in milliseconds since epoch. */
155 creationTime: string | undefined;
156 /** Last-modified time in milliseconds since epoch. */
157 lastModifiedTime: string | undefined;
158 },
159 never,
160 Providers
161>;
162
163/**
164 * A Google BigQuery dataset — a container for tables, views, and routines.
165 *
166 * Changing `datasetId` or `location` replaces the dataset. Labels,
167 * description, expiration, collation, encryption, and access update in
168 * place. Destroy fails if the dataset still contains tables unless
169 * `forceDestroy` is set.
170 *
171 * ### Creating a Dataset
172 * **Example:** Generated name
173 * ```typescript
174 * const dataset = yield* GCP.BigQuery.Dataset("Analytics", {});
175 * ```
176 *
177 * **Example:** Explicit id, location, and labels
178 * ```typescript
179 * const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
180 * datasetId: "app_analytics",
181 * location: "US-CENTRAL1",
182 * labels: { env: "prod" },
183 * description: "application events",
184 * forceDestroy: true,
185 * });
186 * ```
187 *
188 * ### Querying
189 * **Example:** Run a GoogleSQL query against the dataset
190 * ```typescript
191 * const query = yield* GCP.BigQuery.Query(dataset);
192 * const result = yield* query({ query: "SELECT 1 AS n" });
193 * ```
194 *
195 * @resource
196 * @category BigQuery
197 */
198export const Dataset = Resource<Dataset>("GCP.BigQuery.Dataset");
199
200export class DatasetNotResolved extends Data.TaggedError(
201 "GCP.BigQuery.DatasetNotResolved",
202)<{
203 name: string;
204}> {}
205
206const userLabels = (
207 labels: Record<string, string | undefined> | null | undefined,
208): Record<string, string> => stripInternalLabels(tagRecord(labels));
209
210const resourceName = (project: string, datasetId: string) =>
211 `projects/${project}/datasets/${datasetId}`;
212
213const datasetIdOf = (
214 dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem,
215) => {
216 const fromRef = dataset.datasetReference?.datasetId;
217 if (fromRef !== undefined && fromRef.length > 0) return fromRef;
218 const id = dataset.id ?? "";
219 const colon = id.indexOf(":");
220 return colon >= 0 ? id.slice(colon + 1) : id;
221};
222
223const projectOf = (
224 dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem,
225 fallback: string,
226) => {
227 const fromRef = dataset.datasetReference?.projectId;
228 if (fromRef !== undefined && fromRef.length > 0) return fromRef;
229 const id = dataset.id ?? "";
230 const colon = id.indexOf(":");
231 return colon > 0 ? id.slice(0, colon) : fallback;
232};
233
234const toId = (id: string, datasetId: string | undefined, existing?: string) =>
235 Effect.gen(function* () {
236 if (datasetId !== undefined) return datasetId;
237 if (existing !== undefined) return existing;
238 const generated = yield* createPhysicalName({
239 id,
240 maxLength: MAX_DATASET_ID_LENGTH,
241 lowercase: true,
242 delimiter: "_",
243 });
244 return generated.replaceAll("-", "_");
245 });
246
247const toAttrs = (
248 dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem,
249 fallbackProject: string,
250 region: string,
251) => {
252 const datasetId = datasetIdOf(dataset);
253 const project = projectOf(dataset, fallbackProject);
254 const full = dataset as bigquery.Dataset;
255 return {
256 name: resourceName(project, datasetId),
257 datasetId,
258 project,
259 id: full.id ?? `${project}:${datasetId}`,
260 location: dataset.location ?? region,
261 labels: userLabels(dataset.labels),
262 description: full.description,
263 friendlyName: dataset.friendlyName,
264 defaultTableExpirationMs: full.defaultTableExpirationMs,
265 defaultPartitionExpirationMs: full.defaultPartitionExpirationMs,
266 maxTimeTravelHours: full.maxTimeTravelHours,
267 storageBillingModel: full.storageBillingModel,
268 isCaseInsensitive: full.isCaseInsensitive === true,
269 defaultCollation: full.defaultCollation,
270 defaultRoundingMode: full.defaultRoundingMode,
271 kmsKeyName: full.defaultEncryptionConfiguration?.kmsKeyName,
272 access: full.access ?? [],
273 type: dataset.type,
274 selfLink: full.selfLink,
275 etag: full.etag,
276 creationTime: full.creationTime,
277 lastModifiedTime: full.lastModifiedTime,
278 };
279};
280
281const getById = (project: string, datasetId: string) =>
282 bigquery
283 .getDatasets({ projectId: project, datasetId })
284 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
285
286const sameString = (left: string | undefined, right: string | undefined) =>
287 (left ?? "") === (right ?? "");
288
289const accessJson = (access: readonly DatasetAccess[] | undefined) =>
290 JSON.stringify(access ?? []);
291
292export const DatasetProvider = () =>
293 Provider.succeed(Dataset, {
294 stables: ["name", "datasetId", "project", "id", "location", "creationTime"],
295
296 diff: Effect.fn(function* ({ news, olds, output }) {
297 if (!isResolved(news)) return undefined;
298 const previousId = olds?.datasetId ?? output?.datasetId;
299 if (
300 previousId !== undefined &&
301 news.datasetId !== undefined &&
302 news.datasetId !== previousId
303 ) {
304 return { action: "replace" as const, deleteFirst: false };
305 }
306 const previousLocation = olds?.location ?? output?.location;
307 const nextLocation = news.location;
308 if (
309 previousLocation !== undefined &&
310 nextLocation !== undefined &&
311 previousLocation.toUpperCase() !== nextLocation.toUpperCase()
312 ) {
313 const nextId = news.datasetId ?? previousId;
314 return {
315 action: "replace" as const,
316 deleteFirst: nextId !== undefined && nextId === previousId,
317 };
318 }
319 return undefined;
320 }),
321
322 read: Effect.fn(function* ({ id, olds, output }) {
323 const env = yield* GcpEnvironment.current;
324 const datasetId = yield* toId(id, olds?.datasetId, output?.datasetId);
325 const project = output?.project ?? env.project;
326 const existing = yield* getById(project, datasetId);
327 if (existing === undefined) return undefined;
328 const attrs = toAttrs(existing, project, env.region);
329 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels)))
330 ? attrs
331 : Unowned(attrs);
332 }),
333
334 list: () =>
335 Effect.gen(function* () {
336 const env = yield* GcpEnvironment.current;
337 return yield* bigquery.listDatasets
338 .pages({
339 projectId: env.project,
340 maxResults: 1000,
341 all: true,
342 })
343 .pipe(
344 Stream.flatMap((page) => Stream.fromIterable(page.datasets ?? [])),
345 Stream.filter((dataset) =>
346 Object.keys(dataset.labels ?? {}).some((key) =>
347 key.startsWith("alchemy-"),
348 ),
349 ),
350 Stream.map((dataset) => toAttrs(dataset, env.project, env.region)),
351 Stream.runCollect,
352 Effect.map((chunk) => Array.from(chunk)),
353 );
354 }),
355
356 reconcile: Effect.fn(function* ({ id, news, output }) {
357 const env = yield* GcpEnvironment.current;
358 const datasetId = yield* toId(id, news.datasetId, output?.datasetId);
359 const project = output?.project ?? env.project;
360 const location = news.location ?? output?.location ?? env.region;
361 const desiredLabels = {
362 ...toLabels(news.labels),
363 ...(yield* createInternalLabels(id)),
364 };
365 const name = resourceName(project, datasetId);
366
367 let current = yield* getById(project, datasetId);
368
369 if (current === undefined) {
370 const created = yield* bigquery
371 .insertDatasets({
372 projectId: project,
373 body: {
374 datasetReference: { projectId: project, datasetId },
375 location,
376 labels: desiredLabels,
377 description: news.description,
378 friendlyName: news.friendlyName,
379 defaultTableExpirationMs: news.defaultTableExpirationMs,
380 defaultPartitionExpirationMs: news.defaultPartitionExpirationMs,
381 maxTimeTravelHours: news.maxTimeTravelHours,
382 storageBillingModel: news.storageBillingModel,
383 isCaseInsensitive:
384 news.isCaseInsensitive === true ? true : undefined,
385 defaultCollation: news.defaultCollation,
386 defaultRoundingMode: news.defaultRoundingMode,
387 defaultEncryptionConfiguration:
388 news.defaultEncryptionConfiguration,
389 access: news.access,
390 },
391 })
392 .pipe(Effect.catchTag("Conflict", () => getById(project, datasetId)));
393 current = created ?? undefined;
394 }
395
396 if (current === undefined) {
397 return yield* new DatasetNotResolved({ name });
398 }
399
400 const observedLabels = tagRecord(current.labels);
401 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
402 if (removed.length > 0) {
403 // PATCH merges labels; removing one needs a JSON null the typed body
404 // cannot carry, so replace the labels with PUT (get-modify-put).
405 current = yield* bigquery.updateDatasets({
406 projectId: project,
407 datasetId,
408 body: { ...current, labels: desiredLabels },
409 });
410 }
411 const labelsChanged = removed.length === 0 && upsert.length > 0;
412 const descriptionChanged = !sameString(
413 current.description,
414 news.description,
415 );
416 const friendlyNameChanged = !sameString(
417 current.friendlyName,
418 news.friendlyName,
419 );
420 const tableExpirationChanged =
421 news.defaultTableExpirationMs !== undefined &&
422 !sameString(
423 current.defaultTableExpirationMs,
424 news.defaultTableExpirationMs,
425 );
426 const partitionExpirationChanged =
427 news.defaultPartitionExpirationMs !== undefined &&
428 !sameString(
429 current.defaultPartitionExpirationMs,
430 news.defaultPartitionExpirationMs,
431 );
432 const timeTravelChanged =
433 news.maxTimeTravelHours !== undefined &&
434 !sameString(current.maxTimeTravelHours, news.maxTimeTravelHours);
435 const billingChanged =
436 news.storageBillingModel !== undefined &&
437 (current.storageBillingModel ?? "") !== news.storageBillingModel;
438 const caseChanged =
439 (current.isCaseInsensitive === true) !==
440 (news.isCaseInsensitive === true);
441 const collationChanged =
442 news.defaultCollation !== undefined &&
443 !sameString(current.defaultCollation, news.defaultCollation);
444 const roundingChanged =
445 news.defaultRoundingMode !== undefined &&
446 (current.defaultRoundingMode ?? "") !== news.defaultRoundingMode;
447 const encryptionChanged =
448 news.defaultEncryptionConfiguration !== undefined &&
449 (current.defaultEncryptionConfiguration?.kmsKeyName ?? "") !==
450 (news.defaultEncryptionConfiguration.kmsKeyName ?? "");
451 const accessChanged =
452 news.access !== undefined &&
453 accessJson(current.access) !== accessJson(news.access);
454
455 const metadataChanged =
456 labelsChanged ||
457 descriptionChanged ||
458 friendlyNameChanged ||
459 tableExpirationChanged ||
460 partitionExpirationChanged ||
461 timeTravelChanged ||
462 billingChanged ||
463 caseChanged ||
464 collationChanged ||
465 roundingChanged ||
466 encryptionChanged;
467
468 if (metadataChanged || accessChanged) {
469 const body: bigquery.Dataset = {};
470 if (labelsChanged) {
471 body.labels = desiredLabels;
472 }
473 if (descriptionChanged) {
474 body.description = news.description ?? "";
475 }
476 if (friendlyNameChanged) {
477 body.friendlyName = news.friendlyName ?? "";
478 }
479 if (tableExpirationChanged) {
480 body.defaultTableExpirationMs = news.defaultTableExpirationMs;
481 }
482 if (partitionExpirationChanged) {
483 body.defaultPartitionExpirationMs = news.defaultPartitionExpirationMs;
484 }
485 if (timeTravelChanged) {
486 body.maxTimeTravelHours = news.maxTimeTravelHours;
487 }
488 if (billingChanged) {
489 body.storageBillingModel = news.storageBillingModel;
490 }
491 if (caseChanged) {
492 body.isCaseInsensitive = news.isCaseInsensitive === true;
493 }
494 if (collationChanged) {
495 body.defaultCollation = news.defaultCollation;
496 }
497 if (roundingChanged) {
498 body.defaultRoundingMode = news.defaultRoundingMode;
499 }
500 if (encryptionChanged) {
501 body.defaultEncryptionConfiguration =
502 news.defaultEncryptionConfiguration;
503 }
504 if (accessChanged) {
505 body.access = news.access;
506 }
507 current = yield* bigquery.patchDatasets({
508 projectId: project,
509 datasetId,
510 updateMode:
511 accessChanged && metadataChanged
512 ? "UPDATE_FULL"
513 : accessChanged
514 ? "UPDATE_ACL"
515 : "UPDATE_METADATA",
516 body,
517 });
518 }
519
520 return toAttrs(current, project, env.region);
521 }),
522
523 delete: Effect.fn(function* ({ olds, output, force }) {
524 yield* bigquery
525 .deleteDatasets({
526 projectId: output.project,
527 datasetId: output.datasetId,
528 deleteContents: olds.forceDestroy === true || force === true,
529 })
530 .pipe(Effect.catchTag("NotFound", () => Effect.void));
531 }),
532 });