1import * as metastore from "@distilled.cloud/gcp/metastore_v1";
2import * as Effect from "effect/Effect";
3import * as Schedule from "effect/Schedule";
4import { Unowned } from "../../AdoptPolicy.ts";
5import { isResolved } from "../../Diff.ts";
6import * as Provider from "../../Provider.ts";
7import { Resource } from "../../Resource.ts";
8import { tagRecord } from "../../Tags.ts";
9import { GcpEnvironment } from "../Environment.ts";
10import {
11 createInternalLabels,
12 diffLabels,
13 hasAlchemyLabels,
14 toLabels,
15} from "../Labels.ts";
16import type { Providers } from "../Providers.ts";
17import {
18 DEFAULT_HIVE_VERSION,
19 fieldMask,
20 fingerprint,
21 listAtLocation,
22 listLabeledPages,
23 locationParent,
24 normalizeLocation,
25 parseName,
26 replaceOnIdentity,
27 ResourceNotResolved,
28 sameText,
29 toPhysicalId,
30 userLabels,
31 waitForOperation,
32 waitUntilExists,
33 waitUntilGone,
34 waitUntilReady,
35} from "./internal.ts";
36
37export type HiveMetastoreConfig = {
38 /**
39 * Hive metastore schema version. Immutable.
40 * @default "3.1.2"
41 */
42 version?: string;
43 /**
44 * hive-site.xml overrides applied to the primary service.
45 */
46 configOverrides?: Record<string, string>;
47 /**
48 * Endpoint protocol (`THRIFT` or `GRPC`).
49 */
50 endpointProtocol?:
51 | metastore.HiveMetastoreConfigEndpointProtocolEnum
52 | (string & {});
53 /**
54 * Kerberos service-principal configuration.
55 */
56 kerberosConfig?: metastore.KerberosConfig;
57 /**
58 * Auxiliary Hive versions keyed by service name.
59 */
60 auxiliaryVersions?: metastore.AuxiliaryVersionConfigMap;
61};
62
63export type ServiceProps = {
64 /**
65 * Service id (the `{service}` segment of
66 * `projects/{project}/locations/{location}/services/{service}`). If
67 * omitted, a unique RFC1035 name is generated. Must be 2-63 characters,
68 * start with a letter, and end with a letter or number. Immutable —
69 * changing it replaces the service.
70 */
71 serviceId?: string;
72 /**
73 * Region (`us-central1`, …). Immutable — changing it replaces the
74 * service. `US-CENTRAL1` is accepted and normalized to `us-central1`.
75 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
76 */
77 location?: string;
78 /**
79 * Hive metastore software configuration. `version` is immutable.
80 */
81 hiveMetastoreConfig?: HiveMetastoreConfig;
82 /**
83 * Service tier (`DEVELOPER` or `ENTERPRISE`).
84 * @default "DEVELOPER"
85 */
86 tier?: metastore.ServiceTierEnum | (string & {});
87 /**
88 * TCP port for the Hive endpoint.
89 * @default 9083
90 */
91 port?: number;
92 /**
93 * VPC network
94 * (`projects/{project}/global/networks/{network}`). Immutable.
95 */
96 network?: string;
97 /**
98 * Backend database type. Immutable.
99 */
100 databaseType?: metastore.ServiceDatabaseTypeEnum | (string & {});
101 /**
102 * When true, delete is rejected until protection is cleared.
103 * @default false
104 */
105 deletionProtection?: boolean;
106 /**
107 * Release channel. Immutable.
108 */
109 releaseChannel?: metastore.ServiceReleaseChannelEnum | (string & {});
110 /**
111 * Customer-managed encryption. Immutable.
112 */
113 encryptionConfig?: metastore.EncryptionConfig;
114 /**
115 * Consumer-side network configuration. Immutable.
116 */
117 networkConfig?: metastore.NetworkConfig;
118 /**
119 * Scaling configuration.
120 */
121 scalingConfig?: metastore.ScalingConfig;
122 /**
123 * Telemetry / log format. Defaults to JSON.
124 */
125 telemetryConfig?: metastore.TelemetryConfig;
126 /**
127 * Scheduled backup configuration.
128 */
129 scheduledBackup?: metastore.ScheduledBackup;
130 /**
131 * External metadata integrations (Data Catalog).
132 */
133 metadataIntegration?: metastore.MetadataIntegration;
134 /**
135 * One-hour UTC maintenance window. Not used with SPANNER.
136 */
137 maintenanceWindow?: metastore.MaintenanceWindow;
138 /**
139 * User labels. Alchemy ownership labels are merged in automatically.
140 */
141 labels?: Record<string, string>;
142};
143
144export type Service = Resource<
145 "GCP.Metastore.Service",
146 ServiceProps,
147 {
148 /** Full resource name. */
149 name: string;
150 /** Service id (last path segment). */
151 serviceId: string;
152 /** Project id. */
153 project: string;
154 /** Location id. */
155 location: string;
156 /** Hive metastore configuration. */
157 hiveMetastoreConfig: HiveMetastoreConfig | undefined;
158 /** Service tier. */
159 tier: string | undefined;
160 /** Hive endpoint port. */
161 port: number | undefined;
162 /** VPC network. */
163 network: string | undefined;
164 /** Backend database type. */
165 databaseType: string | undefined;
166 /** Whether delete is protected. */
167 deletionProtection: boolean;
168 /** Release channel. */
169 releaseChannel: string | undefined;
170 /** Customer-managed encryption, if any. */
171 encryptionConfig: metastore.EncryptionConfig | undefined;
172 /** Consumer network configuration. */
173 networkConfig: metastore.NetworkConfig | undefined;
174 /** Scaling configuration. */
175 scalingConfig: metastore.ScalingConfig | undefined;
176 /** Telemetry configuration. */
177 telemetryConfig: metastore.TelemetryConfig | undefined;
178 /** Scheduled backup configuration. */
179 scheduledBackup: metastore.ScheduledBackup | undefined;
180 /** External metadata integrations. */
181 metadataIntegration: metastore.MetadataIntegration | undefined;
182 /** Maintenance window. */
183 maintenanceWindow: metastore.MaintenanceWindow | undefined;
184 /** User labels (Alchemy ownership labels stripped). */
185 labels: Record<string, string>;
186 /** Hive endpoint URI. */
187 endpointUri: string | undefined;
188 /** Artifact Cloud Storage URI. */
189 artifactGcsUri: string | undefined;
190 /** Server-reported state. */
191 state: string | undefined;
192 /** Extra status text, if any. */
193 stateMessage: string | undefined;
194 /** Server-generated resource uid. */
195 uid: string | undefined;
196 /** RFC3339 creation timestamp. */
197 createTime: string | undefined;
198 /** RFC3339 last-update timestamp. */
199 updateTime: string | undefined;
200 },
201 never,
202 Providers
203>;
204
205/**
206 * A Dataproc Metastore service — a managed Hive metastore.
207 *
208 * Changing `serviceId`, `location`, Hive `version`, `network`,
209 * `databaseType`, `encryptionConfig`, `networkConfig`, or
210 * `releaseChannel` replaces the service. Labels, port, tier, scaling,
211 * telemetry, scheduled backups, metadata integration, maintenance
212 * window, Hive overrides, and deletion protection update in place.
213 *
214 * ### Creating a Service
215 * **Example:** Developer tier
216 * ```typescript
217 * const service = yield* GCP.Metastore.Service("Hive", {
218 * hiveMetastoreConfig: { version: "3.1.2" },
219 * tier: "DEVELOPER",
220 * labels: { env: "test" },
221 * });
222 * ```
223 *
224 * **Example:** Explicit id
225 * ```typescript
226 * const service = yield* GCP.Metastore.Service("Hive", {
227 * serviceId: "analytics-hive",
228 * hiveMetastoreConfig: { version: "3.1.2" },
229 * port: 9083,
230 * });
231 * ```
232 *
233 * ### Updating a Service
234 * **Example:** Labels and port
235 * ```typescript
236 * // Same logical id, changed props: the engine updates it in place.
237 * const service = yield* GCP.Metastore.Service("Hive", {
238 * hiveMetastoreConfig: { version: "3.1.2" },
239 * port: 9084,
240 * labels: { env: "prod" },
241 * });
242 * ```
243 *
244 * @resource
245 * @category Metastore
246 */
247export const Service = Resource<Service>("GCP.Metastore.Service");
248
249const resourceName = (project: string, location: string, serviceId: string) =>
250 `${locationParent(project, location)}/services/${serviceId}`;
251
252const toHiveConfig = (
253 config: metastore.HiveMetastoreConfig | undefined,
254): HiveMetastoreConfig | undefined =>
255 config === undefined
256 ? undefined
257 : {
258 version: config.version,
259 configOverrides: tagRecord(config.configOverrides),
260 endpointProtocol: config.endpointProtocol,
261 kerberosConfig: config.kerberosConfig,
262 auxiliaryVersions: config.auxiliaryVersions,
263 };
264
265const desiredHiveConfig = (
266 config: HiveMetastoreConfig | undefined,
267): metastore.HiveMetastoreConfig => ({
268 version: config?.version ?? DEFAULT_HIVE_VERSION,
269 configOverrides: config?.configOverrides,
270 endpointProtocol: config?.endpointProtocol,
271 kerberosConfig: config?.kerberosConfig,
272 auxiliaryVersions: config?.auxiliaryVersions,
273});
274
275const toAttrs = (item: metastore.Service, project: string, region: string) => {
276 const name = item.name ?? "";
277 const parsed = parseName(name, "services", region);
278 return {
279 name,
280 serviceId: parsed.id,
281 project: parsed.project || project,
282 location: parsed.location,
283 hiveMetastoreConfig: toHiveConfig(item.hiveMetastoreConfig),
284 tier: item.tier,
285 port: item.port,
286 network: item.network,
287 databaseType: item.databaseType,
288 deletionProtection: item.deletionProtection === true,
289 releaseChannel: item.releaseChannel,
290 encryptionConfig: item.encryptionConfig,
291 networkConfig: item.networkConfig,
292 scalingConfig: item.scalingConfig,
293 telemetryConfig: item.telemetryConfig,
294 scheduledBackup: item.scheduledBackup,
295 metadataIntegration: item.metadataIntegration,
296 maintenanceWindow: item.maintenanceWindow,
297 labels: userLabels(item.labels),
298 endpointUri: item.endpointUri,
299 artifactGcsUri: item.artifactGcsUri,
300 state: item.state,
301 stateMessage: item.stateMessage,
302 uid: item.uid,
303 createTime: item.createTime,
304 updateTime: item.updateTime,
305 };
306};
307
308const getByName = (name: string) =>
309 name.length === 0
310 ? Effect.succeed(undefined)
311 : metastore
312 .getProjectsLocationsServices({ name })
313 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
314
315const listOwned = (project: string, region: string) =>
316 listAtLocation(project, (parent) =>
317 listLabeledPages(
318 metastore.listProjectsLocationsServices.pages({
319 parent,
320 pageSize: 1000,
321 }),
322 (page) => page.services,
323 (item) => item.labels,
324 ),
325 );
326
327export const ServiceProvider = () =>
328 Provider.succeed(Service, {
329 stables: [
330 "name",
331 "serviceId",
332 "project",
333 "location",
334 "network",
335 "databaseType",
336 "releaseChannel",
337 "uid",
338 "createTime",
339 ],
340
341 diff: Effect.fn(function* ({ news, olds, output }) {
342 if (!isResolved(news)) return undefined;
343 const env = yield* GcpEnvironment.current;
344 const previousVersion =
345 olds?.hiveMetastoreConfig?.version ??
346 output?.hiveMetastoreConfig?.version;
347 const nextVersion = news.hiveMetastoreConfig?.version ?? previousVersion;
348 const previousNetwork = olds?.network ?? output?.network;
349 const previousDatabase = olds?.databaseType ?? output?.databaseType;
350 const previousChannel = olds?.releaseChannel ?? output?.releaseChannel;
351 return replaceOnIdentity({
352 previousId: olds?.serviceId ?? output?.serviceId,
353 nextId: news.serviceId ?? olds?.serviceId ?? output?.serviceId,
354 previousLocation: normalizeLocation(
355 olds?.location ?? output?.location,
356 env.region,
357 ),
358 nextLocation: normalizeLocation(
359 news.location ?? olds?.location ?? output?.location,
360 env.region,
361 ),
362 extra:
363 (previousVersion !== undefined &&
364 nextVersion !== undefined &&
365 nextVersion !== previousVersion) ||
366 (previousNetwork !== undefined &&
367 news.network !== undefined &&
368 news.network !== previousNetwork) ||
369 (previousDatabase !== undefined &&
370 news.databaseType !== undefined &&
371 news.databaseType !== previousDatabase) ||
372 (previousChannel !== undefined &&
373 news.releaseChannel !== undefined &&
374 news.releaseChannel !== previousChannel) ||
375 (olds?.encryptionConfig !== undefined &&
376 news.encryptionConfig !== undefined &&
377 fingerprint(news.encryptionConfig) !==
378 fingerprint(olds.encryptionConfig)) ||
379 (olds?.networkConfig !== undefined &&
380 news.networkConfig !== undefined &&
381 fingerprint(news.networkConfig) !==
382 fingerprint(olds.networkConfig)),
383 });
384 }),
385
386 read: Effect.fn(function* ({ id, olds, output }) {
387 const env = yield* GcpEnvironment.current;
388 const serviceId = yield* toPhysicalId(
389 id,
390 olds?.serviceId,
391 output?.serviceId,
392 "service",
393 );
394 const location = normalizeLocation(
395 olds?.location ?? output?.location,
396 env.region,
397 );
398 const name =
399 output?.name ?? resourceName(env.project, location, serviceId);
400 const existing = yield* getByName(name);
401 if (existing === undefined) return undefined;
402 const attrs = toAttrs(existing, env.project, env.region);
403 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels)))
404 ? attrs
405 : Unowned(attrs);
406 }),
407
408 list: () =>
409 Effect.gen(function* () {
410 const env = yield* GcpEnvironment.current;
411 const items = yield* listOwned(env.project, env.region);
412 return items.map((item) => toAttrs(item, env.project, env.region));
413 }),
414
415 reconcile: Effect.fn(function* ({ id, news, output }) {
416 const env = yield* GcpEnvironment.current;
417 const serviceId = yield* toPhysicalId(
418 id,
419 news.serviceId,
420 output?.serviceId,
421 "service",
422 );
423 const location = normalizeLocation(
424 news.location ?? output?.location,
425 env.region,
426 );
427 const name = resourceName(env.project, location, serviceId);
428 const hive = desiredHiveConfig(news.hiveMetastoreConfig);
429 const desiredLabels = {
430 ...toLabels(news.labels),
431 ...(yield* createInternalLabels(id)),
432 };
433
434 let current = yield* getByName(output?.name ?? name);
435
436 if (current === undefined) {
437 const created = yield* metastore
438 .createProjectsLocationsServices({
439 parent: locationParent(env.project, location),
440 serviceId,
441 body: {
442 hiveMetastoreConfig: hive,
443 tier: news.tier,
444 port: news.port,
445 network: news.network,
446 databaseType: news.databaseType,
447 deletionProtection: news.deletionProtection,
448 releaseChannel: news.releaseChannel,
449 encryptionConfig: news.encryptionConfig,
450 networkConfig: news.networkConfig,
451 scalingConfig: news.scalingConfig,
452 telemetryConfig: news.telemetryConfig,
453 scheduledBackup: news.scheduledBackup,
454 metadataIntegration: news.metadataIntegration,
455 maintenanceWindow: news.maintenanceWindow,
456 labels: desiredLabels,
457 },
458 })
459 .pipe(Effect.catchTag("Conflict", () => Effect.succeed(undefined)));
460 if (created !== undefined) {
461 // ALREADY_EXISTS (6): a concurrent create won the race.
462 yield* waitForOperation(created).pipe(
463 Effect.catchTag("GCP.OperationFailed", (error) =>
464 error.code === 6 ? Effect.void : Effect.fail(error),
465 ),
466 );
467 }
468 current = yield* waitUntilExists(getByName(name), name);
469 current = yield* waitUntilReady(
470 getByName(name),
471 name,
472 (item) => item.state,
473 (item) => item.stateMessage,
474 );
475 }
476
477 if (current === undefined) {
478 return yield* new ResourceNotResolved({ name });
479 }
480
481 const observedLabels = tagRecord(current.labels);
482 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
483 const labelsChanged = upsert.length > 0 || removed.length > 0;
484 const hiveChanged =
485 fingerprint({
486 configOverrides: current.hiveMetastoreConfig?.configOverrides,
487 endpointProtocol: current.hiveMetastoreConfig?.endpointProtocol,
488 kerberosConfig: current.hiveMetastoreConfig?.kerberosConfig,
489 auxiliaryVersions: current.hiveMetastoreConfig?.auxiliaryVersions,
490 }) !==
491 fingerprint({
492 configOverrides: hive.configOverrides,
493 endpointProtocol: hive.endpointProtocol,
494 kerberosConfig: hive.kerberosConfig,
495 auxiliaryVersions: hive.auxiliaryVersions,
496 });
497 const mask = fieldMask([
498 labelsChanged && "labels",
499 news.port !== undefined && news.port !== current.port && "port",
500 news.tier !== undefined && !sameText(current.tier, news.tier) && "tier",
501 news.deletionProtection !== undefined &&
502 (current.deletionProtection === true) !==
503 (news.deletionProtection === true) &&
504 "deletionProtection",
505 news.scalingConfig !== undefined &&
506 fingerprint(current.scalingConfig) !==
507 fingerprint(news.scalingConfig) &&
508 "scalingConfig",
509 news.telemetryConfig !== undefined &&
510 fingerprint(current.telemetryConfig) !==
511 fingerprint(news.telemetryConfig) &&
512 "telemetryConfig",
513 news.scheduledBackup !== undefined &&
514 fingerprint(current.scheduledBackup) !==
515 fingerprint(news.scheduledBackup) &&
516 "scheduledBackup",
517 news.metadataIntegration !== undefined &&
518 fingerprint(current.metadataIntegration) !==
519 fingerprint(news.metadataIntegration) &&
520 "metadataIntegration",
521 news.maintenanceWindow !== undefined &&
522 fingerprint(current.maintenanceWindow) !==
523 fingerprint(news.maintenanceWindow) &&
524 "maintenanceWindow",
525 hiveChanged && "hiveMetastoreConfig",
526 ]);
527
528 if (mask.length > 0) {
529 const operation = yield* metastore.patchProjectsLocationsServices({
530 name: current.name ?? name,
531 updateMask: mask,
532 body: {
533 labels: desiredLabels,
534 port: news.port,
535 tier: news.tier,
536 deletionProtection: news.deletionProtection,
537 scalingConfig: news.scalingConfig,
538 telemetryConfig: news.telemetryConfig,
539 scheduledBackup: news.scheduledBackup,
540 metadataIntegration: news.metadataIntegration,
541 maintenanceWindow: news.maintenanceWindow,
542 hiveMetastoreConfig: hive,
543 },
544 });
545 yield* waitForOperation(operation);
546 current = yield* waitUntilReady(
547 getByName(current.name ?? name),
548 current.name ?? name,
549 (item) => item.state,
550 (item) => item.stateMessage,
551 );
552 }
553
554 if (current === undefined) {
555 return yield* new ResourceNotResolved({ name });
556 }
557
558 return toAttrs(current, env.project, env.region);
559 }),
560
561 delete: Effect.fn(function* ({ output }) {
562 const existing = yield* getByName(output.name);
563 if (existing === undefined) return;
564 if (existing.deletionProtection === true) {
565 const unlocked = yield* metastore
566 .patchProjectsLocationsServices({
567 name: output.name,
568 updateMask: "deletionProtection",
569 body: { deletionProtection: false },
570 })
571 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
572 if (unlocked !== undefined) {
573 // NOT_FOUND (5): already gone.
574 yield* waitForOperation(unlocked).pipe(
575 Effect.catchTag("GCP.OperationFailed", (error) =>
576 error.code === 5 ? Effect.void : Effect.fail(error),
577 ),
578 );
579 }
580 }
581 const operation = yield* metastore
582 .deleteProjectsLocationsServices({ name: output.name })
583 .pipe(
584 Effect.retry({
585 while: (error) => error._tag === "Conflict",
586 times: 8,
587 schedule: Schedule.spaced("2 seconds"),
588 }),
589 Effect.catchTag("NotFound", () => Effect.succeed(undefined)),
590 );
591 if (operation !== undefined) {
592 // NOT_FOUND (5): already gone.
593 yield* waitForOperation(operation).pipe(
594 Effect.catchTag("GCP.OperationFailed", (error) =>
595 error.code === 5 ? Effect.void : Effect.fail(error),
596 ),
597 );
598 }
599 yield* waitUntilGone(getByName(output.name), output.name);
600 }),
601 });