1import * as GCP from "@/GCP";
2import * as Output from "@/Output";
3import * as Test from "@/Test/Alchemy";
4import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
5import { expect } from "alchemy-test";
6import * as Effect from "effect/Effect";
7import { MinimumLogLevel } from "effect/References";
8import * as Schedule from "effect/Schedule";
9import { withDataplexSlot } from "./quota.ts";
10
11const { test } = Test.make({ providers: GCP.providers() });
12
13const logLevel = Effect.provideService(
14 MinimumLogLevel,
15 process.env.DEBUG ? "Debug" : "Info",
16);
17
18const waitUntilGone = (name: string) =>
19 dataplex.getProjectsLocationsLakesZonesEntities({ name, view: "BASIC" }).pipe(
20 Effect.as("found" as const),
21 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
22 Effect.repeat({
23 schedule: Schedule.spaced("2 seconds"),
24 until: (status) => status === "gone",
25 times: 10,
26 }),
27 );
28
29// A lake (~2.5 min) plus a bucket asset (~1.5 min), then teardown of both,
30// runs past 5 minutes.
31test.provider.skipIf(!process.env.GCP_TEST_SLOW || !!process.env.FAST)(
32 "create, update, and delete a lake entity",
33 (stack) =>
34 Effect.gen(function* () {
35 yield* stack.destroy();
36
37 const created = yield* stack.deploy(
38 Effect.gen(function* () {
39 const bucket = yield* GCP.Storage.Bucket("EntityData", {
40 location: "US-CENTRAL1",
41 forceDestroy: true,
42 });
43 const lake = yield* GCP.Dataplex.Lake("Warehouse", {
44 location: "us-central1",
45 labels: { env: "test" },
46 });
47 const zone = yield* GCP.Dataplex.LakesZone("Landing", {
48 lake: lake.name,
49 type: "RAW",
50 locationType: "SINGLE_REGION",
51 discoverySpec: { enabled: false },
52 labels: { env: "test" },
53 });
54 const asset = yield* GCP.Dataplex.LakesAsset("RawBucket", {
55 zone: zone.name,
56 resourceSpec: {
57 type: "STORAGE_BUCKET",
58 name: Output.interpolate`projects/${bucket.projectNumber}/buckets/${bucket.bucketName}`,
59 },
60 discoverySpec: { enabled: false },
61 labels: { env: "test" },
62 });
63 const entity = yield* GCP.Dataplex.LakesEntity("Events", {
64 zone: zone.name,
65 asset: asset.assetId,
66 type: "TABLE",
67 system: "CLOUD_STORAGE",
68 dataPath: Output.interpolate`gs://${bucket.bucketName}/events`,
69 format: { format: "PARQUET" },
70 displayName: "events a",
71 description: "event table",
72 schema: {
73 userManaged: true,
74 fields: [{ name: "id", type: "STRING", mode: "REQUIRED" }],
75 },
76 });
77 return { bucket, lake, zone, asset, entity };
78 }),
79 );
80
81 expect(created.entity.name).toContain("/entities/");
82 expect(created.entity.entityId).toEqual(expect.any(String));
83 expect(created.entity.type).toEqual("TABLE");
84 expect(created.entity.system).toEqual("CLOUD_STORAGE");
85 expect(created.entity.asset).toEqual(created.asset.assetId);
86 expect(created.entity.description).toEqual("event table");
87
88 const fetched = yield* dataplex.getProjectsLocationsLakesZonesEntities({
89 name: created.entity.name,
90 view: "FULL",
91 });
92 expect(fetched.name).toEqual(created.entity.name);
93 expect(fetched.type).toEqual("TABLE");
94 expect(fetched.description).toEqual("event table");
95
96 const updated = yield* stack.deploy(
97 Effect.gen(function* () {
98 const bucket = yield* GCP.Storage.Bucket("EntityData", {
99 bucketName: created.bucket.bucketName,
100 location: "US-CENTRAL1",
101 forceDestroy: true,
102 });
103 const lake = yield* GCP.Dataplex.Lake("Warehouse", {
104 lakeId: created.lake.lakeId,
105 location: "us-central1",
106 labels: { env: "test" },
107 });
108 const zone = yield* GCP.Dataplex.LakesZone("Landing", {
109 lake: lake.name,
110 zoneId: created.zone.zoneId,
111 type: "RAW",
112 locationType: "SINGLE_REGION",
113 discoverySpec: { enabled: false },
114 labels: { env: "test" },
115 });
116 const asset = yield* GCP.Dataplex.LakesAsset("RawBucket", {
117 zone: zone.name,
118 assetId: created.asset.assetId,
119 resourceSpec: {
120 type: "STORAGE_BUCKET",
121 name: Output.interpolate`projects/${bucket.projectNumber}/buckets/${bucket.bucketName}`,
122 },
123 discoverySpec: { enabled: false },
124 labels: { env: "test" },
125 });
126 const entity = yield* GCP.Dataplex.LakesEntity("Events", {
127 zone: zone.name,
128 entityId: created.entity.entityId,
129 asset: asset.assetId,
130 type: "TABLE",
131 system: "CLOUD_STORAGE",
132 dataPath: Output.interpolate`gs://${bucket.bucketName}/events`,
133 format: { format: "PARQUET" },
134 displayName: "events b",
135 description: "event table b",
136 schema: {
137 userManaged: true,
138 fields: [
139 { name: "id", type: "STRING", mode: "REQUIRED" },
140 { name: "payload", type: "STRING", mode: "NULLABLE" },
141 ],
142 },
143 });
144 return { bucket, lake, zone, asset, entity };
145 }),
146 );
147
148 expect(updated.entity.name).toEqual(created.entity.name);
149 expect(updated.entity.displayName).toEqual("events b");
150 expect(updated.entity.description).toEqual("event table b");
151
152 yield* stack.destroy();
153 const gone = yield* waitUntilGone(created.entity.name);
154 expect(gone).toEqual("gone");
155 }).pipe(logLevel, withDataplexSlot),
156 { tags: ["provider:gcp", "provider:gcp:dataplex", "live"], timeout: 900_000 },
157);