1import * as GCP from "@/GCP";
2import * as Output from "@/Output";
3import * as Test from "@/Test/Alchemy";
4import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
5import { expect } from "alchemy-test";
6import * as Effect from "effect/Effect";
7import { MinimumLogLevel } from "effect/References";
8import * as Schedule from "effect/Schedule";
9import { withDataplexSlot } from "./quota.ts";
10
11const { test } = Test.make({ providers: GCP.providers() });
12
13const logLevel = Effect.provideService(
14 MinimumLogLevel,
15 process.env.DEBUG ? "Debug" : "Info",
16);
17
18const waitUntilGone = (name: string) =>
19 dataplex.getProjectsLocationsLakesZonesAssets({ name }).pipe(
20 Effect.as("found" as const),
21 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
22 Effect.repeat({
23 schedule: Schedule.spaced("2 seconds"),
24 until: (status) => status === "gone",
25 times: 10,
26 }),
27 );
28
29// A lake (~2.5 min) plus a bucket asset (~1.5 min), then teardown of both,
30// runs past 5 minutes.
31test.provider.skipIf(!process.env.GCP_TEST_SLOW || !!process.env.FAST)(
32 "create, update, and delete a lake asset",
33 (stack) =>
34 Effect.gen(function* () {
35 yield* stack.destroy();
36
37 const created = yield* stack.deploy(
38 Effect.gen(function* () {
39 const bucket = yield* GCP.Storage.Bucket("LandingData", {
40 location: "US-CENTRAL1",
41 forceDestroy: true,
42 labels: { env: "test" },
43 });
44 const lake = yield* GCP.Dataplex.Lake("Warehouse", {
45 location: "us-central1",
46 labels: { env: "test" },
47 });
48 const zone = yield* GCP.Dataplex.LakesZone("Landing", {
49 lake: lake.name,
50 type: "RAW",
51 locationType: "SINGLE_REGION",
52 discoverySpec: { enabled: false },
53 labels: { env: "test" },
54 });
55 const asset = yield* GCP.Dataplex.LakesAsset("RawBucket", {
56 zone: zone.name,
57 displayName: "landing bucket",
58 labels: { env: "test" },
59 resourceSpec: {
60 type: "STORAGE_BUCKET",
61 name: Output.interpolate`projects/${bucket.projectNumber}/buckets/${bucket.bucketName}`,
62 },
63 discoverySpec: { enabled: false },
64 });
65 return { bucket, lake, zone, asset };
66 }),
67 );
68
69 expect(created.asset.name).toContain("/assets/");
70 expect(created.asset.assetId).toEqual(expect.any(String));
71 expect(created.asset.zone).toEqual(created.zone.name);
72 expect(created.asset.resourceType).toEqual("STORAGE_BUCKET");
73 expect(created.asset.labels).toMatchObject({ env: "test" });
74
75 const fetched = yield* dataplex.getProjectsLocationsLakesZonesAssets({
76 name: created.asset.name,
77 });
78 expect(fetched.name).toEqual(created.asset.name);
79 expect(fetched.labels?.env).toEqual("test");
80 expect(fetched.resourceSpec?.type).toEqual("STORAGE_BUCKET");
81
82 const updated = yield* stack.deploy(
83 Effect.gen(function* () {
84 const bucket = yield* GCP.Storage.Bucket("LandingData", {
85 bucketName: created.bucket.bucketName,
86 location: "US-CENTRAL1",
87 forceDestroy: true,
88 labels: { env: "test" },
89 });
90 const lake = yield* GCP.Dataplex.Lake("Warehouse", {
91 lakeId: created.lake.lakeId,
92 location: "us-central1",
93 labels: { env: "test" },
94 });
95 const zone = yield* GCP.Dataplex.LakesZone("Landing", {
96 lake: lake.name,
97 zoneId: created.zone.zoneId,
98 type: "RAW",
99 locationType: "SINGLE_REGION",
100 discoverySpec: { enabled: false },
101 labels: { env: "test" },
102 });
103 const asset = yield* GCP.Dataplex.LakesAsset("RawBucket", {
104 zone: zone.name,
105 assetId: created.asset.assetId,
106 displayName: "landing bucket b",
107 description: "updated asset",
108 labels: { env: "prod", team: "data" },
109 resourceSpec: {
110 type: "STORAGE_BUCKET",
111 name: Output.interpolate`projects/${bucket.projectNumber}/buckets/${bucket.bucketName}`,
112 },
113 discoverySpec: { enabled: false },
114 });
115 return { bucket, lake, zone, asset };
116 }),
117 );
118
119 expect(updated.asset.name).toEqual(created.asset.name);
120 expect(updated.asset.displayName).toEqual("landing bucket b");
121 expect(updated.asset.description).toEqual("updated asset");
122 expect(updated.asset.labels).toMatchObject({ env: "prod", team: "data" });
123
124 yield* stack.destroy();
125 const gone = yield* waitUntilGone(created.asset.name);
126 expect(gone).toEqual("gone");
127 }).pipe(logLevel, withDataplexSlot),
128 { tags: ["provider:gcp", "provider:gcp:dataplex", "live"], timeout: 900_000 },
129);