1import * as GCP from "@/GCP";
2import * as Test from "@/Test/Alchemy";
3import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
4import { expect } from "alchemy-test";
5import * as Effect from "effect/Effect";
6import { MinimumLogLevel } from "effect/References";
7import * as Schedule from "effect/Schedule";
8
9const { test } = Test.make({ providers: GCP.providers() });
10
11const logLevel = Effect.provideService(
12 MinimumLogLevel,
13 process.env.DEBUG ? "Debug" : "Info",
14);
15
16const waitUntilGone = (projectId: string, datasetId: string, tableId: string) =>
17 bigquery
18 .getTables({
19 projectId,
20 datasetId,
21 tableId,
22 })
23 .pipe(
24 Effect.as("found" as const),
25 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
26 Effect.repeat({
27 schedule: Schedule.spaced("1 second"),
28 until: (status) => status === "gone",
29 times: 10,
30 }),
31 );
32
33test.provider(
34 "create, update, and delete a table",
35 (stack) =>
36 Effect.gen(function* () {
37 yield* stack.destroy();
38
39 const created = yield* stack.deploy(
40 Effect.gen(function* () {
41 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
42 location: "US-CENTRAL1",
43 forceDestroy: true,
44 });
45 return yield* GCP.BigQuery.Table("Events", {
46 datasetId: dataset.datasetId,
47 schema: [
48 { name: "id", type: "STRING" },
49 { name: "created_at", type: "TIMESTAMP" },
50 ],
51 labels: { env: "test" },
52 description: "order events",
53 timePartitioning: { type: "DAY", field: "created_at" },
54 clustering: { fields: ["id"] },
55 });
56 }),
57 );
58
59 expect(created.tableId).toEqual(expect.any(String));
60 expect(created.tableId).toMatch(/^[a-zA-Z0-9_]+$/);
61 expect(created.datasetId).toEqual(expect.any(String));
62 expect(created.project).toEqual(expect.any(String));
63 expect(created.type).toEqual("TABLE");
64 expect(created.labels).toMatchObject({ env: "test" });
65 expect(created.description).toEqual("order events");
66 expect(created.timePartitioning).toMatchObject({
67 type: "DAY",
68 field: "created_at",
69 });
70 expect(created.clustering).toEqual({ fields: ["id"] });
71 expect(created.schema).toEqual(
72 expect.arrayContaining([
73 expect.objectContaining({ name: "id", type: "STRING" }),
74 expect.objectContaining({ name: "created_at", type: "TIMESTAMP" }),
75 ]),
76 );
77
78 const fetched = yield* bigquery.getTables({
79 projectId: created.project,
80 datasetId: created.datasetId,
81 tableId: created.tableId,
82 view: "FULL",
83 });
84 expect(fetched.tableReference?.tableId).toEqual(created.tableId);
85 expect(fetched.labels?.env).toEqual("test");
86 expect(fetched.labels?.["alchemy-id"]).toEqual(expect.any(String));
87 expect(fetched.description).toEqual("order events");
88 expect(fetched.timePartitioning?.type).toEqual("DAY");
89 expect(fetched.timePartitioning?.field).toEqual("created_at");
90 expect(fetched.clustering?.fields).toEqual(["id"]);
91
92 const inserted = yield* bigquery.insertAllTabledata({
93 projectId: created.project,
94 datasetId: created.datasetId,
95 tableId: created.tableId,
96 body: {
97 rows: [{ json: { id: "row-1", created_at: "2024-01-01T00:00:00Z" } }],
98 },
99 });
100 expect(inserted.insertErrors ?? []).toEqual([]);
101
102 const listed = yield* bigquery
103 .listTabledata({
104 projectId: created.project,
105 datasetId: created.datasetId,
106 tableId: created.tableId,
107 maxResults: 10,
108 })
109 .pipe(
110 Effect.repeat({
111 schedule: Schedule.spaced("1 second"),
112 until: (page) => (page.rows ?? []).length > 0,
113 times: 10,
114 }),
115 );
116 expect((listed.rows ?? []).length).toBeGreaterThan(0);
117
118 const updated = yield* stack.deploy(
119 Effect.gen(function* () {
120 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
121 datasetId: created.datasetId,
122 location: "US-CENTRAL1",
123 forceDestroy: true,
124 });
125 return yield* GCP.BigQuery.Table("Events", {
126 datasetId: dataset.datasetId,
127 tableId: created.tableId,
128 schema: [
129 { name: "id", type: "STRING" },
130 { name: "created_at", type: "TIMESTAMP" },
131 { name: "name", type: "STRING" },
132 ],
133 labels: { env: "prod", role: "events" },
134 description: "order events v2",
135 friendlyName: "Order Events",
136 timePartitioning: {
137 type: "DAY",
138 field: "created_at",
139 expirationMs: "2592000000",
140 },
141 clustering: { fields: ["id"] },
142 });
143 }),
144 );
145
146 expect(updated.name).toEqual(created.name);
147 expect(updated.tableId).toEqual(created.tableId);
148 expect(updated.labels).toMatchObject({ env: "prod", role: "events" });
149 expect(updated.description).toEqual("order events v2");
150 expect(updated.friendlyName).toEqual("Order Events");
151 expect(updated.schema).toEqual(
152 expect.arrayContaining([
153 expect.objectContaining({ name: "id", type: "STRING" }),
154 expect.objectContaining({ name: "created_at", type: "TIMESTAMP" }),
155 expect.objectContaining({ name: "name", type: "STRING" }),
156 ]),
157 );
158 expect(updated.timePartitioning).toMatchObject({
159 type: "DAY",
160 field: "created_at",
161 expirationMs: "2592000000",
162 });
163
164 const fetchedUpdate = yield* bigquery.getTables({
165 projectId: created.project,
166 datasetId: created.datasetId,
167 tableId: created.tableId,
168 view: "FULL",
169 });
170 expect(fetchedUpdate.labels?.env).toEqual("prod");
171 expect(fetchedUpdate.labels?.role).toEqual("events");
172 expect(fetchedUpdate.description).toEqual("order events v2");
173 expect(fetchedUpdate.friendlyName).toEqual("Order Events");
174 expect(fetchedUpdate.timePartitioning?.expirationMs).toEqual(
175 "2592000000",
176 );
177 expect(
178 (fetchedUpdate.schema?.fields ?? []).map((field) => field.name),
179 ).toEqual(expect.arrayContaining(["id", "created_at", "name"]));
180
181 yield* stack.destroy();
182
183 const gone = yield* waitUntilGone(
184 created.project,
185 created.datasetId,
186 created.tableId,
187 );
188 expect(gone).toEqual("gone");
189 }).pipe(logLevel),
190 { tags: ["provider:gcp", "provider:gcp:bigquery", "live"], timeout: 90_000 },
191);