1import * as GCP from "@/GCP";
2import * as Test from "@/Test/Alchemy";
3import * as Core from "@/Test/Core";
4import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
5import * as resourcemanager from "@distilled.cloud/gcp/cloudresourcemanager_v3";
6import { describe, expect } from "alchemy-test";
7import * as Effect from "effect/Effect";
8import { dockerAvailable, expectProbe } from "../bindingHost.ts";
9import BigQueryBindingsHost, {
10 Analytics,
11 Inserted,
12 Listed,
13 Queried,
14 ReadRows,
15 ReadWrite,
16 Written,
17} from "./fixtures/bindings-host.ts";
18
19const testOptions = { providers: GCP.providers() };
20const { test, beforeAll, afterAll } = Test.make(testOptions);
21const sharedStack = Core.scratchStack(testOptions, "BigQueryBindings");
22
23let baseUrl: string;
24let hostAccount: string;
25let dataset: { project: string; datasetId: string; location: string };
26let tables: {
27 queried: string;
28 listed: string;
29 inserted: string;
30 readRows: string;
31 written: string;
32 readWrite: string;
33};
34
35/** Run a GoogleSQL statement as the deploying principal. */
36const runQuery = (sql: string) =>
37 bigquery
38 .queryJobs({
39 projectId: dataset.project,
40 body: {
41 query: sql,
42 useLegacySql: false,
43 location: dataset.location,
44 defaultDataset: {
45 projectId: dataset.project,
46 datasetId: dataset.datasetId,
47 },
48 timeoutMs: 60_000,
49 },
50 })
51 .pipe(
52 Effect.map((response) => {
53 expect(response.jobComplete).toEqual(true);
54 return (response.rows ?? []).map((row) => row.f?.[0]?.v);
55 }),
56 );
57
58/**
59 * Seed one row with DML: committed storage is immediately visible to
60 * tabledata.list (streamed rows may still sit in the buffer).
61 */
62const seed = (table: string) =>
63 runQuery(`INSERT INTO ${table} (id) VALUES ('seed')`);
64
65/** Every project-level role (and its IAM Condition) `account` holds. */
66const projectGrantsOf = (account: string) =>
67 resourcemanager
68 .getIamPolicyProjects({
69 resource: `projects/${dataset.project}`,
70 body: { options: { requestedPolicyVersion: 3 } },
71 })
72 .pipe(
73 Effect.map((policy) =>
74 (policy.bindings ?? [])
75 .filter((binding) =>
76 (binding.members ?? []).includes(`serviceAccount:${account}`),
77 )
78 .map((binding) => ({
79 role: binding.role,
80 condition: binding.condition?.expression,
81 })),
82 ),
83 );
84
85/** bigquery.jobs.create is only grantable on the project. */
86const JOB_USER = [{ role: "roles/bigquery.jobUser", condition: undefined }];
87
88/** Roles `account` holds on a table's own IAM policy. */
89const tableRolesOf = (table: string, account: string) =>
90 bigquery
91 .getIamPolicyTables({
92 resource: table,
93 body: { options: { requestedPolicyVersion: 1 } },
94 })
95 .pipe(
96 Effect.map((policy) =>
97 (policy.bindings ?? [])
98 .filter((binding) =>
99 (binding.members ?? []).includes(`serviceAccount:${account}`),
100 )
101 .map((binding) => binding.role),
102 ),
103 );
104
105const LEGACY_DATASET_ROLES: Record<string, string> = {
106 READER: "roles/bigquery.dataViewer",
107 WRITER: "roles/bigquery.dataEditor",
108 OWNER: "roles/bigquery.dataOwner",
109};
110
111describe.skipIf(!dockerAvailable)(
112 "BigQuery Bindings",
113 {
114 tags: ["provider:gcp", "provider:gcp:bigquery", "provider:gcp:run", "live"],
115 },
116 () => {
117 beforeAll(
118 Effect.gen(function* () {
119 yield* sharedStack.destroy();
120 const out = yield* sharedStack.deploy(
121 Effect.gen(function* () {
122 const host = yield* BigQueryBindingsHost;
123 const analytics = yield* Analytics;
124 return {
125 uri: host.uri,
126 serviceAccount: host.serviceAccount,
127 dataset: {
128 project: analytics.project,
129 datasetId: analytics.datasetId,
130 location: analytics.location,
131 },
132 tables: {
133 // Not bound by any table binding: deployed for Query.
134 queried: (yield* Queried).name,
135 listed: (yield* Listed).name,
136 inserted: (yield* Inserted).name,
137 readRows: (yield* ReadRows).name,
138 written: (yield* Written).name,
139 readWrite: (yield* ReadWrite).name,
140 },
141 };
142 }),
143 );
144 baseUrl = out.uri!;
145 hostAccount = out.serviceAccount!;
146 dataset = out.dataset;
147 tables = out.tables;
148 }),
149 { timeout: 900_000 },
150 );
151
152 afterAll(sharedStack.destroy(), { timeout: 600_000 });
153
154 describe("Query", () => {
155 test.provider(
156 "queries the dataset as the host's service account, reading only that dataset",
157 (_stack) =>
158 Effect.gen(function* () {
159 yield* seed("queried");
160 const response = yield* expectProbe<bigquery.QueryResponse>(
161 baseUrl,
162 "query",
163 );
164 expect(response.jobComplete).toEqual(true);
165 expect((response.rows ?? []).map((row) => row.f?.[0]?.v)).toEqual([
166 "seed",
167 ]);
168
169 const live = yield* bigquery.getDatasets({
170 projectId: dataset.project,
171 datasetId: dataset.datasetId,
172 });
173 const datasetRoles = (live.access ?? [])
174 .filter((item) => item.userByEmail === hostAccount)
175 .map((item) => LEGACY_DATASET_ROLES[item.role!] ?? item.role);
176 expect(datasetRoles).toEqual(["roles/bigquery.dataViewer"]);
177 expect(yield* projectGrantsOf(hostAccount)).toEqual(JOB_USER);
178 }),
179 {
180 tags: ["provider:gcp", "provider:gcp:bigquery", "live"],
181 timeout: 600_000,
182 },
183 );
184 });
185
186 describe("InsertAll", () => {
187 test.provider(
188 "streams a row as the host's service account, granted on the table only",
189 (_stack) =>
190 Effect.gen(function* () {
191 const response = yield* expectProbe<{
192 insertErrors?: unknown[];
193 }>(baseUrl, "insertAll");
194 expect(response.insertErrors ?? []).toEqual([]);
195 expect(yield* runQuery("SELECT id FROM inserted")).toEqual([
196 "insert-all",
197 ]);
198 expect(yield* tableRolesOf(tables.inserted, hostAccount)).toEqual([
199 "roles/bigquery.dataEditor",
200 ]);
201 }),
202 {
203 tags: ["provider:gcp", "provider:gcp:bigquery", "live"],
204 timeout: 600_000,
205 },
206 );
207 });
208
209 describe("ListTabledata", () => {
210 test.provider(
211 "lists rows as the host's service account, granted on the table only",
212 (_stack) =>
213 Effect.gen(function* () {
214 yield* seed("listed");
215 const page = yield* expectProbe<bigquery.TableDataList>(
216 baseUrl,
217 "listTabledata",
218 );
219 expect((page.rows ?? []).map((row) => row.f?.[0]?.v)).toEqual([
220 "seed",
221 ]);
222 expect(yield* tableRolesOf(tables.listed, hostAccount)).toEqual([
223 "roles/bigquery.dataViewer",
224 ]);
225 }),
226 {
227 tags: ["provider:gcp", "provider:gcp:bigquery", "live"],
228 timeout: 600_000,
229 },
230 );
231 });
232
233 describe("ReadTable", () => {
234 test.provider(
235 "lists and queries the table as the host's service account",
236 (_stack) =>
237 Effect.gen(function* () {
238 yield* seed("read_rows");
239 const out = yield* expectProbe<{
240 listed: Array<{ id: string }>;
241 queried: Array<{ id: string }>;
242 }>(baseUrl, "readTable");
243 expect(out.listed).toEqual([{ id: "seed" }]);
244 expect(out.queried).toEqual([{ id: "seed" }]);
245 expect(yield* tableRolesOf(tables.readRows, hostAccount)).toEqual([
246 "roles/bigquery.dataViewer",
247 ]);
248 expect(yield* projectGrantsOf(hostAccount)).toEqual(JOB_USER);
249 }),
250 {
251 tags: ["provider:gcp", "provider:gcp:bigquery", "live"],
252 timeout: 600_000,
253 },
254 );
255 });
256
257 describe("WriteTable", () => {
258 test.provider(
259 "inserts a row as the host's service account, granted on the table only",
260 (_stack) =>
261 Effect.gen(function* () {
262 yield* expectProbe(baseUrl, "writeTable");
263 expect(yield* runQuery("SELECT id FROM written")).toEqual([
264 "written",
265 ]);
266 expect(yield* tableRolesOf(tables.written, hostAccount)).toEqual([
267 "roles/bigquery.dataEditor",
268 ]);
269 }),
270 {
271 tags: ["provider:gcp", "provider:gcp:bigquery", "live"],
272 timeout: 600_000,
273 },
274 );
275 });
276
277 describe("ReadWriteTable", () => {
278 test.provider(
279 "inserts and reads back a row as the host's service account",
280 (_stack) =>
281 Effect.gen(function* () {
282 const rows = yield* expectProbe<Array<{ id: string }>>(
283 baseUrl,
284 "readWriteTable",
285 );
286 expect(rows).toEqual([{ id: "read-write" }]);
287 expect(yield* runQuery("SELECT id FROM read_write")).toEqual([
288 "read-write",
289 ]);
290 expect(yield* tableRolesOf(tables.readWrite, hostAccount)).toEqual([
291 "roles/bigquery.dataEditor",
292 ]);
293 expect(yield* projectGrantsOf(hostAccount)).toEqual(JOB_USER);
294 }),
295 {
296 tags: ["provider:gcp", "provider:gcp:bigquery", "live"],
297 timeout: 600_000,
298 },
299 );
300 });
301 },
302);