1import * as GCP from "@/GCP";
2import * as Test from "@/Test/Alchemy";
3import * as bqdt from "@distilled.cloud/gcp/bigquerydatatransfer_v1";
4import { expect } from "alchemy-test";
5import * as Effect from "effect/Effect";
6import { MinimumLogLevel } from "effect/References";
7import * as Schedule from "effect/Schedule";
8import { GcpEnvironment } from "@/GCP/Environment";
9
10const { test } = Test.make({ providers: GCP.providers() });
11
12const logLevel = Effect.provideService(
13 MinimumLogLevel,
14 process.env.DEBUG ? "Debug" : "Info",
15);
16
17const LOCATION = "us-central1";
18
19const waitUntilGone = (name: string) =>
20 bqdt.getProjectsLocationsTransferConfigs({ name }).pipe(
21 Effect.as("found" as const),
22 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
23 Effect.repeat({
24 schedule: Schedule.spaced("1 second"),
25 until: (status) => status === "gone",
26 times: 10,
27 }),
28 );
29
30test.provider(
31 "getProjectsLocationsTransferConfigs on a missing config fails with NotFound",
32 (stack) =>
33 Effect.gen(function* () {
34 const { project } = yield* GcpEnvironment.current;
35 yield* stack.destroy();
36
37 const error = yield* Effect.flip(
38 bqdt.getProjectsLocationsTransferConfigs({
39 name: `projects/${project}/locations/${LOCATION}/transferConfigs/00000000-0000-0000-0000-000000000000`,
40 }),
41 );
42 expect(error._tag).toBe("NotFound");
43
44 yield* stack.destroy();
45 }).pipe(logLevel),
46 {
47 tags: ["provider:gcp", "provider:gcp:bigquerydatatransfer", "live"],
48 timeout: 90_000,
49 },
50);
51
52test.provider(
53 "create, update, and delete a transfer config",
54 (stack) =>
55 Effect.gen(function* () {
56 yield* stack.destroy();
57
58 const created = yield* stack.deploy(
59 Effect.gen(function* () {
60 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
61 location: "US-CENTRAL1",
62 forceDestroy: true,
63 });
64 return yield* GCP.BigQueryDataTransfer.TransferConfig("Nightly", {
65 location: "us-central1",
66 dataSourceId: "scheduled_query",
67 destinationDatasetId: dataset.datasetId,
68 displayName: "nightly",
69 schedule: "every 24 hours",
70 scheduleOptions: { disableAutoScheduling: true },
71 params: {
72 query: "SELECT 1 AS n",
73 destination_table_name_template: "nightly",
74 write_disposition: "WRITE_TRUNCATE",
75 },
76 });
77 }),
78 );
79
80 expect(created.name).toContain("/transferConfigs/");
81 expect(created.transferConfigId).toEqual(expect.any(String));
82 expect(created.location).toEqual("us-central1");
83 expect(created.dataSourceId).toEqual("scheduled_query");
84 expect(created.displayName).toEqual("nightly");
85 expect(created.disabled).toEqual(false);
86 expect(created.scheduleOptions?.disableAutoScheduling).toEqual(true);
87 expect(created.params).toMatchObject({
88 query: "SELECT 1 AS n",
89 destination_table_name_template: "nightly",
90 write_disposition: "WRITE_TRUNCATE",
91 });
92
93 const fetched = yield* bqdt.getProjectsLocationsTransferConfigs({
94 name: created.name,
95 });
96 expect(fetched.name).toEqual(created.name);
97 expect(fetched.dataSourceId).toEqual("scheduled_query");
98 expect(fetched.displayName).toContain("alchemy-id=");
99 expect(fetched.displayName).toContain("nightly");
100 expect(fetched.scheduleOptions?.disableAutoScheduling).toEqual(true);
101 expect(fetched.params).toMatchObject({
102 query: "SELECT 1 AS n",
103 destination_table_name_template: "nightly",
104 });
105
106 const updated = yield* stack.deploy(
107 Effect.gen(function* () {
108 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
109 location: "US-CENTRAL1",
110 forceDestroy: true,
111 });
112 return yield* GCP.BigQueryDataTransfer.TransferConfig("Nightly", {
113 location: "us-central1",
114 dataSourceId: "scheduled_query",
115 destinationDatasetId: dataset.datasetId,
116 displayName: "nightly v2",
117 schedule: "every 12 hours",
118 scheduleOptions: { disableAutoScheduling: true },
119 disabled: true,
120 params: {
121 query: "SELECT 2 AS n",
122 destination_table_name_template: "nightly",
123 write_disposition: "WRITE_TRUNCATE",
124 },
125 });
126 }),
127 );
128
129 expect(updated.name).toEqual(created.name);
130 expect(updated.transferConfigId).toEqual(created.transferConfigId);
131 expect(updated.displayName).toEqual("nightly v2");
132 expect(updated.disabled).toEqual(true);
133 expect(updated.scheduleOptions?.disableAutoScheduling).toEqual(true);
134 expect(updated.params).toMatchObject({ query: "SELECT 2 AS n" });
135
136 const fetchedUpdate = yield* bqdt.getProjectsLocationsTransferConfigs({
137 name: updated.name,
138 });
139 expect(fetchedUpdate.displayName).toContain("nightly v2");
140 expect(fetchedUpdate.disabled).toEqual(true);
141 expect(fetchedUpdate.params).toMatchObject({ query: "SELECT 2 AS n" });
142
143 yield* stack.destroy();
144
145 const gone = yield* waitUntilGone(updated.name);
146 expect(gone).toEqual("gone");
147 }).pipe(logLevel),
148 {
149 tags: ["provider:gcp", "provider:gcp:bigquerydatatransfer", "live"],
150 timeout: 120_000,
151 },
152);