1import * as GCP from "@/GCP";
2import * as Test from "@/Test/Alchemy";
3import * as dataproc from "@distilled.cloud/gcp/dataproc_v1";
4import { expect } from "alchemy-test";
5import * as Effect from "effect/Effect";
6import { MinimumLogLevel } from "effect/References";
7import * as Schedule from "effect/Schedule";
8import { GcpEnvironment } from "@/GCP/Environment";
9
10const { test } = Test.make({ providers: GCP.providers() });
11
12const logLevel = Effect.provideService(
13 MinimumLogLevel,
14 process.env.DEBUG ? "Debug" : "Info",
15);
16
17// Each serverless workload reserves 12 vCPUs and hundreds of GB of Hyperdisk;
18// next to the rest of the suite the testing project's quotas run out
19// (`Quota 'HDB_TOTAL_GB' exceeded. Limit: 500.0 in region us-central1`,
20// `Insufficient 'CPUS_ALL_REGIONS' quota`). Passes solo in ~2 minutes; set
21// GCP_TEST_DATAPROC_SERVERLESS=1 on a project with headroom.
22const runLifecycle =
23 !!process.env.GCP_TEST_DATAPROC_SERVERLESS && !process.env.FAST;
24
25const waitUntilGone = (name: string) =>
26 dataproc.getProjectsLocationsBatches({ name }).pipe(
27 Effect.as("found" as const),
28 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
29 Effect.repeat({
30 schedule: Schedule.spaced("3 seconds"),
31 until: (status) => status === "gone",
32 times: 10,
33 }),
34 );
35
36test.provider(
37 "getProjectsLocationsBatches on a missing batch fails with NotFound",
38 (stack) =>
39 Effect.gen(function* () {
40 const { project } = yield* GcpEnvironment.current;
41 yield* stack.destroy();
42
43 const error = yield* Effect.flip(
44 dataproc.getProjectsLocationsBatches({
45 name: `projects/${project}/locations/us-central1/batches/alchemy-dataproc-missing`,
46 }),
47 );
48 expect(error._tag).toEqual("NotFound");
49
50 yield* stack.destroy();
51 }).pipe(logLevel),
52 { tags: ["provider:gcp", "provider:gcp:dataproc", "live"], timeout: 90_000 },
53);
54
55test.provider.skipIf(!runLifecycle)(
56 "create and delete a spark batch",
57 (stack) =>
58 Effect.gen(function* () {
59 yield* stack.destroy();
60
61 const created = yield* stack.deploy(
62 Effect.gen(function* () {
63 return yield* GCP.Dataproc.Batch("Pi", {
64 location: "us-central1",
65 sparkBatch: {
66 mainClass: "org.apache.spark.examples.SparkPi",
67 jarFileUris: [
68 "file:///usr/lib/spark/examples/jars/spark-examples.jar",
69 ],
70 args: ["1"],
71 },
72 environmentConfig: { executionConfig: { ttl: "600s" } },
73 labels: { env: "test" },
74 });
75 }),
76 );
77
78 expect(created.name).toContain("/batches/");
79 expect(created.batchId).toEqual(expect.any(String));
80 expect(created.location).toEqual("us-central1");
81 expect(created.labels).toMatchObject({ env: "test" });
82 expect(created.state).toEqual(expect.any(String));
83
84 const fetched = yield* dataproc.getProjectsLocationsBatches({
85 name: created.name,
86 });
87 expect(fetched.name).toEqual(created.name);
88 expect(fetched.labels?.env).toEqual("test");
89 expect(fetched.sparkBatch?.mainClass).toEqual(
90 "org.apache.spark.examples.SparkPi",
91 );
92
93 yield* stack.destroy();
94
95 const gone = yield* waitUntilGone(created.name);
96 expect(gone).toEqual("gone");
97 }).pipe(logLevel),
98 { tags: ["provider:gcp", "provider:gcp:dataproc", "live"], timeout: 600_000 },
99);