1import * as GCP from "@/GCP";
2import * as Test from "@/Test/Alchemy";
3import * as datalineage from "@distilled.cloud/gcp/datalineage_v1";
4import { expect } from "alchemy-test";
5import * as Effect from "effect/Effect";
6import { MinimumLogLevel } from "effect/References";
7import * as Schedule from "effect/Schedule";
8
9const { test } = Test.make({ providers: GCP.providers() });
10
11const logLevel = Effect.provideService(
12 MinimumLogLevel,
13 process.env.DEBUG ? "Debug" : "Info",
14);
15
16const waitUntilGone = (name: string) =>
17 datalineage.getProjectsLocationsProcessesRuns({ name }).pipe(
18 Effect.as("found" as const),
19 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
20 Effect.repeat({
21 schedule: Schedule.spaced("1 second"),
22 until: (status) => status === "gone",
23 times: 10,
24 }),
25 );
26
27test.provider.skipIf(!!process.env.FAST)(
28 "create, update, and delete a lineage run",
29 (stack) =>
30 Effect.gen(function* () {
31 yield* stack.destroy();
32
33 const created = yield* stack.deploy(
34 Effect.gen(function* () {
35 const process = yield* GCP.DataLineage.Process("Etl", {
36 location: "us-central1",
37 displayName: "run parent",
38 attributes: { env: "test" },
39 });
40 const run = yield* GCP.DataLineage.ProcessesRun("Nightly", {
41 process: process.name,
42 startTime: "2024-01-01T00:00:00Z",
43 state: "STARTED",
44 displayName: "nightly a",
45 attributes: { env: "test" },
46 });
47 return { process, run };
48 }),
49 );
50
51 expect(created.run.name).toContain("/runs/");
52 expect(created.run.runId).toEqual(expect.any(String));
53 expect(created.run.process).toEqual(created.process.name);
54 expect(created.run.state).toEqual("STARTED");
55 expect(created.run.displayName).toEqual("nightly a");
56 expect(created.run.attributes).toMatchObject({ env: "test" });
57
58 const fetched = yield* datalineage.getProjectsLocationsProcessesRuns({
59 name: created.run.name,
60 });
61 expect(fetched.name).toEqual(created.run.name);
62 expect(fetched.state).toEqual("STARTED");
63 expect(
64 Object.keys(fetched.attributes ?? {}).some((key) =>
65 key.startsWith("alchemy-"),
66 ),
67 ).toEqual(true);
68
69 const updated = yield* stack.deploy(
70 Effect.gen(function* () {
71 const process = yield* GCP.DataLineage.Process("Etl", {
72 processId: created.process.processId,
73 location: "us-central1",
74 displayName: "run parent",
75 attributes: { env: "test" },
76 });
77 const run = yield* GCP.DataLineage.ProcessesRun("Nightly", {
78 process: process.name,
79 runId: created.run.runId,
80 startTime: "2024-01-01T00:00:00Z",
81 endTime: "2024-01-01T01:00:00Z",
82 state: "COMPLETED",
83 displayName: "nightly b",
84 attributes: { env: "prod" },
85 });
86 return { process, run };
87 }),
88 );
89
90 expect(updated.run.name).toEqual(created.run.name);
91 expect(updated.run.state).toEqual("COMPLETED");
92 expect(updated.run.endTime).toBeDefined();
93 expect(updated.run.displayName).toEqual("nightly b");
94 expect(updated.run.attributes).toMatchObject({ env: "prod" });
95
96 yield* stack.destroy();
97 const gone = yield* waitUntilGone(created.run.name);
98 expect(gone).toEqual("gone");
99 }).pipe(logLevel),
100 {
101 tags: ["provider:gcp", "provider:gcp:datalineage", "live"],
102 timeout: 90_000,
103 },
104);