1import * as AWS from "@/AWS";
2import { Dataset, Ruleset } from "@/AWS/DataBrew";
3import { Bucket } from "@/AWS/S3";
4import * as Test from "@/Test/Alchemy";
5import * as databrew from "@distilled.cloud/aws/databrew";
6import { expect } from "alchemy-test";
7import * as Effect from "effect/Effect";
8
9const { test } = Test.make({ providers: AWS.providers() });
10
11const getRuleset = (name: string) =>
12 databrew
13 .describeRuleset({ Name: name })
14 .pipe(
15 Effect.catchTag("ResourceNotFoundException", () =>
16 Effect.succeed(undefined),
17 ),
18 );
19
20test.provider(
21 "create, update, delete DataBrew ruleset",
22 (stack) =>
23 Effect.gen(function* () {
24 yield* stack.destroy();
25
26 const created = yield* stack.deploy(
27 Effect.gen(function* () {
28 const bucket = yield* Bucket("RulesetBucket", { forceDestroy: true });
29 const dataset = yield* Dataset("Source", {
30 format: "CSV",
31 input: {
32 s3InputDefinition: {
33 bucket: bucket.bucketName,
34 key: "raw/data.csv",
35 },
36 },
37 });
38 const ruleset = yield* Ruleset("Quality", {
39 description: "basic data quality",
40 targetArn: dataset.datasetArn,
41 rules: [
42 {
43 name: "no-missing-ids",
44 checkExpression: "AGG(MISSING_VALUES_PERCENTAGE) == :val1",
45 substitutionMap: { ":val1": "0" },
46 columnSelectors: [{ name: "id" }],
47 },
48 ],
49 tags: { Environment: "test" },
50 });
51 return { bucket, dataset, ruleset };
52 }),
53 );
54
55 expect(created.ruleset.rulesetName).toBeDefined();
56 expect(created.ruleset.rulesetArn).toContain(
57 `:ruleset/${created.ruleset.rulesetName}`,
58 );
59 expect(created.ruleset.targetArn).toEqual(created.dataset.datasetArn);
60
61 // out-of-band verification
62 const observed = yield* getRuleset(created.ruleset.rulesetName);
63 expect(observed?.TargetArn).toEqual(created.dataset.datasetArn);
64 expect(observed?.Rules?.length).toEqual(1);
65 expect(observed?.Rules?.[0]?.Name).toEqual("no-missing-ids");
66 expect(observed?.Tags?.["alchemy::id"]).toBeDefined();
67
68 // update: add a second rule
69 yield* stack.deploy(
70 Effect.gen(function* () {
71 const bucket = yield* Bucket("RulesetBucket", { forceDestroy: true });
72 const dataset = yield* Dataset("Source", {
73 format: "CSV",
74 input: {
75 s3InputDefinition: {
76 bucket: bucket.bucketName,
77 key: "raw/data.csv",
78 },
79 },
80 });
81 const ruleset = yield* Ruleset("Quality", {
82 description: "basic data quality v2",
83 targetArn: dataset.datasetArn,
84 rules: [
85 {
86 name: "no-missing-ids",
87 checkExpression: "AGG(MISSING_VALUES_PERCENTAGE) == :val1",
88 substitutionMap: { ":val1": "0" },
89 columnSelectors: [{ name: "id" }],
90 },
91 {
92 name: "row-count",
93 checkExpression: "AGG(DUPLICATE_ROWS_COUNT) == :val1",
94 substitutionMap: { ":val1": "0" },
95 },
96 ],
97 tags: { Environment: "test" },
98 });
99 return { ruleset };
100 }),
101 );
102
103 const reobserved = yield* getRuleset(created.ruleset.rulesetName);
104 expect(reobserved?.Description).toEqual("basic data quality v2");
105 expect(reobserved?.Rules?.length).toEqual(2);
106
107 yield* stack.destroy();
108 const gone = yield* getRuleset(created.ruleset.rulesetName);
109 expect(gone).toBeUndefined();
110 }),
111 {
112 tags: ["provider:aws", "provider:aws:databrew", "provider:aws:s3", "live"],
113 timeout: 120_000,
114 },
115);