1import * as databrew from "@distilled.cloud/aws/databrew";
2import * as Effect from "effect/Effect";
3import * as Stream from "effect/Stream";
4import { Unowned } from "../../AdoptPolicy.ts";
5import { isResolved } from "../../Diff.ts";
6import { createPhysicalName } from "../../PhysicalName.ts";
7import * as Provider from "../../Provider.ts";
8import { Resource } from "../../Resource.ts";
9import { createInternalTags, hasAlchemyTags } from "../../Tags.ts";
10import { AWSEnvironment } from "../Environment.ts";
11import type { Providers } from "../Providers.ts";
12import {
13 cleanMap,
14 databrewArn,
15 fetchObservedTags,
16 syncTags,
17} from "./internal.ts";
18
19/** A single data-quality rule (a check expression + optional threshold). */
20export interface RulesetRule {
21 /** The rule name (unique within the ruleset). */
22 name: string;
23 /** Skip the rule without deleting it. @default false */
24 disabled?: boolean;
25 /**
26 * The check expression, e.g. `:col1 is_between :val1 and :val2` or
27 * `AGG(MISSING_VALUES_PERCENTAGE) < :val1`.
28 */
29 checkExpression: string;
30 /** Variable substitutions for the expression, keyed by `:name`. */
31 substitutionMap?: Record<string, string>;
32 /** Pass/fail threshold for row-level checks. */
33 threshold?: {
34 /** The threshold value. */
35 value: number;
36 /** Comparison type. @default "GREATER_THAN_OR_EQUAL" */
37 type?:
38 | "GREATER_THAN_OR_EQUAL"
39 | "LESS_THAN_OR_EQUAL"
40 | "GREATER_THAN"
41 | "LESS_THAN"
42 | (string & {});
43 /** Whether `value` is a row count or a percentage. @default "COUNT" */
44 unit?: "COUNT" | "PERCENTAGE" | (string & {});
45 };
46 /** Columns the rule applies to (by name or regex). */
47 columnSelectors?: {
48 /** A regular expression matching column names. */
49 regex?: string;
50 /** An exact column name. */
51 name?: string;
52 }[];
53}
54
55export interface RulesetProps {
56 /**
57 * Name of the ruleset. If omitted, a unique name is generated. Changing
58 * the name replaces the ruleset.
59 * @default a generated physical name
60 */
61 rulesetName?: string;
62 /**
63 * A description of the ruleset.
64 */
65 description?: string;
66 /**
67 * The ARN of the DataBrew dataset the ruleset validates. Changing it
68 * replaces the ruleset.
69 */
70 targetArn: string;
71 /**
72 * The data-quality rules.
73 */
74 rules: RulesetRule[];
75 /**
76 * Tags to apply to the ruleset. Merged with internal Alchemy tags.
77 */
78 tags?: Record<string, string>;
79}
80
81export interface Ruleset extends Resource<
82 "AWS.DataBrew.Ruleset",
83 RulesetProps,
84 {
85 /** Name of the ruleset. */
86 rulesetName: string;
87 /** ARN of the ruleset. */
88 rulesetArn: string;
89 /** ARN of the dataset the ruleset validates. */
90 targetArn: string;
91 },
92 {},
93 Providers
94> {}
95
96/**
97 * An AWS Glue DataBrew ruleset — a set of data-quality rules bound to a
98 * dataset. Attach it to a profile job via `validationConfigurations` to
99 * produce pass/fail validation results alongside the data profile.
100 * ### Creating Rulesets
101 * **Example:** Data-Quality Rules for a Dataset
102 * ```typescript
103 * import * as AWS from "alchemy/AWS";
104 *
105 * const ruleset = yield* AWS.DataBrew.Ruleset("Quality", {
106 * targetArn: dataset.datasetArn,
107 * rules: [
108 * {
109 * name: "no-missing-ids",
110 * checkExpression: "AGG(MISSING_VALUES_PERCENTAGE) == :val1",
111 * substitutionMap: { ":val1": "0" },
112 * columnSelectors: [{ name: "id" }],
113 * },
114 * ],
115 * });
116 * ```
117 *
118 * **Example:** Validate in a Profile Job
119 * ```typescript
120 * const profile = yield* AWS.DataBrew.Job("Profile", {
121 * type: "PROFILE",
122 * datasetName: dataset.datasetName,
123 * role: role.roleArn,
124 * outputLocation: { bucket: bucket.bucketName, key: "profiles/" },
125 * validationConfigurations: [{ rulesetArn: ruleset.rulesetArn }],
126 * });
127 * ```
128 *
129 * @resource
130 */
131export const Ruleset = Resource<Ruleset>("AWS.DataBrew.Ruleset");
132
133const buildRules = (rules: RulesetRule[]) =>
134 rules.map((rule) => ({
135 Name: rule.name,
136 Disabled: rule.disabled,
137 CheckExpression: rule.checkExpression,
138 SubstitutionMap: rule.substitutionMap,
139 Threshold: rule.threshold
140 ? {
141 Value: rule.threshold.value,
142 Type: rule.threshold.type,
143 Unit: rule.threshold.unit,
144 }
145 : undefined,
146 ColumnSelectors: rule.columnSelectors?.map((s) => ({
147 Regex: s.regex,
148 Name: s.name,
149 })),
150 }));
151
152export const RulesetProvider = () =>
153 Provider.effect(
154 Ruleset,
155 Effect.gen(function* () {
156 const createName = Effect.fn(function* (
157 id: string,
158 props: { rulesetName?: string | undefined },
159 ) {
160 return (
161 props.rulesetName ??
162 (yield* createPhysicalName({ id, maxLength: 255 }))
163 );
164 });
165
166 const observe = Effect.fn(function* (name: string) {
167 return yield* databrew
168 .describeRuleset({ Name: name })
169 .pipe(
170 Effect.catchTag("ResourceNotFoundException", () =>
171 Effect.succeed(undefined),
172 ),
173 );
174 });
175
176 return Ruleset.Provider.of({
177 stables: ["rulesetName", "rulesetArn", "targetArn"],
178
179 list: () =>
180 Effect.gen(function* () {
181 const { accountId, region } = yield* AWSEnvironment.current;
182 const pages = yield* databrew.listRulesets
183 .pages({})
184 .pipe(Stream.runCollect);
185 return Array.from(pages)
186 .flatMap((page) => page.Rulesets ?? [])
187 .map((r) => ({
188 rulesetName: r.Name,
189 rulesetArn:
190 r.ResourceArn ??
191 databrewArn(region, accountId, "ruleset", r.Name),
192 targetArn: r.TargetArn ?? "",
193 }));
194 }),
195
196 read: Effect.fn(function* ({ id, olds, output }) {
197 const { accountId, region } = yield* AWSEnvironment.current;
198 const name =
199 output?.rulesetName ?? (yield* createName(id, olds ?? {}));
200 const ruleset = yield* observe(name);
201 if (ruleset === undefined) return undefined;
202 const arn =
203 ruleset.ResourceArn ??
204 databrewArn(region, accountId, "ruleset", name);
205 const attrs = {
206 rulesetName: name,
207 rulesetArn: arn,
208 targetArn: ruleset.TargetArn ?? "",
209 };
210 const tags = cleanMap(ruleset.Tags);
211 return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
212 }),
213
214 diff: Effect.fn(function* ({ id, news, olds }) {
215 if (!isResolved(news)) return undefined;
216 const oldName = yield* createName(id, olds);
217 const newName = yield* createName(id, news);
218 if (oldName !== newName) return { action: "replace" } as const;
219 // UpdateRuleset only accepts Description + Rules.
220 if (olds.targetArn !== news.targetArn) {
221 return { action: "replace" } as const;
222 }
223 }),
224
225 reconcile: Effect.fn(function* ({ id, news, output, session }) {
226 const { accountId, region } = yield* AWSEnvironment.current;
227 const name = output?.rulesetName ?? (yield* createName(id, news));
228 const internalTags = yield* createInternalTags(id);
229 const desiredTags = { ...news.tags, ...internalTags };
230
231 // 1. OBSERVE
232 const ruleset = yield* observe(name);
233
234 // 2. ENSURE / 3. SYNC — UpdateRuleset is a full PUT of the rules
235 if (ruleset === undefined) {
236 yield* databrew
237 .createRuleset({
238 Name: name,
239 Description: news.description,
240 TargetArn: news.targetArn,
241 Rules: buildRules(news.rules),
242 Tags: desiredTags,
243 })
244 .pipe(Effect.catchTag("ConflictException", () => Effect.void));
245 } else {
246 yield* databrew.updateRuleset({
247 Name: name,
248 Description: news.description,
249 Rules: buildRules(news.rules),
250 });
251 }
252
253 const arn =
254 ruleset?.ResourceArn ??
255 databrewArn(region, accountId, "ruleset", name);
256
257 // 3b. SYNC TAGS against observed cloud tags
258 const observedTags = yield* fetchObservedTags(arn);
259 yield* syncTags(arn, observedTags, desiredTags);
260
261 yield* session.note(name);
262 return {
263 rulesetName: name,
264 rulesetArn: arn,
265 targetArn: news.targetArn,
266 };
267 }),
268
269 delete: Effect.fn(function* ({ output }) {
270 yield* databrew
271 .deleteRuleset({ Name: output.rulesetName })
272 .pipe(
273 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
274 );
275 }),
276 });
277 }),
278 );