1import * as AWS from "@/AWS";
2import { Database, Table } from "@/AWS/Glue";
3import * as Test from "@/Test/Alchemy";
4import * as glue from "@distilled.cloud/aws/glue";
5import { expect } from "alchemy-test";
6import * as Effect from "effect/Effect";
7
8const { test } = Test.make({ providers: AWS.providers() });
9
10const getTable = (databaseName: string, name: string) =>
11 glue.getTable({ DatabaseName: databaseName, Name: name }).pipe(
12 Effect.map((r) => r.Table),
13 Effect.catchTag("EntityNotFoundException", () => Effect.succeed(undefined)),
14 );
15
16test.provider(
17 "create, update, delete Glue table over a database",
18 (stack) =>
19 Effect.gen(function* () {
20 yield* stack.destroy();
21
22 // create database + parquet table
23 const created = yield* stack.deploy(
24 Effect.gen(function* () {
25 const database = yield* Database("AnalyticsDb", {});
26 const table = yield* Table("Events", {
27 databaseName: database.databaseName,
28 tableType: "EXTERNAL_TABLE",
29 storageDescriptor: {
30 location: "s3://example-bucket/events/",
31 inputFormat:
32 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
33 outputFormat:
34 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
35 serdeInfo: {
36 serializationLibrary:
37 "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
38 },
39 columns: [
40 { name: "id", type: "string" },
41 { name: "amount", type: "double" },
42 ],
43 },
44 partitionKeys: [{ name: "dt", type: "string" }],
45 parameters: { classification: "parquet" },
46 });
47 return { database, table };
48 }),
49 );
50
51 expect(created.table.tableName).toBeDefined();
52 expect(created.table.databaseName).toEqual(created.database.databaseName);
53 expect(created.table.tableArn).toContain(
54 `:table/${created.database.databaseName}/${created.table.tableName}`,
55 );
56
57 // out-of-band verification
58 const observed = yield* getTable(
59 created.database.databaseName,
60 created.table.tableName,
61 );
62 expect(observed?.Name).toEqual(created.table.tableName);
63 expect(observed?.TableType).toEqual("EXTERNAL_TABLE");
64 expect(observed?.StorageDescriptor?.Location).toEqual(
65 "s3://example-bucket/events/",
66 );
67 expect(observed?.StorageDescriptor?.Columns?.map((c) => c.Name)).toEqual([
68 "id",
69 "amount",
70 ]);
71 expect(observed?.PartitionKeys?.map((c) => c.Name)).toEqual(["dt"]);
72 expect(observed?.Parameters?.classification).toEqual("parquet");
73 expect(observed?.Parameters?.["alchemy::id"]).toBeDefined();
74
75 // update: add a column + change a parameter
76 const updated = yield* stack.deploy(
77 Effect.gen(function* () {
78 const database = yield* Database("AnalyticsDb", {});
79 const table = yield* Table("Events", {
80 databaseName: database.databaseName,
81 tableType: "EXTERNAL_TABLE",
82 storageDescriptor: {
83 location: "s3://example-bucket/events-v2/",
84 inputFormat:
85 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
86 outputFormat:
87 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
88 serdeInfo: {
89 serializationLibrary:
90 "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
91 },
92 columns: [
93 { name: "id", type: "string" },
94 { name: "amount", type: "double" },
95 { name: "currency", type: "string" },
96 ],
97 },
98 partitionKeys: [{ name: "dt", type: "string" }],
99 parameters: { classification: "parquet", owner: "analytics" },
100 });
101 return { database, table };
102 }),
103 );
104
105 expect(updated.table.tableName).toEqual(created.table.tableName);
106 const reobserved = yield* getTable(
107 created.database.databaseName,
108 created.table.tableName,
109 );
110 expect(reobserved?.StorageDescriptor?.Location).toEqual(
111 "s3://example-bucket/events-v2/",
112 );
113 expect(
114 reobserved?.StorageDescriptor?.Columns?.map((c) => c.Name),
115 ).toEqual(["id", "amount", "currency"]);
116 expect(reobserved?.Parameters?.owner).toEqual("analytics");
117
118 // delete
119 yield* stack.destroy();
120 const gone = yield* getTable(
121 created.database.databaseName,
122 created.table.tableName,
123 );
124 expect(gone).toBeUndefined();
125 }),
126 { tags: ["provider:aws", "provider:aws:glue", "live"] },
127);