1import * as bigtable from "@distilled.cloud/gcp/bigtableadmin_v2";
2import * as Effect from "effect/Effect";
3import * as Stream from "effect/Stream";
4import { createPhysicalName } from "../../PhysicalName.ts";
5import { waitForOperation as waitForGcpOperation } from "../Operation.ts";
6
7export const DEFAULT_ZONE = "us-central1-b";
8export const DEFAULT_STORAGE = "HDD";
9export const DEFAULT_SERVE_NODES = 1;
10export const DEFAULT_INSTANCE_TYPE = "PRODUCTION";
11export const DEFAULT_EDITION = "ENTERPRISE";
12export const DEFAULT_CLUSTER_ID = "cluster";
13export const MAX_INSTANCE_ID_LENGTH = 33;
14export const MIN_INSTANCE_ID_LENGTH = 6;
15export const MAX_CLUSTER_ID_LENGTH = 30;
16export const MIN_CLUSTER_ID_LENGTH = 6;
17export const MAX_APP_PROFILE_ID_LENGTH = 50;
18export const MAX_TABLE_ID_LENGTH = 50;
19export const MAX_BACKUP_ID_LENGTH = 50;
20export const MAX_LOGICAL_VIEW_ID_LENGTH = 50;
21export const MAX_MATERIALIZED_VIEW_ID_LENGTH = 50;
22export const MAX_AUTHORIZED_VIEW_ID_LENGTH = 50;
23export const MAX_SCHEMA_BUNDLE_ID_LENGTH = 50;
24
25export const lastSegment = (value: string) => {
26 const trimmed = value.replace(/\/+$/, "");
27 const parts = trimmed.split("/");
28 return parts[parts.length - 1] || trimmed;
29};
30
31export const rfc1035 = (
32 name: string,
33 maxLength: number,
34 minLength = 1,
35): string => {
36 let next = name
37 .toLowerCase()
38 .replace(/[^a-z0-9-]/g, "-")
39 .replace(/-+/g, "-")
40 .replace(/^-+|-+$/g, "");
41 if (!/^[a-z]/.test(next)) next = `b${next}`;
42 next = next.slice(0, maxLength).replace(/-+$/g, "");
43 if (next.length === 0) next = "bigtable";
44 if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, maxLength - 1)}0`;
45 if (next.length < minLength) {
46 next = `${next}${"x".repeat(minLength)}`.slice(0, minLength);
47 }
48 return next.slice(0, maxLength);
49};
50
51export const toPhysicalId = (
52 id: string,
53 explicit: string | undefined,
54 existing: string | undefined,
55 maxLength: number,
56 minLength = 1,
57) =>
58 Effect.gen(function* () {
59 if (explicit !== undefined) return explicit;
60 if (existing !== undefined) return existing;
61 return rfc1035(
62 yield* createPhysicalName({
63 id,
64 maxLength,
65 lowercase: true,
66 }),
67 maxLength,
68 minLength,
69 );
70 });
71
72export const instanceName = (project: string, instanceId: string) =>
73 `projects/${project}/instances/${instanceId}`;
74
75export const clusterName = (
76 project: string,
77 instanceId: string,
78 clusterId: string,
79) => `${instanceName(project, instanceId)}/clusters/${clusterId}`;
80
81export const appProfileName = (
82 project: string,
83 instanceId: string,
84 appProfileId: string,
85) => `${instanceName(project, instanceId)}/appProfiles/${appProfileId}`;
86
87export const tableName = (
88 project: string,
89 instanceId: string,
90 tableId: string,
91) => `${instanceName(project, instanceId)}/tables/${tableId}`;
92
93export const backupName = (
94 project: string,
95 instanceId: string,
96 clusterId: string,
97 backupId: string,
98) => `${clusterName(project, instanceId, clusterId)}/backups/${backupId}`;
99
100export const logicalViewName = (
101 project: string,
102 instanceId: string,
103 logicalViewId: string,
104) => `${instanceName(project, instanceId)}/logicalViews/${logicalViewId}`;
105
106export const materializedViewName = (
107 project: string,
108 instanceId: string,
109 materializedViewId: string,
110) =>
111 `${instanceName(project, instanceId)}/materializedViews/${materializedViewId}`;
112
113export const authorizedViewName = (
114 project: string,
115 instanceId: string,
116 tableId: string,
117 authorizedViewId: string,
118) =>
119 `${tableName(project, instanceId, tableId)}/authorizedViews/${authorizedViewId}`;
120
121export const schemaBundleName = (
122 project: string,
123 instanceId: string,
124 tableId: string,
125 schemaBundleId: string,
126) =>
127 `${tableName(project, instanceId, tableId)}/schemaBundles/${schemaBundleId}`;
128
129export const parseResourceName = (name: string) => {
130 const parts = name.split("/").filter((part) => part.length > 0);
131 const indexOf = (segment: string) => parts.lastIndexOf(segment);
132 const after = (segment: string) => {
133 const at = indexOf(segment);
134 return at >= 0 && parts[at + 1] ? parts[at + 1]! : "";
135 };
136 const project = after("projects");
137 const instanceId = after("instances");
138 const clusterId = after("clusters");
139 const tableId = after("tables");
140 return {
141 project,
142 instanceId,
143 clusterId,
144 appProfileId: after("appProfiles"),
145 tableId,
146 backupId: after("backups"),
147 logicalViewId: after("logicalViews"),
148 materializedViewId: after("materializedViews"),
149 authorizedViewId: after("authorizedViews"),
150 schemaBundleId: after("schemaBundles"),
151 instance: project && instanceId ? instanceName(project, instanceId) : "",
152 cluster:
153 project && instanceId && clusterId
154 ? clusterName(project, instanceId, clusterId)
155 : "",
156 table:
157 project && instanceId && tableId
158 ? tableName(project, instanceId, tableId)
159 : "",
160 };
161};
162
163export const instanceNameOf = (project: string, instance: string) => {
164 if (instance.includes("/instances/")) {
165 const parsed = parseResourceName(instance);
166 return instanceName(parsed.project || project, parsed.instanceId);
167 }
168 return instanceName(project, lastSegment(instance));
169};
170
171export const instanceIdOf = (value: string) =>
172 value.includes("/instances/")
173 ? parseResourceName(value).instanceId
174 : lastSegment(value);
175
176export const clusterIdOf = (value: string) =>
177 value.includes("/clusters/")
178 ? parseResourceName(value).clusterId
179 : lastSegment(value);
180
181export const tableIdOf = (value: string) =>
182 value.includes("/tables/")
183 ? parseResourceName(value).tableId
184 : lastSegment(value);
185
186export const clusterNameOf = (
187 project: string,
188 instance: string,
189 cluster: string,
190) => {
191 if (cluster.includes("/clusters/")) {
192 const parsed = parseResourceName(cluster);
193 return clusterName(
194 parsed.project || project,
195 parsed.instanceId || instanceIdOf(instance),
196 parsed.clusterId,
197 );
198 }
199 return clusterName(project, instanceIdOf(instance), lastSegment(cluster));
200};
201
202export const tableNameOf = (
203 project: string,
204 instance: string,
205 table: string,
206) => {
207 if (table.includes("/tables/")) {
208 const parsed = parseResourceName(table);
209 return tableName(
210 parsed.project || project,
211 parsed.instanceId || instanceIdOf(instance),
212 parsed.tableId,
213 );
214 }
215 return `${instanceNameOf(project, instance)}/tables/${lastSegment(table)}`;
216};
217
218export const clusterLocation = (project: string, location: string) => {
219 if (location.includes("/locations/")) {
220 const zone = lastSegment(location);
221 const parsed = parseResourceName(location);
222 return `projects/${parsed.project || project}/locations/${zone}`;
223 }
224 return `projects/${project}/locations/${lastSegment(location).toLowerCase()}`;
225};
226
227export const zoneOf = (location: string | undefined) =>
228 lastSegment(location ?? DEFAULT_ZONE).toLowerCase();
229
230export const hasAlchemyPrefix = (
231 labels: Record<string, string | undefined> | null | undefined,
232) => Object.keys(labels ?? {}).some((key) => key.startsWith("alchemy-"));
233
234/**
235 * Wait for a Bigtable admin operation. Cluster resizes and materialized
236 * views are the slowest (several minutes). With `alreadyExistsOk`, an
237 * ALREADY_EXISTS (code 6) result counts as success (create race).
238 */
239export const waitForOperation = (
240 operation: bigtable.Operation,
241 options?: { alreadyExistsOk?: boolean },
242) =>
243 waitForGcpOperation(operation, (name) => bigtable.getOperations({ name }), {
244 budget: "20 minutes",
245 }).pipe(
246 Effect.catchIf(
247 (error) =>
248 options?.alreadyExistsOk === true &&
249 error._tag === "GCP.OperationFailed" &&
250 error.code === 6,
251 () => Effect.succeed(operation),
252 ),
253 );
254
255export const getInstanceByName = (name: string) =>
256 bigtable
257 .getProjectsInstances({ name })
258 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
259
260export const collectPages = <Page, A, E, R>(
261 pages: Stream.Stream<Page, E, R>,
262 items: (page: Page) => readonly A[] | undefined,
263) =>
264 pages.pipe(
265 Stream.flatMap((page) => Stream.fromIterable(items(page) ?? [])),
266 Stream.runCollect,
267 Effect.map((chunk) => Array.from(chunk)),
268 );
269
270export const listAlchemyInstances = (project: string) =>
271 collectPages(
272 bigtable.listProjectsInstances.pages({ parent: `projects/${project}` }),
273 (page) => page.instances,
274 ).pipe(
275 Effect.map((instances) =>
276 instances.filter((instance) => hasAlchemyPrefix(instance.labels)),
277 ),
278 );
279
280export const parentOwned = (instanceNameValue: string) =>
281 getInstanceByName(instanceNameValue).pipe(
282 Effect.map((instance) =>
283 instance === undefined ? true : hasAlchemyPrefix(instance.labels),
284 ),
285 );
286
287export const listAlchemyTables = (project: string) =>
288 Effect.gen(function* () {
289 const instances = (yield* listAlchemyInstances(project)).filter(
290 (instance): instance is bigtable.Instance & { name: string } =>
291 typeof instance.name === "string" && instance.name.length > 0,
292 );
293 const pages = yield* Effect.forEach(
294 instances,
295 (instance) =>
296 collectPages(
297 bigtable.listProjectsInstancesTables.pages({
298 parent: instance.name,
299 pageSize: 1000,
300 }),
301 (page) => page.tables,
302 ).pipe(
303 // The instance was deleted between the two list calls.
304 Effect.catchTag("NotFound", () =>
305 Effect.succeed([] as bigtable.Table[]),
306 ),
307 ),
308 { concurrency: 4 },
309 );
310 return pages.flat();
311 });