1import * as GCP from "@/GCP";
2import { GcpEnvironment } from "@/GCP/Environment";
3import * as Kubernetes from "@/Kubernetes";
4import type {
5 IdentityState,
6 RegistryState,
7} from "@/Kubernetes/ClusterAdapter.ts";
8import type { Connection } from "@/Kubernetes/Connection.ts";
9import { connectCluster, readObject } from "@/Kubernetes/internal/client.ts";
10import type { KubernetesObjectRef } from "@/Kubernetes/internal/objects.ts";
11import * as Test from "@/Test/Alchemy";
12import * as Core from "@/Test/Core";
13import * as artifactregistry from "@distilled.cloud/gcp/artifactregistry_v1";
14import * as resourcemanager from "@distilled.cloud/gcp/cloudresourcemanager_v3";
15import * as compute from "@distilled.cloud/gcp/compute_v1";
16import * as container from "@distilled.cloud/gcp/container_v1";
17import * as storage from "@distilled.cloud/gcp/storage_v1";
18import { describe, expect } from "alchemy-test";
19import * as Data from "effect/Data";
20import * as Effect from "effect/Effect";
21import * as Layer from "effect/Layer";
22import * as Schedule from "effect/Schedule";
23import * as HttpClient from "effect/http/HttpClient";
24import * as HttpClientRequest from "effect/http/HttpClientRequest";
25import { spawnSync } from "node:child_process";
26import GkeSmokeApiLive, { GkeSmokeApi } from "./fixtures/smoke-api.ts";
27import GkeSmokeJob from "./fixtures/smoke-job.ts";
28import {
29 JOB_MARKER_BODY,
30 JOB_MARKER_KEY,
31 SMOKE_NAMESPACE,
32 SmokeBucket,
33 SmokeCluster,
34 SmokeNamespace,
35} from "./fixtures/smoke-resources.ts";
36import { SMOKE_REGION } from "../zones.ts";
37
38/**
39 * GKE flagship smoke — the GCP counterpart of the ECS Task / EKS path:
40 *
41 * Autopilot `GCP.Container.Cluster` → an Effect-native
42 * `Kubernetes.Deployment` behind a GKE LoadBalancer, bound to a Cloud
43 * Storage bucket (`GCP.Storage.ReadWriteBucket`) through Workload
44 * Identity → an inline-effect `Kubernetes.Job` that writes a marker object
45 * through `GCP.Storage.WriteBucket`. Every side effect is verified
46 * out-of-band via distilled, and teardown is proven clean.
47 *
48 * Autopilot creation alone takes ~10 minutes, so the smoke only runs with
49 * `GCP_TEST_SLOW=1` (and never under `--fast`). The images are built
50 * locally, so it also needs a Docker daemon.
51 */
52
53const dockerAvailable = (() => {
54 try {
55 return (
56 spawnSync("docker", ["info"], { stdio: "ignore", timeout: 15_000 })
57 .status === 0
58 );
59 } catch {
60 return false;
61 }
62})();
63
64const skip =
65 !process.env.GCP_TEST_SLOW || !!process.env.FAST || !dockerAvailable;
66
67const testOptions = {
68 providers: Layer.mergeAll(GCP.providers(), Kubernetes.providers()),
69};
70const { test, beforeAll, afterAll } = Test.make(testOptions);
71// Durable (file-namespaced) state: a hard-killed run leaves resumable rows
72// so the next run's up-front destroy reclaims the cluster.
73const sharedStack = Core.scratchStack(
74 testOptions,
75 "GkeSmoke",
76 "test/GCP/Container/Cluster.smoke.test.ts",
77);
78
79interface StackOutputs {
80 clusterName: string;
81 connection: Connection;
82 bucketName: string;
83 url: string | undefined;
84 apiServiceAccount: string;
85 apiIdentity: IdentityState | undefined;
86 apiRegistry: RegistryState | undefined;
87 jobServiceAccount: string;
88 jobIdentity: IdentityState | undefined;
89 jobRegistry: RegistryState | undefined;
90 jobObjects: KubernetesObjectRef[];
91}
92
93let outputs: StackOutputs;
94let baseUrl: string;
95
96class NotReady extends Data.TaggedError("NotReady")<{
97 readonly detail: string;
98}> {}
99
100class StillExists extends Data.TaggedError("StillExists")<{
101 readonly what: string;
102}> {}
103
104const repositoryOf = (registry: RegistryState | undefined) =>
105 registry?.kind === "gcp-artifact-registry" ? registry.name : undefined;
106
107const memberOf = (identity: IdentityState | undefined) =>
108 identity?.kind === "gcp-workload-identity" ? identity.member : undefined;
109
110const deployProgram = Effect.gen(function* () {
111 const cluster = yield* SmokeCluster;
112 const bucket = yield* SmokeBucket;
113 yield* SmokeNamespace;
114 const api = yield* GkeSmokeApi;
115 const job = yield* GkeSmokeJob;
116 return {
117 clusterName: cluster.name,
118 connection: cluster.connection,
119 bucketName: bucket.bucketName,
120 url: api.url,
121 apiServiceAccount: api.serviceAccountName,
122 apiIdentity: api.identity,
123 apiRegistry: api.registry,
124 jobServiceAccount: job.serviceAccountName,
125 jobIdentity: job.identity,
126 jobRegistry: job.registry,
127 jobObjects: job.kubernetesObjects,
128 };
129}).pipe(Effect.provide(GkeSmokeApiLive));
130
131/** GET/PUT returning status + text; transport errors propagate. */
132const call = (request: HttpClientRequest.HttpClientRequest) =>
133 HttpClient.execute(request).pipe(
134 Effect.flatMap((response) =>
135 response.text.pipe(
136 Effect.map((text) => ({ status: response.status, text })),
137 ),
138 ),
139 );
140
141/**
142 * Re-run `attempt` every 10s until `ready` accepts its value. Transport
143 * errors count as not ready — a fresh load balancer refuses connections
144 * until its backends pass health checks.
145 */
146const until = <A, E, R>(
147 attempt: Effect.Effect<A, E, R>,
148 ready: (value: A) => boolean,
149 times: number,
150) =>
151 attempt.pipe(
152 Effect.filterOrFail(
153 ready,
154 (value) => new NotReady({ detail: JSON.stringify(value) }),
155 ),
156 Effect.tapError((error) => Effect.logDebug(`not ready: ${String(error)}`)),
157 Effect.retry({ schedule: Schedule.spaced("10 seconds"), times }),
158 );
159
160/** Bounded wait until an out-of-band probe reports the resource gone. */
161const waitUntilGone = <E, R>(
162 what: string,
163 probe: Effect.Effect<boolean, E, R>,
164) =>
165 probe.pipe(
166 Effect.flatMap((gone) =>
167 gone ? Effect.void : Effect.fail(new StillExists({ what })),
168 ),
169 Effect.retry({
170 while: (e) => e instanceof StillExists,
171 schedule: Schedule.max([
172 Schedule.fixed("5 seconds"),
173 Schedule.recurs(24),
174 ]),
175 }),
176 );
177
178/** The KSA Workload Identity principals, derived independently. */
179const expectedPrincipals = Effect.gen(function* () {
180 const { project } = yield* GcpEnvironment.current;
181 const { name } = yield* resourcemanager.getProjects({
182 name: `projects/${project}`,
183 });
184 const number = (name ?? "").split("/").pop()!;
185 const principal = (ksa: string) =>
186 `principal://iam.googleapis.com/projects/${number}/locations/global/` +
187 `workloadIdentityPools/${project}.svc.id.goog/subject/ns/${SMOKE_NAMESPACE}/sa/${ksa}`;
188 return {
189 api: principal(outputs.apiServiceAccount),
190 job: principal(outputs.jobServiceAccount),
191 };
192});
193
194/** Load-balancer plumbing GKE created for Services in the smoke namespace. */
195const namespaceLoadBalancers = Effect.gen(function* () {
196 const { project } = yield* GcpEnvironment.current;
197 const marker = `"kubernetes.io/service-name":"${SMOKE_NAMESPACE}/`;
198 const rules = yield* compute.listForwardingRules({
199 project,
200 region: SMOKE_REGION,
201 });
202 const backends = yield* compute.listRegionBackendServices({
203 project,
204 region: SMOKE_REGION,
205 });
206 return [
207 ...(rules.items ?? []).filter((r) =>
208 (r.description ?? "").includes(marker),
209 ),
210 ...(backends.items ?? []).filter((b) =>
211 (b.description ?? "").includes(marker),
212 ),
213 ].map((item) => item.name);
214});
215
216describe.skipIf(skip).sequential(
217 "GKE smoke",
218 {
219 tags: [
220 "provider:gcp",
221 "provider:gcp:container",
222 "provider:gcp:storage",
223 "provider:gcp:artifactregistry",
224 "provider:kubernetes",
225 "live",
226 ],
227 },
228 () => {
229 beforeAll(
230 Effect.gen(function* () {
231 yield* Effect.logInfo("GKE smoke: destroying previous stack");
232 yield* sharedStack.destroy();
233
234 yield* Effect.logInfo("GKE smoke: deploying stack (~10 min)");
235 outputs = (yield* sharedStack.deploy(deployProgram)) as StackOutputs;
236 expect(outputs.url).toMatch(/^http:\/\/[^/]+:3000$/);
237 baseUrl = outputs.url!.replace(/\/+$/, "");
238
239 yield* Effect.logInfo(`GKE smoke: probing ${baseUrl}/health`);
240 const health = yield* until(
241 call(HttpClientRequest.get(`${baseUrl}/health`)),
242 (r) => r.status === 200,
243 60,
244 );
245 expect(JSON.parse(health.text)).toEqual({ ok: true });
246 }),
247 { timeout: 1_800_000 },
248 );
249
250 afterAll(sharedStack.destroy(), { timeout: 1_800_000 });
251
252 test.provider(
253 "Workload Identity grants land on each KSA principal (bucket IAM)",
254 (_stack) =>
255 Effect.gen(function* () {
256 const principals = yield* expectedPrincipals;
257 // The adapter's recorded members are the same principals.
258 expect(memberOf(outputs.apiIdentity)).toBe(principals.api);
259 expect(memberOf(outputs.jobIdentity)).toBe(principals.job);
260
261 const policy = yield* storage.getIamPolicyBuckets({
262 bucket: outputs.bucketName,
263 optionsRequestedPolicyVersion: 3,
264 });
265 const rolesOf = (member: string) =>
266 (policy.bindings ?? [])
267 .filter((b) => (b.members ?? []).includes(member))
268 .map((b) => b.role);
269 // Least privilege, on the bucket only.
270 expect(rolesOf(principals.api)).toEqual(["roles/storage.objectUser"]);
271 expect(rolesOf(principals.job)).toEqual(["roles/storage.objectUser"]);
272
273 // Nothing was granted at the project level.
274 const { project } = yield* GcpEnvironment.current;
275 const projectPolicy = yield* resourcemanager.getIamPolicyProjects({
276 resource: `projects/${project}`,
277 body: { options: { requestedPolicyVersion: 3 } },
278 });
279 expect(
280 (projectPolicy.bindings ?? []).filter((b) =>
281 (b.members ?? []).some(
282 (m) => m === principals.api || m === principals.job,
283 ),
284 ),
285 ).toEqual([]);
286 }),
287 {
288 tags: ["provider:gcp", "provider:gcp:container", "live"],
289 timeout: 120_000,
290 },
291 );
292
293 test.provider(
294 "the Deployment reads and writes the bucket from the pod through the LoadBalancer",
295 (_stack) =>
296 Effect.gen(function* () {
297 const key = "api/round-trip.txt";
298 const body = yield* Effect.sync(
299 () =>
300 `written through the GKE load balancer ${crypto.randomUUID()}`,
301 );
302
303 // A fresh bucket grant may take a moment to reach Storage; the
304 // pod answers 500 until then.
305 const put = yield* until(
306 call(
307 HttpClientRequest.put(`${baseUrl}/objects/${key}`).pipe(
308 HttpClientRequest.bodyText(body, "text/plain"),
309 ),
310 ),
311 (r) => r.status !== 500,
312 30,
313 );
314 expect(put.status).toBe(200);
315 expect(JSON.parse(put.text)).toEqual({ key, read: body });
316
317 // Out-of-band: the object is real.
318 const object = yield* storage.getObjects({
319 bucket: outputs.bucketName,
320 object: key,
321 });
322 expect(object.size).toBe(
323 String(new TextEncoder().encode(body).length),
324 );
325 expect(object.contentType).toBe("text/plain");
326
327 const missing = yield* call(
328 HttpClientRequest.get(`${baseUrl}/objects/nope/missing.txt`),
329 );
330 expect(missing.status).toBe(404);
331 }),
332 {
333 tags: ["provider:gcp", "provider:gcp:container", "live"],
334 timeout: 600_000,
335 },
336 );
337
338 test.provider(
339 "the inline-effect Job completes and its marker exists",
340 (_stack) =>
341 Effect.gen(function* () {
342 const ref = outputs.jobObjects.find((o) => o.kind === "Job");
343 expect(ref).toBeDefined();
344 const transport = yield* connectCluster(outputs.connection);
345
346 // Autopilot provisions a node for the pod on demand; the pod
347 // retries under the Job's backoff while the grant propagates.
348 const job = yield* until(
349 readObject({ transport, object: ref! }).pipe(
350 Effect.map(
351 (o) =>
352 (o as { status?: { succeeded?: number; failed?: number } })
353 .status ?? {},
354 ),
355 ),
356 (status) => (status.succeeded ?? 0) >= 1,
357 60,
358 );
359 expect(job.succeeded).toBe(1);
360
361 // Out-of-band: the marker object the Job wrote.
362 const marker = yield* storage.getObjects({
363 bucket: outputs.bucketName,
364 object: JOB_MARKER_KEY,
365 });
366 expect(marker.size).toBe(
367 String(new TextEncoder().encode(JOB_MARKER_BODY).length),
368 );
369
370 // …and the Deployment's read binding serves it back.
371 const read = yield* call(
372 HttpClientRequest.get(`${baseUrl}/objects/${JOB_MARKER_KEY}`),
373 );
374 expect(read.status).toBe(200);
375 expect(read.text).toBe(JOB_MARKER_BODY);
376 }),
377 {
378 tags: ["provider:gcp", "provider:gcp:container", "live"],
379 timeout: 720_000,
380 },
381 );
382
383 test.provider(
384 "destroy leaves no cluster, load balancer, bucket, grants, or repository",
385 (_stack) =>
386 Effect.gen(function* () {
387 const principals = yield* expectedPrincipals;
388 const repositories = [outputs.apiRegistry, outputs.jobRegistry]
389 .map(repositoryOf)
390 .filter((name) => name !== undefined);
391 expect(repositories).toHaveLength(2);
392
393 yield* sharedStack.destroy();
394
395 yield* waitUntilGone(
396 "cluster",
397 container
398 .getProjectsLocationsClusters({ name: outputs.clusterName })
399 .pipe(
400 Effect.as(false),
401 Effect.catchTag("NotFound", () => Effect.succeed(true)),
402 ),
403 );
404 yield* waitUntilGone(
405 "bucket",
406 storage.getBuckets({ bucket: outputs.bucketName }).pipe(
407 Effect.as(false),
408 Effect.catchTag("NotFound", () => Effect.succeed(true)),
409 ),
410 );
411 for (const name of repositories) {
412 yield* waitUntilGone(
413 `repository ${name}`,
414 artifactregistry.getProjectsLocationsRepositories({ name }).pipe(
415 Effect.as(false),
416 Effect.catchTag("NotFound", () => Effect.succeed(true)),
417 ),
418 );
419 }
420 // The LoadBalancer Service was drained before the cluster went,
421 // so GKE left no forwarding rule or backend service behind.
422 yield* waitUntilGone(
423 "load balancer",
424 namespaceLoadBalancers.pipe(
425 Effect.map((left) => left.length === 0),
426 ),
427 );
428 // No grant outlives the workloads at the project level either.
429 const { project } = yield* GcpEnvironment.current;
430 const projectPolicy = yield* resourcemanager.getIamPolicyProjects({
431 resource: `projects/${project}`,
432 body: { options: { requestedPolicyVersion: 3 } },
433 });
434 expect(
435 (projectPolicy.bindings ?? []).filter((b) =>
436 (b.members ?? []).some(
437 (m) => m === principals.api || m === principals.job,
438 ),
439 ),
440 ).toEqual([]);
441 }),
442 {
443 tags: ["provider:gcp", "provider:gcp:container", "live"],
444 timeout: 1_200_000,
445 retry: 0,
446 },
447 );
448 },
449);