1import * as GCP from "@/GCP";
2import { GcpEnvironment } from "@/GCP/Environment";
3import { DestroyError } from "@/Apply";
4import * as Core from "@/Test/Core";
5import * as Test from "@/Test/Alchemy";
6import * as cloudfunctions from "@distilled.cloud/gcp/cloudfunctions_v2";
7import * as compute from "@distilled.cloud/gcp/compute_v1";
8import * as pubsub from "@distilled.cloud/gcp/pubsub_v1";
9import * as storage from "@distilled.cloud/gcp/storage_v1";
10import { describe, expect } from "alchemy-test";
11import * as Cause from "effect/Cause";
12import * as Data from "effect/Data";
13import * as Effect from "effect/Effect";
14import * as Schedule from "effect/Schedule";
15import * as HttpClient from "effect/http/HttpClient";
16import EgressFunction, {
17 APP_CIDR,
18 AppSubnet,
19 DATA_CIDR,
20 DataSubnet,
21 EGRESS_TAG,
22 PODS_CIDR,
23 Probes,
24 type ProbeResult,
25 REGION,
26 Results,
27 SERVICES_CIDR,
28 SmokeNetwork,
29} from "./fixtures/egress-function.ts";
30
31/**
32 * GCP counterpart of the AWS VPC smoke: a custom-mode VPC with two regional
33 * subnetworks (one with secondary ranges), a Cloud Router hosting Cloud NAT
34 * on a reserved static IP, ingress/egress firewall rules (allow-internal,
35 * deny-all egress overridden by a tagged tcp:443 allow), a custom route, and
36 * an internal-only Effect-native Cloud Function using Direct VPC egress.
37 * The function is driven through Pub/Sub, probes the internet, and records
38 * the egress IP it observed — which must be the NAT's static address.
39 */
40
41const testOptions = { providers: GCP.providers() };
42const { test, beforeAll, afterAll } = Test.make(testOptions);
43// Durable state: a subnet held by Cloud Run (below) stays resumable, so a
44// later run's destroy drains it instead of orphaning it.
45const sharedStack = Core.scratchStack(
46 testOptions,
47 "NetworkSmoke",
48 "test/GCP/Compute/Network.smoke.test.ts",
49);
50
51// Cloud Run keeps the Direct VPC egress IPs it reserved in a subnet
52// (`serverless-ipv4-*` addresses, purpose SERVERLESS) for up to 1-2 hours
53// after the function is deleted, and the subnet cannot be deleted until they
54// are released. Only that documented hold is tolerated: it fails the
55// subnet's delete and skips the network's; every other failure surfaces.
56// https://cloud.google.com/run/docs/configuring/vpc-direct-vpc
57const HELD_BY_CLOUD_RUN =
58 /already being used by \S+\/addresses\/serverless-ipv4-/;
59
60const destroyStack = sharedStack.destroy().pipe(
61 Effect.as([] as ReadonlyArray<string>),
62 Effect.catch((error: unknown) =>
63 error instanceof DestroyError &&
64 error.failures.every(
65 (failure) =>
66 failure.resourceType === "GCP.Compute.Subnetwork" &&
67 HELD_BY_CLOUD_RUN.test(Cause.pretty(failure.cause)),
68 ) &&
69 error.blocked.every(
70 (blocked) => blocked.resourceType === "GCP.Compute.Network",
71 )
72 ? Effect.succeed([
73 ...error.failures.map((failure) => failure.logicalId),
74 ...error.blocked.map((blocked) => blocked.logicalId),
75 ])
76 : Effect.fail(error),
77 ),
78);
79
80// private.googleapis.com — the canonical custom route for Private Google
81// Access without the default internet route.
82const PRIVATE_GOOGLE_APIS = "199.36.153.8/30";
83
84interface StackOutputs {
85 networkName: string;
86 appSubnetName: string;
87 dataSubnetName: string;
88 routerName: string;
89 natName: string;
90 addressName: string;
91 address: string;
92 addressSelfLink: string;
93 allowInternalName: string;
94 denyEgressName: string;
95 allowHttpsEgressName: string;
96 routeName: string;
97 functionName: string;
98 functionUrl: string;
99 bucketName: string;
100 topicName: string;
101}
102
103let outputs: StackOutputs;
104
105class StillExists extends Data.TaggedError("StillExists")<{
106 readonly what: string;
107}> {}
108
109class ProbeNotRecorded extends Data.TaggedError("ProbeNotRecorded")<{
110 readonly object: string;
111}> {}
112
113// Bounded wait until an out-of-band probe reports the resource gone.
114const waitUntilGone = <E, R>(
115 what: string,
116 probe: Effect.Effect<boolean, E, R>,
117) =>
118 probe.pipe(
119 Effect.flatMap((gone) =>
120 gone ? Effect.void : Effect.fail(new StillExists({ what })),
121 ),
122 Effect.retry({
123 while: (e) => e instanceof StillExists,
124 schedule: Schedule.max([
125 Schedule.fixed("3 seconds"),
126 Schedule.recurs(30),
127 ]),
128 }),
129 );
130
131const NAT_NAME = "smoke-egress";
132
133const deployProgram = Effect.gen(function* () {
134 const network = yield* SmokeNetwork;
135 const appSubnet = yield* AppSubnet;
136 const dataSubnet = yield* DataSubnet;
137
138 const egressIp = yield* GCP.Compute.Address("EgressIp", {
139 region: REGION,
140 addressType: "EXTERNAL",
141 description: "cloud nat egress",
142 });
143
144 const router = yield* GCP.Compute.Router("NatRouter", {
145 region: REGION,
146 network: network.networkName,
147 description: "network smoke nat",
148 nats: [
149 {
150 name: NAT_NAME,
151 sourceSubnetworkIpRangesToNat: "LIST_OF_SUBNETWORKS",
152 subnetworks: [{ name: appSubnet.subnetworkName }],
153 natIpAllocateOption: "MANUAL_ONLY",
154 natIps: [egressIp.selfLink.as<string>()],
155 logConfig: { enable: true, filter: "ERRORS_ONLY" },
156 },
157 ],
158 });
159
160 const allowInternal = yield* GCP.Compute.Firewall("AllowInternal", {
161 network: network.networkName,
162 direction: "INGRESS",
163 priority: 1000,
164 sourceRanges: [APP_CIDR, DATA_CIDR, PODS_CIDR, SERVICES_CIDR],
165 allowed: [{ protocol: "tcp" }, { protocol: "udp" }, { protocol: "icmp" }],
166 });
167
168 const denyEgress = yield* GCP.Compute.Firewall("DenyAllEgress", {
169 network: network.networkName,
170 direction: "EGRESS",
171 priority: 65000,
172 destinationRanges: ["0.0.0.0/0"],
173 denied: [{ protocol: "all" }],
174 });
175
176 const allowHttpsEgress = yield* GCP.Compute.Firewall("AllowHttpsEgress", {
177 network: network.networkName,
178 direction: "EGRESS",
179 priority: 1000,
180 destinationRanges: ["0.0.0.0/0"],
181 targetTags: [EGRESS_TAG],
182 allowed: [{ protocol: "tcp", ports: ["443"] }],
183 });
184
185 const route = yield* GCP.Compute.Route("PrivateGoogleApis", {
186 network: network.networkName,
187 destRange: PRIVATE_GOOGLE_APIS,
188 nextHopGateway: "default-internet-gateway",
189 priority: 900,
190 description: "private.googleapis.com",
191 });
192
193 const fn = yield* EgressFunction;
194 const bucket = yield* Results;
195 const topic = yield* Probes;
196
197 return {
198 networkName: network.networkName,
199 appSubnetName: appSubnet.subnetworkName,
200 dataSubnetName: dataSubnet.subnetworkName,
201 routerName: router.routerName,
202 natName: NAT_NAME,
203 addressName: egressIp.addressName,
204 address: egressIp.address,
205 addressSelfLink: egressIp.selfLink,
206 allowInternalName: allowInternal.firewallName,
207 denyEgressName: denyEgress.firewallName,
208 allowHttpsEgressName: allowHttpsEgress.firewallName,
209 routeName: route.routeName,
210 functionName: fn.name,
211 functionUrl: fn.url,
212 bucketName: bucket.bucketName,
213 topicName: topic.name,
214 };
215});
216
217const lastSegment = (value: string | undefined) => value?.split("/").at(-1);
218
219describe.skipIf(!!process.env.FAST).sequential(
220 "Network smoke",
221 {
222 tags: [
223 "provider:gcp",
224 "provider:gcp:compute",
225 "provider:gcp:cloudfunctions",
226 "provider:gcp:pubsub",
227 "provider:gcp:storage",
228 "live",
229 ],
230 },
231 () => {
232 beforeAll(
233 Effect.gen(function* () {
234 yield* Effect.logInfo("Network smoke: destroying previous stack");
235 const held = yield* destroyStack;
236 if (held.length > 0) {
237 yield* Effect.logInfo(
238 `Network smoke: still held by Cloud Run, reconciled in place: ${held.join(", ")}`,
239 );
240 }
241
242 yield* Effect.logInfo("Network smoke: deploying stack");
243 outputs = (yield* sharedStack.deploy(deployProgram)) as StackOutputs;
244 }),
245 { timeout: 900_000 },
246 );
247
248 afterAll(destroyStack, { timeout: 600_000 });
249
250 test.provider(
251 "custom-mode network, subnetworks, and route match the declaration",
252 (_stack) =>
253 Effect.gen(function* () {
254 const { project } = yield* GcpEnvironment.current;
255
256 const network = yield* compute.getNetworks({
257 project,
258 network: outputs.networkName,
259 });
260 expect(network.autoCreateSubnetworks).toBe(false);
261 expect(network.routingConfig?.routingMode).toBe("REGIONAL");
262 expect((network.subnetworks ?? []).map(lastSegment).sort()).toEqual(
263 [outputs.appSubnetName, outputs.dataSubnetName].sort(),
264 );
265
266 const app = yield* compute.getSubnetworks({
267 project,
268 region: REGION,
269 subnetwork: outputs.appSubnetName,
270 });
271 expect(app.ipCidrRange).toBe(APP_CIDR);
272 expect(app.privateIpGoogleAccess).toBe(true);
273 expect(lastSegment(app.network)).toBe(outputs.networkName);
274
275 const data = yield* compute.getSubnetworks({
276 project,
277 region: REGION,
278 subnetwork: outputs.dataSubnetName,
279 });
280 expect(data.ipCidrRange).toBe(DATA_CIDR);
281 expect(
282 (data.secondaryIpRanges ?? [])
283 .map((r) => ({
284 rangeName: r.rangeName,
285 ipCidrRange: r.ipCidrRange,
286 }))
287 .sort((a, b) => a.rangeName!.localeCompare(b.rangeName!)),
288 ).toEqual([
289 { rangeName: "pods", ipCidrRange: PODS_CIDR },
290 { rangeName: "services", ipCidrRange: SERVICES_CIDR },
291 ]);
292
293 const route = yield* compute.getRoutes({
294 project,
295 route: outputs.routeName,
296 });
297 expect(route.destRange).toBe(PRIVATE_GOOGLE_APIS);
298 expect(route.priority).toBe(900);
299 expect(lastSegment(route.network)).toBe(outputs.networkName);
300 expect(lastSegment(route.nextHopGateway)).toBe(
301 "default-internet-gateway",
302 );
303 }),
304 {
305 tags: ["provider:gcp", "provider:gcp:compute", "live"],
306 timeout: 60_000,
307 },
308 );
309
310 test.provider(
311 "firewall: allow-internal ingress, deny-all egress, tagged tcp:443 override",
312 (_stack) =>
313 Effect.gen(function* () {
314 const { project } = yield* GcpEnvironment.current;
315 const get = (firewall: string) =>
316 compute.getFirewalls({ project, firewall });
317
318 const internal = yield* get(outputs.allowInternalName);
319 expect(internal.direction).toBe("INGRESS");
320 expect(lastSegment(internal.network)).toBe(outputs.networkName);
321 expect([...(internal.sourceRanges ?? [])].sort()).toEqual(
322 [APP_CIDR, DATA_CIDR, PODS_CIDR, SERVICES_CIDR].sort(),
323 );
324 expect(
325 (internal.allowed ?? []).map((rule) => rule.IPProtocol).sort(),
326 ).toEqual(["icmp", "tcp", "udp"]);
327
328 const deny = yield* get(outputs.denyEgressName);
329 expect(deny.direction).toBe("EGRESS");
330 expect(deny.priority).toBe(65000);
331 expect(deny.destinationRanges).toEqual(["0.0.0.0/0"]);
332 expect(deny.denied?.map((rule) => rule.IPProtocol)).toEqual(["all"]);
333
334 const https = yield* get(outputs.allowHttpsEgressName);
335 expect(https.direction).toBe("EGRESS");
336 expect(https.priority).toBe(1000);
337 expect(https.targetTags).toEqual([EGRESS_TAG]);
338 expect(https.allowed).toEqual([
339 { IPProtocol: "tcp", ports: ["443"] },
340 ]);
341 }),
342 {
343 tags: ["provider:gcp", "provider:gcp:compute", "live"],
344 timeout: 60_000,
345 },
346 );
347
348 test.provider(
349 "Cloud Router hosts NAT for the app subnet on the static address",
350 (_stack) =>
351 Effect.gen(function* () {
352 const { project } = yield* GcpEnvironment.current;
353
354 const router = yield* compute.getRouters({
355 project,
356 region: REGION,
357 router: outputs.routerName,
358 });
359 expect(lastSegment(router.network)).toBe(outputs.networkName);
360 expect(router.nats).toHaveLength(1);
361 const nat = router.nats![0]!;
362 expect(nat.name).toBe(outputs.natName);
363 expect(nat.sourceSubnetworkIpRangesToNat).toBe("LIST_OF_SUBNETWORKS");
364 expect(nat.subnetworks?.map((s) => lastSegment(s.name))).toEqual([
365 outputs.appSubnetName,
366 ]);
367 expect(nat.natIpAllocateOption).toBe("MANUAL_ONLY");
368 expect(nat.natIps?.map(lastSegment)).toEqual([outputs.addressName]);
369 expect(nat.logConfig).toEqual({
370 enable: true,
371 filter: "ERRORS_ONLY",
372 });
373
374 const address = yield* compute.getAddresses({
375 project,
376 region: REGION,
377 address: outputs.addressName,
378 });
379 expect(address.address).toBe(outputs.address);
380 expect(address.addressType).toBe("EXTERNAL");
381 expect(address.status).toBe("IN_USE");
382 }),
383 {
384 tags: ["provider:gcp", "provider:gcp:compute", "live"],
385 timeout: 60_000,
386 },
387 );
388
389 test.provider(
390 "function is internal-only and egresses directly into the app subnet",
391 (_stack) =>
392 Effect.gen(function* () {
393 const fn = yield* cloudfunctions.getProjectsLocationsFunctions({
394 name: outputs.functionName,
395 });
396 expect(fn.state).toBe("ACTIVE");
397 expect(fn.serviceConfig?.ingressSettings).toBe("ALLOW_INTERNAL_ONLY");
398 expect(fn.serviceConfig?.directVpcEgress).toBe(
399 "VPC_EGRESS_ALL_TRAFFIC",
400 );
401 const nic = fn.serviceConfig?.directVpcNetworkInterface?.[0];
402 expect(lastSegment(nic?.network)).toBe(outputs.networkName);
403 expect(lastSegment(nic?.subnetwork)).toBe(outputs.appSubnetName);
404 expect(nic?.tags).toEqual([EGRESS_TAG]);
405
406 // Internal-only ingress: the public URL refuses outside callers.
407 const response = yield* HttpClient.get(outputs.functionUrl);
408 expect([403, 404]).toContain(response.status);
409 }),
410 {
411 tags: ["provider:gcp", "provider:gcp:compute", "live"],
412 timeout: 60_000,
413 },
414 );
415
416 test.provider(
417 "a Pub/Sub-driven probe egresses through Cloud NAT and the firewall",
418 (_stack) =>
419 Effect.gen(function* () {
420 const published = yield* pubsub.publishProjectsTopics({
421 topic: outputs.topicName,
422 body: { messages: [{ data: btoa("probe") }] },
423 });
424 const messageId = published.messageIds?.[0];
425 expect(messageId).toEqual(expect.any(String));
426 const object = `probe-${messageId}`;
427
428 // Cold start + Direct VPC egress interface attach + push delivery.
429 const probe: ProbeResult = yield* storage
430 .getObjects({ bucket: outputs.bucketName, object })
431 .pipe(
432 Effect.map((found): ProbeResult =>
433 Object.fromEntries(
434 Object.entries(found.metadata ?? {}).filter(
435 (entry): entry is [string, string] =>
436 entry[1] !== undefined,
437 ),
438 ),
439 ),
440 Effect.catchTag("NotFound", () =>
441 Effect.fail(new ProbeNotRecorded({ object })),
442 ),
443 Effect.retry({
444 while: (e) => e._tag === "ProbeNotRecorded",
445 schedule: Schedule.max([
446 Schedule.fixed("5 seconds"),
447 Schedule.recurs(36),
448 ]),
449 }),
450 );
451
452 expect(probe.ipError).toBeUndefined();
453 // The public IP the internet saw is the NAT's static address.
454 expect(probe.ip).toBe(outputs.address);
455 // tcp:80 falls through to the deny-all egress rule.
456 expect(probe.port80Blocked).toBe("true");
457 }),
458 {
459 tags: ["provider:gcp", "provider:gcp:compute", "live"],
460 timeout: 300_000,
461 },
462 );
463
464 test.provider(
465 "destroy removes the function, NAT router, firewalls, route, subnets, and network",
466 (_stack) =>
467 Effect.gen(function* () {
468 const { project } = yield* GcpEnvironment.current;
469 const held = yield* destroyStack;
470 // Only the function's egress subnet (and its network) may linger.
471 expect(
472 held.every((id) => id === "AppSubnet" || id === "SmokeNetwork"),
473 ).toBe(true);
474 if (held.includes("AppSubnet")) {
475 // Out-of-band: the only users of the subnet are Cloud Run's
476 // serverless IP reservations.
477 const addresses = yield* compute.listAddresses({
478 project,
479 region: REGION,
480 filter: `subnetwork eq .*/subnetworks/${outputs.appSubnetName}$`,
481 });
482 expect(addresses.items?.length ?? 0).toBeGreaterThan(0);
483 for (const address of addresses.items ?? []) {
484 expect(address.purpose).toBe("SERVERLESS");
485 expect(address.name).toMatch(/^serverless-ipv4-/);
486 }
487 }
488
489 const notFound = () => Effect.succeed(true);
490
491 yield* waitUntilGone(
492 "function",
493 cloudfunctions
494 .getProjectsLocationsFunctions({
495 name: outputs.functionName,
496 })
497 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
498 );
499 yield* waitUntilGone(
500 "router",
501 compute
502 .getRouters({
503 project,
504 region: REGION,
505 router: outputs.routerName,
506 })
507 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
508 );
509 yield* waitUntilGone(
510 "address",
511 compute
512 .getAddresses({
513 project,
514 region: REGION,
515 address: outputs.addressName,
516 })
517 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
518 );
519 for (const firewall of [
520 outputs.allowInternalName,
521 outputs.denyEgressName,
522 outputs.allowHttpsEgressName,
523 ]) {
524 yield* waitUntilGone(
525 `firewall ${firewall}`,
526 compute
527 .getFirewalls({ project, firewall })
528 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
529 );
530 }
531 yield* waitUntilGone(
532 "route",
533 compute
534 .getRoutes({ project, route: outputs.routeName })
535 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
536 );
537 for (const subnetwork of [
538 ...(held.includes("AppSubnet") ? [] : [outputs.appSubnetName]),
539 outputs.dataSubnetName,
540 ]) {
541 yield* waitUntilGone(
542 `subnetwork ${subnetwork}`,
543 compute
544 .getSubnetworks({ project, region: REGION, subnetwork })
545 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
546 );
547 }
548 if (!held.includes("SmokeNetwork")) {
549 yield* waitUntilGone(
550 "network",
551 compute
552 .getNetworks({ project, network: outputs.networkName })
553 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
554 );
555 }
556 yield* waitUntilGone(
557 "bucket",
558 storage
559 .getBuckets({ bucket: outputs.bucketName })
560 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
561 );
562 yield* waitUntilGone(
563 "topic",
564 pubsub
565 .getProjectsTopics({ topic: outputs.topicName })
566 .pipe(Effect.as(false), Effect.catchTag("NotFound", notFound)),
567 );
568 }),
569 {
570 tags: ["provider:gcp", "provider:gcp:compute", "live"],
571 timeout: 600_000,
572 },
573 );
574 },
575);