1import { adopt, Unowned } from "@/AdoptPolicy";
2import { InstanceId } from "@/InstanceId";
3import { createPhysicalName } from "@/PhysicalName";
4import * as Provider from "@/Provider";
5import { Branch } from "@/Neon/Branch";
6import { Credential } from "@/Neon/Credential";
7import { Bucket, bucketStorageClient, type BucketProps } from "@/Neon/Bucket";
8import { storageBodyBytes } from "@/Neon/Object";
9import { makeStorageClient } from "@/Neon/Storage";
10import { Project } from "@/Neon/Project";
11import { providers } from "@/Neon/Providers";
12import * as Test from "@/Test/Alchemy";
13import * as SDK from "@distilled.cloud/neon";
14import { expect } from "alchemy-test";
15import * as Effect from "effect/Effect";
16import * as Result from "effect/Result";
17import * as HttpClient from "effect/http/HttpClient";
18
19const { test } = Test.make({ providers: providers() });
20
21test.provider(
22 "never-created buckets without branch identity are absent",
23 (stack) =>
24 Effect.gen(function* () {
25 yield* stack.destroy();
26 const provider = yield* Provider.findProvider(Bucket);
27 const cases: BucketProps[] = [
28 { branch: { projectId: "", branchId: "" } },
29 { branch: { projectId: "project", branchId: "" } },
30 { branch: { projectId: "", branchId: "branch" } },
31 { project: { projectId: "" } },
32 ];
33 for (const olds of cases) {
34 expect(
35 yield* provider.read!({
36 id: "Incomplete",
37 fqn: "Incomplete",
38 instanceId: "never-created",
39 olds,
40 output: undefined,
41 }),
42 ).toBeUndefined();
43 }
44 yield* stack.destroy();
45 }),
46 { tags: ["provider:neon", "provider:neon:bucket", "live"] },
47);
48
49test.provider(
50 "bucket recovery observes tags even without cached identity or explicit name",
51 (stack) =>
52 Effect.gen(function* () {
53 yield* stack.destroy();
54 const instanceId = "0123456789abcdef0123456789abcdef";
55 const bucket = yield* stack.deploy(
56 Effect.gen(function* () {
57 const project = yield* Project("RecoveryProject", {
58 region: "aws-us-east-2",
59 });
60 const name = yield* createPhysicalName({
61 id: "Recovered",
62 instanceId,
63 maxLength: 63,
64 lowercase: true,
65 });
66 return yield* Bucket("Recovered", {
67 project,
68 name,
69 forceDestroy: true,
70 });
71 }),
72 );
73 const client = yield* bucketStorageClient(bucket);
74 const provider = yield* Provider.findProvider(Bucket);
75 const olds: BucketProps = {
76 branch: { projectId: bucket.projectId, branchId: bucket.branchId },
77 credential: bucket.credential,
78 forceDestroy: true,
79 };
80 const read = provider.read!({
81 id: "Recovered",
82 fqn: "Recovered",
83 instanceId,
84 olds,
85 output: undefined,
86 }).pipe(Effect.provideService(InstanceId, instanceId));
87 yield* Effect.gen(function* () {
88 expect(Unowned.is(yield* read)).toBe(false);
89 expect((yield* read)?.bucketName).toBe(bucket.bucketName);
90 yield* client.putTags({});
91 expect(Unowned.is(yield* read)).toBe(true);
92 yield* client.putTags({ "alchemy::stack": "another-stack" });
93 expect(Unowned.is(yield* read)).toBe(true);
94 expect(
95 Unowned.is(
96 yield* provider.read!({
97 id: "Recovered",
98 fqn: "Recovered",
99 instanceId,
100 olds,
101 output: bucket,
102 }),
103 ),
104 ).toBe(true);
105 }).pipe(Effect.ensuring(client.putTags(bucket.tags).pipe(Effect.orDie)));
106 yield* stack.destroy();
107 expect(
108 yield* SDK.getProject({ project_id: bucket.projectId }).pipe(
109 Effect.as(false),
110 Effect.catchTag("NotFound", () => Effect.succeed(true)),
111 ),
112 ).toBe(true);
113 yield* stack.destroy();
114 }),
115 {
116 tags: [
117 "provider:neon",
118 "provider:neon:bucket",
119 "provider:neon:project",
120 "live",
121 ],
122 timeout: 120_000,
123 },
124);
125
126test.provider(
127 "bucket tags CORS anonymous access and multipart cleanup",
128 (stack) =>
129 Effect.gen(function* () {
130 yield* stack.destroy();
131 const deploy = (updated: boolean) =>
132 stack.deploy(
133 Effect.gen(function* () {
134 const project = yield* Project("BucketProject", {
135 region: "aws-us-east-2",
136 });
137 const bucket = yield* Bucket("Private", {
138 project,
139 tags: updated
140 ? { phase: "updated" }
141 : { phase: "initial", removed: "yes" },
142 cors: updated
143 ? []
144 : [
145 {
146 AllowedOrigins: ["https://example.com"],
147 AllowedMethods: ["GET", "PUT"],
148 },
149 ],
150 forceDestroy: true,
151 });
152 const publicBucket = yield* Bucket("Public", {
153 project,
154 access: "public_read",
155 forceDestroy: true,
156 });
157 return { project, bucket, publicBucket };
158 }),
159 );
160 const first = yield* deploy(false);
161 const listing = yield* SDK.listProjectBranchBuckets({
162 project_id: first.bucket.projectId,
163 branch_id: first.bucket.branchId,
164 });
165 expect(
166 listing.buckets.some(
167 (bucket) => bucket.name === first.bucket.bucketName,
168 ),
169 ).toBe(true);
170 const client = yield* bucketStorageClient(first.bucket);
171 yield* client.put("private.txt", "private");
172 const publicClient = yield* bucketStorageClient(first.publicBucket);
173 yield* publicClient.put("public.txt", "public");
174 const http = yield* HttpClient.HttpClient;
175 const privateRead = yield* http.get(
176 `${first.bucket.endpoint}/${first.bucket.bucketName}/private.txt`,
177 );
178 expect([401, 403]).toContain(privateRead.status);
179 const publicRead = yield* http.get(
180 `${first.publicBucket.endpoint}/${first.publicBucket.bucketName}/public.txt`,
181 );
182 expect(publicRead.status).toBe(200);
183 expect(yield* publicRead.text).toBe("public");
184 const updated = yield* deploy(true);
185 expect(updated.bucket.bucketName).toBe(first.bucket.bucketName);
186 const observed = yield* (yield* bucketStorageClient(
187 updated.bucket,
188 )).getTags();
189 expect(observed.TagSet).toContainEqual({
190 Key: "phase",
191 Value: "updated",
192 });
193 expect(observed.TagSet?.some((tag) => tag.Key === "removed")).toBe(false);
194 expect((yield* client.getCors()).CORSRules ?? []).toEqual([]);
195 const upload = yield* client.createMultipartUpload("unfinished.bin");
196 expect(upload.UploadId).toBeDefined();
197 yield* client.uploadPart(
198 "unfinished.bin",
199 upload.UploadId!,
200 1,
201 new Uint8Array([1, 2, 3]),
202 );
203 const completed = yield* client.createMultipartUpload("completed.bin");
204 const part = yield* client.uploadPart(
205 "completed.bin",
206 completed.UploadId!,
207 1,
208 new Uint8Array([4, 5, 6]),
209 );
210 expect(
211 (yield* client.listParts("completed.bin", completed.UploadId!)).Parts
212 ?.length,
213 ).toBe(1);
214 yield* client.completeMultipartUpload(
215 "completed.bin",
216 completed.UploadId!,
217 [{ ETag: part.ETag, PartNumber: 1 }],
218 );
219 expect((yield* client.head("completed.bin"))?.ContentLength).toBe(3);
220 yield* stack.destroy();
221 yield* stack.destroy();
222 }),
223 {
224 tags: [
225 "provider:neon",
226 "provider:neon:bucket",
227 "provider:neon:project",
228 "live",
229 ],
230 timeout: 120_000,
231 },
232);
233
234test.provider(
235 "nonempty bucket deletion and visibility updates fail without destroying data",
236 (stack) =>
237 Effect.gen(function* () {
238 yield* stack.destroy();
239 const deploy = (
240 forceDestroy = false,
241 access: "private" | "public_read" = "private",
242 ) =>
243 stack.deploy(
244 Effect.gen(function* () {
245 const project = yield* Project("SafeBucketProject", {
246 region: "aws-us-east-2",
247 });
248 return yield* Bucket("SafeBucket", {
249 project,
250 forceDestroy,
251 access,
252 });
253 }),
254 );
255 const bucket = yield* deploy();
256 const client = yield* bucketStorageClient(bucket);
257 yield* client.put("retained.txt", "preserve me");
258 const visibility = yield* deploy(false, "public_read").pipe(
259 Effect.result,
260 );
261 expect(Result.isFailure(visibility)).toBe(true);
262 const listing = yield* SDK.listProjectBranchBuckets({
263 project_id: bucket.projectId,
264 branch_id: bucket.branchId,
265 });
266 expect(
267 listing.buckets.find((item) => item.name === bucket.bucketName)
268 ?.access_level,
269 ).toBe("private");
270 expect((yield* client.head("retained.txt"))?.ContentLength).toBe(11);
271 yield* deploy();
272 const deletion = yield* stack.destroy().pipe(Effect.result);
273 expect(Result.isFailure(deletion)).toBe(true);
274 expect((yield* client.head("retained.txt"))?.ContentLength).toBe(11);
275 yield* deploy(true);
276 yield* stack.destroy();
277 yield* stack.destroy();
278 }),
279 {
280 tags: [
281 "provider:neon",
282 "provider:neon:bucket",
283 "provider:neon:project",
284 "live",
285 ],
286 timeout: 120_000,
287 },
288);
289
290test.provider(
291 "child storage inherits files while writes and deletes remain branch-local",
292 (stack) =>
293 Effect.gen(function* () {
294 yield* stack.destroy();
295 const parentProgram = Effect.gen(function* () {
296 const project = yield* Project("StorageLineageProject", {
297 region: "aws-us-east-2",
298 });
299 const bucket = yield* Bucket("ParentBucket", {
300 project,
301 tags: { lineage: "parent" },
302 cors: [
303 {
304 AllowedOrigins: ["https://parent.example.com"],
305 AllowedMethods: ["GET"],
306 },
307 ],
308 forceDestroy: true,
309 });
310 return { project, bucket };
311 });
312 const parent = yield* stack.deploy(parentProgram);
313 const outcome = yield* Effect.gen(function* () {
314 const parentClient = yield* bucketStorageClient(parent.bucket);
315 yield* parentClient.put("inherited.txt", "parent");
316 const parentTags = yield* parentClient.getTags();
317 const parentCors = yield* parentClient.getCors();
318 const branchProgram = Effect.gen(function* () {
319 const { project, bucket } = yield* parentProgram;
320 const branch = yield* Branch("ChildBranch", { project });
321 return { branch, bucket };
322 });
323 const { branch } = yield* stack.deploy(branchProgram);
324 const provider = yield* Provider.findProvider(Bucket);
325 expect(
326 Unowned.is(
327 yield* provider.read!({
328 id: "ChildBucket",
329 fqn: "ChildBucket",
330 instanceId: "inherited-child",
331 olds: {
332 branch,
333 name: parent.bucket.bucketName,
334 credential: parent.bucket.credential,
335 },
336 output: undefined,
337 }),
338 ),
339 ).toBe(true);
340 const deployChild = (updated: boolean) =>
341 stack.deploy(
342 Effect.gen(function* () {
343 const { branch, bucket: parentBucket } = yield* branchProgram;
344 const bucket = yield* Bucket("ChildBucket", {
345 branch,
346 name: parentBucket.bucketName,
347 tags: { lineage: updated ? "updated-child" : "child" },
348 cors: updated
349 ? []
350 : [
351 {
352 AllowedOrigins: ["https://child.example.com"],
353 AllowedMethods: ["GET"],
354 },
355 ],
356 forceDestroy: true,
357 }).pipe(adopt(true));
358 return { branch, bucket };
359 }),
360 );
361 const child = yield* deployChild(false);
362 expect(child.bucket.tags["alchemy::branch"]).toBe(
363 child.branch.branchId,
364 );
365 expect(child.bucket.tags.lineage).toBe("child");
366 expect(child.bucket.cors).toEqual([
367 {
368 AllowedOrigins: ["https://child.example.com"],
369 AllowedMethods: ["GET"],
370 },
371 ]);
372 const childClient = yield* bucketStorageClient(child.bucket);
373 const inherited = yield* storageBodyBytes(
374 (yield* childClient.get("inherited.txt"))?.Body,
375 );
376 expect(
377 yield* Effect.sync(() => new TextDecoder().decode(inherited)),
378 ).toBe("parent");
379 expect(yield* parentClient.getTags()).toEqual(parentTags);
380 expect(yield* parentClient.getCors()).toEqual(parentCors);
381 const updated = yield* deployChild(true);
382 expect(updated.bucket.bucketName).toBe(child.bucket.bucketName);
383 expect(updated.bucket.tags.lineage).toBe("updated-child");
384 expect(updated.bucket.cors).toEqual([]);
385 expect(yield* parentClient.getTags()).toEqual(parentTags);
386 expect(yield* parentClient.getCors()).toEqual(parentCors);
387 const ancestorClient = yield* makeStorageClient(
388 {
389 endpoint: child.bucket.endpoint,
390 region: child.bucket.region,
391 accessKeyId: parent.bucket.credential.tokenId,
392 secretAccessKey: parent.bucket.credential.s3SecretAccessKey,
393 },
394 child.bucket.bucketName,
395 );
396 expect(
397 (yield* ancestorClient.head("inherited.txt"))?.ContentLength,
398 ).toBe(6);
399 yield* childClient.put("inherited.txt", "child");
400 const original = yield* storageBodyBytes(
401 (yield* parentClient.get("inherited.txt"))?.Body,
402 );
403 expect(
404 yield* Effect.sync(() => new TextDecoder().decode(original)),
405 ).toBe("parent");
406 yield* childClient.delete("inherited.txt");
407 expect(yield* childClient.get("inherited.txt")).toBeUndefined();
408 expect((yield* parentClient.head("inherited.txt"))?.ContentLength).toBe(
409 6,
410 );
411 yield* stack.deploy(parentProgram);
412 expect((yield* parentClient.head("inherited.txt"))?.ContentLength).toBe(
413 6,
414 );
415 expect(yield* parentClient.getTags()).toEqual(parentTags);
416 expect(yield* parentClient.getCors()).toEqual(parentCors);
417 expect(
418 yield* SDK.getProjectBranch({
419 project_id: branch.projectId,
420 branch_id: branch.branchId,
421 }).pipe(
422 Effect.as(false),
423 Effect.catchTag("NotFound", () => Effect.succeed(true)),
424 ),
425 ).toBe(true);
426 }).pipe(Effect.result);
427 yield* stack.destroy();
428 expect(
429 yield* SDK.getProject({ project_id: parent.project.projectId }).pipe(
430 Effect.as(false),
431 Effect.catchTag("NotFound", () => Effect.succeed(true)),
432 ),
433 ).toBe(true);
434 yield* Effect.log("Inherited storage fixture project is absent");
435 yield* stack.destroy();
436 if (Result.isFailure(outcome)) return yield* Effect.fail(outcome.failure);
437 }),
438 {
439 tags: [
440 "provider:neon",
441 "provider:neon:branch",
442 "provider:neon:bucket",
443 "provider:neon:object",
444 "provider:neon:project",
445 "live",
446 ],
447 timeout: 120_000,
448 },
449);
450
451test.provider(
452 "inherited S3 data remains isolated and inherited tag rejection is typed",
453 (stack) =>
454 Effect.gen(function* () {
455 yield* stack.destroy();
456 const parentProgram = Effect.gen(function* () {
457 const project = yield* Project("StorageDataLineageProject", {
458 region: "aws-us-east-2",
459 });
460 const bucket = yield* Bucket("SourceBucket", {
461 project,
462 forceDestroy: true,
463 });
464 return { project, bucket };
465 });
466 const parent = yield* stack.deploy(parentProgram);
467 const parentClient = yield* bucketStorageClient(parent.bucket);
468 yield* parentClient.put("inherited.txt", "parent");
469 const child = yield* stack.deploy(
470 Effect.gen(function* () {
471 const { project } = yield* parentProgram;
472 const branch = yield* Branch("DataChild", { project });
473 const reader = yield* Credential("ChildReader", {
474 branch,
475 scopes: ["storage:read"],
476 });
477 return { branch, reader };
478 }),
479 );
480 const storage = yield* SDK.getProjectBranchStorage({
481 project_id: child.branch.projectId,
482 branch_id: child.branch.branchId,
483 });
484 const config = {
485 endpoint: storage.s3_endpoint,
486 region: storage.region,
487 accessKeyId: parent.bucket.credential.tokenId,
488 secretAccessKey: parent.bucket.credential.s3SecretAccessKey,
489 };
490 const client = yield* makeStorageClient(config, parent.bucket.bucketName);
491 const reader = yield* makeStorageClient(
492 {
493 ...config,
494 accessKeyId: child.reader.tokenId,
495 secretAccessKey: child.reader.s3SecretAccessKey,
496 },
497 parent.bucket.bucketName,
498 );
499 const bytes = yield* storageBodyBytes(
500 (yield* reader.get("inherited.txt"))?.Body,
501 );
502 expect(yield* Effect.sync(() => new TextDecoder().decode(bytes))).toBe(
503 "parent",
504 );
505 const denied = yield* reader
506 .put("denied.txt", "must not write")
507 .pipe(Effect.result);
508 expect(Result.isFailure(denied)).toBe(true);
509 if (Result.isFailure(denied))
510 expect(denied.failure._tag).toBe("AccessDeniedException");
511 const tagging = yield* client
512 .putTags({ phase: "child" })
513 .pipe(Effect.result);
514 expect(Result.isFailure(tagging)).toBe(true);
515 if (Result.isFailure(tagging))
516 expect(tagging.failure._tag).toBe("NoSuchBucket");
517 yield* client.put("inherited.txt", "child");
518 expect((yield* parentClient.head("inherited.txt"))?.ContentLength).toBe(
519 6,
520 );
521 yield* client.delete("inherited.txt");
522 expect(yield* client.get("inherited.txt")).toBeUndefined();
523 const original = yield* storageBodyBytes(
524 (yield* parentClient.get("inherited.txt"))?.Body,
525 );
526 expect(yield* Effect.sync(() => new TextDecoder().decode(original))).toBe(
527 "parent",
528 );
529 yield* stack.deploy(parentProgram);
530 expect((yield* parentClient.head("inherited.txt"))?.ContentLength).toBe(
531 6,
532 );
533 yield* stack.destroy();
534 yield* stack.destroy();
535 }),
536 {
537 tags: [
538 "provider:neon",
539 "provider:neon:branch",
540 "provider:neon:bucket",
541 "provider:neon:credential",
542 "provider:neon:object",
543 "provider:neon:project",
544 "live",
545 ],
546 timeout: 120_000,
547 },
548);
549
550test.provider(
551 "forceDestroy paginates more than one thousand objects",
552 (stack) =>
553 Effect.gen(function* () {
554 yield* stack.destroy();
555 const bucket = yield* stack.deploy(
556 Effect.gen(function* () {
557 const project = yield* Project("PaginatedBucketProject", {
558 region: "aws-us-east-2",
559 });
560 return yield* Bucket("PaginatedBucket", {
561 project,
562 forceDestroy: true,
563 });
564 }),
565 );
566 const client = yield* bucketStorageClient(bucket);
567 yield* Effect.forEach(
568 Array.from({ length: 1001 }, (_, index) => index),
569 (index) => client.put(`page/${String(index).padStart(4, "0")}`, "x"),
570 { concurrency: 24 },
571 );
572 const first = yield* client.list({ limit: 1000 });
573 expect(first.IsTruncated).toBe(true);
574 expect(first.Contents).toHaveLength(1000);
575 yield* stack.destroy();
576 yield* stack.destroy();
577 }),
578 {
579 tags: [
580 "provider:neon",
581 "provider:neon:bucket",
582 "provider:neon:project",
583 "live",
584 ],
585 timeout: 120_000,
586 },
587);