1import * as AWS from "@/AWS";
2import * as Core from "@/Test/Core";
3import * as Test from "@/Test/Alchemy";
4import * as IAM from "@distilled.cloud/aws/iam";
5import * as Lambda from "@distilled.cloud/aws/lambda";
6import * as S3 from "@distilled.cloud/aws/s3";
7import { describe, expect } from "alchemy-test";
8import * as Data from "effect/Data";
9import * as Effect from "effect/Effect";
10import * as Schedule from "effect/Schedule";
11import * as Schema from "effect/Schema";
12import * as Stream from "effect/Stream";
13import * as HttpClient from "effect/http/HttpClient";
14import * as HttpClientRequest from "effect/http/HttpClientRequest";
15import S3VersionedMultipartFunctionLive, {
16 S3VersionedMultipartFunction,
17} from "./fixtures/versioned-multipart-handler.ts";
18
19const options = { providers: AWS.providers(), dev: false };
20const { test, beforeAll, afterAll } = Test.make(options);
21const stack = Core.scratchStack(
22 options,
23 "S3VersionedMultipart",
24 "test/AWS/S3/VersionedMultipart.test.ts",
25);
26const actions = {
27 CreateMultipartUpload: ["s3:PutObject"],
28 UploadPart: ["s3:PutObject"],
29 UploadPartCopy: ["s3:PutObject", "s3:GetObject", "s3:GetObjectVersion"],
30 UploadPartCopySource: ["s3:GetObject", "s3:GetObjectVersion"],
31 UploadPartCopyUnboundSource: [],
32 ListParts: ["s3:ListMultipartUploadParts"],
33 ListMultipartUploads: ["s3:ListBucketMultipartUploads"],
34 CompleteMultipartUpload: ["s3:PutObject"],
35 AbortMultipartUpload: ["s3:AbortMultipartUpload"],
36} as const;
37type Binding = keyof typeof actions;
38const bucketInfo = Schema.Array(
39 Schema.Struct({
40 binding: Schema.Literals([
41 "CreateMultipartUpload",
42 "UploadPart",
43 "UploadPartCopy",
44 "UploadPartCopySource",
45 "UploadPartCopyUnboundSource",
46 "ListParts",
47 "ListMultipartUploads",
48 "CompleteMultipartUpload",
49 "AbortMultipartUpload",
50 ]),
51 bucketName: Schema.String,
52 bucketArn: Schema.String,
53 }),
54);
55let buckets: typeof bucketInfo.Type = [];
56let baseUrl: string;
57let roleName: string;
58let functionName: string;
59
60class FixtureNotReady extends Data.TaggedError("FixtureNotReady") {}
61class UnexpectedFixtureStatus extends Data.TaggedError(
62 "UnexpectedFixtureStatus",
63)<{
64 readonly status: number;
65}> {}
66class BucketStillExists extends Data.TaggedError("BucketStillExists") {}
67
68const bucketFor = (binding: Binding) => {
69 const bucket = buckets.find((bucket) => bucket.binding === binding);
70 if (!bucket) throw new Error(`Missing fixture bucket for ${binding}`);
71 return bucket.bucketName;
72};
73
74const post = <A, I>(
75 path: string,
76 body: object,
77 schema: Schema.Codec<A, I>,
78 expectedStatus = 200,
79) =>
80 Effect.gen(function* () {
81 const response = yield* HttpClient.execute(
82 HttpClientRequest.post(`${baseUrl}${path}`).pipe(
83 HttpClientRequest.bodyJsonUnsafe(body),
84 ),
85 );
86 if (response.status !== expectedStatus) {
87 return yield* Effect.fail(
88 new Error(`${path}: HTTP ${response.status}: ${yield* response.text}`),
89 );
90 }
91 return yield* Schema.decodeUnknownEffect(schema)(yield* response.json);
92 });
93
94const policyDocument = Schema.fromJsonString(
95 Schema.Struct({
96 Statement: Schema.Array(
97 Schema.Struct({
98 Effect: Schema.String,
99 Action: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
100 Resource: Schema.Union([Schema.String, Schema.Array(Schema.String)]),
101 }),
102 ),
103 }),
104);
105
106const assertPermissions = Effect.gen(function* () {
107 const configuration = yield* Lambda.getFunctionConfiguration({
108 FunctionName: functionName,
109 });
110 expect(configuration.FunctionName).toBe(functionName);
111 expect(configuration.Role?.endsWith(`/${roleName}`)).toBe(true);
112 const attached = yield* IAM.listAttachedRolePolicies
113 .pages({ RoleName: roleName })
114 .pipe(Stream.runCollect);
115 expect(
116 attached
117 .flatMap((page) => page.AttachedPolicies ?? [])
118 .map((policy) => policy.PolicyArn),
119 ).toEqual([
120 "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
121 ]);
122 const pages = yield* IAM.listRolePolicies
123 .pages({ RoleName: roleName })
124 .pipe(Stream.runCollect);
125 const policies = yield* Effect.forEach(
126 pages.flatMap((page) => page.PolicyNames),
127 (PolicyName) =>
128 Effect.gen(function* () {
129 const policy = yield* IAM.getRolePolicy({
130 RoleName: roleName,
131 PolicyName,
132 });
133 const decoded = yield* Effect.try(() =>
134 decodeURIComponent(policy.PolicyDocument),
135 );
136 return yield* Schema.decodeUnknownEffect(policyDocument)(decoded);
137 }),
138 );
139 const actual = policies
140 .flatMap((policy) =>
141 policy.Statement.flatMap((statement) => {
142 const actionList =
143 typeof statement.Action === "string"
144 ? [statement.Action]
145 : statement.Action;
146 const resources =
147 typeof statement.Resource === "string"
148 ? [statement.Resource]
149 : statement.Resource;
150 return actionList.flatMap((action) =>
151 resources.map(
152 (resource) => `${statement.Effect} ${action} ${resource}`,
153 ),
154 );
155 }),
156 )
157 .sort();
158 const expected = buckets
159 .flatMap((bucket) => [
160 ...actions[bucket.binding].map(
161 (action) =>
162 `Allow ${action} ${bucket.bucketArn}${bucket.binding === "ListMultipartUploads" ? "" : "/*"}`,
163 ),
164 ...(bucket.binding === "UploadPartCopySource"
165 ? [`Allow s3:ListBucket ${bucket.bucketArn}`]
166 : []),
167 ])
168 .sort();
169 expect(actual).toEqual(expected);
170 for (const bucket of buckets) {
171 expect(
172 (yield* S3.getBucketVersioning({ Bucket: bucket.bucketName })).Status,
173 ).toBe("Enabled");
174 }
175});
176
177const assertBucketDeleted = (Bucket: string) =>
178 S3.headBucket({ Bucket }).pipe(
179 Effect.flatMap(() => Effect.fail(new BucketStillExists())),
180 Effect.retry({
181 while: (error) => error._tag === "BucketStillExists",
182 schedule: Schedule.spaced("1 second"),
183 times: 10,
184 }),
185 Effect.catchTag("NotFound", () => Effect.void),
186 );
187
188const abortPending = (Bucket: string, Key: string, UploadId: string) =>
189 S3.abortMultipartUpload({ Bucket, Key, UploadId }).pipe(
190 Effect.catchTag("NoSuchUpload", () => Effect.void),
191 Effect.orDie,
192 );
193
194const withUpload = <A, E, R>(
195 Bucket: string,
196 Key: string,
197 run: (UploadId: string) => Effect.Effect<A, E, R>,
198) =>
199 Effect.gen(function* () {
200 const created = yield* S3.createMultipartUpload({ Bucket, Key });
201 expect(created.UploadId).toBeTruthy();
202 return yield* run(created.UploadId!).pipe(
203 Effect.ensuring(abortPending(Bucket, Key, created.UploadId!)),
204 );
205 });
206
207const seedVersions = Effect.fn(function* (Bucket: string, Key: string) {
208 const oldBody = "original version";
209 const currentBody = "current version with a different payload";
210 const old = yield* S3.putObject({ Bucket, Key, Body: oldBody });
211 const current = yield* S3.putObject({ Bucket, Key, Body: currentBody });
212 expect(old.VersionId).toBeTruthy();
213 expect(current.VersionId).toBeTruthy();
214 expect(old.VersionId).not.toBe("null");
215 expect(current.VersionId).not.toBe(old.VersionId);
216 return {
217 old: old.VersionId!,
218 current: current.VersionId!,
219 oldBody,
220 currentBody,
221 };
222});
223
224const assertBody = Effect.fn(function* (
225 Bucket: string,
226 Key: string,
227 VersionId: string | undefined,
228 body: string,
229) {
230 const object = yield* S3.getObject({ Bucket, Key, VersionId });
231 if (VersionId !== undefined) expect(object.VersionId).toBe(VersionId);
232 expect(object.Body).toBeDefined();
233 expect(yield* object.Body!.pipe(Stream.decodeText, Stream.mkString)).toBe(
234 body,
235 );
236 return object.VersionId;
237});
238
239const assertVersions = Effect.fn(function* (
240 Bucket: string,
241 Key: string,
242 versions: {
243 old: string;
244 current: string;
245 oldBody: string;
246 currentBody: string;
247 },
248) {
249 expect(yield* assertBody(Bucket, Key, undefined, versions.currentBody)).toBe(
250 versions.current,
251 );
252 yield* assertBody(Bucket, Key, versions.old, versions.oldBody);
253 const listed = yield* S3.listObjectVersions({ Bucket, Prefix: Key });
254 expect(
255 listed.Versions?.filter((version) => version.Key === Key)
256 .map((version) => ({
257 id: version.VersionId,
258 latest: version.IsLatest,
259 }))
260 .sort((a, b) => a.id!.localeCompare(b.id!)),
261 ).toEqual(
262 [
263 { id: versions.old, latest: false },
264 { id: versions.current, latest: true },
265 ].sort((a, b) => a.id.localeCompare(b.id)),
266 );
267});
268
269const minimumPartSize = 5 * 1024 * 1024;
270
271const assertCompletedVersion = Effect.fn(function* (
272 Bucket: string,
273 Key: string,
274 versions: {
275 old: string;
276 current: string;
277 oldBody: string;
278 currentBody: string;
279 },
280 VersionId: string,
281 Body: string,
282) {
283 expect(VersionId).toBeTruthy();
284 expect(VersionId).not.toBe("null");
285 expect(VersionId).not.toBe(versions.old);
286 expect(VersionId).not.toBe(versions.current);
287 expect(yield* assertBody(Bucket, Key, undefined, Body)).toBe(VersionId);
288 yield* assertBody(Bucket, Key, VersionId, Body);
289 yield* assertBody(Bucket, Key, versions.old, versions.oldBody);
290 yield* assertBody(Bucket, Key, versions.current, versions.currentBody);
291 const listed = yield* S3.listObjectVersions({ Bucket, Prefix: Key });
292 expect(
293 listed.Versions?.filter((version) => version.Key === Key)
294 .map((version) => ({
295 id: version.VersionId,
296 latest: version.IsLatest,
297 }))
298 .sort((a, b) => a.id!.localeCompare(b.id!)),
299 ).toEqual(
300 [
301 { id: versions.old, latest: false },
302 { id: versions.current, latest: false },
303 { id: VersionId, latest: true },
304 ].sort((a, b) => a.id.localeCompare(b.id)),
305 );
306 expect(
307 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ?? [],
308 ).toEqual([]);
309});
310
311const uploadedPart = Schema.Struct({ ETag: Schema.String });
312const completedUpload = Schema.Struct({
313 VersionId: Schema.String,
314 ETag: Schema.String,
315});
316const noSuchUpload = Schema.Struct({ tag: Schema.Literal("NoSuchUpload") });
317
318beforeAll(
319 Effect.gen(function* () {
320 yield* stack.destroy();
321 const deployed = yield* stack.deploy(
322 S3VersionedMultipartFunction.pipe(
323 Effect.provide(S3VersionedMultipartFunctionLive),
324 ),
325 );
326 expect(deployed.functionUrl).toBeTruthy();
327 baseUrl = deployed.functionUrl!.replace(/\/+$/, "");
328 roleName = deployed.roleName;
329 functionName = deployed.functionName;
330 buckets = yield* HttpClient.get(`${baseUrl}/info`).pipe(
331 Effect.flatMap((response) =>
332 Effect.gen(function* () {
333 if (response.status === 503 || response.status === 502) {
334 return yield* Effect.fail(new FixtureNotReady());
335 }
336 if (response.status === 200) return yield* response.json;
337 return yield* Effect.fail(
338 new UnexpectedFixtureStatus({ status: response.status }),
339 );
340 }),
341 ),
342 Effect.flatMap(Schema.decodeUnknownEffect(bucketInfo)),
343 Effect.retry({
344 while: (error) =>
345 error._tag === "FixtureNotReady" ||
346 (error._tag === "HttpClientError" &&
347 error.reason._tag === "TransportError"),
348 schedule: Schedule.spaced("2 seconds"),
349 times: 10,
350 }),
351 );
352 expect(buckets.map((bucket) => bucket.binding).sort()).toEqual(
353 Object.keys(actions).sort(),
354 );
355 expect(new Set(buckets.map((bucket) => bucket.bucketName)).size).toBe(9);
356 yield* Core.withProviders(
357 assertPermissions,
358 options,
359 "S3VersionedMultipart",
360 );
361 }),
362 { timeout: 120_000, retry: 0 },
363);
364
365afterAll(
366 Effect.gen(function* () {
367 yield* stack.destroy();
368 yield* Core.withProviders(
369 Effect.forEach(
370 buckets,
371 (bucket) => assertBucketDeleted(bucket.bucketName),
372 { discard: true },
373 ),
374 options,
375 "S3VersionedMultipart",
376 );
377 }),
378 { timeout: 120_000, retry: 0 },
379);
380
381describe(
382 "CreateMultipartUpload",
383 {
384 tags: [
385 "provider:aws",
386 "provider:aws:iam",
387 "provider:aws:lambda",
388 "provider:aws:s3",
389 "live",
390 ],
391 },
392 () => {
393 test.provider(
394 "creates a pending upload without replacing either existing version",
395 () =>
396 Effect.gen(function* () {
397 const Bucket = bucketFor("CreateMultipartUpload");
398 const Key = "versioned-multipart/create.txt";
399 const versions = yield* seedVersions(Bucket, Key);
400 const created = yield* post(
401 "/create",
402 { Key, ContentType: "text/markdown" },
403 Schema.Struct({
404 Bucket: Schema.String,
405 Key: Schema.String,
406 UploadId: Schema.String,
407 }),
408 );
409 yield* Effect.gen(function* () {
410 expect(created.Bucket).toBe(Bucket);
411 expect(created.Key).toBe(Key);
412 expect(created.UploadId).toBeTruthy();
413 const pending = yield* S3.listMultipartUploads({
414 Bucket,
415 Prefix: Key,
416 });
417 expect(pending.Uploads?.map((upload) => upload.UploadId)).toContain(
418 created.UploadId,
419 );
420 yield* assertVersions(Bucket, Key, versions);
421 const part = yield* S3.uploadPart({
422 Bucket,
423 Key,
424 UploadId: created.UploadId,
425 PartNumber: 1,
426 Body: "created through Lambda",
427 });
428 const completed = yield* S3.completeMultipartUpload({
429 Bucket,
430 Key,
431 UploadId: created.UploadId,
432 MultipartUpload: { Parts: [{ PartNumber: 1, ETag: part.ETag! }] },
433 });
434 expect(completed.VersionId).toBeTruthy();
435 const head = yield* S3.headObject({
436 Bucket,
437 Key,
438 VersionId: completed.VersionId,
439 });
440 expect(head.ContentType).toBe("text/markdown");
441 yield* assertBody(Bucket, Key, versions.old, versions.oldBody);
442 yield* assertBody(
443 Bucket,
444 Key,
445 versions.current,
446 versions.currentBody,
447 );
448 }).pipe(Effect.ensuring(abortPending(Bucket, Key, created.UploadId)));
449 }),
450 { timeout: 120_000, retry: 0 },
451 );
452 },
453);
454
455describe(
456 "UploadPart",
457 {
458 tags: [
459 "provider:aws",
460 "provider:aws:iam",
461 "provider:aws:lambda",
462 "provider:aws:s3",
463 "live",
464 ],
465 },
466 () => {
467 test.provider(
468 "rejects an out-of-band aborted upload with typed NoSuchUpload",
469 () =>
470 Effect.gen(function* () {
471 const Bucket = bucketFor("UploadPart");
472 const Key = "versioned-multipart/upload-aborted.txt";
473 const versions = yield* seedVersions(Bucket, Key);
474 yield* withUpload(Bucket, Key, (UploadId) =>
475 Effect.gen(function* () {
476 yield* S3.abortMultipartUpload({ Bucket, Key, UploadId });
477 expect(
478 yield* post(
479 "/upload",
480 { Key, UploadId, PartNumber: 1, Body: "rejected part" },
481 noSuchUpload,
482 404,
483 ),
484 ).toEqual({ tag: "NoSuchUpload" });
485 yield* assertVersions(Bucket, Key, versions);
486 expect(
487 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key }))
488 .Uploads ?? [],
489 ).toEqual([]);
490 }),
491 );
492 }),
493 { timeout: 120_000, retry: 0 },
494 );
495
496 test.provider(
497 "uploads a part through its isolated binding without changing existing versions",
498 () =>
499 Effect.gen(function* () {
500 const Bucket = bucketFor("UploadPart");
501 const Key = "versioned-multipart/upload.txt";
502 const versions = yield* seedVersions(Bucket, Key);
503 yield* withUpload(Bucket, Key, (UploadId) =>
504 Effect.gen(function* () {
505 const part = yield* post(
506 "/upload",
507 {
508 Key,
509 UploadId,
510 PartNumber: 1,
511 Body: "a",
512 Repeat: minimumPartSize,
513 },
514 uploadedPart,
515 );
516 const initial = yield* S3.listParts({ Bucket, Key, UploadId });
517 expect(
518 initial.Parts?.map((entry) => ({
519 part: entry.PartNumber,
520 size: entry.Size,
521 etag: entry.ETag,
522 })),
523 ).toEqual([{ part: 1, size: minimumPartSize, etag: part.ETag }]);
524 yield* assertVersions(Bucket, Key, versions);
525 const replacement = yield* post(
526 "/upload",
527 {
528 Key,
529 UploadId,
530 PartNumber: 1,
531 Body: "b",
532 Repeat: minimumPartSize,
533 },
534 uploadedPart,
535 );
536 expect(replacement.ETag).not.toBe(part.ETag);
537 const tail = "final part uploaded through Lambda";
538 const final = yield* post(
539 "/upload",
540 { Key, UploadId, PartNumber: 2, Body: tail },
541 uploadedPart,
542 );
543 const listed = yield* S3.listParts({ Bucket, Key, UploadId });
544 expect(
545 listed.Parts?.map((entry) => ({
546 part: entry.PartNumber,
547 size: entry.Size,
548 etag: entry.ETag,
549 })),
550 ).toEqual([
551 { part: 1, size: minimumPartSize, etag: replacement.ETag },
552 { part: 2, size: tail.length, etag: final.ETag },
553 ]);
554 yield* assertVersions(Bucket, Key, versions);
555 const completed = yield* S3.completeMultipartUpload({
556 Bucket,
557 Key,
558 UploadId,
559 MultipartUpload: {
560 Parts: [
561 { PartNumber: 1, ETag: replacement.ETag },
562 { PartNumber: 2, ETag: final.ETag },
563 ],
564 },
565 });
566 const Body = yield* Effect.sync(
567 () => "b".repeat(minimumPartSize) + tail,
568 );
569 yield* assertCompletedVersion(
570 Bucket,
571 Key,
572 versions,
573 completed.VersionId!,
574 Body,
575 );
576 }),
577 );
578 }),
579 { timeout: 120_000, retry: 0 },
580 );
581 },
582);
583
584describe(
585 "UploadPartCopy",
586 {
587 tags: [
588 "provider:aws",
589 "provider:aws:iam",
590 "provider:aws:lambda",
591 "provider:aws:s3",
592 "live",
593 ],
594 },
595 () => {
596 test.provider(
597 "rejects an out-of-band aborted upload with typed NoSuchUpload",
598 () =>
599 Effect.gen(function* () {
600 const Bucket = bucketFor("UploadPartCopy");
601 const Key = "versioned-multipart/copy-aborted/destination.txt";
602 const sourceBucket = bucketFor("UploadPartCopySource");
603 const sourceKey = "versioned-multipart/copy-aborted/source.txt";
604 const sourceVersions = yield* seedVersions(sourceBucket, sourceKey);
605 const destinationVersions = yield* seedVersions(Bucket, Key);
606 const CopySource = yield* Effect.sync(
607 () =>
608 `${sourceBucket}/${sourceKey}?versionId=${encodeURIComponent(sourceVersions.old)}`,
609 );
610 yield* withUpload(Bucket, Key, (UploadId) =>
611 Effect.gen(function* () {
612 yield* S3.abortMultipartUpload({ Bucket, Key, UploadId });
613 expect(
614 yield* post(
615 "/copy",
616 { Key, UploadId, PartNumber: 1, CopySource },
617 noSuchUpload,
618 404,
619 ),
620 ).toEqual({ tag: "NoSuchUpload" });
621 yield* assertVersions(sourceBucket, sourceKey, sourceVersions);
622 yield* assertVersions(Bucket, Key, destinationVersions);
623 expect(
624 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key }))
625 .Uploads ?? [],
626 ).toEqual([]);
627 }),
628 );
629 }),
630 { timeout: 120_000, retry: 0 },
631 );
632
633 test.provider(
634 "rejects a deleted source version with typed NoSuchVersion",
635 () =>
636 Effect.gen(function* () {
637 const Bucket = bucketFor("UploadPartCopy");
638 const Key =
639 "versioned-multipart/copy-missing-version/destination.txt";
640 const sourceBucket = bucketFor("UploadPartCopySource");
641 const sourceKey =
642 "versioned-multipart/copy-missing-version/source.txt";
643 const sourceVersions = yield* seedVersions(sourceBucket, sourceKey);
644 const destinationVersions = yield* seedVersions(Bucket, Key);
645 const removed = yield* S3.putObject({
646 Bucket: sourceBucket,
647 Key: sourceKey,
648 Body: "permanently deleted source version",
649 });
650 expect(removed.VersionId).toBeTruthy();
651 yield* S3.deleteObject({
652 Bucket: sourceBucket,
653 Key: sourceKey,
654 VersionId: removed.VersionId!,
655 });
656 const CopySource = yield* Effect.sync(
657 () =>
658 `${sourceBucket}/${sourceKey}?versionId=${encodeURIComponent(removed.VersionId!)}`,
659 );
660 yield* withUpload(Bucket, Key, (UploadId) =>
661 Effect.gen(function* () {
662 expect(
663 yield* post(
664 "/copy",
665 { Key, UploadId, PartNumber: 1, CopySource },
666 Schema.Struct({ tag: Schema.Literal("NoSuchVersion") }),
667 404,
668 ),
669 ).toEqual({ tag: "NoSuchVersion" });
670 expect(
671 (yield* S3.listParts({ Bucket, Key, UploadId })).Parts ?? [],
672 ).toEqual([]);
673 yield* assertVersions(sourceBucket, sourceKey, sourceVersions);
674 yield* assertVersions(Bucket, Key, destinationVersions);
675 }),
676 );
677 expect(
678 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ??
679 [],
680 ).toEqual([]);
681 }),
682 { timeout: 120_000, retry: 0 },
683 );
684
685 test.provider(
686 "rejects an explicitly selected source delete marker with typed InvalidRequest",
687 () =>
688 Effect.gen(function* () {
689 const Bucket = bucketFor("UploadPartCopy");
690 const Key = "versioned-multipart/copy-delete-marker/destination.txt";
691 const sourceBucket = bucketFor("UploadPartCopySource");
692 const sourceKey = "versioned-multipart/copy-delete-marker/source.txt";
693 const sourceVersions = yield* seedVersions(sourceBucket, sourceKey);
694 const destinationVersions = yield* seedVersions(Bucket, Key);
695 const marker = yield* S3.deleteObject({
696 Bucket: sourceBucket,
697 Key: sourceKey,
698 });
699 expect(marker.DeleteMarker).toBe(true);
700 expect(marker.VersionId).toBeTruthy();
701 const sourceState = yield* S3.listObjectVersions({
702 Bucket: sourceBucket,
703 Prefix: sourceKey,
704 });
705 expect(
706 sourceState.DeleteMarkers?.map((entry) => ({
707 id: entry.VersionId,
708 latest: entry.IsLatest,
709 })),
710 ).toEqual([{ id: marker.VersionId, latest: true }]);
711 const CopySource = yield* Effect.sync(
712 () =>
713 `${sourceBucket}/${sourceKey}?versionId=${encodeURIComponent(marker.VersionId!)}`,
714 );
715 yield* withUpload(Bucket, Key, (UploadId) =>
716 Effect.gen(function* () {
717 expect(
718 yield* post(
719 "/copy",
720 { Key, UploadId, PartNumber: 1, CopySource },
721 Schema.Struct({ tag: Schema.Literal("InvalidRequest") }),
722 400,
723 ),
724 ).toEqual({ tag: "InvalidRequest" });
725 expect(
726 (yield* S3.listParts({ Bucket, Key, UploadId })).Parts ?? [],
727 ).toEqual([]);
728 yield* assertBody(
729 sourceBucket,
730 sourceKey,
731 sourceVersions.old,
732 sourceVersions.oldBody,
733 );
734 yield* assertBody(
735 sourceBucket,
736 sourceKey,
737 sourceVersions.current,
738 sourceVersions.currentBody,
739 );
740 const after = yield* S3.listObjectVersions({
741 Bucket: sourceBucket,
742 Prefix: sourceKey,
743 });
744 expect(after.Versions).toEqual(sourceState.Versions);
745 expect(after.DeleteMarkers).toEqual(sourceState.DeleteMarkers);
746 yield* assertVersions(Bucket, Key, destinationVersions);
747 }),
748 );
749 expect(
750 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ??
751 [],
752 ).toEqual([]);
753 }),
754 { timeout: 120_000, retry: 0 },
755 );
756
757 test.provider(
758 "copies an encoded source key's old version rather than its differently sized latest version",
759 () =>
760 Effect.gen(function* () {
761 const Bucket = bucketFor("UploadPartCopy");
762 const sourceKey =
763 "versioned-multipart/source space/+plus%percent/雪?#.txt";
764 const Key = "versioned-multipart/destination +%/copy.txt";
765 const versions = yield* seedVersions(Bucket, sourceKey);
766 const CopySource = yield* Effect.sync(
767 () =>
768 `${Bucket}/${sourceKey.split("/").map(encodeURIComponent).join("/")}?versionId=${encodeURIComponent(versions.old)}`,
769 );
770 yield* withUpload(Bucket, Key, (UploadId) =>
771 Effect.gen(function* () {
772 const copied = yield* post(
773 "/copy",
774 { Key, UploadId, PartNumber: 1, CopySource },
775 Schema.Struct({
776 CopySourceVersionId: Schema.String,
777 CopyPartResult: uploadedPart,
778 }),
779 );
780 expect(copied.CopySourceVersionId).toBe(versions.old);
781 expect(copied.CopySourceVersionId).not.toBe(versions.current);
782 const listed = yield* S3.listParts({ Bucket, Key, UploadId });
783 expect(
784 listed.Parts?.map((part) => ({
785 size: part.Size,
786 etag: part.ETag,
787 })),
788 ).toEqual([
789 {
790 size: versions.oldBody.length,
791 etag: copied.CopyPartResult.ETag,
792 },
793 ]);
794 const completed = yield* S3.completeMultipartUpload({
795 Bucket,
796 Key,
797 UploadId,
798 MultipartUpload: {
799 Parts: [{ PartNumber: 1, ETag: copied.CopyPartResult.ETag }],
800 },
801 });
802 expect(completed.VersionId).toBeTruthy();
803 yield* assertBody(
804 Bucket,
805 Key,
806 completed.VersionId!,
807 versions.oldBody,
808 );
809 yield* assertVersions(Bucket, sourceKey, versions);
810 }),
811 );
812 }),
813 { timeout: 120_000, retry: 0 },
814 );
815
816 test.provider(
817 "copies an old version from a separate GetObject-only source bucket",
818 () =>
819 Effect.gen(function* () {
820 const Bucket = bucketFor("UploadPartCopy");
821 const sourceBucket = bucketFor("UploadPartCopySource");
822 const sourceKey =
823 "versioned-multipart/cross-bucket/source +%/雪?#.txt";
824 const Key = "versioned-multipart/cross-bucket/destination.txt";
825 expect(sourceBucket).not.toBe(Bucket);
826 const versions = yield* seedVersions(sourceBucket, sourceKey);
827 const CopySource = yield* Effect.sync(
828 () =>
829 `${sourceBucket}/${sourceKey.split("/").map(encodeURIComponent).join("/")}?versionId=${encodeURIComponent(versions.old)}`,
830 );
831 yield* withUpload(Bucket, Key, (UploadId) =>
832 Effect.gen(function* () {
833 const copied = yield* post(
834 "/copy",
835 { Key, UploadId, PartNumber: 1, CopySource },
836 Schema.Struct({
837 CopySourceVersionId: Schema.String,
838 CopyPartResult: uploadedPart,
839 }),
840 );
841 expect(copied.CopySourceVersionId).toBe(versions.old);
842 expect(copied.CopySourceVersionId).not.toBe(versions.current);
843 const completed = yield* S3.completeMultipartUpload({
844 Bucket,
845 Key,
846 UploadId,
847 MultipartUpload: {
848 Parts: [{ PartNumber: 1, ETag: copied.CopyPartResult.ETag }],
849 },
850 });
851 expect(completed.VersionId).toBeTruthy();
852 expect(completed.VersionId).not.toBe("null");
853 yield* assertBody(
854 Bucket,
855 Key,
856 completed.VersionId!,
857 versions.oldBody,
858 );
859 expect(
860 yield* assertBody(Bucket, Key, undefined, versions.oldBody),
861 ).toBe(completed.VersionId);
862 const head = yield* S3.headObject({
863 Bucket,
864 Key,
865 VersionId: completed.VersionId!,
866 });
867 expect(head.VersionId).toBe(completed.VersionId);
868 expect(head.ETag).toBe(completed.ETag);
869 expect(head.ContentLength).toBe(versions.oldBody.length);
870 yield* assertVersions(sourceBucket, sourceKey, versions);
871 }),
872 );
873 }),
874 { timeout: 120_000, retry: 0 },
875 );
876
877 test.provider(
878 "rejects an old version in an unbound source bucket with typed AccessDeniedException",
879 () =>
880 Effect.gen(function* () {
881 const Bucket = bucketFor("UploadPartCopy");
882 const sourceBucket = bucketFor("UploadPartCopyUnboundSource");
883 const sourceKey = "versioned-multipart/unbound/source +%.txt";
884 const Key = "versioned-multipart/unbound/destination.txt";
885 expect(sourceBucket).not.toBe(Bucket);
886 expect(sourceBucket).not.toBe(bucketFor("UploadPartCopySource"));
887 const sourceVersions = yield* seedVersions(sourceBucket, sourceKey);
888 const destinationVersions = yield* seedVersions(Bucket, Key);
889 const CopySource = yield* Effect.sync(
890 () =>
891 `${sourceBucket}/${sourceKey.split("/").map(encodeURIComponent).join("/")}?versionId=${encodeURIComponent(sourceVersions.old)}`,
892 );
893 yield* withUpload(Bucket, Key, (UploadId) =>
894 Effect.gen(function* () {
895 expect(
896 yield* post(
897 "/copy",
898 { Key, UploadId, PartNumber: 1, CopySource },
899 Schema.Struct({
900 tag: Schema.Literal("AccessDeniedException"),
901 }),
902 403,
903 ),
904 ).toEqual({ tag: "AccessDeniedException" });
905 expect(
906 (yield* S3.listParts({ Bucket, Key, UploadId })).Parts ?? [],
907 ).toEqual([]);
908 yield* assertVersions(sourceBucket, sourceKey, sourceVersions);
909 yield* assertVersions(Bucket, Key, destinationVersions);
910 }),
911 );
912 }),
913 { timeout: 120_000, retry: 0 },
914 );
915 },
916);
917
918describe(
919 "ListParts",
920 {
921 tags: [
922 "provider:aws",
923 "provider:aws:iam",
924 "provider:aws:lambda",
925 "provider:aws:s3",
926 "live",
927 ],
928 },
929 () => {
930 test.provider(
931 "rejects an out-of-band aborted upload with typed NoSuchUpload",
932 () =>
933 Effect.gen(function* () {
934 const Bucket = bucketFor("ListParts");
935 const Key = "versioned-multipart/parts-aborted.txt";
936 const versions = yield* seedVersions(Bucket, Key);
937 yield* withUpload(Bucket, Key, (UploadId) =>
938 Effect.gen(function* () {
939 yield* S3.uploadPart({
940 Bucket,
941 Key,
942 UploadId,
943 PartNumber: 1,
944 Body: "aborted part",
945 });
946 yield* S3.abortMultipartUpload({ Bucket, Key, UploadId });
947 expect(
948 yield* post("/parts", { Key, UploadId }, noSuchUpload, 404),
949 ).toEqual({ tag: "NoSuchUpload" });
950 yield* assertVersions(Bucket, Key, versions);
951 expect(
952 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key }))
953 .Uploads ?? [],
954 ).toEqual([]);
955 }),
956 );
957 }),
958 { timeout: 120_000, retry: 0 },
959 );
960
961 test.provider(
962 "lists and paginates SDK-seeded parts without touching object versions",
963 () =>
964 Effect.gen(function* () {
965 const Bucket = bucketFor("ListParts");
966 const Key = "versioned-multipart/parts.txt";
967 const versions = yield* seedVersions(Bucket, Key);
968 yield* withUpload(Bucket, Key, (UploadId) =>
969 Effect.gen(function* () {
970 const first = yield* S3.uploadPart({
971 Bucket,
972 Key,
973 UploadId,
974 PartNumber: 1,
975 Body: "first",
976 });
977 const second = yield* S3.uploadPart({
978 Bucket,
979 Key,
980 UploadId,
981 PartNumber: 2,
982 Body: "second part",
983 });
984 const page = Schema.Struct({
985 IsTruncated: Schema.Boolean,
986 NextPartNumberMarker: Schema.optional(Schema.String),
987 Parts: Schema.Array(
988 Schema.Struct({
989 PartNumber: Schema.Number,
990 ETag: Schema.String,
991 Size: Schema.Number,
992 }),
993 ),
994 });
995 const firstPage = yield* post(
996 "/parts",
997 { Key, UploadId, MaxParts: 1 },
998 page,
999 );
1000 expect(firstPage.IsTruncated).toBe(true);
1001 expect(firstPage.Parts).toEqual([
1002 { PartNumber: 1, ETag: first.ETag, Size: 5 },
1003 ]);
1004 expect(firstPage.NextPartNumberMarker).toBeTruthy();
1005 const secondPage = yield* post(
1006 "/parts",
1007 {
1008 Key,
1009 UploadId,
1010 MaxParts: 1,
1011 PartNumberMarker: firstPage.NextPartNumberMarker,
1012 },
1013 page,
1014 );
1015 expect(secondPage.IsTruncated).toBe(false);
1016 expect(secondPage.Parts).toEqual([
1017 { PartNumber: 2, ETag: second.ETag, Size: 11 },
1018 ]);
1019 yield* assertVersions(Bucket, Key, versions);
1020 }),
1021 );
1022 }),
1023 { timeout: 120_000, retry: 0 },
1024 );
1025 },
1026);
1027
1028describe(
1029 "ListMultipartUploads",
1030 {
1031 tags: [
1032 "provider:aws",
1033 "provider:aws:iam",
1034 "provider:aws:lambda",
1035 "provider:aws:s3",
1036 "live",
1037 ],
1038 },
1039 () => {
1040 test.provider(
1041 "lists only matching pending uploads, not committed versions or other prefixes",
1042 () =>
1043 Effect.gen(function* () {
1044 const Bucket = bucketFor("ListMultipartUploads");
1045 const Key = "versioned-multipart/list/matching.txt";
1046 const versions = yield* seedVersions(Bucket, Key);
1047 const Prefix = "versioned-multipart/list/";
1048 const nextKey = `${Prefix}next.txt`;
1049 const excludedKey = "versioned-multipart/other/excluded.txt";
1050 const page = Schema.Struct({
1051 IsTruncated: Schema.Boolean,
1052 NextKeyMarker: Schema.optional(Schema.String),
1053 NextUploadIdMarker: Schema.optional(Schema.String),
1054 Uploads: Schema.Array(
1055 Schema.Struct({
1056 Key: Schema.String,
1057 UploadId: Schema.String,
1058 }),
1059 ),
1060 });
1061 yield* withUpload(Bucket, Key, (UploadId) =>
1062 withUpload(Bucket, Key, (siblingId) =>
1063 withUpload(Bucket, nextKey, (nextId) =>
1064 withUpload(Bucket, excludedKey, () =>
1065 Effect.gen(function* () {
1066 const listed = yield* post("/uploads", { Prefix }, page);
1067 const expected = [
1068 { Key, UploadId },
1069 { Key, UploadId: siblingId },
1070 { Key: nextKey, UploadId: nextId },
1071 ];
1072 expect(
1073 [...listed.Uploads].sort((a, b) =>
1074 a.UploadId.localeCompare(b.UploadId),
1075 ),
1076 ).toEqual(
1077 [...expected].sort((a, b) =>
1078 a.UploadId.localeCompare(b.UploadId),
1079 ),
1080 );
1081 const first = yield* post(
1082 "/uploads",
1083 { Prefix, MaxUploads: 1 },
1084 page,
1085 );
1086 expect(first.IsTruncated).toBe(true);
1087 expect(first.Uploads).toHaveLength(1);
1088 expect(first.Uploads[0].Key).toBe(Key);
1089 expect(first.NextKeyMarker).toBe(Key);
1090 expect(first.NextUploadIdMarker).toBe(
1091 first.Uploads[0].UploadId,
1092 );
1093 const second = yield* post(
1094 "/uploads",
1095 {
1096 Prefix,
1097 MaxUploads: 1,
1098 KeyMarker: first.NextKeyMarker,
1099 UploadIdMarker: first.NextUploadIdMarker,
1100 },
1101 page,
1102 );
1103 expect(second.IsTruncated).toBe(true);
1104 expect(second.Uploads).toHaveLength(1);
1105 expect(second.Uploads[0].Key).toBe(Key);
1106 expect(second.NextKeyMarker).toBe(Key);
1107 expect(second.NextUploadIdMarker).toBe(
1108 second.Uploads[0].UploadId,
1109 );
1110 expect(
1111 [
1112 first.Uploads[0].UploadId,
1113 second.Uploads[0].UploadId,
1114 ].sort(),
1115 ).toEqual([UploadId, siblingId].sort());
1116 const third = yield* post(
1117 "/uploads",
1118 {
1119 Prefix,
1120 MaxUploads: 1,
1121 KeyMarker: second.NextKeyMarker,
1122 UploadIdMarker: second.NextUploadIdMarker,
1123 },
1124 page,
1125 );
1126 expect(third.IsTruncated).toBe(false);
1127 expect(third.Uploads).toEqual([
1128 { Key: nextKey, UploadId: nextId },
1129 ]);
1130 yield* assertVersions(Bucket, Key, versions);
1131 }),
1132 ),
1133 ),
1134 ),
1135 );
1136 expect(
1137 (yield* S3.listMultipartUploads({ Bucket })).Uploads ?? [],
1138 ).toEqual([]);
1139 }),
1140 { timeout: 120_000, retry: 0 },
1141 );
1142 },
1143);
1144
1145describe(
1146 "CompleteMultipartUpload",
1147 {
1148 tags: [
1149 "provider:aws",
1150 "provider:aws:iam",
1151 "provider:aws:lambda",
1152 "provider:aws:s3",
1153 "live",
1154 ],
1155 },
1156 () => {
1157 test.provider(
1158 "rejects an out-of-band aborted upload with typed NoSuchUpload",
1159 () =>
1160 Effect.gen(function* () {
1161 const Bucket = bucketFor("CompleteMultipartUpload");
1162 const Key = "versioned-multipart/complete-aborted.txt";
1163 const versions = yield* seedVersions(Bucket, Key);
1164 yield* withUpload(Bucket, Key, (UploadId) =>
1165 Effect.gen(function* () {
1166 const part = yield* S3.uploadPart({
1167 Bucket,
1168 Key,
1169 UploadId,
1170 PartNumber: 1,
1171 Body: "aborted part",
1172 });
1173 expect(part.ETag).toBeTruthy();
1174 yield* S3.abortMultipartUpload({ Bucket, Key, UploadId });
1175 expect(
1176 yield* post(
1177 "/complete",
1178 {
1179 Key,
1180 UploadId,
1181 MultipartUpload: {
1182 Parts: [{ PartNumber: 1, ETag: part.ETag! }],
1183 },
1184 },
1185 noSuchUpload,
1186 404,
1187 ),
1188 ).toEqual({ tag: "NoSuchUpload" });
1189 yield* assertVersions(Bucket, Key, versions);
1190 expect(
1191 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key }))
1192 .Uploads ?? [],
1193 ).toEqual([]);
1194 }),
1195 );
1196 }),
1197 { timeout: 120_000, retry: 0 },
1198 );
1199
1200 test.provider(
1201 "returns a new version while preserving both previous versions",
1202 () =>
1203 Effect.gen(function* () {
1204 const Bucket = bucketFor("CompleteMultipartUpload");
1205 const Key = "versioned-multipart/complete.txt";
1206 const versions = yield* seedVersions(Bucket, Key);
1207 yield* withUpload(Bucket, Key, (UploadId) =>
1208 Effect.gen(function* () {
1209 const firstBody = yield* Effect.sync(() =>
1210 "c".repeat(minimumPartSize),
1211 );
1212 const tail = "new completed multipart version";
1213 const Body = firstBody + tail;
1214 const part = yield* S3.uploadPart({
1215 Bucket,
1216 Key,
1217 UploadId,
1218 PartNumber: 1,
1219 Body: firstBody,
1220 });
1221 const final = yield* S3.uploadPart({
1222 Bucket,
1223 Key,
1224 UploadId,
1225 PartNumber: 2,
1226 Body: tail,
1227 });
1228 const completed = yield* post(
1229 "/complete",
1230 {
1231 Key,
1232 UploadId,
1233 MultipartUpload: {
1234 Parts: [
1235 { PartNumber: 1, ETag: part.ETag! },
1236 { PartNumber: 2, ETag: final.ETag! },
1237 ],
1238 },
1239 },
1240 completedUpload,
1241 );
1242 yield* assertCompletedVersion(
1243 Bucket,
1244 Key,
1245 versions,
1246 completed.VersionId,
1247 Body,
1248 );
1249 }),
1250 );
1251 }),
1252 { timeout: 120_000, retry: 0 },
1253 );
1254 },
1255);
1256
1257describe(
1258 "AbortMultipartUpload",
1259 {
1260 tags: [
1261 "provider:aws",
1262 "provider:aws:iam",
1263 "provider:aws:lambda",
1264 "provider:aws:s3",
1265 "live",
1266 ],
1267 },
1268 () => {
1269 test.provider(
1270 "leaves versions and unrelated uploads intact when aborting inactive upload IDs",
1271 () =>
1272 Effect.gen(function* () {
1273 const Bucket = bucketFor("AbortMultipartUpload");
1274 const Key = "versioned-multipart/invalid-upload-ids.txt";
1275 const sourceKey = "versioned-multipart/invalid-upload-id-source.txt";
1276 const versions = yield* seedVersions(Bucket, Key);
1277 const assertInactiveAbort = (UploadId: string) =>
1278 Effect.gen(function* () {
1279 const response = yield* HttpClient.execute(
1280 HttpClientRequest.post(`${baseUrl}/abort`).pipe(
1281 HttpClientRequest.bodyJsonUnsafe({ Key, UploadId }),
1282 ),
1283 );
1284 expect([200, 404]).toContain(response.status);
1285 expect(yield* response.json).toEqual(
1286 response.status === 200
1287 ? { aborted: true }
1288 : { tag: "NoSuchUpload" },
1289 );
1290 });
1291 // Use an issued ID under a different key, not a malformed upload ID.
1292 yield* withUpload(Bucket, sourceKey, (UploadId) =>
1293 Effect.gen(function* () {
1294 yield* assertInactiveAbort(UploadId);
1295 const pending = yield* S3.listMultipartUploads({
1296 Bucket,
1297 Prefix: sourceKey,
1298 });
1299 expect(pending.Uploads?.map((upload) => upload.UploadId)).toEqual(
1300 [UploadId],
1301 );
1302 yield* assertVersions(Bucket, Key, versions);
1303 }),
1304 );
1305 yield* withUpload(Bucket, Key, (UploadId) =>
1306 Effect.gen(function* () {
1307 yield* S3.uploadPart({
1308 Bucket,
1309 Key,
1310 UploadId,
1311 PartNumber: 1,
1312 Body: "aborted upload bytes",
1313 });
1314 yield* post(
1315 "/abort",
1316 { Key, UploadId },
1317 Schema.Struct({ aborted: Schema.Literal(true) }),
1318 );
1319 yield* assertInactiveAbort(UploadId);
1320 yield* assertVersions(Bucket, Key, versions);
1321 }),
1322 );
1323 yield* withUpload(Bucket, Key, (UploadId) =>
1324 Effect.gen(function* () {
1325 const Body = "completed upload survives a rejected abort";
1326 const part = yield* S3.uploadPart({
1327 Bucket,
1328 Key,
1329 UploadId,
1330 PartNumber: 1,
1331 Body,
1332 });
1333 const completed = yield* S3.completeMultipartUpload({
1334 Bucket,
1335 Key,
1336 UploadId,
1337 MultipartUpload: {
1338 Parts: [{ PartNumber: 1, ETag: part.ETag! }],
1339 },
1340 });
1341 yield* assertInactiveAbort(UploadId);
1342 yield* assertCompletedVersion(
1343 Bucket,
1344 Key,
1345 versions,
1346 completed.VersionId!,
1347 Body,
1348 );
1349 }),
1350 );
1351 expect(
1352 (yield* S3.listMultipartUploads({ Bucket, Prefix: Key })).Uploads ??
1353 [],
1354 ).toEqual([]);
1355 expect(
1356 (yield* S3.listMultipartUploads({ Bucket, Prefix: sourceKey }))
1357 .Uploads ?? [],
1358 ).toEqual([]);
1359 }),
1360 { timeout: 120_000, retry: 0 },
1361 );
1362
1363 test.provider(
1364 "removes only the selected pending upload, preserving sibling upload and object versions",
1365 () =>
1366 Effect.gen(function* () {
1367 const Bucket = bucketFor("AbortMultipartUpload");
1368 const Key = "versioned-multipart/abort.txt";
1369 const versions = yield* seedVersions(Bucket, Key);
1370 yield* withUpload(Bucket, Key, (UploadId) =>
1371 withUpload(Bucket, Key, (siblingId) =>
1372 Effect.gen(function* () {
1373 yield* S3.uploadPart({
1374 Bucket,
1375 Key,
1376 UploadId,
1377 PartNumber: 1,
1378 Body: "discard this part",
1379 });
1380 const sibling = yield* S3.uploadPart({
1381 Bucket,
1382 Key,
1383 UploadId: siblingId,
1384 PartNumber: 1,
1385 Body: "keep this part",
1386 });
1387 expect(
1388 yield* post(
1389 "/abort",
1390 { Key, UploadId },
1391 Schema.Struct({ aborted: Schema.Boolean }),
1392 ),
1393 ).toEqual({ aborted: true });
1394 const pending = yield* S3.listMultipartUploads({
1395 Bucket,
1396 Prefix: Key,
1397 });
1398 expect(
1399 pending.Uploads?.map((upload) => upload.UploadId),
1400 ).toEqual([siblingId]);
1401 expect(
1402 (yield* S3.listParts({
1403 Bucket,
1404 Key,
1405 UploadId: siblingId,
1406 })).Parts?.map((part) => part.ETag),
1407 ).toEqual([sibling.ETag]);
1408 yield* assertVersions(Bucket, Key, versions);
1409 }),
1410 ),
1411 );
1412 }),
1413 { timeout: 120_000, retry: 0 },
1414 );
1415 },
1416);