1import * as s3vectors from "@distilled.cloud/aws/s3vectors";
2import * as Effect from "effect/Effect";
3import { Unowned } from "../../AdoptPolicy.ts";
4import { isResolved } from "../../Diff.ts";
5import { createPhysicalName } from "../../PhysicalName.ts";
6import * as Provider from "../../Provider.ts";
7import { Resource } from "../../Resource.ts";
8import { createInternalTags, diffTags, hasAlchemyTags } from "../../Tags.ts";
9import type { Providers } from "../Providers.ts";
10
11export interface IndexProps {
12 /**
13 * Name of the vector bucket that holds this index. Pass
14 * `bucket.vectorBucketName` from a {@link VectorBucket}.
15 *
16 * Changing it replaces the index.
17 */
18 vectorBucketName: string;
19 /**
20 * Name of the index (3-63 chars, lowercase). If omitted, a unique name is
21 * generated from the app, stage, and logical id.
22 *
23 * Changing the name replaces the index.
24 * @default ${app}-${stage}-${id}
25 */
26 indexName?: string;
27 /**
28 * The data type of the vectors stored in this index. Immutable — changing
29 * it replaces the index.
30 * @default "float32"
31 */
32 dataType?: "float32";
33 /**
34 * The number of dimensions of the vectors stored in this index. Immutable —
35 * changing it replaces the index.
36 */
37 dimension: number;
38 /**
39 * The distance metric used for similarity queries. Immutable — changing it
40 * replaces the index.
41 */
42 distanceMetric: "euclidean" | "cosine";
43 /**
44 * Metadata keys that are stored but NOT filterable at query time. Immutable
45 * — changing it replaces the index.
46 */
47 nonFilterableMetadataKeys?: string[];
48 /**
49 * Tags to apply to the index. Merged with internal Alchemy tags.
50 */
51 tags?: Record<string, string>;
52}
53
54export interface Index extends Resource<
55 "AWS.S3Vectors.Index",
56 IndexProps,
57 {
58 /**
59 * Name of the vector bucket containing the index.
60 */
61 vectorBucketName: string;
62 /**
63 * Name of the index.
64 */
65 indexName: string;
66 /**
67 * ARN of the index.
68 */
69 indexArn: string;
70 },
71 never,
72 Providers
73> {}
74
75/**
76 * A vector index inside an S3 Vectors {@link VectorBucket} — stores vectors of
77 * a fixed dimension and answers similarity queries under a distance metric.
78 *
79 * The index shape (data type, dimension, distance metric, non-filterable
80 * metadata keys) is fixed at create time; changing any of them replaces the
81 * index.
82 *
83 * ### Creating an Index
84 * **Example:** Cosine-Similarity Index
85 * ```typescript
86 * import * as S3Vectors from "alchemy/AWS/S3Vectors";
87 *
88 * const bucket = yield* S3Vectors.VectorBucket("Embeddings", {});
89 * const index = yield* S3Vectors.Index("Docs", {
90 * vectorBucketName: bucket.vectorBucketName,
91 * dimension: 1024,
92 * distanceMetric: "cosine",
93 * });
94 * ```
95 *
96 * @resource
97 */
98export const Index = Resource<Index>("AWS.S3Vectors.Index");
99
100const shapeKey = (props: IndexProps) =>
101 JSON.stringify({
102 dataType: props.dataType ?? "float32",
103 dimension: props.dimension,
104 distanceMetric: props.distanceMetric,
105 nonFilterableMetadataKeys: [
106 ...(props.nonFilterableMetadataKeys ?? []),
107 ].sort(),
108 });
109
110export const IndexProvider = () =>
111 Provider.effect(
112 Index,
113 Effect.gen(function* () {
114 const createName = Effect.fn(function* (
115 id: string,
116 props: Pick<IndexProps, "indexName">,
117 ) {
118 return (
119 props.indexName ??
120 (yield* createPhysicalName({ id, maxLength: 63 })).toLowerCase()
121 );
122 });
123
124 const observe = (vectorBucketName: string, indexName: string) =>
125 s3vectors.getIndex({ vectorBucketName, indexName }).pipe(
126 Effect.map((r) => r.index),
127 Effect.catchTag("NotFoundException", () => Effect.succeed(undefined)),
128 );
129
130 const observedTags = (resourceArn: string) =>
131 s3vectors.listTagsForResource({ resourceArn }).pipe(
132 Effect.map(
133 (r) =>
134 Object.fromEntries(
135 Object.entries(r.tags ?? {}).filter(([, v]) => v !== undefined),
136 ) as Record<string, string>,
137 ),
138 Effect.catchTag("NotFoundException", () =>
139 Effect.succeed({} as Record<string, string>),
140 ),
141 );
142
143 return Index.Provider.of({
144 stables: ["vectorBucketName", "indexName", "indexArn"],
145 // Indexes are keyed by a parent vector bucket; global enumeration would
146 // require fanning out over every bucket. Return empty — the engine
147 // reads owned indexes via read() using the cached output identity.
148 list: () => Effect.succeed([]),
149 read: Effect.fn(function* ({ id, olds, output }) {
150 const vectorBucketName =
151 output?.vectorBucketName ?? olds?.vectorBucketName;
152 if (vectorBucketName === undefined) return undefined;
153 const name = output?.indexName ?? (yield* createName(id, olds ?? {}));
154 const found = yield* observe(vectorBucketName, name);
155 if (!found) return undefined;
156 const attrs = {
157 vectorBucketName: found.vectorBucketName,
158 indexName: found.indexName,
159 indexArn: found.indexArn,
160 };
161 const tags = yield* observedTags(found.indexArn);
162 return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
163 }),
164 diff: Effect.fn(function* ({ id, news, olds }) {
165 if (!isResolved(news)) return undefined;
166 if (
167 olds?.vectorBucketName !== undefined &&
168 olds.vectorBucketName !== news.vectorBucketName
169 ) {
170 return { action: "replace" } as const;
171 }
172 const oldName = yield* createName(id, olds ?? {});
173 const newName = yield* createName(id, news ?? {});
174 if (oldName !== newName) return { action: "replace" } as const;
175 // The index shape is immutable — any change replaces.
176 if (olds !== undefined && shapeKey(olds) !== shapeKey(news)) {
177 return { action: "replace" } as const;
178 }
179 // fall through: engine default update (tags only)
180 }),
181 reconcile: Effect.fn(function* ({ id, news, output, session }) {
182 const vectorBucketName = news.vectorBucketName;
183 const name = output?.indexName ?? (yield* createName(id, news));
184 const internalTags = yield* createInternalTags(id);
185 const desiredTags = { ...news.tags, ...internalTags };
186
187 // 1. OBSERVE — cloud state is authoritative.
188 let live = yield* observe(vectorBucketName, name);
189
190 // 2. ENSURE — create when missing; a concurrent create surfaces as
191 // ConflictException, which we treat as a race and re-observe.
192 if (live === undefined) {
193 yield* s3vectors
194 .createIndex({
195 vectorBucketName,
196 indexName: name,
197 dataType: news.dataType ?? "float32",
198 dimension: news.dimension,
199 distanceMetric: news.distanceMetric,
200 metadataConfiguration: news.nonFilterableMetadataKeys
201 ? {
202 nonFilterableMetadataKeys: news.nonFilterableMetadataKeys,
203 }
204 : undefined,
205 tags: desiredTags,
206 })
207 .pipe(Effect.catchTag("ConflictException", () => Effect.void));
208 live = yield* observe(vectorBucketName, name);
209 }
210
211 const indexArn = live?.indexArn ?? output?.indexArn;
212
213 // 3. SYNC TAGS — diff against OBSERVED cloud tags.
214 if (indexArn !== undefined) {
215 const currentTags = yield* observedTags(indexArn);
216 const { upsert, removed } = diffTags(currentTags, desiredTags);
217 if (upsert.length > 0) {
218 yield* s3vectors.tagResource({
219 resourceArn: indexArn,
220 tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
221 });
222 }
223 if (removed.length > 0) {
224 yield* s3vectors.untagResource({
225 resourceArn: indexArn,
226 tagKeys: removed,
227 });
228 }
229 }
230
231 yield* session.note(name);
232 return { vectorBucketName, indexName: name, indexArn: indexArn! };
233 }),
234 delete: Effect.fn(function* ({ output }) {
235 yield* s3vectors
236 .deleteIndex({
237 vectorBucketName: output.vectorBucketName,
238 indexName: output.indexName,
239 })
240 .pipe(Effect.catchTag("NotFoundException", () => Effect.void));
241 }),
242 });
243 }),
244 );