1import * as aiGateway from "@distilled.cloud/cloudflare/ai-gateway";
2import * as Effect from "effect/Effect";
3import * as Predicate from "effect/Predicate";
4import * as Stream from "effect/Stream";
5import { deepEqual, isResolved } from "../../Diff.ts";
6import { createPhysicalName } from "../../PhysicalName.ts";
7import * as Provider from "../../Provider.ts";
8import { Resource } from "../../Resource.ts";
9import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
10import type { Providers } from "../Providers.ts";
11
12const TypeId = "Cloudflare.AI.DynamicRouting" as const;
13type TypeId = typeof TypeId;
14
15/**
16 * Reference to another element in the route graph.
17 */
18export type RouteEdge = {
19 /**
20 * The `id` of the element the edge points at.
21 */
22 elementId: string;
23};
24
25/**
26 * Entry point of the route graph. Every route has exactly one.
27 */
28export type RouteStartElement = {
29 /**
30 * Unique element identifier within the route graph.
31 */
32 id: string;
33 type: "start";
34 /**
35 * The element executed first.
36 */
37 outputs: { next: RouteEdge };
38};
39
40/**
41 * Branches the request based on conditions over request metadata.
42 */
43export type RouteConditionalElement = {
44 /**
45 * Unique element identifier within the route graph.
46 */
47 id: string;
48 type: "conditional";
49 /**
50 * Conditions evaluated against the request.
51 */
52 properties: { conditions?: unknown };
53 /**
54 * Edges taken when the conditions evaluate true / false.
55 */
56 outputs: { true: RouteEdge; false: RouteEdge };
57};
58
59/**
60 * Splits traffic across multiple edges by percentage.
61 */
62export type RoutePercentageElement = {
63 /**
64 * Unique element identifier within the route graph.
65 */
66 id: string;
67 type: "percentage";
68 /**
69 * Percentage-weighted edges.
70 */
71 outputs: Record<string, unknown>;
72};
73
74/**
75 * Applies a rate limit; requests over the limit take the fallback edge.
76 */
77export type RouteRateElement = {
78 /**
79 * Unique element identifier within the route graph.
80 */
81 id: string;
82 type: "rate";
83 properties: {
84 /**
85 * Key the limit is bucketed by.
86 */
87 key: string;
88 /**
89 * Maximum count/cost allowed inside the window.
90 */
91 limit: number;
92 /**
93 * Whether the limit counts requests or cost.
94 */
95 limitType: "count" | "cost";
96 /**
97 * Window size in seconds.
98 */
99 window: number;
100 };
101 /**
102 * Edges taken when under (success) or over (fallback) the limit.
103 */
104 outputs: { success: RouteEdge; fallback: RouteEdge };
105};
106
107/**
108 * Sends the request to a provider/model; failures take the fallback edge.
109 */
110export type RouteModelElement = {
111 /**
112 * Unique element identifier within the route graph.
113 */
114 id: string;
115 type: "model";
116 properties: {
117 /**
118 * Provider slug (e.g. `workers-ai`, `openai`, `anthropic`).
119 */
120 provider: string;
121 /**
122 * Model identifier (e.g. `@cf/meta/llama-3.1-8b-instruct`).
123 */
124 model: string;
125 /**
126 * Number of retries before taking the fallback edge.
127 */
128 retries: number;
129 /**
130 * Request timeout in milliseconds.
131 */
132 timeout: number;
133 };
134 /**
135 * Edges taken on success / failure.
136 */
137 outputs: { success: RouteEdge; fallback: RouteEdge };
138};
139
140/**
141 * Terminal element of the route graph.
142 */
143export type RouteEndElement = {
144 /**
145 * Unique element identifier within the route graph.
146 */
147 id: string;
148 type: "end";
149 /**
150 * Always empty.
151 */
152 outputs: Record<string, never>;
153};
154
155/**
156 * A node in an AI Gateway dynamic routing graph. A well-formed graph starts
157 * at a `start` element and every path terminates at an `end` element.
158 */
159export type RouteElement =
160 | RouteStartElement
161 | RouteConditionalElement
162 | RoutePercentageElement
163 | RouteRateElement
164 | RouteModelElement
165 | RouteEndElement;
166
167export type GatewayDynamicRoutingProps = {
168 /**
169 * The AI Gateway the route belongs to. Changing the gateway triggers a
170 * replacement.
171 */
172 gatewayId: string;
173 /**
174 * Route name, unique within the gateway. If omitted, a unique name is
175 * generated from the app, stage, and logical ID. Renames are applied in
176 * place.
177 * @default ${app}-${stage}-${id}
178 */
179 name?: string;
180 /**
181 * The element graph describing how requests are routed. Changing the
182 * graph creates a new route version and deploys it.
183 */
184 elements: RouteElement[];
185};
186
187export type GatewayDynamicRoutingAttributes = {
188 /**
189 * Server-generated route identifier. Stable across updates.
190 */
191 routeId: string;
192 /**
193 * The Cloudflare account the route belongs to.
194 */
195 accountId: string;
196 /**
197 * The AI Gateway the route belongs to.
198 */
199 gatewayId: string;
200 /**
201 * Route name.
202 */
203 name: string;
204 /**
205 * The element graph of the currently deployed route version.
206 */
207 elements: RouteElement[];
208 /**
209 * Identifier of the currently deployed route version.
210 */
211 versionId: string;
212 /**
213 * Identifier of the active deployment.
214 */
215 deploymentId: string;
216 /**
217 * When the route was created.
218 */
219 createdAt: string;
220 /**
221 * When the route was last modified.
222 */
223 modifiedAt: string;
224};
225
226export type GatewayDynamicRouting = Resource<
227 TypeId,
228 GatewayDynamicRoutingProps,
229 GatewayDynamicRoutingAttributes,
230 never,
231 Providers
232>;
233
234/**
235 * A dynamic routing configuration ("route") on a Cloudflare.AI. Gateway.
236 *
237 * Dynamic routing models request handling as a graph of elements — start,
238 * conditional, percentage split, rate limit, model, and end nodes — so a
239 * single gateway endpoint can A/B test models, enforce per-user budgets, and
240 * fall back between providers without app changes.
241 *
242 * Cloudflare versions route configurations: changing `elements` creates a
243 * new version and deploys it; the reconciler also re-deploys when the live
244 * deployed version drifts from the desired graph. Renames are applied in
245 * place; only moving the route to a different gateway forces a replacement.
246 * ### Creating a Route
247 * **Example:** Route all traffic to one model
248 * ```typescript
249 * const gateway = yield* Cloudflare.AI.Gateway("Gateway");
250 *
251 * const route = yield* Cloudflare.AI.GatewayDynamicRouting("Llama", {
252 * gatewayId: gateway.gatewayId,
253 * elements: [
254 * { id: "start", type: "start", outputs: { next: { elementId: "model" } } },
255 * {
256 * id: "model",
257 * type: "model",
258 * properties: {
259 * provider: "workers-ai",
260 * model: "@cf/meta/llama-3.1-8b-instruct",
261 * retries: 1,
262 * timeout: 30000,
263 * },
264 * outputs: {
265 * success: { elementId: "end" },
266 * fallback: { elementId: "end" },
267 * },
268 * },
269 * { id: "end", type: "end", outputs: {} },
270 * ],
271 * });
272 * ```
273 *
274 * ### Updating a Route
275 * **Example:** Change the model — creates and deploys a new version
276 * ```typescript
277 * const route = yield* Cloudflare.AI.GatewayDynamicRouting("Llama", {
278 * gatewayId: gateway.gatewayId,
279 * elements: [
280 * { id: "start", type: "start", outputs: { next: { elementId: "model" } } },
281 * {
282 * id: "model",
283 * type: "model",
284 * properties: {
285 * provider: "workers-ai",
286 * model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
287 * retries: 2,
288 * timeout: 60000,
289 * },
290 * outputs: {
291 * success: { elementId: "end" },
292 * fallback: { elementId: "end" },
293 * },
294 * },
295 * { id: "end", type: "end", outputs: {} },
296 * ],
297 * });
298 * ```
299 *
300 * @see https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/
301 *
302 * @resource
303 * @product AI Gateway
304 * @category AI
305 */
306export const GatewayDynamicRouting = Resource<GatewayDynamicRouting>(TypeId, {
307 aliases: ["Cloudflare.AiGateway.DynamicRouting"],
308});
309
310/**
311 * Returns true if the given value is a GatewayDynamicRouting resource.
312 */
313export const isDynamicRouting = (
314 value: unknown,
315): value is GatewayDynamicRouting =>
316 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
317
318export const DynamicRoutingProvider = () =>
319 Provider.succeed(GatewayDynamicRouting, {
320 stables: ["routeId", "accountId", "gatewayId", "createdAt"],
321 diff: Effect.fn(function* ({ olds, news, output }) {
322 if (!isResolved(news)) return undefined;
323 const { accountId } = yield* yield* CloudflareEnvironment;
324 if ((output?.accountId ?? accountId) !== accountId) {
325 return { action: "replace" } as const;
326 }
327 // The gateway is a path parameter — a route cannot move between
328 // gateways in place. By diff time both sides are resolved strings.
329 const oldGatewayId = output?.gatewayId ?? olds?.gatewayId;
330 if (
331 typeof oldGatewayId === "string" &&
332 typeof news.gatewayId === "string" &&
333 oldGatewayId !== news.gatewayId
334 ) {
335 return { action: "replace" } as const;
336 }
337 return undefined;
338 }),
339 read: Effect.fn(function* ({ id, olds, output }) {
340 const { accountId } = yield* yield* CloudflareEnvironment;
341 const acct = output?.accountId ?? accountId;
342 const gatewayId =
343 output?.gatewayId ?? (olds?.gatewayId as string | undefined);
344 if (gatewayId === undefined) return undefined;
345
346 if (output?.routeId) {
347 const observed = yield* getRoute(acct, gatewayId, output.routeId);
348 return observed ? toAttributes(observed, acct) : undefined;
349 }
350 // Cold read — recover from lost state by matching the deterministic
351 // physical name. Route names are unique within a gateway, so an exact
352 // match identifies the route.
353 const name = yield* createRouteName(id, olds?.name);
354 const match = yield* findByName(acct, gatewayId, name);
355 if (match) {
356 const observed = yield* getRoute(acct, gatewayId, match.id);
357 return observed ? toAttributes(observed, acct) : undefined;
358 }
359 return undefined;
360 }),
361 reconcile: Effect.fn(function* ({ id, news, output }) {
362 const { accountId } = yield* yield* CloudflareEnvironment;
363 const gatewayId = news.gatewayId as string;
364 const name = yield* createRouteName(id, news.name);
365 const desired = news.elements ?? [];
366
367 // Observe — the routeId cached on `output` is a hint, not a
368 // guarantee: a missing route falls through and we recreate.
369 const observed = output?.routeId
370 ? yield* getRoute(
371 output.accountId ?? accountId,
372 gatewayId,
373 output.routeId,
374 )
375 : undefined;
376
377 // Ensure — create if missing. Route names are unique per gateway:
378 // tolerate the `RouteAlreadyExists` race (a peer reconciler created
379 // it concurrently, or state persistence failed after a previous
380 // create) by recovering the existing route by name.
381 const routeId =
382 observed?.id ??
383 (yield* aiGateway
384 .createDynamicRouting({
385 accountId,
386 gatewayId,
387 name,
388 elements: desired,
389 })
390 .pipe(
391 Effect.catchTag("RouteAlreadyExists", (error) =>
392 findByName(accountId, gatewayId, name).pipe(
393 Effect.flatMap((match) =>
394 match ? Effect.succeed(match) : Effect.fail(error),
395 ),
396 ),
397 ),
398 Effect.map((route) => route.id),
399 ));
400
401 // Sync — observed cloud state is the diff baseline. Creation above
402 // auto-deploys version 1, so a fresh `get` covers all paths.
403 const current = yield* aiGateway.getDynamicRouting({
404 accountId,
405 gatewayId,
406 id: routeId,
407 });
408
409 if (current.name !== name) {
410 yield* aiGateway
411 .patchDynamicRouting({
412 accountId,
413 gatewayId,
414 id: routeId,
415 name,
416 })
417 .pipe(
418 // Cloudflare does not cascade-delete a gateway's routes when the
419 // gateway itself is deleted: recreating a gateway with the same
420 // id resurrects its old route rows, and a rename onto one of
421 // those ghosts trips the backing `UNIQUE(routes.name,
422 // routes.gateway_id)` constraint (leaked as a 500, typed as
423 // `RouteAlreadyExists` via a distilled patch). Route names are
424 // unique and deterministic, so the holder is a stale duplicate
425 // of this logical resource — delete it and retry the rename.
426 Effect.catchTag("RouteAlreadyExists", (error) =>
427 Effect.gen(function* () {
428 const holder = yield* findByName(accountId, gatewayId, name);
429 if (holder === undefined || holder.id === routeId) {
430 return yield* Effect.fail(error);
431 }
432 yield* aiGateway
433 .deleteDynamicRouting({
434 accountId,
435 gatewayId,
436 id: holder.id,
437 })
438 .pipe(Effect.catchTag("RouteNotFound", () => Effect.void));
439 yield* aiGateway.patchDynamicRouting({
440 accountId,
441 gatewayId,
442 id: routeId,
443 name,
444 });
445 }),
446 ),
447 );
448 }
449
450 // `get` returns the *deployed* version's graph, so an undeployed
451 // draft version naturally shows up as drift and gets re-deployed.
452 const observedElements = elementsOf(current);
453 if (!deepEqual(observedElements, desired)) {
454 const version = yield* aiGateway.createVersionDynamicRouting({
455 accountId,
456 gatewayId,
457 id: routeId,
458 elements: desired,
459 });
460 yield* aiGateway.createDeploymentDynamicRouting({
461 accountId,
462 gatewayId,
463 id: routeId,
464 versionId: version.versionId,
465 });
466 }
467
468 // Return — re-read the final state so attributes reflect the
469 // deployed version, not stale intermediate responses.
470 const final = yield* aiGateway.getDynamicRouting({
471 accountId,
472 gatewayId,
473 id: routeId,
474 });
475 return toAttributes(final, accountId);
476 }),
477 delete: Effect.fn(function* ({ output }) {
478 yield* aiGateway
479 .deleteDynamicRouting({
480 accountId: output.accountId,
481 gatewayId: output.gatewayId,
482 id: output.routeId,
483 })
484 .pipe(
485 // Route already gone — or the whole parent gateway is gone.
486 Effect.catchTag("RouteNotFound", () => Effect.void),
487 Effect.catchTag("GatewayNotFound", () => Effect.void),
488 );
489 }),
490 list: Effect.fn(function* () {
491 const { accountId } = yield* yield* CloudflareEnvironment;
492 // Routes are scoped under a gateway and there is no account-wide
493 // route list, so fan out: enumerate every account gateway, then
494 // exhaustively list each gateway's routes.
495 const gateways = yield* aiGateway.listAiGateways
496 .pages({ accountId })
497 .pipe(
498 Stream.runCollect,
499 Effect.map((chunk) =>
500 Array.from(chunk).flatMap((page) => page.result ?? []),
501 ),
502 );
503 const rows = yield* Effect.forEach(
504 gateways,
505 (gateway) =>
506 listRoutes(accountId, gateway.id).pipe(
507 // A gateway removed between enumeration and its route list is
508 // gone — skip it rather than failing the whole listing.
509 Effect.catchTag("GatewayNotFound", () =>
510 Effect.succeed([] as GatewayDynamicRoutingAttributes[]),
511 ),
512 ),
513 { concurrency: 10 },
514 );
515 return rows.flat();
516 }),
517 });
518
519/**
520 * Exhaustively list every route in a gateway. `listDynamicRoutings` is not a
521 * paginated distilled operation (the Cloudflare response carries only
522 * `page`/`per_page`, no total), so page manually until a short page signals
523 * the end.
524 *
525 * The list endpoint omits each route's element graph — its items carry only
526 * deployment/version metadata, no `version.data` — so hydrate every route with
527 * a per-route `get` to populate `elements` into the exact `read` shape.
528 */
529const listRoutes = (accountId: string, gatewayId: string) =>
530 Effect.gen(function* () {
531 const perPage = 50;
532 const ids: string[] = [];
533 for (let page = 1; ; page++) {
534 const response = yield* aiGateway.listDynamicRoutings({
535 accountId,
536 gatewayId,
537 page,
538 perPage,
539 });
540 const routes = response.data.routes;
541 for (const route of routes) {
542 ids.push(route.id);
543 }
544 if (routes.length < perPage) break;
545 }
546 const hydrated = yield* Effect.forEach(
547 ids,
548 (id) => getRoute(accountId, gatewayId, id),
549 { concurrency: 10 },
550 );
551 return hydrated
552 .filter((r) => r !== undefined)
553 .map((route) => toAttributes(route, accountId));
554 });
555
556/**
557 * Read a route by id, mapping "gone" (`RouteNotFound`, Cloudflare error code
558 * 7005, or `GatewayNotFound`, code 7002, when the parent gateway was
559 * deleted) to `undefined`.
560 */
561const getRoute = (accountId: string, gatewayId: string, id: string) =>
562 aiGateway.getDynamicRouting({ accountId, gatewayId, id }).pipe(
563 Effect.catchTag("RouteNotFound", () => Effect.succeed(undefined)),
564 Effect.catchTag("GatewayNotFound", () => Effect.succeed(undefined)),
565 );
566
567/**
568 * Find a route by exact name. Route names are unique within a gateway. A
569 * missing parent gateway means the route is gone too.
570 */
571const findByName = (accountId: string, gatewayId: string, name: string) =>
572 aiGateway.listDynamicRoutings({ accountId, gatewayId, perPage: 50 }).pipe(
573 Effect.map((list) => list.data.routes.find((r) => r.name === name)),
574 Effect.catchTag("GatewayNotFound", () => Effect.succeed(undefined)),
575 );
576
577const createRouteName = (id: string, name: string | undefined) =>
578 Effect.gen(function* () {
579 return name ?? (yield* createPhysicalName({ id, lowercase: true }));
580 });
581
582/**
583 * The deployed element graph. Cloudflare returns it as `version.data` (the
584 * distilled schema types it `unknown` because the public spec wrongly
585 * declares it a string).
586 */
587const elementsOf = (
588 route: aiGateway.GetDynamicRoutingResponse,
589): RouteElement[] => (route.version.data ?? []) as RouteElement[];
590
591const toAttributes = (
592 route: aiGateway.GetDynamicRoutingResponse,
593 accountId: string,
594): GatewayDynamicRoutingAttributes => ({
595 routeId: route.id,
596 accountId,
597 gatewayId: route.gatewayId,
598 name: route.name,
599 elements: elementsOf(route),
600 versionId: route.deployment.versionId,
601 deploymentId: route.deployment.deploymentId,
602 createdAt: route.createdAt,
603 modifiedAt: route.modifiedAt,
604});