authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-24 18:10:10-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-24 18:10:10-07:00
logd23eab13d9d91697c5d4e7a458213b3f6473bc96
tree59238ae10eda8a75e4481941546a8964c05e301b
parent46f3c2ca2c99ceab0547452ac863232944621fce
signature Signed by PGP key B5690EEEBB952194

fix(relay): export tunnel cleanup counters to Axiom (#13528)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

3 files changed, 68 insertions(+), 2 deletions(-)

infra/relay/src/environments/ManagedEndpointReaper.test.ts+49
......@@ -4,6 +4,7 @@ import * as Effect from "effect/Effect";
44import * as Layer from "effect/Layer";
55import * as Redacted from "effect/Redacted";
66import * as TestClock from "effect/testing/TestClock";
7import * as Tracer from "effect/Tracer";
78
89import * as RelayConfiguration from "../Config.ts";
910import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
......@@ -574,6 +575,54 @@ describe("ManagedEndpointReaper", () => {
574575 }).pipe(Effect.provide(state.layer));
575576 });
576577
578 it.effect("records dry-run counters on the sweep span", () => {
579 const spans: Array<Tracer.NativeSpan> = [];
580 const tracer = Tracer.make({
581 span: (options) => {
582 const span = new Tracer.NativeSpan(options);
583 spans.push(span);
584 return span;
585 },
586 });
587 const expired = [
588 tunnel({
589 id: "recoverable",
590 suffix: "aaaaaaaaaaaaaaaa",
591 status: "down",
592 timestamp: "2026-08-25T11:00:00.000Z",
593 }),
594 tunnel({
595 id: "legacy",
596 suffix: "bbbbbbbbbbbbbbbb",
597 status: "down",
598 timestamp: "2026-08-25T11:00:00.000Z",
599 }),
600 ];
601 const state = harness({
602 cleanupMode: "dry-run",
603 tunnels: expired,
604 allocations: [
605 allocation({ tunnelId: "recoverable", recoveryEnabled: true }),
606 allocation({ tunnelId: "legacy", recoveryEnabled: false }),
607 ],
608 });
609
610 return Effect.gen(function* () {
611 yield* TestClock.setTime(NOW_MILLIS);
612 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
613 yield* reaper.sweep;
614 const sweepSpan = spans.find((span) => span.name === "relay.managed_endpoint_reaper.sweep");
615 expect(Object.fromEntries(sweepSpan?.attributes ?? [])).toMatchObject({
616 "relay.managed_endpoint_reaper.mode": "dry-run",
617 "relay.managed_endpoint_reaper.scanned": 2,
618 "relay.managed_endpoint_reaper.wouldDelete": 1,
619 "relay.managed_endpoint_reaper.skippedLegacy": 1,
620 "relay.managed_endpoint_reaper.deleted": 0,
621 });
622 expect(state.deleted).toEqual([]);
623 }).pipe(Effect.provide(state.layer), Effect.withTracer(tracer));
624 });
625
577626 it.effect("does no Cloudflare work while cleanup is off", () => {
578627 const state = harness({
579628 cleanupMode: "off",
infra/relay/src/environments/ManagedEndpointReaper.ts+14-1
......@@ -287,7 +287,20 @@ export const make = Effect.gen(function* () {
287287 failed,
288288 truncated,
289289 };
290 }).pipe(Effect.withSpan("relay.managed_endpoint_reaper.sweep"));
290 }).pipe(
291 // Dry-run rollout reads these counters from the exported span.
292 Effect.tap((result) =>
293 Effect.annotateCurrentSpan(
294 Object.fromEntries(
295 Object.entries(result).map(([key, value]) => [
296 `relay.managed_endpoint_reaper.${key}`,
297 value,
298 ]),
299 ),
300 ),
301 ),
302 Effect.withSpan("relay.managed_endpoint_reaper.sweep"),
303 );
291304
292305 return ManagedEndpointReaper.of({ sweep });
293306});
infra/relay/src/worker.ts+5-1
......@@ -358,7 +358,11 @@ export const ApiLive = Api.make(
358358 ),
359359 ],
360360 { concurrency: 2, discard: true },
361 ).pipe(Effect.withSpan("relay.cron.prune_expired_state"), Effect.provide(runtimeLayer)),
361 ).pipe(
362 Effect.withSpan("relay.cron.prune_expired_state"),
363 // Export cron spans to Axiom like HTTP spans; the scope flushes them before the run ends.
364 Effect.provide(Layer.merge(runtimeLayer, relayTraceLayer)),
365 ),
362366 );
363367
364368 const fetch = Layer.merge(