authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-05-14 08:41:02+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-05-14 08:41:02+02:00
log4120e94598c7e4237ce9ecfd53d0449eeb0e0d40
tree2c1397d748dd93f3cdbc92656178654eccf5c45f
parent34bb18c8cf2327338bc14a216537328bd58a69fc
signature Signed by PGP key B5690EEEBB952194

Back off VCS remote refresh failures (#2686)


2 files changed, 76 insertions(+), 22 deletions(-)

apps/server/src/vcs/VcsStatusBroadcaster.test.ts+23
......@@ -310,6 +310,29 @@ describe("VcsStatusBroadcaster", () => {
310310 }).pipe(Effect.provide(makeTestLayer(state)));
311311 });
312312
313 it("backs off remote refresh failures exponentially and honors larger configured intervals", () => {
314 assert.equal(
315 Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(1, Duration.seconds(1))),
316 30_000,
317 );
318 assert.equal(
319 Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(2, Duration.seconds(1))),
320 60_000,
321 );
322 assert.equal(
323 Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(3, Duration.seconds(1))),
324 120_000,
325 );
326 assert.equal(
327 Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(1, Duration.minutes(5))),
328 300_000,
329 );
330 assert.equal(
331 Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(20, Duration.seconds(1))),
332 900_000,
333 );
334 });
335
313336 it.effect("stops the remote poller after the last stream subscriber disconnects", () => {
314337 const state = {
315338 currentLocalStatus: baseLocalStatus,
apps/server/src/vcs/VcsStatusBroadcaster.ts+53-22
......@@ -7,6 +7,7 @@ import * as FileSystem from "effect/FileSystem";
77import * as Layer from "effect/Layer";
88import * as PubSub from "effect/PubSub";
99import * as Ref from "effect/Ref";
10import * as Schedule from "effect/Schedule";
1011import * as Scope from "effect/Scope";
1112import * as Stream from "effect/Stream";
1213import * as SynchronizedRef from "effect/SynchronizedRef";
......@@ -23,6 +24,8 @@ import { mergeGitStatusParts } from "@t3tools/shared/git";
2324import * as GitWorkflowService from "../git/GitWorkflowService.ts";
2425
2526const DEFAULT_VCS_STATUS_REFRESH_INTERVAL = Duration.seconds(30);
27const VCS_STATUS_REFRESH_FAILURE_BASE_DELAY = Duration.seconds(30);
28const VCS_STATUS_REFRESH_FAILURE_MAX_DELAY = Duration.minutes(15);
2629
2730interface VcsStatusChange {
2831 readonly cwd: string;
......@@ -48,6 +51,20 @@ interface StreamStatusOptions {
4851 readonly automaticRemoteRefreshInterval?: Effect.Effect<Duration.Duration, never>;
4952}
5053
54export function remoteRefreshFailureDelay(
55 consecutiveFailures: number,
56 configuredInterval: Duration.Duration,
57) {
58 const exponent = Math.max(0, consecutiveFailures - 1);
59 const backoffMs =
60 Duration.toMillis(VCS_STATUS_REFRESH_FAILURE_BASE_DELAY) * Math.pow(2, exponent);
61 const cappedBackoff = Duration.min(
62 Duration.millis(backoffMs),
63 VCS_STATUS_REFRESH_FAILURE_MAX_DELAY,
64 );
65 return Duration.max(configuredInterval, cappedBackoff);
66}
67
5168export interface VcsStatusBroadcasterShape {
5269 readonly getStatus: (
5370 input: VcsStatusInput,
......@@ -241,32 +258,46 @@ export const layer = Layer.effect(
241258 cwd: string,
242259 automaticRemoteRefreshInterval: Effect.Effect<Duration.Duration, never>,
243260 ) => {
244 const logRefreshFailure = (error: GitManagerServiceError) =>
245 Effect.logWarning("VCS remote status refresh failed", {
246 cwd,
247 detail: error.message,
261 return Effect.gen(function* () {
262 const consecutiveFailuresRef = yield* Ref.make(0);
263 const refreshRemoteStatusIfEnabled = Effect.gen(function* () {
264 const configuredInterval = yield* automaticRemoteRefreshInterval;
265 const activeInterval = Duration.isZero(configuredInterval)
266 ? DEFAULT_VCS_STATUS_REFRESH_INTERVAL
267 : configuredInterval;
268 if (Duration.isZero(configuredInterval)) {
269 return activeInterval;
270 }
271
272 const exit = yield* refreshRemoteStatus(cwd).pipe(Effect.exit);
273 if (Exit.isSuccess(exit)) {
274 yield* Ref.set(consecutiveFailuresRef, 0);
275 return activeInterval;
276 }
277
278 const consecutiveFailures = yield* Ref.updateAndGet(
279 consecutiveFailuresRef,
280 (count) => count + 1,
281 );
282 const nextDelay = remoteRefreshFailureDelay(consecutiveFailures, activeInterval);
283 yield* Effect.logWarning("VCS remote status refresh failed", {
284 cwd,
285 detail: exit.cause.toString(),
286 consecutiveFailures,
287 nextDelayMs: Duration.toMillis(nextDelay),
288 });
289 return nextDelay;
248290 });
249 const refreshRemoteStatusIfEnabled = automaticRemoteRefreshInterval.pipe(
250 Effect.flatMap((interval) =>
251 Duration.isZero(interval) ? Effect.void : refreshRemoteStatus(cwd).pipe(Effect.asVoid),
252 ),
253 );
254 const sleepForConfiguredInterval = automaticRemoteRefreshInterval.pipe(
255 Effect.flatMap((interval) =>
256 Effect.sleep(Duration.isZero(interval) ? DEFAULT_VCS_STATUS_REFRESH_INTERVAL : interval),
257 ),
258 );
259291
260 return refreshRemoteStatusIfEnabled.pipe(
261 Effect.catch(logRefreshFailure),
262 Effect.andThen(
263 Effect.forever(
264 sleepForConfiguredInterval.pipe(
265 Effect.andThen(refreshRemoteStatusIfEnabled.pipe(Effect.catch(logRefreshFailure))),
292 return yield* refreshRemoteStatusIfEnabled.pipe(
293 Effect.repeat(
294 Schedule.identity<Duration.Duration>().pipe(
295 Schedule.addDelay((delay) => Effect.succeed(delay)),
266296 ),
267297 ),
268 ),
269 );
298 Effect.asVoid,
299 );
300 });
270301 };
271302
272303 const retainRemotePoller = Effect.fn("VcsStatusBroadcaster.retainRemotePoller")(function* (