Commit 4120e94598c7e4237ce9ecfd53d0449eeb0e0d40 | the-knife

author	Julius Marminge <julius0216@outlook.com> 2026-05-14 08:41:02+02:00
committer	GitHub <noreply@github.com> 2026-05-14 08:41:02+02:00
log	4120e94598c7e4237ce9ecfd53d0449eeb0e0d40
tree	2c1397d748dd93f3cdbc92656178654eccf5c45f
parent	34bb18c8cf2327338bc14a216537328bd58a69fc
signature	Signed by PGP key B5690EEEBB952194

Back off VCS remote refresh failures (#2686)

2 files changed, 76 insertions(+), 22 deletions(-)

apps
- server
  - src
    vcs
    VcsStatusBroadcaster.test.ts
    VcsStatusBroadcaster.ts

apps/server/src/vcs/VcsStatusBroadcaster.test.ts+23

...	@@ -310,6 +310,29 @@ describe("VcsStatusBroadcaster", () => {	...	@@ -310,6 +310,29 @@ describe("VcsStatusBroadcaster", () => {
310	}).pipe(Effect.provide(makeTestLayer(state)));	310	}).pipe(Effect.provide(makeTestLayer(state)));
311	});	311	});
312		312
		313	it("backs off remote refresh failures exponentially and honors larger configured intervals", () => {
		314	assert.equal(
		315	Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(1, Duration.seconds(1))),
		316	30_000,
		317	);
		318	assert.equal(
		319	Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(2, Duration.seconds(1))),
		320	60_000,
		321	);
		322	assert.equal(
		323	Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(3, Duration.seconds(1))),
		324	120_000,
		325	);
		326	assert.equal(
		327	Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(1, Duration.minutes(5))),
		328	300_000,
		329	);
		330	assert.equal(
		331	Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(20, Duration.seconds(1))),
		332	900_000,
		333	);
		334	});
		335
313	it.effect("stops the remote poller after the last stream subscriber disconnects", () => {	336	it.effect("stops the remote poller after the last stream subscriber disconnects", () => {
314	const state = {	337	const state = {
315	currentLocalStatus: baseLocalStatus,	338	currentLocalStatus: baseLocalStatus,

apps/server/src/vcs/VcsStatusBroadcaster.ts+53-22

 @@ -7,6 +7,7 @@ import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
 import * as PubSub from "effect/PubSub";
 import * as Ref from "effect/Ref";
+import * as Schedule from "effect/Schedule";
 import * as Scope from "effect/Scope";
 import * as Stream from "effect/Stream";
 import * as SynchronizedRef from "effect/SynchronizedRef";
 @@ -23,6 +24,8 @@ import { mergeGitStatusParts } from "@t3tools/shared/git";
 import * as GitWorkflowService from "../git/GitWorkflowService.ts";
 const DEFAULT_VCS_STATUS_REFRESH_INTERVAL = Duration.seconds(30);
+const VCS_STATUS_REFRESH_FAILURE_BASE_DELAY = Duration.seconds(30);
+const VCS_STATUS_REFRESH_FAILURE_MAX_DELAY = Duration.minutes(15);
 interface VcsStatusChange {
   readonly cwd: string;
 @@ -48,6 +51,20 @@ interface StreamStatusOptions {
   readonly automaticRemoteRefreshInterval?: Effect.Effect<Duration.Duration, never>;
 }
+export function remoteRefreshFailureDelay(
+  consecutiveFailures: number,
+  configuredInterval: Duration.Duration,
+) {
+  const exponent = Math.max(0, consecutiveFailures - 1);
+  const backoffMs =
+    Duration.toMillis(VCS_STATUS_REFRESH_FAILURE_BASE_DELAY) * Math.pow(2, exponent);
+  const cappedBackoff = Duration.min(
+    Duration.millis(backoffMs),
+    VCS_STATUS_REFRESH_FAILURE_MAX_DELAY,
+  );
+  return Duration.max(configuredInterval, cappedBackoff);
+}
 export interface VcsStatusBroadcasterShape {
   readonly getStatus: (
     input: VcsStatusInput,
 @@ -241,32 +258,46 @@ export const layer = Layer.effect(
       cwd: string,
       automaticRemoteRefreshInterval: Effect.Effect<Duration.Duration, never>,
     ) => {
-      const logRefreshFailure = (error: GitManagerServiceError) =>
+      return Effect.gen(function* () {
-        Effect.logWarning("VCS remote status refresh failed", {
+        const consecutiveFailuresRef = yield* Ref.make(0);
-          cwd,
+        const refreshRemoteStatusIfEnabled = Effect.gen(function* () {
-          detail: error.message,
+          const configuredInterval = yield* automaticRemoteRefreshInterval;
+          const activeInterval = Duration.isZero(configuredInterval)
+            ? DEFAULT_VCS_STATUS_REFRESH_INTERVAL
+            : configuredInterval;
+          if (Duration.isZero(configuredInterval)) {
+            return activeInterval;
+          }
+          const exit = yield* refreshRemoteStatus(cwd).pipe(Effect.exit);
+          if (Exit.isSuccess(exit)) {
+            yield* Ref.set(consecutiveFailuresRef, 0);
+            return activeInterval;
+          }
+          const consecutiveFailures = yield* Ref.updateAndGet(
+            consecutiveFailuresRef,
+            (count) => count + 1,
+          );
+          const nextDelay = remoteRefreshFailureDelay(consecutiveFailures, activeInterval);
+          yield* Effect.logWarning("VCS remote status refresh failed", {
+            cwd,
+            detail: exit.cause.toString(),
+            consecutiveFailures,
+            nextDelayMs: Duration.toMillis(nextDelay),
+          });
+          return nextDelay;
         });
-      const refreshRemoteStatusIfEnabled = automaticRemoteRefreshInterval.pipe(
-        Effect.flatMap((interval) =>
-          Duration.isZero(interval) ? Effect.void : refreshRemoteStatus(cwd).pipe(Effect.asVoid),
-        ),
-      );
-      const sleepForConfiguredInterval = automaticRemoteRefreshInterval.pipe(
-        Effect.flatMap((interval) =>
-          Effect.sleep(Duration.isZero(interval) ? DEFAULT_VCS_STATUS_REFRESH_INTERVAL : interval),
-        ),
-      );
-      return refreshRemoteStatusIfEnabled.pipe(
+        return yield* refreshRemoteStatusIfEnabled.pipe(
-        Effect.catch(logRefreshFailure),
+          Effect.repeat(
-        Effect.andThen(
+            Schedule.identity<Duration.Duration>().pipe(
-          Effect.forever(
+              Schedule.addDelay((delay) => Effect.succeed(delay)),
-            sleepForConfiguredInterval.pipe(
-              Effect.andThen(refreshRemoteStatusIfEnabled.pipe(Effect.catch(logRefreshFailure))),
             ),
           ),
-        ),
+          Effect.asVoid,
-      );
+        );
+      });
     };
     const retainRemotePoller = Effect.fn("VcsStatusBroadcaster.retainRemotePoller")(function* (