refactor(gateway): simplify restart flow and expand lock tests

src/cli/gateway-cli/run-loop.test.ts

@@ -57,62 +57,86 @@ function removeNewSignalListeners(
   }
 }
 
+async function withIsolatedSignals(run: () => Promise<void>) {
+  const beforeSigterm = new Set(
+    process.listeners("SIGTERM") as Array<(...args: unknown[]) => void>,
+  );
+  const beforeSigint = new Set(process.listeners("SIGINT") as Array<(...args: unknown[]) => void>);
+  const beforeSigusr1 = new Set(
+    process.listeners("SIGUSR1") as Array<(...args: unknown[]) => void>,
+  );
+  try {
+    await run();
+  } finally {
+    removeNewSignalListeners("SIGTERM", beforeSigterm);
+    removeNewSignalListeners("SIGINT", beforeSigint);
+    removeNewSignalListeners("SIGUSR1", beforeSigusr1);
+  }
+}
+
+function createRuntimeWithExitSignal(exitCallOrder?: string[]) {
+  let resolveExit: (code: number) => void = () => {};
+  const exited = new Promise<number>((resolve) => {
+    resolveExit = resolve;
+  });
+  const runtime = {
+    log: vi.fn(),
+    error: vi.fn(),
+    exit: vi.fn((code: number) => {
+      exitCallOrder?.push("exit");
+      resolveExit(code);
+    }),
+  };
+  return { runtime, exited };
+}
+
 describe("runGatewayLoop", () => {
   it("restarts after SIGUSR1 even when drain times out, and resets lanes for the new iteration", async () => {
     vi.clearAllMocks();
-    getActiveTaskCount.mockReturnValueOnce(2).mockReturnValueOnce(0);
-    waitForActiveTasks.mockResolvedValueOnce({ drained: false });
 
-    type StartServer = () => Promise<{
-      close: (opts: { reason: string; restartExpectedMs: number | null }) => Promise<void>;
-    }>;
+    await withIsolatedSignals(async () => {
+      getActiveTaskCount.mockReturnValueOnce(2).mockReturnValueOnce(0);
+      waitForActiveTasks.mockResolvedValueOnce({ drained: false });
 
-    const closeFirst = vi.fn(async () => {});
-    const closeSecond = vi.fn(async () => {});
+      type StartServer = () => Promise<{
+        close: (opts: { reason: string; restartExpectedMs: number | null }) => Promise<void>;
+      }>;
 
-    const start = vi.fn<StartServer>();
-    let resolveFirst: (() => void) | null = null;
-    const startedFirst = new Promise<void>((resolve) => {
-      resolveFirst = resolve;
-    });
-    start.mockImplementationOnce(async () => {
-      resolveFirst?.();
-      return { close: closeFirst };
-    });
+      const closeFirst = vi.fn(async () => {});
+      const closeSecond = vi.fn(async () => {});
 
-    let resolveSecond: (() => void) | null = null;
-    const startedSecond = new Promise<void>((resolve) => {
-      resolveSecond = resolve;
-    });
-    start.mockImplementationOnce(async () => {
-      resolveSecond?.();
-      return { close: closeSecond };
-    });
+      const start = vi.fn<StartServer>();
+      let resolveFirst: (() => void) | null = null;
+      const startedFirst = new Promise<void>((resolve) => {
+        resolveFirst = resolve;
+      });
+      start.mockImplementationOnce(async () => {
+        resolveFirst?.();
+        return { close: closeFirst };
+      });
 
-    start.mockRejectedValueOnce(new Error("stop-loop"));
+      let resolveSecond: (() => void) | null = null;
+      const startedSecond = new Promise<void>((resolve) => {
+        resolveSecond = resolve;
+      });
+      start.mockImplementationOnce(async () => {
+        resolveSecond?.();
+        return { close: closeSecond };
+      });
 
-    const beforeSigterm = new Set(
-      process.listeners("SIGTERM") as Array<(...args: unknown[]) => void>,
-    );
-    const beforeSigint = new Set(
-      process.listeners("SIGINT") as Array<(...args: unknown[]) => void>,
-    );
-    const beforeSigusr1 = new Set(
-      process.listeners("SIGUSR1") as Array<(...args: unknown[]) => void>,
-    );
+      start.mockRejectedValueOnce(new Error("stop-loop"));
 
-    const { runGatewayLoop } = await import("./run-loop.js");
-    const runtime = {
-      log: vi.fn(),
-      error: vi.fn(),
-      exit: vi.fn(),
-    };
-    const loopPromise = runGatewayLoop({
-      start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
-      runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
-    });
+      const { runGatewayLoop } = await import("./run-loop.js");
+      const runtime = {
+        log: vi.fn(),
+        error: vi.fn(),
+        exit: vi.fn(),
+      };
+      const loopPromise = runGatewayLoop({
+        start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
+        runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
+      });
 
-    try {
       await startedFirst;
       expect(start).toHaveBeenCalledTimes(1);
       await new Promise<void>((resolve) => setImmediate(resolve));
@@ -142,86 +166,105 @@ describe("runGatewayLoop", () => {
       expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledTimes(2);
       expect(resetAllLanes).toHaveBeenCalledTimes(2);
       expect(acquireGatewayLock).toHaveBeenCalledTimes(3);
-    } finally {
-      removeNewSignalListeners("SIGTERM", beforeSigterm);
-      removeNewSignalListeners("SIGINT", beforeSigint);
-      removeNewSignalListeners("SIGUSR1", beforeSigusr1);
-    }
+    });
   });
 
   it("releases the lock before exiting on spawned restart", async () => {
     vi.clearAllMocks();
 
-    const lockRelease = vi.fn(async () => {});
-    acquireGatewayLock.mockResolvedValueOnce({
-      release: lockRelease,
-    });
+    await withIsolatedSignals(async () => {
+      const lockRelease = vi.fn(async () => {});
+      acquireGatewayLock.mockResolvedValueOnce({
+        release: lockRelease,
+      });
 
-    // Override process-respawn to return "spawned" mode
-    restartGatewayProcessWithFreshPid.mockReturnValueOnce({
-      mode: "spawned",
-      pid: 9999,
-    });
+      // Override process-respawn to return "spawned" mode
+      restartGatewayProcessWithFreshPid.mockReturnValueOnce({
+        mode: "spawned",
+        pid: 9999,
+      });
 
-    const close = vi.fn(async () => {});
-    let resolveStarted: (() => void) | null = null;
-    const started = new Promise<void>((resolve) => {
-      resolveStarted = resolve;
-    });
+      const close = vi.fn(async () => {});
+      let resolveStarted: (() => void) | null = null;
+      const started = new Promise<void>((resolve) => {
+        resolveStarted = resolve;
+      });
 
-    const start = vi.fn(async () => {
-      resolveStarted?.();
-      return { close };
-    });
+      const start = vi.fn(async () => {
+        resolveStarted?.();
+        return { close };
+      });
 
-    const exitCallOrder: string[] = [];
-    const runtime = {
-      log: vi.fn(),
-      error: vi.fn(),
-      exit: vi.fn(() => {
-        exitCallOrder.push("exit");
-      }),
-    };
+      const exitCallOrder: string[] = [];
+      const { runtime, exited } = createRuntimeWithExitSignal(exitCallOrder);
+      lockRelease.mockImplementation(async () => {
+        exitCallOrder.push("lockRelease");
+      });
 
-    lockRelease.mockImplementation(async () => {
-      exitCallOrder.push("lockRelease");
-    });
+      vi.resetModules();
+      const { runGatewayLoop } = await import("./run-loop.js");
+      const _loopPromise = runGatewayLoop({
+        start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
+        runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
+      });
 
-    const beforeSigterm = new Set(
-      process.listeners("SIGTERM") as Array<(...args: unknown[]) => void>,
-    );
-    const beforeSigint = new Set(
-      process.listeners("SIGINT") as Array<(...args: unknown[]) => void>,
-    );
-    const beforeSigusr1 = new Set(
-      process.listeners("SIGUSR1") as Array<(...args: unknown[]) => void>,
-    );
-
-    vi.resetModules();
-    const { runGatewayLoop } = await import("./run-loop.js");
-    const _loopPromise = runGatewayLoop({
-      start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
-      runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
-    });
-
-    try {
       await started;
       await new Promise<void>((resolve) => setImmediate(resolve));
 
       process.emit("SIGUSR1");
 
-      // Wait for the shutdown path to complete
-      await new Promise<void>((resolve) => setTimeout(resolve, 100));
-
+      await exited;
       expect(lockRelease).toHaveBeenCalled();
       expect(runtime.exit).toHaveBeenCalledWith(0);
-      // Lock must be released BEFORE exit
       expect(exitCallOrder).toEqual(["lockRelease", "exit"]);
-    } finally {
-      removeNewSignalListeners("SIGTERM", beforeSigterm);
-      removeNewSignalListeners("SIGINT", beforeSigint);
-      removeNewSignalListeners("SIGUSR1", beforeSigusr1);
-    }
+    });
+  });
+
+  it("exits when lock reacquire fails during in-process restart fallback", async () => {
+    vi.clearAllMocks();
+
+    await withIsolatedSignals(async () => {
+      const lockRelease = vi.fn(async () => {});
+      acquireGatewayLock
+        .mockResolvedValueOnce({
+          release: lockRelease,
+        })
+        .mockRejectedValueOnce(new Error("lock timeout"));
+
+      restartGatewayProcessWithFreshPid.mockReturnValueOnce({
+        mode: "disabled",
+      });
+
+      const close = vi.fn(async () => {});
+      let resolveStarted: (() => void) | null = null;
+      const started = new Promise<void>((resolve) => {
+        resolveStarted = resolve;
+      });
+      const start = vi.fn(async () => {
+        resolveStarted?.();
+        return { close };
+      });
+
+      const { runtime, exited } = createRuntimeWithExitSignal();
+
+      vi.resetModules();
+      const { runGatewayLoop } = await import("./run-loop.js");
+      const _loopPromise = runGatewayLoop({
+        start: start as unknown as Parameters<typeof runGatewayLoop>[0]["start"],
+        runtime: runtime as unknown as Parameters<typeof runGatewayLoop>[0]["runtime"],
+      });
+
+      await started;
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      process.emit("SIGUSR1");
+
+      await expect(exited).resolves.toBe(1);
+      expect(acquireGatewayLock).toHaveBeenCalledTimes(2);
+      expect(start).toHaveBeenCalledTimes(1);
+      expect(gatewayLog.error).toHaveBeenCalledWith(
+        expect.stringContaining("failed to reacquire gateway lock for in-process restart"),
+      );
+    });
   });
 });