perf(test): reduce hot-suite setup and duplicate test work

This commit is contained in:
Peter Steinberger
2026-02-13 23:30:35 +00:00
parent ab4a08a82a
commit e794ef0478
5 changed files with 65 additions and 239 deletions

View File

@@ -281,7 +281,7 @@ describe("memory indexing with OpenAI batches", () => {
expect(batchCreates).toBe(2);
});
it("falls back to non-batch on failure and resets failures after success", async () => {
it("tracks batch failures, resets on success, and disables after repeated failures", async () => {
const content = ["flaky", "batch"].join("\n\n");
await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-09.md"), content);
@@ -376,12 +376,14 @@ describe("memory indexing with OpenAI batches", () => {
}
manager = result.manager;
// First failure: fallback to regular embeddings and increment failure count.
await manager.sync({ force: true });
expect(embedBatch).toHaveBeenCalled();
let status = manager.status();
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(1);
// Success should reset failure count.
embedBatch.mockClear();
mode = "ok";
await fs.writeFile(
@@ -393,110 +395,33 @@ describe("memory indexing with OpenAI batches", () => {
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(0);
expect(embedBatch).not.toHaveBeenCalled();
});
it("disables batch after repeated failures and skips batch thereafter", async () => {
const content = ["repeat", "failures"].join("\n\n");
await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-10.md"), content);
let uploadedRequests: Array<{ custom_id?: string }> = [];
const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
const url =
typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
if (url.endsWith("/files")) {
const body = init?.body;
if (!(body instanceof FormData)) {
throw new Error("expected FormData upload");
}
for (const [key, value] of body.entries()) {
if (key !== "file") {
continue;
}
if (typeof value === "string") {
uploadedRequests = value
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line) as { custom_id?: string });
} else {
const text = await value.text();
uploadedRequests = text
.split("\n")
.filter(Boolean)
.map((line) => JSON.parse(line) as { custom_id?: string });
}
}
return new Response(JSON.stringify({ id: "file_1" }), {
status: 200,
headers: { "Content-Type": "application/json" },
});
}
if (url.endsWith("/batches")) {
return new Response("batch failed", { status: 500 });
}
if (url.endsWith("/files/file_out/content")) {
const lines = uploadedRequests.map((request, index) =>
JSON.stringify({
custom_id: request.custom_id,
response: {
status_code: 200,
body: { data: [{ embedding: [index + 1, 0, 0], index: 0 }] },
},
}),
);
return new Response(lines.join("\n"), {
status: 200,
headers: { "Content-Type": "application/jsonl" },
});
}
throw new Error(`unexpected fetch ${url}`);
});
vi.stubGlobal("fetch", fetchMock);
const cfg = {
agents: {
defaults: {
workspace: workspaceDir,
memorySearch: {
provider: "openai",
model: "text-embedding-3-small",
store: { path: indexPath },
sync: { watch: false, onSessionStart: false, onSearch: false },
query: { minScore: 0 },
remote: { batch: { enabled: true, wait: true, pollIntervalMs: 1 } },
},
},
list: [{ id: "main", default: true }],
},
};
const result = await getMemorySearchManager({ cfg, agentId: "main" });
expect(result.manager).not.toBeNull();
if (!result.manager) {
throw new Error("manager missing");
}
manager = result.manager;
// Two more failures after reset should disable remote batching.
mode = "fail";
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fail-a"].join("\n\n"),
);
await manager.sync({ force: true });
let status = manager.status();
status = manager.status();
expect(status.batch?.enabled).toBe(true);
expect(status.batch?.failures).toBe(1);
embedBatch.mockClear();
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-10.md"),
["repeat", "failures", "again"].join("\n\n"),
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fail-b"].join("\n\n"),
);
await manager.sync({ force: true });
status = manager.status();
expect(status.batch?.enabled).toBe(false);
expect(status.batch?.failures).toBeGreaterThanOrEqual(2);
// Once disabled, batch endpoints are skipped and fallback embeddings run directly.
const fetchCalls = fetchMock.mock.calls.length;
embedBatch.mockClear();
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-10.md"),
["repeat", "failures", "fallback"].join("\n\n"),
path.join(workspaceDir, "memory", "2026-01-09.md"),
["flaky", "batch", "fallback"].join("\n\n"),
);
await manager.sync({ force: true });
expect(fetchMock.mock.calls.length).toBe(fetchCalls);