2026-01-26 22:05:31 -05:00
|
|
|
import crypto from "node:crypto";
|
|
|
|
|
import fs from "node:fs/promises";
|
|
|
|
|
import os from "node:os";
|
|
|
|
|
import path from "node:path";
|
2026-01-09 19:59:45 +01:00
|
|
|
import { describe, expect, it, vi } from "vitest";
|
2026-01-30 03:15:10 +01:00
|
|
|
import type { OpenClawConfig } from "../config/config.js";
|
2026-01-26 22:05:31 -05:00
|
|
|
import type { AuthProfileStore } from "./auth-profiles.js";
|
|
|
|
|
import { saveAuthProfileStore } from "./auth-profiles.js";
|
|
|
|
|
import { AUTH_STORE_VERSION } from "./auth-profiles/constants.js";
|
2026-01-09 19:59:45 +01:00
|
|
|
import { runWithModelFallback } from "./model-fallback.js";
|
|
|
|
|
|
2026-01-30 03:15:10 +01:00
|
|
|
function makeCfg(overrides: Partial<OpenClawConfig> = {}): OpenClawConfig {
|
2026-01-09 19:59:45 +01:00
|
|
|
return {
|
|
|
|
|
agents: {
|
|
|
|
|
defaults: {
|
|
|
|
|
model: {
|
|
|
|
|
primary: "openai/gpt-4.1-mini",
|
|
|
|
|
fallbacks: ["anthropic/claude-haiku-3-5"],
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
...overrides,
|
2026-01-30 03:15:10 +01:00
|
|
|
} as OpenClawConfig;
|
2026-01-09 19:59:45 +01:00
|
|
|
}
|
|
|
|
|
|
2026-02-18 05:04:51 +00:00
|
|
|
function makeFallbacksOnlyCfg(): OpenClawConfig {
|
|
|
|
|
return {
|
|
|
|
|
agents: {
|
|
|
|
|
defaults: {
|
|
|
|
|
model: {
|
|
|
|
|
fallbacks: ["openai/gpt-5.2"],
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
} as OpenClawConfig;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function makeProviderFallbackCfg(provider: string): OpenClawConfig {
|
|
|
|
|
return makeCfg({
|
|
|
|
|
agents: {
|
|
|
|
|
defaults: {
|
|
|
|
|
model: {
|
|
|
|
|
primary: `${provider}/m1`,
|
|
|
|
|
fallbacks: ["fallback/ok-model"],
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function withTempAuthStore<T>(
|
|
|
|
|
store: AuthProfileStore,
|
|
|
|
|
run: (tempDir: string) => Promise<T>,
|
|
|
|
|
): Promise<T> {
|
|
|
|
|
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-"));
|
|
|
|
|
saveAuthProfileStore(store, tempDir);
|
|
|
|
|
try {
|
|
|
|
|
return await run(tempDir);
|
|
|
|
|
} finally {
|
|
|
|
|
await fs.rm(tempDir, { recursive: true, force: true });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function runWithStoredAuth(params: {
|
|
|
|
|
cfg: OpenClawConfig;
|
|
|
|
|
store: AuthProfileStore;
|
|
|
|
|
provider: string;
|
|
|
|
|
run: (provider: string, model: string) => Promise<string>;
|
|
|
|
|
}) {
|
|
|
|
|
return withTempAuthStore(params.store, async (tempDir) =>
|
|
|
|
|
runWithModelFallback({
|
|
|
|
|
cfg: params.cfg,
|
|
|
|
|
provider: params.provider,
|
|
|
|
|
model: "m1",
|
|
|
|
|
agentDir: tempDir,
|
|
|
|
|
run: params.run,
|
|
|
|
|
}),
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2026-02-16 14:52:09 +00:00
|
|
|
async function expectFallsBackToHaiku(params: {
|
|
|
|
|
provider: string;
|
|
|
|
|
model: string;
|
|
|
|
|
firstError: Error;
|
|
|
|
|
}) {
|
|
|
|
|
const cfg = makeCfg();
|
|
|
|
|
const run = vi.fn().mockRejectedValueOnce(params.firstError).mockResolvedValueOnce("ok");
|
|
|
|
|
|
|
|
|
|
const result = await runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: params.provider,
|
|
|
|
|
model: params.model,
|
|
|
|
|
run,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(result.result).toBe("ok");
|
|
|
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
|
|
|
expect(run.mock.calls[1]?.[0]).toBe("anthropic");
|
|
|
|
|
expect(run.mock.calls[1]?.[1]).toBe("claude-haiku-3-5");
|
|
|
|
|
}
|
|
|
|
|
|
2026-01-09 19:59:45 +01:00
|
|
|
describe("runWithModelFallback", () => {
|
Agents: add nested subagent orchestration controls and reduce subagent token waste (#14447)
* Agents: add subagent orchestration controls
* Agents: add subagent orchestration controls (WIP uncommitted changes)
* feat(subagents): add depth-based spawn gating for sub-sub-agents
* feat(subagents): tool policy, registry, and announce chain for nested agents
* feat(subagents): system prompt, docs, changelog for nested sub-agents
* fix(subagents): prevent model fallback override, show model during active runs, and block context overflow fallback
Bug 1: When a session has an explicit model override (e.g., gpt/openai-codex),
the fallback candidate logic in resolveFallbackCandidates silently appended the
global primary model (opus) as a backstop. On reinjection/steer with a transient
error, the session could fall back to opus which has a smaller context window
and crash. Fix: when storedModelOverride is set, pass fallbacksOverride ?? []
instead of undefined, preventing the implicit primary backstop.
Bug 2: Active subagents showed 'model n/a' in /subagents list because
resolveModelDisplay only read entry.model/modelProvider (populated after run
completes). Fix: fall back to modelOverride/providerOverride fields which are
populated at spawn time via sessions.patch.
Bug 3: Context overflow errors (prompt too long, context_length_exceeded) could
theoretically escape runEmbeddedPiAgent and be treated as failover candidates
in runWithModelFallback, causing a switch to a model with a smaller context
window. Fix: in runWithModelFallback, detect context overflow errors via
isLikelyContextOverflowError and rethrow them immediately instead of trying the
next model candidate.
* fix(subagents): track spawn depth in session store and fix announce routing for nested agents
* Fix compaction status tracking and dedupe overflow compaction triggers
* fix(subagents): enforce depth block via session store and implement cascade kill
* fix: inject group chat context into system prompt
* fix(subagents): always write model to session store at spawn time
* Preserve spawnDepth when agent handler rewrites session entry
* fix(subagents): suppress announce on steer-restart
* fix(subagents): fallback spawned session model to runtime default
* fix(subagents): enforce spawn depth when caller key resolves by sessionId
* feat(subagents): implement active-first ordering for numeric targets and enhance task display
- Added a test to verify that subagents with numeric targets follow an active-first list ordering.
- Updated `resolveSubagentTarget` to sort subagent runs based on active status and recent activity.
- Enhanced task display in command responses to prevent truncation of long task descriptions.
- Introduced new utility functions for compacting task text and managing subagent run states.
* fix(subagents): show model for active runs via run record fallback
When the spawned model matches the agent's default model, the session
store's override fields are intentionally cleared (isDefault: true).
The model/modelProvider fields are only populated after the run
completes. This left active subagents showing 'model n/a'.
Fix: store the resolved model on SubagentRunRecord at registration
time, and use it as a fallback in both display paths (subagents tool
and /subagents command) when the session store entry has no model info.
Changes:
- SubagentRunRecord: add optional model field
- registerSubagentRun: accept and persist model param
- sessions-spawn-tool: pass resolvedModel to registerSubagentRun
- subagents-tool: pass run record model as fallback to resolveModelDisplay
- commands-subagents: pass run record model as fallback to resolveModelDisplay
* feat(chat): implement session key resolution and reset on sidebar navigation
- Added functions to resolve the main session key and reset chat state when switching sessions from the sidebar.
- Updated the `renderTab` function to handle session key changes when navigating to the chat tab.
- Introduced a test to verify that the session resets to "main" when opening chat from the sidebar navigation.
* fix: subagent timeout=0 passthrough and fallback prompt duplication
Bug 1: runTimeoutSeconds=0 now means 'no timeout' instead of applying 600s default
- sessions-spawn-tool: default to undefined (not 0) when neither timeout param
is provided; use != null check so explicit 0 passes through to gateway
- agent.ts: accept 0 as valid timeout (resolveAgentTimeoutMs already handles
0 → MAX_SAFE_TIMEOUT_MS)
Bug 2: model fallback no longer re-injects the original prompt as a duplicate
- agent.ts: track fallback attempt index; on retries use a short continuation
message instead of the full original prompt since the session file already
contains it from the first attempt
- Also skip re-sending images on fallback retries (already in session)
* feat(subagents): truncate long task descriptions in subagents command output
- Introduced a new utility function to format task previews, limiting their length to improve readability.
- Updated the command handler to use the new formatting function, ensuring task descriptions are truncated appropriately.
- Adjusted related tests to verify that long task descriptions are now truncated in the output.
* refactor(subagents): update subagent registry path resolution and improve command output formatting
- Replaced direct import of STATE_DIR with a utility function to resolve the state directory dynamically.
- Enhanced the formatting of command output for active and recent subagents, adding separators for better readability.
- Updated related tests to reflect changes in command output structure.
* fix(subagent): default sessions_spawn to no timeout when runTimeoutSeconds omitted
The previous fix (75a791106) correctly handled the case where
runTimeoutSeconds was explicitly set to 0 ("no timeout"). However,
when models omit the parameter entirely (which is common since the
schema marks it as optional), runTimeoutSeconds resolved to undefined.
undefined flowed through the chain as:
sessions_spawn → timeout: undefined (since undefined != null is false)
→ gateway agent handler → agentCommand opts.timeout: undefined
→ resolveAgentTimeoutMs({ overrideSeconds: undefined })
→ DEFAULT_AGENT_TIMEOUT_SECONDS (600s = 10 minutes)
This caused subagents to be killed at exactly 10 minutes even though
the user's intent (via TOOLS.md) was for subagents to run without a
timeout.
Fix: default runTimeoutSeconds to 0 (no timeout) when neither
runTimeoutSeconds nor timeoutSeconds is provided by the caller.
Subagent spawns are long-running by design and should not inherit the
600s agent-command default timeout.
* fix(subagent): accept timeout=0 in agent-via-gateway path (second 600s default)
* fix: thread timeout override through getReplyFromConfig dispatch path
getReplyFromConfig called resolveAgentTimeoutMs({ cfg }) with no override,
always falling back to the config default (600s). Add timeoutOverrideSeconds
to GetReplyOptions and pass it through as overrideSeconds so callers of the
dispatch chain can specify a custom timeout (0 = no timeout).
This complements the existing timeout threading in agentCommand and the
cron isolated-agent runner, which already pass overrideSeconds correctly.
* feat(model-fallback): normalize OpenAI Codex model references and enhance fallback handling
- Added normalization for OpenAI Codex model references, specifically converting "gpt-5.3-codex" to "openai-codex" before execution.
- Updated the `resolveFallbackCandidates` function to utilize the new normalization logic.
- Enhanced tests to verify the correct behavior of model normalization and fallback mechanisms.
- Introduced a new test case to ensure that the normalization process works as expected for various input formats.
* feat(tests): add unit tests for steer failure behavior in openclaw-tools
- Introduced a new test file to validate the behavior of subagents when steer replacement dispatch fails.
- Implemented tests to ensure that the announce behavior is restored correctly and that the suppression reason is cleared as expected.
- Enhanced the subagent registry with a new function to clear steer restart suppression.
- Updated related components to support the new test scenarios.
* fix(subagents): replace stop command with kill in slash commands and documentation
- Updated the `/subagents` command to replace `stop` with `kill` for consistency in controlling sub-agent runs.
- Modified related documentation to reflect the change in command usage.
- Removed legacy timeoutSeconds references from the sessions-spawn-tool schema and tests to streamline timeout handling.
- Enhanced tests to ensure correct behavior of the updated commands and their interactions.
* feat(tests): add unit tests for readLatestAssistantReply function
- Introduced a new test file for the `readLatestAssistantReply` function to validate its behavior with various message scenarios.
- Implemented tests to ensure the function correctly retrieves the latest assistant message and handles cases where the latest message has no text.
- Mocked the gateway call to simulate different message histories for comprehensive testing.
* feat(tests): enhance subagent kill-all cascade tests and announce formatting
- Added a new test to verify that the `kill-all` command cascades through ended parents to active descendants in subagents.
- Updated the subagent announce formatting tests to reflect changes in message structure, including the replacement of "Findings:" with "Result:" and the addition of new expectations for message content.
- Improved the handling of long findings and stats in the announce formatting logic to ensure concise output.
- Refactored related functions to enhance clarity and maintainability in the subagent registry and tools.
* refactor(subagent): update announce formatting and remove unused constants
- Modified the subagent announce formatting to replace "Findings:" with "Result:" and adjusted related expectations in tests.
- Removed constants for maximum announce findings characters and summary words, simplifying the announcement logic.
- Updated the handling of findings to retain full content instead of truncating, ensuring more informative outputs.
- Cleaned up unused imports in the commands-subagents file to enhance code clarity.
* feat(tests): enhance billing error handling in user-facing text
- Added tests to ensure that normal text mentioning billing plans is not rewritten, preserving user context.
- Updated the `isBillingErrorMessage` and `sanitizeUserFacingText` functions to improve handling of billing-related messages.
- Introduced new test cases for various scenarios involving billing messages to ensure accurate processing and output.
- Enhanced the subagent announce flow to correctly manage active descendant runs, preventing premature announcements.
* feat(subagent): enhance workflow guidance and auto-announcement clarity
- Added a new guideline in the subagent system prompt to emphasize trust in push-based completion, discouraging busy polling for status updates.
- Updated documentation to clarify that sub-agents will automatically announce their results, improving user understanding of the workflow.
- Enhanced tests to verify the new guidance on avoiding polling loops and to ensure the accuracy of the updated prompts.
* fix(cron): avoid announcing interim subagent spawn acks
* chore: clean post-rebase imports
* fix(cron): fall back to child replies when parent stays interim
* fix(subagents): make active-run guidance advisory
* fix(subagents): update announce flow to handle active descendants and enhance test coverage
- Modified the announce flow to defer announcements when active descendant runs are present, ensuring accurate status reporting.
- Updated tests to verify the new behavior, including scenarios where no fallback requester is available and ensuring proper handling of finished subagents.
- Enhanced the announce formatting to include an `expectFinal` flag for better clarity in the announcement process.
* fix(subagents): enhance announce flow and formatting for user updates
- Updated the announce flow to provide clearer instructions for user updates based on active subagent runs and requester context.
- Refactored the announcement logic to improve clarity and ensure internal context remains private.
- Enhanced tests to verify the new message expectations and formatting, including updated prompts for user-facing updates.
- Introduced a new function to build reply instructions based on session context, improving the overall announcement process.
* fix: resolve prep blockers and changelog placement (#14447) (thanks @tyler6204)
* fix: restore cron delivery-plan import after rebase (#14447) (thanks @tyler6204)
* fix: resolve test failures from rebase conflicts (#14447) (thanks @tyler6204)
* fix: apply formatting after rebase (#14447) (thanks @tyler6204)
2026-02-14 22:03:45 -08:00
|
|
|
it("normalizes openai gpt-5.3 codex to openai-codex before running", async () => {
|
|
|
|
|
const cfg = makeCfg();
|
|
|
|
|
const run = vi.fn().mockResolvedValueOnce("ok");
|
|
|
|
|
|
|
|
|
|
const result = await runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-5.3-codex",
|
|
|
|
|
run,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(result.result).toBe("ok");
|
|
|
|
|
expect(run).toHaveBeenCalledTimes(1);
|
|
|
|
|
expect(run).toHaveBeenCalledWith("openai-codex", "gpt-5.3-codex");
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-09 19:59:45 +01:00
|
|
|
it("does not fall back on non-auth errors", async () => {
|
|
|
|
|
const cfg = makeCfg();
|
2026-01-14 14:31:43 +00:00
|
|
|
const run = vi.fn().mockRejectedValueOnce(new Error("bad request")).mockResolvedValueOnce("ok");
|
2026-01-09 19:59:45 +01:00
|
|
|
|
|
|
|
|
await expect(
|
|
|
|
|
runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
|
|
|
|
run,
|
|
|
|
|
}),
|
|
|
|
|
).rejects.toThrow("bad request");
|
|
|
|
|
expect(run).toHaveBeenCalledTimes(1);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("falls back on auth errors", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-09 19:59:45 +01:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: Object.assign(new Error("nope"), { status: 401 }),
|
2026-01-09 21:17:00 +01:00
|
|
|
});
|
2026-02-12 00:42:33 -03:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("falls back on transient HTTP 5xx errors", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-02-12 00:42:33 -03:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: new Error(
|
|
|
|
|
"521 <!DOCTYPE html><html><head><title>Web server is down</title></head><body>Cloudflare</body></html>",
|
|
|
|
|
),
|
2026-02-12 00:42:33 -03:00
|
|
|
});
|
2026-01-09 21:17:00 +01:00
|
|
|
});
|
|
|
|
|
|
2026-01-09 21:31:13 +01:00
|
|
|
it("falls back on 402 payment required", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-09 21:31:13 +01:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: Object.assign(new Error("payment required"), { status: 402 }),
|
2026-01-09 21:31:13 +01:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-09 21:17:00 +01:00
|
|
|
it("falls back on billing errors", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-09 21:17:00 +01:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: new Error(
|
|
|
|
|
"LLM request rejected: Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.",
|
|
|
|
|
),
|
2026-01-09 19:59:45 +01:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-12 22:48:37 -05:00
|
|
|
it("falls back on credential validation errors", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-12 22:48:37 -05:00
|
|
|
provider: "anthropic",
|
|
|
|
|
model: "claude-opus-4",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: new Error('No credentials found for profile "anthropic:default".'),
|
2026-01-12 22:48:37 -05:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-26 22:05:31 -05:00
|
|
|
it("skips providers when all profiles are in cooldown", async () => {
|
|
|
|
|
const provider = `cooldown-test-${crypto.randomUUID()}`;
|
|
|
|
|
const profileId = `${provider}:default`;
|
|
|
|
|
|
|
|
|
|
const store: AuthProfileStore = {
|
|
|
|
|
version: AUTH_STORE_VERSION,
|
|
|
|
|
profiles: {
|
|
|
|
|
[profileId]: {
|
|
|
|
|
type: "api_key",
|
|
|
|
|
provider,
|
|
|
|
|
key: "test-key",
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
usageStats: {
|
|
|
|
|
[profileId]: {
|
2026-02-18 05:04:51 +00:00
|
|
|
cooldownUntil: Date.now() + 5 * 60_000,
|
2026-01-26 22:05:31 -05:00
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
|
2026-02-18 05:04:51 +00:00
|
|
|
const cfg = makeProviderFallbackCfg(provider);
|
2026-01-26 22:05:31 -05:00
|
|
|
const run = vi.fn().mockImplementation(async (providerId, modelId) => {
|
2026-01-31 16:19:20 +09:00
|
|
|
if (providerId === "fallback") {
|
|
|
|
|
return "ok";
|
|
|
|
|
}
|
2026-01-26 22:05:31 -05:00
|
|
|
throw new Error(`unexpected provider: ${providerId}/${modelId}`);
|
|
|
|
|
});
|
|
|
|
|
|
2026-02-18 05:04:51 +00:00
|
|
|
const result = await runWithStoredAuth({
|
|
|
|
|
cfg,
|
|
|
|
|
store,
|
|
|
|
|
provider,
|
|
|
|
|
run,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(result.result).toBe("ok");
|
|
|
|
|
expect(run.mock.calls).toEqual([["fallback", "ok-model"]]);
|
|
|
|
|
expect(result.attempts[0]?.reason).toBe("rate_limit");
|
2026-01-26 22:05:31 -05:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("does not skip when any profile is available", async () => {
|
|
|
|
|
const provider = `cooldown-mixed-${crypto.randomUUID()}`;
|
|
|
|
|
const profileA = `${provider}:a`;
|
|
|
|
|
const profileB = `${provider}:b`;
|
|
|
|
|
|
|
|
|
|
const store: AuthProfileStore = {
|
|
|
|
|
version: AUTH_STORE_VERSION,
|
|
|
|
|
profiles: {
|
|
|
|
|
[profileA]: {
|
|
|
|
|
type: "api_key",
|
|
|
|
|
provider,
|
|
|
|
|
key: "key-a",
|
|
|
|
|
},
|
|
|
|
|
[profileB]: {
|
|
|
|
|
type: "api_key",
|
|
|
|
|
provider,
|
|
|
|
|
key: "key-b",
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
usageStats: {
|
|
|
|
|
[profileA]: {
|
|
|
|
|
cooldownUntil: Date.now() + 60_000,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
|
2026-02-18 05:04:51 +00:00
|
|
|
const cfg = makeProviderFallbackCfg(provider);
|
2026-01-26 22:05:31 -05:00
|
|
|
const run = vi.fn().mockImplementation(async (providerId) => {
|
2026-01-31 16:19:20 +09:00
|
|
|
if (providerId === provider) {
|
|
|
|
|
return "ok";
|
|
|
|
|
}
|
2026-01-26 22:05:31 -05:00
|
|
|
return "unexpected";
|
|
|
|
|
});
|
|
|
|
|
|
2026-02-18 05:04:51 +00:00
|
|
|
const result = await runWithStoredAuth({
|
|
|
|
|
cfg,
|
|
|
|
|
store,
|
|
|
|
|
provider,
|
|
|
|
|
run,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(result.result).toBe("ok");
|
|
|
|
|
expect(run.mock.calls).toEqual([[provider, "m1"]]);
|
|
|
|
|
expect(result.attempts).toEqual([]);
|
2026-01-26 22:05:31 -05:00
|
|
|
});
|
|
|
|
|
|
2026-01-09 14:59:02 +01:00
|
|
|
it("does not append configured primary when fallbacksOverride is set", async () => {
|
|
|
|
|
const cfg = makeCfg({
|
|
|
|
|
agents: {
|
|
|
|
|
defaults: {
|
|
|
|
|
model: {
|
|
|
|
|
primary: "openai/gpt-4.1-mini",
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
const run = vi
|
|
|
|
|
.fn()
|
2026-01-14 14:31:43 +00:00
|
|
|
.mockImplementation(() => Promise.reject(Object.assign(new Error("nope"), { status: 401 })));
|
2026-01-09 14:59:02 +01:00
|
|
|
|
|
|
|
|
await expect(
|
|
|
|
|
runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: "anthropic",
|
|
|
|
|
model: "claude-opus-4-5",
|
|
|
|
|
fallbacksOverride: ["anthropic/claude-haiku-3-5"],
|
|
|
|
|
run,
|
|
|
|
|
}),
|
|
|
|
|
).rejects.toThrow("All models failed");
|
|
|
|
|
|
|
|
|
|
expect(run.mock.calls).toEqual([
|
|
|
|
|
["anthropic", "claude-opus-4-5"],
|
|
|
|
|
["anthropic", "claude-haiku-3-5"],
|
|
|
|
|
]);
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-09 16:10:53 +01:00
|
|
|
it("uses fallbacksOverride instead of agents.defaults.model.fallbacks", async () => {
|
2026-02-18 05:04:51 +00:00
|
|
|
const cfg = makeFallbacksOnlyCfg();
|
2026-01-09 16:10:53 +01:00
|
|
|
|
|
|
|
|
const calls: Array<{ provider: string; model: string }> = [];
|
|
|
|
|
|
|
|
|
|
const res = await runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: "anthropic",
|
|
|
|
|
model: "claude-opus-4-5",
|
|
|
|
|
fallbacksOverride: ["openai/gpt-4.1"],
|
|
|
|
|
run: async (provider, model) => {
|
|
|
|
|
calls.push({ provider, model });
|
|
|
|
|
if (provider === "anthropic") {
|
2026-01-13 06:49:26 +00:00
|
|
|
throw Object.assign(new Error("nope"), { status: 401 });
|
2026-01-09 16:10:53 +01:00
|
|
|
}
|
|
|
|
|
if (provider === "openai" && model === "gpt-4.1") {
|
|
|
|
|
return "ok";
|
|
|
|
|
}
|
|
|
|
|
throw new Error(`unexpected candidate: ${provider}/${model}`);
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(res.result).toBe("ok");
|
|
|
|
|
expect(calls).toEqual([
|
|
|
|
|
{ provider: "anthropic", model: "claude-opus-4-5" },
|
|
|
|
|
{ provider: "openai", model: "gpt-4.1" },
|
|
|
|
|
]);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("treats an empty fallbacksOverride as disabling global fallbacks", async () => {
|
2026-02-18 05:04:51 +00:00
|
|
|
const cfg = makeFallbacksOnlyCfg();
|
2026-01-09 16:10:53 +01:00
|
|
|
|
|
|
|
|
const calls: Array<{ provider: string; model: string }> = [];
|
|
|
|
|
|
|
|
|
|
await expect(
|
|
|
|
|
runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: "anthropic",
|
|
|
|
|
model: "claude-opus-4-5",
|
|
|
|
|
fallbacksOverride: [],
|
|
|
|
|
run: async (provider, model) => {
|
|
|
|
|
calls.push({ provider, model });
|
|
|
|
|
throw new Error("primary failed");
|
|
|
|
|
},
|
|
|
|
|
}),
|
|
|
|
|
).rejects.toThrow("primary failed");
|
|
|
|
|
|
2026-01-14 14:31:43 +00:00
|
|
|
expect(calls).toEqual([{ provider: "anthropic", model: "claude-opus-4-5" }]);
|
2026-01-09 16:10:53 +01:00
|
|
|
});
|
|
|
|
|
|
2026-01-15 11:58:15 +00:00
|
|
|
it("defaults provider/model when missing (regression #946)", async () => {
|
|
|
|
|
const cfg = makeCfg({
|
|
|
|
|
agents: {
|
|
|
|
|
defaults: {
|
|
|
|
|
model: {
|
|
|
|
|
primary: "openai/gpt-4.1-mini",
|
|
|
|
|
fallbacks: [],
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const calls: Array<{ provider: string; model: string }> = [];
|
|
|
|
|
|
|
|
|
|
const result = await runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: undefined as unknown as string,
|
|
|
|
|
model: undefined as unknown as string,
|
|
|
|
|
run: async (provider, model) => {
|
|
|
|
|
calls.push({ provider, model });
|
|
|
|
|
return "ok";
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(result.result).toBe("ok");
|
|
|
|
|
expect(calls).toEqual([{ provider: "openai", model: "gpt-4.1-mini" }]);
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-13 04:02:47 +00:00
|
|
|
it("falls back on missing API key errors", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-13 04:02:47 +00:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: new Error("No API key found for profile openai."),
|
2026-01-13 04:12:02 +00:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("falls back on lowercase credential errors", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-13 04:12:02 +00:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: new Error("no api key found for profile openai"),
|
2026-01-13 04:02:47 +00:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-18 07:52:19 +00:00
|
|
|
it("falls back on timeout abort errors", async () => {
|
|
|
|
|
const timeoutCause = Object.assign(new Error("request timed out"), { name: "TimeoutError" });
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-18 07:52:19 +00:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: Object.assign(new Error("aborted"), { name: "AbortError", cause: timeoutCause }),
|
2026-01-18 07:52:19 +00:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
it("falls back on abort errors with timeout reasons", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-18 07:52:19 +00:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: Object.assign(new Error("aborted"), {
|
|
|
|
|
name: "AbortError",
|
|
|
|
|
reason: "deadline exceeded",
|
|
|
|
|
}),
|
2026-01-18 07:52:19 +00:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-02-16 20:59:44 -05:00
|
|
|
it("falls back on abort errors with reason: abort", async () => {
|
|
|
|
|
await expectFallsBackToHaiku({
|
|
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
|
|
|
|
firstError: Object.assign(new Error("aborted"), {
|
|
|
|
|
name: "AbortError",
|
|
|
|
|
reason: "reason: abort",
|
|
|
|
|
}),
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-20 19:23:06 +00:00
|
|
|
it("falls back when message says aborted but error is a timeout", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-20 19:23:06 +00:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: Object.assign(new Error("request aborted"), { code: "ETIMEDOUT" }),
|
2026-01-20 19:23:06 +00:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-24 06:27:24 -05:00
|
|
|
it("falls back on provider abort errors with request-aborted messages", async () => {
|
2026-02-16 14:52:09 +00:00
|
|
|
await expectFallsBackToHaiku({
|
2026-01-24 06:27:24 -05:00
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
2026-02-16 14:52:09 +00:00
|
|
|
firstError: Object.assign(new Error("Request was aborted"), { name: "AbortError" }),
|
2026-01-24 06:27:24 -05:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-18 07:52:19 +00:00
|
|
|
it("does not fall back on user aborts", async () => {
|
|
|
|
|
const cfg = makeCfg();
|
|
|
|
|
const run = vi
|
|
|
|
|
.fn()
|
|
|
|
|
.mockRejectedValueOnce(Object.assign(new Error("aborted"), { name: "AbortError" }))
|
|
|
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
|
|
|
|
|
|
await expect(
|
|
|
|
|
runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: "openai",
|
|
|
|
|
model: "gpt-4.1-mini",
|
|
|
|
|
run,
|
|
|
|
|
}),
|
|
|
|
|
).rejects.toThrow("aborted");
|
|
|
|
|
|
|
|
|
|
expect(run).toHaveBeenCalledTimes(1);
|
|
|
|
|
});
|
|
|
|
|
|
2026-01-09 19:59:45 +01:00
|
|
|
it("appends the configured primary as a last fallback", async () => {
|
|
|
|
|
const cfg = makeCfg({
|
|
|
|
|
agents: {
|
|
|
|
|
defaults: {
|
|
|
|
|
model: {
|
|
|
|
|
primary: "openai/gpt-4.1-mini",
|
|
|
|
|
fallbacks: [],
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
const run = vi
|
|
|
|
|
.fn()
|
2026-01-14 14:31:43 +00:00
|
|
|
.mockRejectedValueOnce(Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }))
|
2026-01-09 19:59:45 +01:00
|
|
|
.mockResolvedValueOnce("ok");
|
|
|
|
|
|
|
|
|
|
const result = await runWithModelFallback({
|
|
|
|
|
cfg,
|
|
|
|
|
provider: "openrouter",
|
|
|
|
|
model: "meta-llama/llama-3.3-70b:free",
|
|
|
|
|
run,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
expect(result.result).toBe("ok");
|
|
|
|
|
expect(run).toHaveBeenCalledTimes(2);
|
|
|
|
|
expect(result.provider).toBe("openai");
|
|
|
|
|
expect(result.model).toBe("gpt-4.1-mini");
|
|
|
|
|
});
|
|
|
|
|
});
|