import type { BrowserFormField } from "../client-actions-core.js"; import type { BrowserRouteContext } from "../server-context.js"; import { type ActKind, isActKind, parseClickButton, parseClickModifiers, } from "./agent.act.shared.js"; import { handleRouteError, readBody, requirePwAi, resolveProfileContext, SELECTOR_UNSUPPORTED_MESSAGE, } from "./agent.shared.js"; import { DEFAULT_DOWNLOAD_DIR, DEFAULT_UPLOAD_DIR, resolvePathWithinRoot, resolvePathsWithinRoot, } from "./path-output.js"; import type { BrowserRouteRegistrar } from "./types.js"; import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js"; export function registerBrowserAgentActRoutes( app: BrowserRouteRegistrar, ctx: BrowserRouteContext, ) { app.post("/act", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) { return; } const body = readBody(req); const kindRaw = toStringOrEmpty(body.kind); if (!isActKind(kindRaw)) { return jsonError(res, 400, "kind is required"); } const kind: ActKind = kindRaw; const targetId = toStringOrEmpty(body.targetId) || undefined; if (Object.hasOwn(body, "selector") && kind !== "wait") { return jsonError(res, 400, SELECTOR_UNSUPPORTED_MESSAGE); } try { const tab = await profileCtx.ensureTabAvailable(targetId); const cdpUrl = profileCtx.profile.cdpUrl; const pw = await requirePwAi(res, `act:${kind}`); if (!pw) { return; } const evaluateEnabled = ctx.state().resolved.evaluateEnabled; switch (kind) { case "click": { const ref = toStringOrEmpty(body.ref); if (!ref) { return jsonError(res, 400, "ref is required"); } const doubleClick = toBoolean(body.doubleClick) ?? false; const timeoutMs = toNumber(body.timeoutMs); const buttonRaw = toStringOrEmpty(body.button) || ""; const button = buttonRaw ? parseClickButton(buttonRaw) : undefined; if (buttonRaw && !button) { return jsonError(res, 400, "button must be left|right|middle"); } const modifiersRaw = toStringArray(body.modifiers) ?? []; const parsedModifiers = parseClickModifiers(modifiersRaw); if (parsedModifiers.error) { return jsonError(res, 400, parsedModifiers.error); } const modifiers = parsedModifiers.modifiers; const clickRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, ref, doubleClick, }; if (button) { clickRequest.button = button; } if (modifiers) { clickRequest.modifiers = modifiers; } if (timeoutMs) { clickRequest.timeoutMs = timeoutMs; } await pw.clickViaPlaywright(clickRequest); return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); } case "type": { const ref = toStringOrEmpty(body.ref); if (!ref) { return jsonError(res, 400, "ref is required"); } if (typeof body.text !== "string") { return jsonError(res, 400, "text is required"); } const text = body.text; const submit = toBoolean(body.submit) ?? false; const slowly = toBoolean(body.slowly) ?? false; const timeoutMs = toNumber(body.timeoutMs); const typeRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, ref, text, submit, slowly, }; if (timeoutMs) { typeRequest.timeoutMs = timeoutMs; } await pw.typeViaPlaywright(typeRequest); return res.json({ ok: true, targetId: tab.targetId }); } case "press": { const key = toStringOrEmpty(body.key); if (!key) { return jsonError(res, 400, "key is required"); } const delayMs = toNumber(body.delayMs); await pw.pressKeyViaPlaywright({ cdpUrl, targetId: tab.targetId, key, delayMs: delayMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } case "hover": { const ref = toStringOrEmpty(body.ref); if (!ref) { return jsonError(res, 400, "ref is required"); } const timeoutMs = toNumber(body.timeoutMs); await pw.hoverViaPlaywright({ cdpUrl, targetId: tab.targetId, ref, timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } case "scrollIntoView": { const ref = toStringOrEmpty(body.ref); if (!ref) { return jsonError(res, 400, "ref is required"); } const timeoutMs = toNumber(body.timeoutMs); const scrollRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, ref, }; if (timeoutMs) { scrollRequest.timeoutMs = timeoutMs; } await pw.scrollIntoViewViaPlaywright(scrollRequest); return res.json({ ok: true, targetId: tab.targetId }); } case "drag": { const startRef = toStringOrEmpty(body.startRef); const endRef = toStringOrEmpty(body.endRef); if (!startRef || !endRef) { return jsonError(res, 400, "startRef and endRef are required"); } const timeoutMs = toNumber(body.timeoutMs); await pw.dragViaPlaywright({ cdpUrl, targetId: tab.targetId, startRef, endRef, timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } case "select": { const ref = toStringOrEmpty(body.ref); const values = toStringArray(body.values); if (!ref || !values?.length) { return jsonError(res, 400, "ref and values are required"); } const timeoutMs = toNumber(body.timeoutMs); await pw.selectOptionViaPlaywright({ cdpUrl, targetId: tab.targetId, ref, values, timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } case "fill": { const rawFields = Array.isArray(body.fields) ? body.fields : []; const fields = rawFields .map((field) => { if (!field || typeof field !== "object") { return null; } const rec = field as Record; const ref = toStringOrEmpty(rec.ref); const type = toStringOrEmpty(rec.type); if (!ref || !type) { return null; } const value = typeof rec.value === "string" || typeof rec.value === "number" || typeof rec.value === "boolean" ? rec.value : undefined; const parsed: BrowserFormField = value === undefined ? { ref, type } : { ref, type, value }; return parsed; }) .filter((field): field is BrowserFormField => field !== null); if (!fields.length) { return jsonError(res, 400, "fields are required"); } const timeoutMs = toNumber(body.timeoutMs); await pw.fillFormViaPlaywright({ cdpUrl, targetId: tab.targetId, fields, timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } case "resize": { const width = toNumber(body.width); const height = toNumber(body.height); if (!width || !height) { return jsonError(res, 400, "width and height are required"); } await pw.resizeViewportViaPlaywright({ cdpUrl, targetId: tab.targetId, width, height, }); return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); } case "wait": { const timeMs = toNumber(body.timeMs); const text = toStringOrEmpty(body.text) || undefined; const textGone = toStringOrEmpty(body.textGone) || undefined; const selector = toStringOrEmpty(body.selector) || undefined; const url = toStringOrEmpty(body.url) || undefined; const loadStateRaw = toStringOrEmpty(body.loadState); const loadState = loadStateRaw === "load" || loadStateRaw === "domcontentloaded" || loadStateRaw === "networkidle" ? loadStateRaw : undefined; const fn = toStringOrEmpty(body.fn) || undefined; const timeoutMs = toNumber(body.timeoutMs) ?? undefined; if (fn && !evaluateEnabled) { return jsonError( res, 403, [ "wait --fn is disabled by config (browser.evaluateEnabled=false).", "Docs: /gateway/configuration#browser-openclaw-managed-browser", ].join("\n"), ); } if ( timeMs === undefined && !text && !textGone && !selector && !url && !loadState && !fn ) { return jsonError( res, 400, "wait requires at least one of: timeMs, text, textGone, selector, url, loadState, fn", ); } await pw.waitForViaPlaywright({ cdpUrl, targetId: tab.targetId, timeMs, text, textGone, selector, url, loadState, fn, timeoutMs, }); return res.json({ ok: true, targetId: tab.targetId }); } case "evaluate": { if (!evaluateEnabled) { return jsonError( res, 403, [ "act:evaluate is disabled by config (browser.evaluateEnabled=false).", "Docs: /gateway/configuration#browser-openclaw-managed-browser", ].join("\n"), ); } const fn = toStringOrEmpty(body.fn); if (!fn) { return jsonError(res, 400, "fn is required"); } const ref = toStringOrEmpty(body.ref) || undefined; const evalTimeoutMs = toNumber(body.timeoutMs); const evalRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, fn, ref, signal: req.signal, }; if (evalTimeoutMs !== undefined) { evalRequest.timeoutMs = evalTimeoutMs; } const result = await pw.evaluateViaPlaywright(evalRequest); return res.json({ ok: true, targetId: tab.targetId, url: tab.url, result, }); } case "close": { await pw.closePageViaPlaywright({ cdpUrl, targetId: tab.targetId }); return res.json({ ok: true, targetId: tab.targetId }); } default: { return jsonError(res, 400, "unsupported kind"); } } } catch (err) { handleRouteError(ctx, res, err); } }); app.post("/hooks/file-chooser", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) { return; } const body = readBody(req); const targetId = toStringOrEmpty(body.targetId) || undefined; const ref = toStringOrEmpty(body.ref) || undefined; const inputRef = toStringOrEmpty(body.inputRef) || undefined; const element = toStringOrEmpty(body.element) || undefined; const paths = toStringArray(body.paths) ?? []; const timeoutMs = toNumber(body.timeoutMs); if (!paths.length) { return jsonError(res, 400, "paths are required"); } try { const uploadPathsResult = resolvePathsWithinRoot({ rootDir: DEFAULT_UPLOAD_DIR, requestedPaths: paths, scopeLabel: `uploads directory (${DEFAULT_UPLOAD_DIR})`, }); if (!uploadPathsResult.ok) { res.status(400).json({ error: uploadPathsResult.error }); return; } const resolvedPaths = uploadPathsResult.paths; const tab = await profileCtx.ensureTabAvailable(targetId); const pw = await requirePwAi(res, "file chooser hook"); if (!pw) { return; } if (inputRef || element) { if (ref) { return jsonError(res, 400, "ref cannot be combined with inputRef/element"); } await pw.setInputFilesViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, inputRef, element, paths: resolvedPaths, }); } else { await pw.armFileUploadViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, paths: resolvedPaths, timeoutMs: timeoutMs ?? undefined, }); if (ref) { await pw.clickViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, ref, }); } } res.json({ ok: true }); } catch (err) { handleRouteError(ctx, res, err); } }); app.post("/hooks/dialog", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) { return; } const body = readBody(req); const targetId = toStringOrEmpty(body.targetId) || undefined; const accept = toBoolean(body.accept); const promptText = toStringOrEmpty(body.promptText) || undefined; const timeoutMs = toNumber(body.timeoutMs); if (accept === undefined) { return jsonError(res, 400, "accept is required"); } try { const tab = await profileCtx.ensureTabAvailable(targetId); const pw = await requirePwAi(res, "dialog hook"); if (!pw) { return; } await pw.armDialogViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, accept, promptText, timeoutMs: timeoutMs ?? undefined, }); res.json({ ok: true }); } catch (err) { handleRouteError(ctx, res, err); } }); app.post("/wait/download", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) { return; } const body = readBody(req); const targetId = toStringOrEmpty(body.targetId) || undefined; const out = toStringOrEmpty(body.path) || ""; const timeoutMs = toNumber(body.timeoutMs); try { const tab = await profileCtx.ensureTabAvailable(targetId); const pw = await requirePwAi(res, "wait for download"); if (!pw) { return; } let downloadPath: string | undefined; if (out.trim()) { const downloadPathResult = resolvePathWithinRoot({ rootDir: DEFAULT_DOWNLOAD_DIR, requestedPath: out, scopeLabel: "downloads directory", }); if (!downloadPathResult.ok) { res.status(400).json({ error: downloadPathResult.error }); return; } downloadPath = downloadPathResult.path; } const result = await pw.waitForDownloadViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, path: downloadPath, timeoutMs: timeoutMs ?? undefined, }); res.json({ ok: true, targetId: tab.targetId, download: result }); } catch (err) { handleRouteError(ctx, res, err); } }); app.post("/download", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) { return; } const body = readBody(req); const targetId = toStringOrEmpty(body.targetId) || undefined; const ref = toStringOrEmpty(body.ref); const out = toStringOrEmpty(body.path); const timeoutMs = toNumber(body.timeoutMs); if (!ref) { return jsonError(res, 400, "ref is required"); } if (!out) { return jsonError(res, 400, "path is required"); } try { const downloadPathResult = resolvePathWithinRoot({ rootDir: DEFAULT_DOWNLOAD_DIR, requestedPath: out, scopeLabel: "downloads directory", }); if (!downloadPathResult.ok) { res.status(400).json({ error: downloadPathResult.error }); return; } const tab = await profileCtx.ensureTabAvailable(targetId); const pw = await requirePwAi(res, "download"); if (!pw) { return; } const result = await pw.downloadViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, ref, path: downloadPathResult.path, timeoutMs: timeoutMs ?? undefined, }); res.json({ ok: true, targetId: tab.targetId, download: result }); } catch (err) { handleRouteError(ctx, res, err); } }); app.post("/response/body", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) { return; } const body = readBody(req); const targetId = toStringOrEmpty(body.targetId) || undefined; const url = toStringOrEmpty(body.url); const timeoutMs = toNumber(body.timeoutMs); const maxChars = toNumber(body.maxChars); if (!url) { return jsonError(res, 400, "url is required"); } try { const tab = await profileCtx.ensureTabAvailable(targetId); const pw = await requirePwAi(res, "response body"); if (!pw) { return; } const result = await pw.responseBodyViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, url, timeoutMs: timeoutMs ?? undefined, maxChars: maxChars ?? undefined, }); res.json({ ok: true, targetId: tab.targetId, response: result }); } catch (err) { handleRouteError(ctx, res, err); } }); app.post("/highlight", async (req, res) => { const profileCtx = resolveProfileContext(req, res, ctx); if (!profileCtx) { return; } const body = readBody(req); const targetId = toStringOrEmpty(body.targetId) || undefined; const ref = toStringOrEmpty(body.ref); if (!ref) { return jsonError(res, 400, "ref is required"); } try { const tab = await profileCtx.ensureTabAvailable(targetId); const pw = await requirePwAi(res, "highlight"); if (!pw) { return; } await pw.highlightViaPlaywright({ cdpUrl: profileCtx.profile.cdpUrl, targetId: tab.targetId, ref, }); res.json({ ok: true, targetId: tab.targetId }); } catch (err) { handleRouteError(ctx, res, err); } }); }