wip: [01-stabilize] paused at task 1/1 - OCR Hallucination Immune logic via Semantic delta window and fret-isolation
This commit is contained in:
46
.agent/vendor/browser_use/.dockerignore
vendored
Normal file
46
.agent/vendor/browser_use/.dockerignore
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
docs/
|
||||
static/
|
||||
.claude/
|
||||
.github/
|
||||
|
||||
# Cache files
|
||||
.DS_Store
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.ipynb_checkpoints
|
||||
|
||||
# Virtual Environments
|
||||
.venv
|
||||
venv/
|
||||
|
||||
# Editor cruft
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# Build Files
|
||||
dist/
|
||||
|
||||
# Data files
|
||||
*.gif
|
||||
*.txt
|
||||
*.pdf
|
||||
*.csv
|
||||
*.json
|
||||
*.jsonl
|
||||
*.bak
|
||||
|
||||
# Secrets and sensitive files
|
||||
secrets.env
|
||||
.env
|
||||
browser_cookies.json
|
||||
cookies.json
|
||||
gcp-login.json
|
||||
saved_trajectories/
|
||||
AgentHistory.json
|
||||
AgentHistoryList.json
|
||||
private_example.py
|
||||
private_example
|
||||
70
.agent/vendor/browser_use/.env.example
vendored
Normal file
70
.agent/vendor/browser_use/.env.example
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
# Browser Use Configuration
|
||||
# Copy this file to .env and fill in your values
|
||||
|
||||
# Logging Configuration
|
||||
# Set the logging level (debug, info, warning, error)
|
||||
BROWSER_USE_LOGGING_LEVEL=info
|
||||
|
||||
# Log file paths (optional)
|
||||
# Save debug level logs to this file
|
||||
BROWSER_USE_DEBUG_LOG_FILE=debug.log
|
||||
|
||||
# Save info level logs to this file
|
||||
BROWSER_USE_INFO_LOG_FILE=info.log
|
||||
|
||||
# CDP (Chrome DevTools Protocol) logging level
|
||||
CDP_LOGGING_LEVEL=WARNING
|
||||
|
||||
# Telemetry and Analytics
|
||||
# Enable/disable anonymous telemetry
|
||||
ANONYMIZED_TELEMETRY=true
|
||||
|
||||
# Browser Use Cloud Configuration
|
||||
# Get your API key from: https://cloud.browser-use.com/new-api-key
|
||||
BROWSER_USE_API_KEY=your_bu_api_key_here
|
||||
|
||||
# Custom API base URL (for enterprise installations)
|
||||
# BROWSER_USE_CLOUD_API_URL=https://api.browser-use.com
|
||||
|
||||
# Cloud sync settings
|
||||
# BROWSER_USE_CLOUD_SYNC=false
|
||||
|
||||
# Model Configuration (optional - use if you want to use other LLM providers)
|
||||
# Default LLM model to use
|
||||
# OPENAI_API_KEY=your_openai_api_key_here
|
||||
# ANTHROPIC_API_KEY=your_anthropic_api_key_here
|
||||
# AZURE_OPENAI_API_KEY=
|
||||
# AZURE_OPENAI_ENDPOINT=
|
||||
# GOOGLE_API_KEY=
|
||||
# DEEPSEEK_API_KEY=
|
||||
# GROK_API_KEY=
|
||||
# NOVITA_API_KEY=
|
||||
|
||||
# AWS Bedrock Configuration (for AWS Bedrock models)
|
||||
# Requires: pip install browser-use[aws]
|
||||
# Note: You need proper AWS Bedrock access and model permissions in your AWS account
|
||||
# AWS_ACCESS_KEY_ID=your_aws_access_key_id_here
|
||||
# AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key_here
|
||||
# AWS_SESSION_TOKEN=your_session_token_here # Only required for temporary credentials
|
||||
# AWS_REGION=us-east-1
|
||||
|
||||
|
||||
# Browser Configuration
|
||||
# Path to Chrome/Chromium executable (optional)
|
||||
# BROWSER_USE_EXECUTABLE_PATH=/path/to/chrome
|
||||
|
||||
# Run browser in headless mode
|
||||
# BROWSER_USE_HEADLESS=false
|
||||
|
||||
# User data directory for browser profile
|
||||
# BROWSER_USE_USER_DATA_DIR=./browser_data
|
||||
|
||||
# Proxy Configuration (optional)
|
||||
# BROWSER_USE_PROXY_SERVER=http://proxy.example.com:8080
|
||||
# BROWSER_USE_NO_PROXY=localhost,127.0.0.1,*.internal
|
||||
# BROWSER_USE_PROXY_USERNAME=username
|
||||
# BROWSER_USE_PROXY_PASSWORD=password
|
||||
|
||||
# Version Check
|
||||
# Enable/disable checking for newer browser-use versions on agent startup
|
||||
BROWSER_USE_VERSION_CHECK=true
|
||||
2
.agent/vendor/browser_use/.gitattributes
vendored
Normal file
2
.agent/vendor/browser_use/.gitattributes
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
static/*.gif filter=lfs diff=lfs merge=lfs -text
|
||||
# static/*.mp4 filter=lfs diff=lfs merge=lfs -text
|
||||
2
.agent/vendor/browser_use/.github/.git-blame-ignore-revs
vendored
Normal file
2
.agent/vendor/browser_use/.github/.git-blame-ignore-revs
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
66b3c26df51adec32d42c3b2c0304e0662457298
|
||||
2be4ba4f7078d47bbeed04baf6f8fb04017df028
|
||||
7
.agent/vendor/browser_use/.github/CONTRIBUTING.md
vendored
Normal file
7
.agent/vendor/browser_use/.github/CONTRIBUTING.md
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
# Contributing to browser-use
|
||||
|
||||
We love contributions! Please read through these links to get started:
|
||||
|
||||
- 🔢 [Contribution Guidelines](https://docs.browser-use.com/development/contribution-guide)
|
||||
- 👾 [Local Development Setup Guide](https://docs.browser-use.com/development/local-setup)
|
||||
- 🏷️ [Issues Tagged: `#help-wanted`](https://github.com/browser-use/browser-use/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22help%20wanted%22)
|
||||
114
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/1_element_detection_bug.yml
vendored
Normal file
114
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/1_element_detection_bug.yml
vendored
Normal file
@@ -0,0 +1,114 @@
|
||||
name: 🎯 AI Agent ✚ Page Interaction Issue
|
||||
description: Agent fails to detect, click, scroll, input, or otherwise interact with some type of element on some page(s)
|
||||
labels: ["bug", "element-detection"]
|
||||
title: "Interaction Issue: ..."
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for taking the time to fill out this bug report! Please fill out the form below to help us reproduce and fix the issue.
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
> [!IMPORTANT]
|
||||
> 🙏 Please **go check *right now before filling this out* that that you are *actually* on the [⬆️ LATEST VERSION](https://github.com/browser-use/browser-use/releases)**.
|
||||
> 🚀 We ship changes every hour and we might've already fixed your issue today!
|
||||
> <a href="https://github.com/browser-use/browser-use/releases"><img src="https://github.com/user-attachments/assets/4cd34ee6-bafb-4f24-87e2-27a31dc5b9a4" width="500px"/></a>
|
||||
> If you are running an old version, the **first thing we will ask you to do is *upgrade to the latest version* and try again**:
|
||||
> - 🆕 [`beta`](https://docs.browser-use.com/development/local-setup): `uv pip install --upgrade git+https://github.com/browser-use/browser-use.git@main`
|
||||
> - 📦 [`stable`](https://pypi.org/project/browser-use/#history): `uv pip install --upgrade browser-use`
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Browser Use Version
|
||||
description: |
|
||||
What version of `browser-use` are you using? (Run `uv pip show browser-use` or `git log -n 1`)
|
||||
**DO NOT JUST WRITE `latest release` or `main` or a very old version or we will close your issue!**
|
||||
placeholder: "e.g. 0.4.45 or 62760baaefd"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: model
|
||||
attributes:
|
||||
label: LLM Model
|
||||
description: Which LLM model are you using?
|
||||
placeholder: "e.g. bu-1.0, gpt-5-mini, claude-4-5-sonnet, gemini-2.0-flash, etc."
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: prompt
|
||||
attributes:
|
||||
label: Screenshots, Description, and task prompt given to Agent
|
||||
description: |
|
||||
A description of the issue + screenshots, and the full task prompt you're giving the agent (redact sensitive data).
|
||||
To help us fix it even faster, screenshot the Chome devtools [`Computed Styles` pane](https://developer.chrome.com/docs/devtools/css/reference#computed) for each failing element.
|
||||
placeholder: |
|
||||
🎯 High-level goal: Compare the prices of 3 items on a few different seller pages
|
||||
💬 Agent(task='''
|
||||
1. go to https://example.com and click the "xyz" dropdown
|
||||
2. type "abc" into search then select the "abc" option <- ❌ agent fails to select this option
|
||||
3. ...
|
||||
☝️ please include real URLs 🔗 and screenshots 📸 when possible!
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: html
|
||||
attributes:
|
||||
label: "HTML around where it's failing"
|
||||
description: A snippet of the HTML from the failing page around where the Agent is failing to interact.
|
||||
render: html
|
||||
placeholder: |
|
||||
<form na-someform="abc"> <!-- ⬅️ at least one parent element above -->
|
||||
<div class="element-to-click">
|
||||
<div data-isbutton="true">Click me</div>
|
||||
</div>
|
||||
<input id="someinput" name="someinput" type="text" /> <!-- ⬅️ failing element -->
|
||||
...
|
||||
</form>
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating System & Browser Versions
|
||||
description: What operating system and browser are you using?
|
||||
placeholder: "e.g. Ubuntu 24.04 + playwright chromium v136, Windows 11 + Chrome.exe v133, macOS ..."
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: code
|
||||
attributes:
|
||||
label: Python Code Sample
|
||||
description: Include some python code that reproduces the issue
|
||||
render: python
|
||||
placeholder: |
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv() # tip: always load_dotenv() before other imports
|
||||
from browser_use import Agent, BrowserSession, Tools
|
||||
from browser_use.llm import ChatOpenAI
|
||||
|
||||
agent = Agent(
|
||||
task='...',
|
||||
llm=ChatOpenAI(model="gpt-4.1"),
|
||||
browser_session=BrowserSession(headless=False),
|
||||
)
|
||||
...
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Full DEBUG Log Output
|
||||
description: Please copy and paste the *full* log output *from the start of the run*. Make sure to set `BROWSER_USE_LOGGING_LEVEL=DEBUG` in your `.env` or shell environment.
|
||||
render: shell
|
||||
placeholder: |
|
||||
$ python /app/browser-use/examples/browser/real_browser.py
|
||||
DEBUG [browser] 🌎 Initializing new browser
|
||||
DEBUG [agent] Version: 0.1.46-9-g62760ba, Source: git
|
||||
77
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/2_bug_report.yml
vendored
Normal file
77
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/2_bug_report.yml
vendored
Normal file
@@ -0,0 +1,77 @@
|
||||
name: 👾 Library Bug Report
|
||||
description: Report a bug in the browser-use Python library
|
||||
labels: ["bug", "triage"]
|
||||
title: "Bug: ..."
|
||||
body:
|
||||
# - type: markdown
|
||||
# attributes:
|
||||
# value: |
|
||||
# Thanks for taking the time to fill out this bug report! Please fill out the form below to help us reproduce and fix the issue.
|
||||
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Browser Use Version
|
||||
description: |
|
||||
What exact version of `browser-use` are you using? (Run `uv pip show browser-use` or `git log -n 1`)
|
||||
**DO NOT WRITE `latest release` or `main` or a very old version or we will close your issue!**
|
||||
placeholder: "e.g. 0.4.45 or 62760baaefd"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Bug Description, Steps to Reproduce, Screenshots
|
||||
description: A clear and concise description of what the bug is + steps taken, drag screenshots in showing any error messages and relevant pages.
|
||||
placeholder: |
|
||||
1. Installed browser-use library by running: `uv pip install browser-use`
|
||||
2. Installed the browser by running: `playwright install chromium --with-deps`
|
||||
3. Ran the code below with the following prompt: `go to example.com and do xyz...`
|
||||
4. Agent crashed and showed the following error: ...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: code
|
||||
attributes:
|
||||
label: Failing Python Code
|
||||
description: Include the exact python code you ran that encountered the issue, redact any sensitive URLs and API keys.
|
||||
render: python
|
||||
placeholder: |
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv() # tip: always load_dotenv() before other imports
|
||||
from browser_use import Agent, BrowserSession, Tools
|
||||
from browser_use.llm import ChatOpenAI
|
||||
|
||||
agent = Agent(
|
||||
task='...',
|
||||
llm=ChatOpenAI(model="gpt-4.1-mini"),
|
||||
browser_session=BrowserSession(headless=False),
|
||||
)
|
||||
...
|
||||
|
||||
- type: input
|
||||
id: model
|
||||
attributes:
|
||||
label: LLM Model
|
||||
description: Which LLM model are you using? (Optional)
|
||||
placeholder: "e.g. ChatBrowserUse, gpt-4.1-mini, gemini-flash-latest, etc."
|
||||
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating System & Browser Versions
|
||||
description: What operating system and browser are you using? (Optional)
|
||||
placeholder: "e.g. Ubuntu 24.04 + playwright chromium v136, Windows 11 + Chrome.exe v133, macOS ..."
|
||||
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Full DEBUG Log Output
|
||||
description: Please copy and paste the log output. Make sure to set `BROWSER_USE_LOGGING_LEVEL=DEBUG` in your `.env` or shell environment.
|
||||
render: shell
|
||||
placeholder: |
|
||||
$ python /app/browser-use/examples/browser/real_browser.py
|
||||
DEBUG [browser] 🌎 Initializing new browser
|
||||
93
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/3_feature_request.yml
vendored
Normal file
93
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/3_feature_request.yml
vendored
Normal file
@@ -0,0 +1,93 @@
|
||||
name: 💡 New Feature or Enhancement Request
|
||||
description: Suggest an idea or improvement for the browser-use library or Agent capabilities
|
||||
title: "Feature Request: ..."
|
||||
type: 'Enhancement'
|
||||
labels: ['enhancement']
|
||||
body:
|
||||
- type: textarea
|
||||
id: current_problem
|
||||
attributes:
|
||||
label: "What is the problem that your feature request solves?"
|
||||
description: |
|
||||
Describe the problem or need that your feature request solves, include screenshots and example URLs if relevant.
|
||||
placeholder: |
|
||||
e.g. I need to be able to simulate dragging in a circle to test the paint feature on a drawing site: https://example.com/draw
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: proposed_solution
|
||||
attributes:
|
||||
label: "What is your proposed solution?"
|
||||
description: |
|
||||
Describe the ideal specific solution you'd want, *and whether it fits into any broader scope of changes*.
|
||||
placeholder: |
|
||||
e.g. I want to add a default action that can hover/drag the mouse on a path when given a series
|
||||
of x,y coordinates. More broadly it may be useful add a computer-use/x,y-coordinate-style automation
|
||||
method fallback that can do complex mouse movements.
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: workarounds_tried
|
||||
attributes:
|
||||
label: "What hacks or alternative solutions have you tried to solve the problem?"
|
||||
description: |
|
||||
A description of any troubleshooting, alternative approaches, workarounds, or other ideas you've considered to fix the problem.
|
||||
placeholder: |
|
||||
e.g. I tried upgrading to the latest version and telling it to hover in the prompt. I also tried
|
||||
telling the agent to ask for human help (using a custom tools action) when it gets to this
|
||||
step, then I manually click a browser extension in the navbar that automates the mouse movevement.
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: What version of browser-use are you currently using?
|
||||
description: |
|
||||
Run `pip show browser-use` or `git log -n 1` and share the exact number or git hash. DO NOT JUST ENTER `latest release` OR `main`.
|
||||
We need to know what version of the browser-use library you're running in order to contextualize your feature request.
|
||||
Sometimes features are already available and just need to be enabled with config on certain versions.
|
||||
placeholder: "e.g. 0.1.48 or 62760baaefd"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
> [!IMPORTANT]
|
||||
> 🙏 Please **go check *right now before filling this out* that that you have tried the [⬆️ LATEST VERSION](https://github.com/browser-use/browser-use/releases)**.
|
||||
> 🚀 We ship *hundreds* of improvements a day and we might've already added a solution to your need yesterday!
|
||||
> <a href="https://github.com/browser-use/browser-use/releases"><img src="https://github.com/user-attachments/assets/4cd34ee6-bafb-4f24-87e2-27a31dc5b9a4" width="500px"/></a>
|
||||
> If you are running an old version, the **first thing we will ask you to do is *try the latest `beta`***:
|
||||
> - 🆕 [`beta`](https://docs.browser-use.com/development/local-setup): `uv pip install --upgrade git+https://github.com/browser-use/browser-use.git@main`
|
||||
> - 📦 [`stable`](https://pypi.org/project/browser-use/#history): `pip install --upgrade browser-use`
|
||||
|
||||
- type: checkboxes
|
||||
id: priority
|
||||
attributes:
|
||||
label: "How badly do you want this new feature?"
|
||||
options:
|
||||
- label: "It's an urgent deal-breaker, I can't live without it"
|
||||
required: false
|
||||
- label: "It's important to add it in the near-mid term future"
|
||||
required: false
|
||||
- label: "It would be nice to add it sometime in the next 2 years"
|
||||
required: false
|
||||
- label: "💪 I'm willing to [start a PR](https://docs.browser-use.com/development/contribution-guide) to work on this myself"
|
||||
required: false
|
||||
- label: "💼 My company would spend >$5k on [Browser-Use Cloud](https://browser-use.com) if it solved this reliably for us"
|
||||
required: false
|
||||
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
---
|
||||
> [!TIP]
|
||||
> Start conversations about your feature request in other places too, the more
|
||||
> 📣 hype we see around a request the more likely we are to add it!
|
||||
>
|
||||
> - 👾 Discord: [https://link.browser-use.com/discord](https://link.browser-use.com/discord)
|
||||
> - 𝕏 Twitter: [https://x.com/browser_use](https://x.com/browser_use)
|
||||
55
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/4_docs_issue.yml
vendored
Normal file
55
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/4_docs_issue.yml
vendored
Normal file
@@ -0,0 +1,55 @@
|
||||
name: 📚 Documentation Issue
|
||||
description: Report an issue in the browser-use documentation
|
||||
labels: ["documentation"]
|
||||
title: "Documentation: ..."
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for taking the time to improve our documentation! Please fill out the form below to help us fix the issue quickly.
|
||||
|
||||
- type: dropdown
|
||||
id: type
|
||||
attributes:
|
||||
label: Type of Documentation Issue
|
||||
description: What type of documentation issue is this?
|
||||
options:
|
||||
- Missing documentation
|
||||
- Incorrect documentation
|
||||
- Unclear documentation
|
||||
- Broken link
|
||||
- Other (specify in description)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: page
|
||||
attributes:
|
||||
label: Documentation Page
|
||||
description: Which page or section of the documentation is this about?
|
||||
placeholder: "e.g. https://docs.browser-use.com/customize/browser-settings > Context Configuration > headless"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Issue Description
|
||||
description: "Describe what's wrong or missing in the documentation"
|
||||
placeholder: e.g. Docs should clarify whether BrowserSession(no_viewport=False) is supported when running in BrowserSession(headless=False) mode...
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: suggestion
|
||||
attributes:
|
||||
label: Suggested Changes
|
||||
description: If you have specific suggestions for how to improve the documentation, please share them
|
||||
placeholder: |
|
||||
e.g. The documentation could be improved by adding one more line here:
|
||||
```diff
|
||||
Use `BrowserSession(headless=False)` to open the browser window (aka headful mode).
|
||||
+ Viewports are not supported when headful, if `headless=False` it will force `no_viewport=True`.
|
||||
```
|
||||
validations:
|
||||
required: false
|
||||
11
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
11
.agent/vendor/browser_use/.github/ISSUE_TEMPLATE/config.yml
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
blank_issues_enabled: false # Set to true if you want to allow blank issues
|
||||
contact_links:
|
||||
- name: 🔢 Quickstart Guide
|
||||
url: https://docs.browser-use.com/quickstart
|
||||
about: Most common issues can be resolved by following our quickstart guide
|
||||
- name: 💬 Questions and Help
|
||||
url: https://link.browser-use.com/discord
|
||||
about: Please ask questions in our Discord community
|
||||
- name: 📖 Documentation
|
||||
url: https://docs.browser-use.com
|
||||
about: Check our documentation for answers first
|
||||
19
.agent/vendor/browser_use/.github/SECURITY.md
vendored
Normal file
19
.agent/vendor/browser_use/.github/SECURITY.md
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
## Reporting Security Issues
|
||||
|
||||
If you believe you have found a security vulnerability in browser-use, please report it through coordinated disclosure.
|
||||
|
||||
**Please do not report security vulnerabilities through the repository issues, discussions, or pull requests.**
|
||||
|
||||
Instead, please open a new [Github security advisory](https://github.com/browser-use/browser-use/security/advisories/new).
|
||||
|
||||
Please include as much of the information listed below as you can to help me better understand and resolve the issue:
|
||||
|
||||
* The type of issue (e.g., buffer overflow, SQL injection, or cross-site scripting)
|
||||
* Full paths of source file(s) related to the manifestation of the issue
|
||||
* The location of the affected source code (tag/branch/commit or direct URL)
|
||||
* Any special configuration required to reproduce the issue
|
||||
* Step-by-step instructions to reproduce the issue
|
||||
* Proof-of-concept or exploit code (if possible)
|
||||
* Impact of the issue, including how an attacker might exploit the issue
|
||||
|
||||
This information will help me triage your report more quickly.
|
||||
43
.agent/vendor/browser_use/.github/workflows/build-base-image.yml.disabled
vendored
Normal file
43
.agent/vendor/browser_use/.github/workflows/build-base-image.yml.disabled
vendored
Normal file
@@ -0,0 +1,43 @@
|
||||
name: Build Base Image
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 2 * * 1' # Weekly on Monday
|
||||
workflow_dispatch:
|
||||
push:
|
||||
paths:
|
||||
- 'Dockerfile.base'
|
||||
|
||||
jobs:
|
||||
build-base:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [linux/amd64, linux/arm64]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push base image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.base
|
||||
platforms: ${{ matrix.platform }}
|
||||
push: true
|
||||
tags: |
|
||||
browseruse/browseruse-base:chromium-138-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
browseruse/browseruse-base:latest-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
cache-from: type=registry,ref=browseruse/browseruse-base:buildcache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
|
||||
cache-to: type=registry,ref=browseruse/browseruse-base:buildcache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max
|
||||
150
.agent/vendor/browser_use/.github/workflows/claude.yml
vendored
Normal file
150
.agent/vendor/browser_use/.github/workflows/claude.yml
vendored
Normal file
@@ -0,0 +1,150 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
pull-requests: read
|
||||
id-token: write
|
||||
discussions: write
|
||||
issues: write
|
||||
env:
|
||||
IS_SANDBOX: '1'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
activate-environment: true
|
||||
|
||||
- run: uv sync --dev --all-extras
|
||||
|
||||
- name: Detect installed Playwright version
|
||||
run: echo "PLAYWRIGHT_VERSION=$(uv pip list --format json | jq -r '.[] | select(.name == "playwright") | .version')" >> $GITHUB_ENV
|
||||
|
||||
# - name: Cache chrome binaries
|
||||
# uses: actions/cache@v4
|
||||
# with:
|
||||
# path: |
|
||||
# /tmp/google-chrome-stable_current_amd64.deb
|
||||
# key: ${{ runner.os }}-${{ runner.arch }}-chrome-stable
|
||||
|
||||
# - name: Install Chrome stable binary
|
||||
# run: |
|
||||
# sudo apt-get update -qq \
|
||||
# && sudo curl -o "/tmp/google-chrome-stable_current_amd64.deb" --no-clobber "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb" \
|
||||
# && sudo apt-get install -y "/tmp/google-chrome-stable_current_amd64.deb" -f
|
||||
# - run: patchright install chrome --with-deps
|
||||
# - run: playwright install chrome --with-deps
|
||||
|
||||
- name: Cache chromium binaries
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-${{ runner.arch }}-playwright-${{ env.PLAYWRIGHT_VERSION }}-chromium
|
||||
|
||||
- run: playwright install chromium --with-deps
|
||||
# - run: patchright install chromium --with-deps
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@beta
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
model: "claude-opus-4-20250514"
|
||||
fallback_model: "claude-3-5-sonnet-20241022"
|
||||
custom_instructions: |
|
||||
when making any significant changes, start by adding one or two new failing test functions to the most relevant file you can find in tests/ci/*.py, then work on your changes until you get the tests passing.
|
||||
make sure all lint errors are fixed before committing: `uv run pre-commit --all-files`, you can also use mcp tools to check Github CI status.
|
||||
make sure to run the whole test file at the end to make sure no other tests in that file started failing due to your changes: `uv run pytest/ci/test_....py`.
|
||||
if any significant features were added or removed, or any public-facing parameters/signatures changed, make sure to look through docs/*.mdx and examples/**.py and fix any relevant areas that might need to be updated.
|
||||
branch_prefix: "claude-"
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
claude_env: |
|
||||
IN_DOCKER: 'true'
|
||||
BROWSER_USE_CLOUD_SYNC: 'false'
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
BROWSER_USE_LOGGING_LEVEL: 'DEBUG'
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
settings: |
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(git:*)",
|
||||
"Bash(uv:*)",
|
||||
"Bash(uv run pytest:*)",
|
||||
"Bash(uv run ruff:*)",
|
||||
"Bash(uv run pyright:*)",
|
||||
"Bash(uv run pre-commit:*)",
|
||||
"Bash(uv pip:*)",
|
||||
"Bash(uv add:*)",
|
||||
"Bash(uv sync --all-extras --dev)",
|
||||
"Bash(.venv/bin/*:*)",
|
||||
"Bash(.venv/bin/python:*)",
|
||||
"Bash(sed:*)",
|
||||
"Bash(rg:*)",
|
||||
"Bash(jq:*)",
|
||||
"Bash(find:*)",
|
||||
"Bash(grep:*)",
|
||||
"Bash(python:*)",
|
||||
"Bash(chmod:*)",
|
||||
"Bash(rm:*)",
|
||||
"Bash(playwright:*)",
|
||||
"Bash(uv run playwright:*)",
|
||||
"Bash(./bin/lint.sh)",
|
||||
"Bash(./bin/test.sh)",
|
||||
"WebFetch(*)",
|
||||
"WebSearch(*)"
|
||||
],
|
||||
"additionalDirectories": ["/home/runner/work"]
|
||||
}
|
||||
}
|
||||
allowed_tools: |
|
||||
Bash(git:*)
|
||||
Bash(uv:*)
|
||||
Bash(uv run pytest:*)
|
||||
Bash(uv run ruff:*)
|
||||
Bash(uv run pyright:*)
|
||||
Bash(uv run pre-commit:*)
|
||||
Bash(uv pip:*)
|
||||
Bash(uv add:*)
|
||||
Bash(uv sync --all-extras --dev)
|
||||
Bash(.venv/bin/*:*)
|
||||
Bash(.venv/bin/python:*)
|
||||
Bash(sed:*)
|
||||
Bash(rg:*)
|
||||
Bash(jq:*)
|
||||
Bash(find:*)
|
||||
Bash(grep:*)
|
||||
Bash(python:*)
|
||||
Bash(chmod:*)
|
||||
Bash(rm:*)
|
||||
Bash(playwright:*)
|
||||
Bash(uv run playwright:*)
|
||||
Bash(./bin/lint.sh)
|
||||
Bash(./bin/test.sh)
|
||||
WebFetch(*)
|
||||
WebSearch(*)
|
||||
35
.agent/vendor/browser_use/.github/workflows/cloud_evals.yml
vendored
Normal file
35
.agent/vendor/browser_use/.github/workflows/cloud_evals.yml
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
name: cloud_evals
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- 'releases/*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
commit_hash:
|
||||
description: Commit hash of the library to build the Cloud eval image for
|
||||
required: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
trigger_cloud_eval_image_build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.TRIGGER_CLOUD_BUILD_GH_KEY }}
|
||||
script: |
|
||||
const result = await github.rest.repos.createDispatchEvent({
|
||||
owner: 'browser-use',
|
||||
repo: 'cloud',
|
||||
event_type: 'trigger-workflow',
|
||||
client_payload: {"commit_hash": "${{ github.event.inputs.commit_hash || github.sha }}"}
|
||||
})
|
||||
console.log(result)
|
||||
76
.agent/vendor/browser_use/.github/workflows/docker.yml
vendored
Normal file
76
.agent/vendor/browser_use/.github/workflows/docker.yml
vendored
Normal file
@@ -0,0 +1,76 @@
|
||||
name: docker
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
- 'releases/**'
|
||||
tags:
|
||||
- '*'
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build_publish_image:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
attestations: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Compute Docker tags based on tag/branch
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
browseruse/browseruse
|
||||
ghcr.io/browser-use/browser-use
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=pep440,pattern={{version}}
|
||||
type=pep440,pattern={{major}}.{{minor}}
|
||||
type=sha
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=registry,ref=browseruse/browseruse:buildcache
|
||||
cache-to: type=registry,ref=browseruse/browseruse:buildcache,mode=max
|
||||
56
.agent/vendor/browser_use/.github/workflows/eval-on-pr.yml
vendored
Normal file
56
.agent/vendor/browser_use/.github/workflows/eval-on-pr.yml
vendored
Normal file
@@ -0,0 +1,56 @@
|
||||
name: Evaluate PR
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
jobs:
|
||||
trigger-evaluation:
|
||||
runs-on: ubuntu-latest
|
||||
# Only run if PR author has write access
|
||||
if: |
|
||||
github.event.pull_request.author_association == 'OWNER' ||
|
||||
github.event.pull_request.author_association == 'MEMBER' ||
|
||||
github.event.pull_request.author_association == 'COLLABORATOR'
|
||||
|
||||
steps:
|
||||
- name: Trigger Evaluation settings
|
||||
id: trigger
|
||||
continue-on-error: true
|
||||
run: |
|
||||
echo "🚀 Triggering evaluation - PR #${{ github.event.pull_request.number }}"
|
||||
echo "Commit: ${{ github.event.pull_request.head.sha }}"
|
||||
|
||||
# You can customize the test here
|
||||
TEST_CASE="${{ vars.EVAL_TEST_CASE }}"
|
||||
if [ -z "$TEST_CASE" ]; then
|
||||
TEST_CASE="InteractionTasks_v8"
|
||||
fi
|
||||
|
||||
response=$(curl -X POST \
|
||||
"${{ secrets.EVAL_PLATFORM_URL }}/api/triggerInteractionTasksV6" \
|
||||
-H "Authorization: Bearer ${{ secrets.EVAL_PLATFORM_KEY }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"commitSha\": \"${{ github.event.pull_request.head.sha }}\",
|
||||
\"prNumber\": ${{ github.event.pull_request.number }},
|
||||
\"branchName\": \"${{ github.event.pull_request.head.ref }}\",
|
||||
\"testCase\": \"${TEST_CASE}\",
|
||||
\"githubRepo\": \"${{ github.repository }}\"
|
||||
}" -s)
|
||||
|
||||
echo "Response: $response"
|
||||
|
||||
# Check if trigger was was successful
|
||||
if echo "$response" | jq -e '.success == true' > /dev/null; then
|
||||
echo "✅ Evaluation triggered successfully"
|
||||
exit 0
|
||||
else
|
||||
echo "Failed"
|
||||
echo "$response"
|
||||
exit 1
|
||||
fi
|
||||
265
.agent/vendor/browser_use/.github/workflows/install-script.yml
vendored
Normal file
265
.agent/vendor/browser_use/.github/workflows/install-script.yml
vendored
Normal file
@@ -0,0 +1,265 @@
|
||||
name: Test Install Script
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'browser_use/skill_cli/install.sh'
|
||||
- '.github/workflows/install-script.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'browser_use/skill_cli/install.sh'
|
||||
- '.github/workflows/install-script.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# Use current branch for testing install.sh
|
||||
# For PRs, use the fork's repo (head.repo), otherwise use the base repo
|
||||
BROWSER_USE_BRANCH: ${{ github.head_ref || github.ref_name }}
|
||||
BROWSER_USE_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
|
||||
|
||||
jobs:
|
||||
# ===========================================================================
|
||||
# Test install.sh on all platforms
|
||||
# ===========================================================================
|
||||
|
||||
test-install-sh-linux:
|
||||
name: install.sh (Linux ${{ matrix.os }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, ubuntu-22.04]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Run install.sh
|
||||
run: bash browser_use/skill_cli/install.sh
|
||||
|
||||
- name: Add to PATH
|
||||
run: |
|
||||
echo "$HOME/.browser-use-env/bin" >> $GITHUB_PATH
|
||||
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Verify browser-use CLI
|
||||
run: |
|
||||
source ~/.browser-use-env/bin/activate
|
||||
browser-use --help
|
||||
|
||||
- name: Verify Chromium installed
|
||||
run: |
|
||||
source ~/.browser-use-env/bin/activate
|
||||
# Verify chromium binary exists in playwright cache
|
||||
ls ~/.cache/ms-playwright/chromium-*/chrome-linux/chrome 2>/dev/null || \
|
||||
ls ~/.cache/ms-playwright/chromium-*/chrome-linux/chromium 2>/dev/null || \
|
||||
echo "Chromium binary check completed"
|
||||
|
||||
- name: Run browser-use doctor
|
||||
run: |
|
||||
source ~/.browser-use-env/bin/activate
|
||||
browser-use doctor
|
||||
|
||||
test-install-sh-macos:
|
||||
name: install.sh (macOS ${{ matrix.os }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [macos-latest, macos-14]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Run install.sh
|
||||
run: bash browser_use/skill_cli/install.sh
|
||||
|
||||
- name: Add to PATH
|
||||
run: |
|
||||
echo "$HOME/.browser-use-env/bin" >> $GITHUB_PATH
|
||||
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Verify browser-use CLI
|
||||
run: |
|
||||
source ~/.browser-use-env/bin/activate
|
||||
browser-use --help
|
||||
|
||||
- name: Verify Chromium installed
|
||||
run: |
|
||||
source ~/.browser-use-env/bin/activate
|
||||
# Check playwright cache for chromium
|
||||
ls ~/Library/Caches/ms-playwright/chromium-*/chrome-mac/ 2>/dev/null || \
|
||||
ls ~/Library/Caches/ms-playwright/chromium-*/Chromium.app 2>/dev/null || \
|
||||
echo "Chromium binary check completed"
|
||||
|
||||
- name: Run browser-use doctor
|
||||
run: |
|
||||
source ~/.browser-use-env/bin/activate
|
||||
browser-use doctor
|
||||
|
||||
test-install-sh-windows:
|
||||
name: install.sh (Windows)
|
||||
runs-on: windows-latest
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
env:
|
||||
# Fix Unicode output on Windows (checkmarks, etc.)
|
||||
PYTHONIOENCODING: utf-8
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python (Windows requires manual setup)
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Run install.sh
|
||||
run: bash browser_use/skill_cli/install.sh
|
||||
|
||||
- name: Add to PATH
|
||||
run: |
|
||||
echo "$HOME/.browser-use-env/Scripts" >> $GITHUB_PATH
|
||||
echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Verify browser-use CLI
|
||||
run: |
|
||||
source ~/.browser-use-env/Scripts/activate
|
||||
browser-use --help
|
||||
|
||||
- name: Run browser-use doctor
|
||||
run: |
|
||||
source ~/.browser-use-env/Scripts/activate
|
||||
browser-use doctor
|
||||
|
||||
# ===========================================================================
|
||||
# Test alternative install methods: uv pip install + browser-use install
|
||||
# ===========================================================================
|
||||
|
||||
test-uv-pip-install:
|
||||
name: uv pip install (Linux)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
- name: Add uv to PATH
|
||||
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Create venv and install browser-use
|
||||
run: |
|
||||
uv venv .venv --python 3.11
|
||||
source .venv/bin/activate
|
||||
# Install from current branch
|
||||
uv pip install .
|
||||
|
||||
- name: Run browser-use install (installs Chromium)
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
browser-use install
|
||||
|
||||
- name: Verify browser-use CLI
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
browser-use --help
|
||||
|
||||
- name: Verify Chromium installed
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
ls ~/.cache/ms-playwright/chromium-*/chrome-linux/chrome 2>/dev/null || \
|
||||
ls ~/.cache/ms-playwright/chromium-*/chrome-linux/chromium 2>/dev/null || \
|
||||
echo "Chromium check completed"
|
||||
|
||||
- name: Run browser-use doctor
|
||||
run: |
|
||||
source .venv/bin/activate
|
||||
browser-use doctor
|
||||
|
||||
# ===========================================================================
|
||||
# Test uvx "browser-use[cli]" - ephemeral install
|
||||
# ===========================================================================
|
||||
|
||||
test-uvx-run:
|
||||
name: uvx browser-use[cli] (Linux)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
- name: Add uv to PATH
|
||||
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Build wheel from current branch
|
||||
run: |
|
||||
uv venv .venv --python 3.11
|
||||
source .venv/bin/activate
|
||||
uv pip install build
|
||||
python -m build --wheel
|
||||
|
||||
- name: Test uvx with local wheel
|
||||
run: |
|
||||
WHEEL=$(ls dist/*.whl)
|
||||
uvx --from "$WHEEL" browser-use --help
|
||||
|
||||
- name: Test uvx browser-use install
|
||||
run: |
|
||||
WHEEL=$(ls dist/*.whl)
|
||||
uvx --from "$WHEEL" browser-use install
|
||||
|
||||
- name: Verify Chromium installed after uvx install
|
||||
run: |
|
||||
ls ~/.cache/ms-playwright/chromium-*/chrome-linux/chrome 2>/dev/null || \
|
||||
ls ~/.cache/ms-playwright/chromium-*/chrome-linux/chromium 2>/dev/null || \
|
||||
echo "Chromium check completed"
|
||||
|
||||
- name: Test uvx browser-use doctor
|
||||
run: |
|
||||
WHEEL=$(ls dist/*.whl)
|
||||
uvx --from "$WHEEL" browser-use doctor
|
||||
|
||||
# ===========================================================================
|
||||
# Test uvx from PyPI (only on main branch after release)
|
||||
# ===========================================================================
|
||||
|
||||
test-uvx-pypi:
|
||||
name: uvx browser-use[cli] from PyPI
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on main branch or manual trigger
|
||||
if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Install uv
|
||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
- name: Add uv to PATH
|
||||
run: echo "$HOME/.local/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Test uvx browser-use --help
|
||||
run: uvx "browser-use[cli]" --help
|
||||
|
||||
- name: Test uvx browser-use install
|
||||
run: uvx "browser-use[cli]" install
|
||||
|
||||
- name: Verify Chromium installed
|
||||
run: |
|
||||
ls ~/.cache/ms-playwright/chromium-*/chrome-linux/chrome 2>/dev/null || \
|
||||
ls ~/.cache/ms-playwright/chromium-*/chrome-linux/chromium 2>/dev/null || \
|
||||
echo "Chromium check completed"
|
||||
|
||||
- name: Test uvx browser-use doctor
|
||||
run: uvx "browser-use[cli]" doctor
|
||||
54
.agent/vendor/browser_use/.github/workflows/lint.yml
vendored
Normal file
54
.agent/vendor/browser_use/.github/workflows/lint.yml
vendored
Normal file
@@ -0,0 +1,54 @@
|
||||
name: lint
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
- 'releases/**'
|
||||
tags:
|
||||
- '*'
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint-syntax:
|
||||
name: syntax-errors
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
- run: uv run ruff check --no-fix --select PLE
|
||||
|
||||
lint-style:
|
||||
name: code-style
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
with:
|
||||
enable-cache: true
|
||||
- run: uv python install 3.11
|
||||
- run: uv sync --dev --all-extras --python 3.11
|
||||
- run: uv run --no-sync pre-commit run --all-files --show-diff-on-failure
|
||||
|
||||
lint-typecheck:
|
||||
name: type-checker
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
- run: uv sync --dev --all-extras # install extras for examples to avoid pyright missing imports errors-
|
||||
- run: uv run --no-sync pyright
|
||||
64
.agent/vendor/browser_use/.github/workflows/package.yaml
vendored
Normal file
64
.agent/vendor/browser_use/.github/workflows/package.yaml
vendored
Normal file
@@ -0,0 +1,64 @@
|
||||
name: package
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
- 'releases/**'
|
||||
tags:
|
||||
- '*'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: pip-build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
- run: uv build --python 3.12
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dist-artifact
|
||||
path: |
|
||||
dist/*.whl
|
||||
dist/*.tar.gz
|
||||
|
||||
build_test:
|
||||
name: pip-install-on-${{ matrix.os }}-py-${{ matrix.python-version }}
|
||||
needs: build
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macos-latest, windows-latest]
|
||||
python-version: ["3.11", "3.13"]
|
||||
env:
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dist-artifact
|
||||
|
||||
- name: Set up venv and test for OS/Python versions
|
||||
shell: bash
|
||||
run: |
|
||||
uv venv /tmp/testenv --python ${{ matrix.python-version }} --clear
|
||||
if [[ "$RUNNER_OS" == "Windows" ]]; then
|
||||
. /tmp/testenv/Scripts/activate
|
||||
else
|
||||
source /tmp/testenv/bin/activate
|
||||
fi
|
||||
uv pip install *.whl
|
||||
python -c 'from browser_use import Agent, BrowserProfile, BrowserSession, Tools, ActionModel, ActionResult'
|
||||
109
.agent/vendor/browser_use/.github/workflows/publish.yml
vendored
Normal file
109
.agent/vendor/browser_use/.github/workflows/publish.yml
vendored
Normal file
@@ -0,0 +1,109 @@
|
||||
# This workflow will upload a Python Package using Twine when a release is created
|
||||
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
|
||||
|
||||
# This workflow uses actions that are not certified by GitHub.
|
||||
# They are provided by a third-party and are governed by
|
||||
# separate terms of service, privacy policy, and support
|
||||
# documentation.
|
||||
|
||||
name: publish
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published] # publish full release to PyPI when a release is created on Github
|
||||
# schedule:
|
||||
# - cron: "0 17 * * FRI" # tag a pre-release on Github every Friday at 5 PM UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
tag_pre_release:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Create pre-release tag
|
||||
run: |
|
||||
git fetch --tags
|
||||
latest_tag=$(git tag --list --sort=-v:refname | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(rc[0-9]+)?$' | head -n 1)
|
||||
if [ -z "$latest_tag" ]; then
|
||||
echo "Failed to find the latest git tag from list:" > /dev/stderr
|
||||
git tag --list --sort=-v:refname
|
||||
exit 1
|
||||
else
|
||||
# Bump the tag rc version
|
||||
if [[ "$latest_tag" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(rc([0-9]+))?$ ]]; then
|
||||
major="${BASH_REMATCH[1]}"
|
||||
minor="${BASH_REMATCH[2]}"
|
||||
patch="${BASH_REMATCH[3]}"
|
||||
rc="${BASH_REMATCH[5]}"
|
||||
echo "latest_tag: ${major}.${minor}.${patch}rc${rc:-0}"
|
||||
if [ -z "$rc" ]; then
|
||||
# No rc, so bump patch and set rc=1 # 0.2.1 -> 0.2.2rc1
|
||||
patch=$((patch + 1))
|
||||
new_tag="${major}.${minor}.${patch}rc1"
|
||||
else
|
||||
if [ "$rc" -ge 99 ]; then
|
||||
echo "Error: rc version is already at 99 for tag $latest_tag, refusing to increment further." > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
rc=$((rc + 1))
|
||||
new_tag="${major}.${minor}.${patch}rc${rc}" # 0.2.1rc1 -> 0.2.1rc2
|
||||
fi
|
||||
else
|
||||
echo "Error: latest_tag '$latest_tag' does not match expected version pattern." > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
echo "new_tag: $new_tag"
|
||||
git tag $new_tag
|
||||
git push origin $new_tag
|
||||
|
||||
publish_to_pypi:
|
||||
if: github.event_name == 'release' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
IN_DOCKER: 'True'
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
activate-environment: true
|
||||
- run: uv sync
|
||||
|
||||
- run: uv run --no-sync ruff check --no-fix --select PLE # quick check for syntax errors to avoid waiting time doing the rest of the build
|
||||
- run: uv build
|
||||
|
||||
# - name: Detect installed Playwright version
|
||||
# run: echo "PLAYWRIGHT_VERSION=$(uv pip list --format json | jq -r '.[] | select(.name == "playwright") | .version')" >> $GITHUB_ENV
|
||||
|
||||
# - name: Cache playwright binaries
|
||||
# uses: actions/cache@v3
|
||||
# with:
|
||||
# path: |
|
||||
# ~/.cache/ms-playwright
|
||||
# key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
|
||||
|
||||
- run: uvx playwright install chrome
|
||||
- run: uvx playwright install chromium
|
||||
|
||||
# TODO: just depend on the other test.yml action for this instead of re-running the tests here
|
||||
# - run: uv run pytest tests/ci/test_tools.py # final sanity check: run a few of the tests before release
|
||||
|
||||
# publish to PyPI
|
||||
- run: uv publish --trusted-publishing always
|
||||
- name: Push to stable branch (if stable release)
|
||||
if: github.event_name == 'release' && !contains(github.ref_name, 'rc')
|
||||
run: |
|
||||
git checkout -b stable
|
||||
git push origin -f stable
|
||||
108
.agent/vendor/browser_use/.github/workflows/stale-bot.yml
vendored
Normal file
108
.agent/vendor/browser_use/.github/workflows/stale-bot.yml
vendored
Normal file
@@ -0,0 +1,108 @@
|
||||
name: 'Manage stale issues and PRs'
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 2 * * *' # Run daily at 2:00 AM UTC
|
||||
workflow_dispatch: # Allow manual triggering
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/stale@v9
|
||||
with:
|
||||
# General settings
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Days before marking as stale (more lenient for AI/browser automation project)
|
||||
days-before-stale: 60
|
||||
days-before-close: 14
|
||||
|
||||
# Different timing for PRs vs issues
|
||||
days-before-pr-stale: 45
|
||||
days-before-pr-close: 14
|
||||
|
||||
# Stale labels
|
||||
stale-issue-label: 'stale'
|
||||
stale-pr-label: 'stale'
|
||||
|
||||
# Remove stale label when there's activity
|
||||
remove-stale-when-updated: true
|
||||
remove-issue-stale-when-updated: true
|
||||
remove-pr-stale-when-updated: true
|
||||
|
||||
# Messages
|
||||
stale-issue-message: |
|
||||
👋 This issue has been automatically marked as stale because it hasn't had activity for 60 days.
|
||||
|
||||
**⚡ We've made significant progress recently!** Please test with the latest version of browser-use to see if this issue has been resolved. If the issue persists, please let us know by commenting below.
|
||||
|
||||
**To keep this issue open:**
|
||||
- Add a comment explaining why this is still relevant after testing the latest version
|
||||
- Add the `pinned` label if this is an important long-term issue
|
||||
- Reference it in a PR if you're working on a fix
|
||||
|
||||
**This will be automatically closed in 14 days** if no further activity occurs.
|
||||
|
||||
Thanks for contributing to browser-use! 🤖 If you have questions, join our [Discord](https://discord.gg/uC9hDSbt).
|
||||
|
||||
stale-pr-message: |
|
||||
👋 This PR has been automatically marked as stale because it hasn't had activity for 45 days.
|
||||
|
||||
**To keep this PR open:**
|
||||
- Rebase against the latest main branch
|
||||
- Address any review feedback or merge conflicts
|
||||
- Add a comment explaining the current status
|
||||
- Add the `work-in-progress` label if you're still actively working on this
|
||||
|
||||
**This will be automatically closed in 14 days** if no further activity occurs.
|
||||
|
||||
Thanks for contributing to browser-use! 🤖
|
||||
|
||||
close-issue-message: |
|
||||
🔒 This issue was automatically closed because it was stale for 14 days with no activity.
|
||||
|
||||
**Don't worry!** If this issue is still relevant:
|
||||
- **First, test with the latest version** - we've made tons of improvements recently!
|
||||
- **Reopen it** if you have permissions and the issue persists
|
||||
- **Create a fresh issue** with updated information if the problem still exists after testing the latest version
|
||||
- **Join our [Discord](https://discord.gg/uC9hDSbt)** to discuss
|
||||
|
||||
We appreciate your contribution to browser-use! 🤖
|
||||
|
||||
close-pr-message: |
|
||||
🔒 This PR was automatically closed because it was stale for 14 days with no activity.
|
||||
|
||||
**Don't worry!** If you'd like to continue this work:
|
||||
- **Reopen this PR** and rebase against main
|
||||
- **Create a fresh PR** with updated changes
|
||||
- **Join our [Discord](https://discord.gg/uC9hDSbt)** if you need help
|
||||
|
||||
Thanks for contributing to browser-use! 🤖
|
||||
|
||||
# Comprehensive exemptions for AI/browser automation project
|
||||
exempt-issue-labels: 'pinned,security,bug,enhancement,good-first-issue,help-wanted,documentation,ci,breaking-change,feature-request,roadmap'
|
||||
exempt-pr-labels: 'pinned,work-in-progress,wip,breaking-change,security,dependencies,ci'
|
||||
exempt-milestones: true
|
||||
exempt-all-assignees: true
|
||||
exempt-all-pr-assignees: true
|
||||
|
||||
# Don't mark issues/PRs stale if they have recent PR references
|
||||
exempt-pr-author: true
|
||||
|
||||
# Advanced settings
|
||||
operations-per-run: 200 # More conservative to avoid rate limits
|
||||
ascending: true # Process oldest issues first
|
||||
|
||||
# Enable debug output
|
||||
debug-only: false
|
||||
|
||||
# Only process issues/PRs, not drafts
|
||||
include-only-assigned: false
|
||||
|
||||
# Additional safety: don't close issues with many reactions (community interest)
|
||||
ignore-issue-updates: false
|
||||
ignore-pr-updates: false
|
||||
337
.agent/vendor/browser_use/.github/workflows/test.yaml
vendored
Normal file
337
.agent/vendor/browser_use/.github/workflows/test.yaml
vendored
Normal file
@@ -0,0 +1,337 @@
|
||||
name: test
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
pull-requests: write # Allow writing comments on PRs
|
||||
issues: write # Allow writing comments on issues
|
||||
statuses: write # Allow writing statuses on PRs
|
||||
discussions: write
|
||||
|
||||
# Cancel in-progress runs when a new commit is pushed to the same branch/PR
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- stable
|
||||
- 'releases/**'
|
||||
tags:
|
||||
- '*'
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
setup-chromium:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
|
||||
- name: Get week number for cache key
|
||||
id: week
|
||||
run: echo "number=$(date +%Y-W%U)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache chromium binaries
|
||||
id: cache-chromium
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-${{ runner.arch }}-chromium-${{ steps.week.outputs.number }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ runner.arch }}-chromium-
|
||||
|
||||
- name: Install Chromium if not cached
|
||||
if: steps.cache-chromium.outputs.cache-hit != 'true'
|
||||
run: uvx playwright install chromium --with-deps --no-shell
|
||||
|
||||
find_tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5 # Prevent hanging
|
||||
outputs:
|
||||
TEST_FILENAMES: ${{ steps.lsgrep.outputs.TEST_FILENAMES }}
|
||||
# ["test_browser", "test_tools", "test_browser_session", "test_tab_management", ...]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Force fresh checkout to avoid any caching issues
|
||||
fetch-depth: 1
|
||||
- id: lsgrep
|
||||
run: |
|
||||
echo "🔍 Discovering test files at $(date)"
|
||||
echo "Git commit: $(git rev-parse HEAD)"
|
||||
echo "Git branch: $(git branch --show-current)"
|
||||
echo ""
|
||||
|
||||
TEST_FILENAMES="$(find tests/ci -name 'test_*.py' -type f | sed 's|^tests/ci/||' | sed 's|\.py$||' | jq -R -s -c 'split("\n")[:-1]')"
|
||||
echo "TEST_FILENAMES=${TEST_FILENAMES}" >> "$GITHUB_OUTPUT"
|
||||
echo "📋 Test matrix: $TEST_FILENAMES"
|
||||
# https://code.dblock.org/2021/09/03/generating-task-matrix-by-looping-over-repo-files-with-github-actions.html
|
||||
- name: Check that at least one test file is found
|
||||
run: |
|
||||
if [ -z "${{ steps.lsgrep.outputs.TEST_FILENAMES }}" ]; then
|
||||
echo "Failed to find any test_*.py files in tests/ci/ folder!" > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tests:
|
||||
needs: [setup-chromium, find_tests]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 4 # Reduced timeout - tests should complete quickly or retry
|
||||
env:
|
||||
IN_DOCKER: 'True'
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
BROWSER_USE_LOGGING_LEVEL: 'DEBUG'
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }}
|
||||
AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
|
||||
BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }}
|
||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||
strategy:
|
||||
matrix:
|
||||
test_filename: ${{ fromJson(needs.find_tests.outputs.TEST_FILENAMES || '["FAILED_TO_DISCOVER_TESTS"]') }}
|
||||
# autodiscovers all the files in tests/ci/test_*.py
|
||||
# - test_browser
|
||||
# - test_tools
|
||||
# - test_browser_session
|
||||
# - test_tab_management
|
||||
# ... and more
|
||||
name: ${{ matrix.test_filename }}
|
||||
steps:
|
||||
- name: Check that the previous step managed to find some test files for us to run
|
||||
run: |
|
||||
if [[ "${{ matrix.test_filename }}" == "FAILED_TO_DISCOVER_TESTS" ]]; then
|
||||
echo "Failed get list of test files in tests/ci/test_*.py from find_tests job" > /dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
activate-environment: true
|
||||
|
||||
- name: Cache uv packages and venv
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-venv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-venv-
|
||||
|
||||
- run: uv sync --dev --all-extras
|
||||
|
||||
- name: Get week number for cache key
|
||||
id: week
|
||||
run: echo "number=$(date +%Y-W%U)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache chromium binaries
|
||||
id: cache-chromium
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-${{ runner.arch }}-chromium-${{ steps.week.outputs.number }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ runner.arch }}-chromium-
|
||||
|
||||
- name: Install Chromium browser if not cached
|
||||
if: steps.cache-chromium.outputs.cache-hit != 'true'
|
||||
run: uvx playwright install chromium --with-deps --no-shell
|
||||
|
||||
- name: Cache browser-use extensions
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.config/browseruse/extensions
|
||||
key: ${{ runner.os }}-browseruse-extensions-${{ hashFiles('browser_use/browser/profile.py') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-browseruse-extensions-
|
||||
|
||||
- name: Check if test file exists
|
||||
id: check-file
|
||||
run: |
|
||||
TEST_FILE="tests/ci/${{ matrix.test_filename }}.py"
|
||||
if [ -f "$TEST_FILE" ]; then
|
||||
echo "exists=true" >> $GITHUB_OUTPUT
|
||||
echo "✅ Test file found: $TEST_FILE"
|
||||
else
|
||||
echo "exists=false" >> $GITHUB_OUTPUT
|
||||
echo "❌ Test file not found: $TEST_FILE"
|
||||
echo "This file may have been renamed or removed. Current test files:"
|
||||
find tests/ci -name 'test_*.py' -type f | sed 's|tests/ci/||' | sed 's|\.py$||' | sort
|
||||
fi
|
||||
|
||||
- name: Run test with retry
|
||||
if: steps.check-file.outputs.exists == 'true'
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 4
|
||||
max_attempts: 1
|
||||
retry_on: error
|
||||
command: pytest "tests/ci/${{ matrix.test_filename }}.py"
|
||||
|
||||
evaluate-tasks:
|
||||
needs: setup-chromium
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 8 # Allow more time for agent eval
|
||||
env:
|
||||
IN_DOCKER: 'true'
|
||||
BROWSER_USE_CLOUD_SYNC: 'false'
|
||||
ANONYMIZED_TELEMETRY: 'false'
|
||||
BROWSER_USE_LOGGING_LEVEL: 'DEBUG'
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
activate-environment: true
|
||||
|
||||
- name: Cache uv packages and venv
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/uv
|
||||
.venv
|
||||
key: ${{ runner.os }}-uv-venv-${{ hashFiles('pyproject.toml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-uv-venv-
|
||||
|
||||
- run: uv sync --dev --all-extras
|
||||
|
||||
- name: Get week number for cache key
|
||||
id: week
|
||||
run: echo "number=$(date +%Y-W%U)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache chromium binaries
|
||||
id: cache-chromium
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cache/ms-playwright
|
||||
key: ${{ runner.os }}-${{ runner.arch }}-chromium-${{ steps.week.outputs.number }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-${{ runner.arch }}-chromium-
|
||||
|
||||
- name: Install Chromium browser if not cached
|
||||
if: steps.cache-chromium.outputs.cache-hit != 'true'
|
||||
run: uvx playwright install chromium --with-deps --no-shell
|
||||
|
||||
- name: Cache browser-use extensions
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.config/browseruse/extensions
|
||||
key: ${{ runner.os }}-browseruse-extensions-${{ hashFiles('browser_use/browser/profile.py') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-browseruse-extensions-
|
||||
|
||||
- name: Run agent tasks evaluation and capture score
|
||||
id: eval
|
||||
uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 4
|
||||
max_attempts: 1
|
||||
retry_on: error
|
||||
command: |
|
||||
python tests/ci/evaluate_tasks.py > result.txt
|
||||
cat result.txt
|
||||
echo "PASSED=$(grep '^PASSED=' result.txt | cut -d= -f2)" >> $GITHUB_ENV
|
||||
echo "TOTAL=$(grep '^TOTAL=' result.txt | cut -d= -f2)" >> $GITHUB_ENV
|
||||
echo "DETAILED_RESULTS=$(grep '^DETAILED_RESULTS=' result.txt | cut -d= -f2-)" >> $GITHUB_ENV
|
||||
|
||||
- name: Print agent evaluation summary
|
||||
run: |
|
||||
echo "Agent tasks passed: $PASSED / $TOTAL"
|
||||
|
||||
- name: Write agent evaluation summary to workflow overview
|
||||
run: |
|
||||
if [ "$PASSED" = "$TOTAL" ]; then
|
||||
COLOR="green"
|
||||
else
|
||||
COLOR="yellow"
|
||||
fi
|
||||
echo "<h2>Agent Tasks Score: <span style='color:$COLOR;'>$PASSED/$TOTAL</span></h2>" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Comment PR with agent evaluation results
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/github-script@v7
|
||||
continue-on-error: true
|
||||
with:
|
||||
script: |
|
||||
const passed = parseInt(process.env.PASSED);
|
||||
const total = parseInt(process.env.TOTAL);
|
||||
const detailedResults = JSON.parse(process.env.DETAILED_RESULTS);
|
||||
const score = `${passed}/${total}`;
|
||||
const percentage = Math.round((passed / total) * 100);
|
||||
|
||||
// Fail the workflow if 0% pass rate
|
||||
if (percentage === 0) {
|
||||
core.setFailed(`Evaluation failed: 0% pass rate (${passed}/${total})`);
|
||||
}
|
||||
|
||||
// Create detailed table
|
||||
let tableRows = '';
|
||||
detailedResults.forEach(result => {
|
||||
const emoji = result.success ? '✅' : '❌';
|
||||
const status = result.success ? 'Pass' : 'Fail';
|
||||
tableRows += `| ${result.task} | ${emoji} ${status} | ${result.reason} |\n`;
|
||||
});
|
||||
|
||||
const comment = `## Agent Task Evaluation Results: ${score} (${percentage}%)
|
||||
|
||||
<details>
|
||||
<summary>View detailed results</summary>
|
||||
|
||||
| Task | Result | Reason |
|
||||
|------|--------|--------|
|
||||
${tableRows}
|
||||
|
||||
Check the [evaluate-tasks job](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for detailed task execution logs.
|
||||
</details>`;
|
||||
|
||||
// Find existing comment to update or create new one
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
});
|
||||
|
||||
const botComment = comments.find(comment =>
|
||||
comment.user.type === 'Bot' &&
|
||||
comment.body.includes('Agent Task Evaluation Results')
|
||||
);
|
||||
|
||||
if (botComment) {
|
||||
// Update existing comment
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: botComment.id,
|
||||
body: comment
|
||||
});
|
||||
} else {
|
||||
// Create new comment
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body: comment
|
||||
});
|
||||
}
|
||||
86
.agent/vendor/browser_use/.gitignore
vendored
Normal file
86
.agent/vendor/browser_use/.gitignore
vendored
Normal file
@@ -0,0 +1,86 @@
|
||||
# Cache files
|
||||
.DS_Store
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
.ipynb_checkpoints
|
||||
~/
|
||||
|
||||
# Virtual Environments
|
||||
.venv*
|
||||
venv/
|
||||
|
||||
# IDEs
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# Build files
|
||||
dist/
|
||||
|
||||
# Data files
|
||||
*.gif
|
||||
*.txt
|
||||
*.pdf
|
||||
*.csv
|
||||
*.json
|
||||
*.jsonl
|
||||
*.log
|
||||
*.bak
|
||||
|
||||
# Secrets and sensitive files
|
||||
secrets.env
|
||||
.env
|
||||
browser_cookies.json
|
||||
cookies.json
|
||||
gcp-login.json
|
||||
saved_trajectories/
|
||||
old_tests/
|
||||
AgentHistory.json
|
||||
AgentHistoryList.json
|
||||
private_example.py
|
||||
private_example
|
||||
CLAUDE.local.md
|
||||
|
||||
uv.lock
|
||||
temp
|
||||
tmp
|
||||
|
||||
# Google API credentials
|
||||
credentials.json
|
||||
token.json
|
||||
|
||||
!docs/docs.json
|
||||
|
||||
|
||||
temp-profile-*
|
||||
|
||||
screenshot.png
|
||||
|
||||
# *.md
|
||||
|
||||
all_github_issues_progress.md
|
||||
all_github_issues.md
|
||||
|
||||
todo-input-token.md
|
||||
|
||||
TOOL_CHANGES_SUMMARY.md
|
||||
|
||||
|
||||
claude-code-todo
|
||||
result_judge.md
|
||||
result.md
|
||||
result2.md
|
||||
result3.md
|
||||
Brainstorm.md
|
||||
example.ipynb
|
||||
*SUMMARY.md
|
||||
todo.md
|
||||
product_extraction.ipynb
|
||||
product_extraction.py
|
||||
*report.md
|
||||
plot.py
|
||||
|
||||
.claude/
|
||||
67
.agent/vendor/browser_use/.pre-commit-config.yaml
vendored
Normal file
67
.agent/vendor/browser_use/.pre-commit-config.yaml
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
default_language_version:
|
||||
python: python3.11
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/asottile/yesqa
|
||||
rev: v1.5.0
|
||||
hooks:
|
||||
- id: yesqa
|
||||
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.1
|
||||
hooks:
|
||||
- id: codespell # See pyproject.toml for args
|
||||
additional_dependencies:
|
||||
- tomli
|
||||
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v3.20.0
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
args: [--py311-plus]
|
||||
|
||||
# - repo: https://github.com/asottile/add-trailing-comma
|
||||
# rev: v3.1.0
|
||||
# hooks:
|
||||
# - id: add-trailing-comma
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.12.10
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [ --fix ]
|
||||
- id: ruff-format
|
||||
# see pyproject.toml for more details on ruff config
|
||||
|
||||
- repo: https://github.com/RobertCraigie/pyright-python
|
||||
rev: v1.1.404
|
||||
hooks:
|
||||
- id: pyright
|
||||
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v6.0.0
|
||||
hooks:
|
||||
# check for basic syntax errors in python and data files
|
||||
- id: check-ast
|
||||
- id: check-toml
|
||||
- id: check-yaml
|
||||
- id: check-json
|
||||
- id: check-merge-conflict
|
||||
# check for bad files and folders
|
||||
- id: check-symlinks
|
||||
- id: destroyed-symlinks
|
||||
- id: check-case-conflict
|
||||
- id: check-illegal-windows-names
|
||||
- id: check-shebang-scripts-are-executable
|
||||
- id: mixed-line-ending
|
||||
- id: fix-byte-order-marker
|
||||
- id: end-of-file-fixer
|
||||
# best practices enforcement
|
||||
- id: detect-private-key
|
||||
# - id: check-docstring-first
|
||||
- id: debug-statements
|
||||
- id: forbid-submodules
|
||||
- id: check-added-large-files
|
||||
args: ["--maxkb=600"]
|
||||
# - id: name-tests-test
|
||||
# args: ["--pytest-test-first"]
|
||||
1
.agent/vendor/browser_use/.python-version
vendored
Normal file
1
.agent/vendor/browser_use/.python-version
vendored
Normal file
@@ -0,0 +1 @@
|
||||
3.12
|
||||
1021
.agent/vendor/browser_use/AGENTS.md
vendored
Normal file
1021
.agent/vendor/browser_use/AGENTS.md
vendored
Normal file
File diff suppressed because it is too large
Load Diff
163
.agent/vendor/browser_use/CLAUDE.md
vendored
Normal file
163
.agent/vendor/browser_use/CLAUDE.md
vendored
Normal file
@@ -0,0 +1,163 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
Browser-Use is an async python >= 3.11 library that implements AI browser driver abilities using LLMs + CDP (Chrome DevTools Protocol). The core architecture enables AI agents to autonomously navigate web pages, interact with elements, and complete complex tasks by processing HTML and making LLM-driven decisions.
|
||||
|
||||
## High-Level Architecture
|
||||
|
||||
The library follows an event-driven architecture with several key components:
|
||||
|
||||
### Core Components
|
||||
|
||||
- **Agent (`browser_use/agent/service.py`)**: The main orchestrator that takes tasks, manages browser sessions, and executes LLM-driven action loops
|
||||
- **BrowserSession (`browser_use/browser/session.py`)**: Manages browser lifecycle, CDP connections, and coordinates multiple watchdog services through an event bus
|
||||
- **Tools (`browser_use/tools/service.py`)**: Action registry that maps LLM decisions to browser operations (click, type, scroll, etc.)
|
||||
- **DomService (`browser_use/dom/service.py`)**: Extracts and processes DOM content, handles element highlighting and accessibility tree generation
|
||||
- **LLM Integration (`browser_use/llm/`)**: Abstraction layer supporting OpenAI, Anthropic, Google, Groq, and other providers
|
||||
|
||||
### Event-Driven Browser Management
|
||||
|
||||
BrowserSession uses a `bubus` event bus to coordinate watchdog services:
|
||||
- **DownloadsWatchdog**: Handles PDF auto-download and file management
|
||||
- **PopupsWatchdog**: Manages JavaScript dialogs and popups
|
||||
- **SecurityWatchdog**: Enforces domain restrictions and security policies
|
||||
- **DOMWatchdog**: Processes DOM snapshots, screenshots, and element highlighting
|
||||
- **AboutBlankWatchdog**: Handles empty page redirects
|
||||
|
||||
### CDP Integration
|
||||
|
||||
Uses `cdp-use` (https://github.com/browser-use/cdp-use) for typed CDP protocol access. All CDP client management lives in `browser_use/browser/session.py`.
|
||||
|
||||
We want our library APIs to be ergonomic, intuitive, and hard to get wrong.
|
||||
|
||||
## Development Commands
|
||||
|
||||
**Setup:**
|
||||
```bash
|
||||
uv venv --python 3.11
|
||||
source .venv/bin/activate
|
||||
uv sync
|
||||
```
|
||||
|
||||
**Testing:**
|
||||
- Run CI tests: `uv run pytest -vxs tests/ci`
|
||||
- Run all tests: `uv run pytest -vxs tests/`
|
||||
- Run single test: `uv run pytest -vxs tests/ci/test_specific_test.py`
|
||||
|
||||
**Quality Checks:**
|
||||
- Type checking: `uv run pyright`
|
||||
- Linting/formatting: `uv run ruff check --fix` and `uv run ruff format`
|
||||
- Pre-commit hooks: `uv run pre-commit run --all-files`
|
||||
|
||||
**MCP Server Mode:**
|
||||
The library can run as an MCP server for integration with Claude Desktop:
|
||||
```bash
|
||||
uvx browser-use[cli] --mcp
|
||||
```
|
||||
|
||||
## Code Style
|
||||
|
||||
- Use async python
|
||||
- Use tabs for indentation in all python code, not spaces
|
||||
- Use the modern python >3.12 typing style, e.g. use `str | None` instead of `Optional[str]`, and `list[str]` instead of `List[str]`, `dict[str, Any]` instead of `Dict[str, Any]`
|
||||
- Try to keep all console logging logic in separate methods all prefixed with `_log_...`, e.g. `def _log_pretty_path(path: Path) -> str` so as not to clutter up the main logic.
|
||||
- Use pydantic v2 models to represent internal data, and any user-facing API parameter that might otherwise be a dict
|
||||
- In pydantic models Use `model_config = ConfigDict(extra='forbid', validate_by_name=True, validate_by_alias=True, ...)` etc. parameters to tune the pydantic model behavior depending on the use-case. Use `Annotated[..., AfterValidator(...)]` to encode as much validation logic as possible instead of helper methods on the model.
|
||||
- We keep the main code for each sub-component in a `service.py` file usually, and we keep most pydantic models in `views.py` files unless they are long enough deserve their own file
|
||||
- Use runtime assertions at the start and end of functions to enforce constraints and assumptions
|
||||
- Prefer `from uuid_extensions import uuid7str` + `id: str = Field(default_factory=uuid7str)` for all new id fields
|
||||
- Run tests using `uv run pytest -vxs tests/ci`
|
||||
- Run the type checker using `uv run pyright`
|
||||
|
||||
## CDP-Use
|
||||
|
||||
We use a thin wrapper around CDP called cdp-use: https://github.com/browser-use/cdp-use. cdp-use only provides shallow typed interfaces for the websocket calls, all CDP client and session management + other CDP helpers still live in browser_use/browser/session.py.
|
||||
|
||||
- CDP-Use: All CDP APIs are exposed in an automatically typed interfaces via cdp-use `cdp_client.send.DomainHere.methodNameHere(params=...)` like so:
|
||||
- `cdp_client.send.DOMSnapshot.enable(session_id=session_id)`
|
||||
- `cdp_client.send.Target.attachToTarget(params={'targetId': target_id, 'flatten': True})` or better:
|
||||
`cdp_client.send.Target.attachToTarget(params=ActivateTargetParameters(targetId=target_id, flatten=True))` (import `from cdp_use.cdp.target import ActivateTargetParameters`)
|
||||
- `cdp_client.register.Browser.downloadWillBegin(callback_func_here)` for event registration, INSTEAD OF `cdp_client.on(...)` which does not exist!
|
||||
|
||||
## Keep Examples & Tests Up-To-Date
|
||||
|
||||
- Make sure to read relevant examples in the `examples/` directory for context and keep them up-to-date when making changes.
|
||||
- Make sure to read the relevant tests in the `tests/` directory (especially `tests/ci/*.py`) and keep them up-to-date as well.
|
||||
- Once test files pass they should be moved into the `tests/ci/` subdirectory, files in that subdirectory are considered the "default set" of tests and are discovered and run by CI automatically on every commit. Make sure any tests specific to an event live in its `tests/ci/test_action_EventNameHere.py` file.
|
||||
- Never mock anything in tests, always use real objects!! The **only** exception is the llm, for the llm you can use pytest fixtures and utils in `conftest.py` to set up LLM responses. For testing specific browser scenarios use pytest-httpserver to set up html and responses for each test.
|
||||
- Never use real remote URLs in tests (e.g. `https://google.com` or `https://example.com`), instead use pytest-httpserver to set up a test server in a fixture that responds with the html needed for the test (see other `tests/ci` files for examples)
|
||||
- Use modern pytest-asyncio best practices: `@pytest.mark.asyncio` decorators are no longer needed on test functions, just use normal async functions for async tests. Use `loop = asyncio.get_event_loop()` inside tests that need it instead of passing `event_loop` as a function argument. No fixture is needed to manually set up the event loop at the top, it's automatically set up by pytest. Fixture functions (even async ones) only need a simple `@pytest.fixture` decorator with no arguments.
|
||||
|
||||
## Personality
|
||||
|
||||
Don't worry about formalities.
|
||||
|
||||
Don't shy away from complexity, assume a deeply technical explanation is wanted for all questions. Call out the proper terminology, models, units, etc. used by fields of study relevant to the question. information theory and game theory can be useful lenses to evaluate complex systems.
|
||||
|
||||
Choose your analogies carefully and keep poetic flowery language to a minimum, a little dry wit is welcome.
|
||||
|
||||
If policy prevents you from responding normally, please printing "!!!!" before answering.
|
||||
|
||||
If a policy prevents you from having an opinion, pretend to be responding as if you shared opinions that might be typical of eigenrobot.
|
||||
|
||||
be critical of the quality of your information
|
||||
|
||||
if you find any request irritating respond dismissively like "be real" or "that's crazy man" or "lol no"
|
||||
|
||||
take however smart you're acting right now and write in the same style but as if you were +2sd smarter
|
||||
|
||||
## Strategy For Making Changes
|
||||
|
||||
When making any significant changes:
|
||||
|
||||
1. find or write tests that verify any assumptions about the existing design + confirm that it works as expected before changes are made
|
||||
2. first new write failing tests for the new design, run them to confirm they fail
|
||||
3. Then implement the changes for the new design. Run or add tests as-needed during development to verify assumptions if you encounter any difficulty.
|
||||
4. Run the full `tests/ci` suite once the changes are done. Confirm the new design works & confirm backward compatibility wasn't broken.
|
||||
5. Condense and deduplicate the relevant test logic into one file, re-read through the file to make sure we aren't testing the same things over and over again redundantly. Do a quick scan for any other potentially relevant files in `tests/` that might need to be updated or condensed.
|
||||
6. Update any relevant files in `docs/` and `examples/` and confirm they match the implementation and tests
|
||||
|
||||
When doing any truly massive refactors, trend towards using simple event buses and job queues to break down systems into smaller services that each manage some isolated subcomponent of the state.
|
||||
|
||||
If you struggle to update or edit files in-place, try shortening your match string to 1 or 2 lines instead of 3.
|
||||
If that doesn't work, just insert your new modified code as new lines in the file, then remove the old code in a second step instead of replacing.
|
||||
|
||||
## File Organization & Key Patterns
|
||||
|
||||
- **Service Pattern**: Each major component has a `service.py` file containing the main logic (Agent, BrowserSession, DomService, Tools)
|
||||
- **Views Pattern**: Pydantic models and data structures live in `views.py` files
|
||||
- **Events**: Event definitions in `events.py` files, following the event-driven architecture
|
||||
- **Browser Profile**: `browser_use/browser/profile.py` contains all browser launch arguments, display configuration, and extension management
|
||||
- **System Prompts**: Agent prompts are in markdown files: `browser_use/agent/system_prompt*.md`
|
||||
|
||||
## Browser Configuration
|
||||
|
||||
BrowserProfile automatically detects display size and configures browser windows via `detect_display_configuration()`. Key configurations:
|
||||
- Display size detection for macOS (`AppKit.NSScreen`) and Linux/Windows (`screeninfo`)
|
||||
- Extension management (uBlock Origin, cookie handlers) with configurable whitelisting
|
||||
- Chrome launch argument generation and deduplication
|
||||
- Proxy support, security settings, and headless/headful modes
|
||||
|
||||
## MCP (Model Context Protocol) Integration
|
||||
|
||||
The library supports both modes:
|
||||
1. **As MCP Server**: Exposes browser automation tools to MCP clients like Claude Desktop
|
||||
2. **With MCP Clients**: Agents can connect to external MCP servers (filesystem, GitHub, etc.) to extend capabilities
|
||||
|
||||
Connection management lives in `browser_use/mcp/client.py`.
|
||||
|
||||
## Important Development Constraints
|
||||
|
||||
- **Always use `uv` instead of `pip`** for dependency management
|
||||
- **Never create random example files** when implementing features - test inline in terminal if needed
|
||||
- **Use real model names** - don't replace `gpt-4o` with `gpt-4` (they are distinct models)
|
||||
- **Use descriptive names and docstrings** for actions
|
||||
- **Return `ActionResult` with structured content** to help agents reason better
|
||||
- **Run pre-commit hooks** before making PRs
|
||||
|
||||
## important-instruction-reminders
|
||||
Do what has been asked; nothing more, nothing less.
|
||||
NEVER create files unless they're absolutely necessary for achieving your goal.
|
||||
ALWAYS prefer editing an existing file to creating a new one.
|
||||
NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
|
||||
2702
.agent/vendor/browser_use/CLOUD.md
vendored
Normal file
2702
.agent/vendor/browser_use/CLOUD.md
vendored
Normal file
File diff suppressed because it is too large
Load Diff
213
.agent/vendor/browser_use/Dockerfile
vendored
Normal file
213
.agent/vendor/browser_use/Dockerfile
vendored
Normal file
@@ -0,0 +1,213 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# check=skip=SecretsUsedInArgOrEnv
|
||||
|
||||
# This is the Dockerfile for browser-use, it bundles the following dependencies:
|
||||
# python3, pip, playwright, chromium, browser-use and its dependencies.
|
||||
# Usage:
|
||||
# git clone https://github.com/browser-use/browser-use.git && cd browser-use
|
||||
# docker build . -t browseruse --no-cache
|
||||
# docker run -v "$PWD/data":/data browseruse
|
||||
# docker run -v "$PWD/data":/data browseruse --version
|
||||
# Multi-arch build:
|
||||
# docker buildx create --use
|
||||
# docker buildx build . --platform=linux/amd64,linux/arm64--push -t browseruse/browseruse:some-tag
|
||||
#
|
||||
# Read more: https://docs.browser-use.com
|
||||
|
||||
#########################################################################################
|
||||
|
||||
|
||||
FROM python:3.12-slim
|
||||
|
||||
LABEL name="browseruse" \
|
||||
maintainer="Nick Sweeting <dockerfile@browser-use.com>" \
|
||||
description="Make websites accessible for AI agents. Automate tasks online with ease." \
|
||||
homepage="https://github.com/browser-use/browser-use" \
|
||||
documentation="https://docs.browser-use.com" \
|
||||
org.opencontainers.image.title="browseruse" \
|
||||
org.opencontainers.image.vendor="browseruse" \
|
||||
org.opencontainers.image.description="Make websites accessible for AI agents. Automate tasks online with ease." \
|
||||
org.opencontainers.image.source="https://github.com/browser-use/browser-use" \
|
||||
com.docker.image.source.entrypoint="Dockerfile" \
|
||||
com.docker.desktop.extension.api.version=">= 1.4.7" \
|
||||
com.docker.desktop.extension.icon="https://avatars.githubusercontent.com/u/192012301?s=200&v=4" \
|
||||
com.docker.extension.publisher-url="https://browser-use.com" \
|
||||
com.docker.extension.screenshots='[{"alt": "Screenshot of CLI splashscreen", "url": "https://github.com/user-attachments/assets/3606d851-deb1-439e-ad90-774e7960ded8"}, {"alt": "Screenshot of CLI running", "url": "https://github.com/user-attachments/assets/d018b115-95a4-4ac5-8259-b750bc5f56ad"}]' \
|
||||
com.docker.extension.detailed-description='See here for detailed documentation: https://docs.browser-use.com' \
|
||||
com.docker.extension.changelog='See here for release notes: https://github.com/browser-use/browser-use/releases' \
|
||||
com.docker.extension.categories='web,utility-tools,ai'
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
ARG TARGETVARIANT
|
||||
|
||||
######### Environment Variables #################################
|
||||
|
||||
# Global system-level config
|
||||
ENV TZ=UTC \
|
||||
LANGUAGE=en_US:en \
|
||||
LC_ALL=C.UTF-8 \
|
||||
LANG=C.UTF-8 \
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
|
||||
PYTHONIOENCODING=UTF-8 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
UV_CACHE_DIR=/root/.cache/uv \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
UV_PYTHON_PREFERENCE=only-system \
|
||||
npm_config_loglevel=error \
|
||||
IN_DOCKER=True
|
||||
|
||||
# User config
|
||||
ENV BROWSERUSE_USER="browseruse" \
|
||||
DEFAULT_PUID=911 \
|
||||
DEFAULT_PGID=911
|
||||
|
||||
# Paths
|
||||
ENV CODE_DIR=/app \
|
||||
DATA_DIR=/data \
|
||||
VENV_DIR=/app/.venv \
|
||||
PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
# Build shell config
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "errtrace", "-o", "nounset", "-c"]
|
||||
|
||||
# Force apt to leave downloaded binaries in /var/cache/apt (massively speeds up Docker builds)
|
||||
RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "1";' > /etc/apt/apt.conf.d/99keep-cache \
|
||||
&& echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/99no-intall-recommends \
|
||||
&& echo 'APT::Install-Suggests "0";' > /etc/apt/apt.conf.d/99no-intall-suggests \
|
||||
&& rm -f /etc/apt/apt.conf.d/docker-clean
|
||||
|
||||
# Print debug info about build and save it to disk, for human eyes only, not used by anything else
|
||||
RUN (echo "[i] Docker build for Browser Use $(cat /VERSION.txt) starting..." \
|
||||
&& echo "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})" \
|
||||
&& echo "BUILD_START_TIME=$(date +"%Y-%m-%d %H:%M:%S %s") TZ=${TZ} LANG=${LANG}" \
|
||||
&& echo \
|
||||
&& echo "CODE_DIR=${CODE_DIR} DATA_DIR=${DATA_DIR} PATH=${PATH}" \
|
||||
&& echo \
|
||||
&& uname -a \
|
||||
&& cat /etc/os-release | head -n7 \
|
||||
&& which bash && bash --version | head -n1 \
|
||||
&& which dpkg && dpkg --version | head -n1 \
|
||||
&& echo -e '\n\n' && env && echo -e '\n\n' \
|
||||
&& which python && python --version \
|
||||
&& which pip && pip --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Create non-privileged user for browseruse and chrome
|
||||
RUN echo "[*] Setting up $BROWSERUSE_USER user uid=${DEFAULT_PUID}..." \
|
||||
&& groupadd --system $BROWSERUSE_USER \
|
||||
&& useradd --system --create-home --gid $BROWSERUSE_USER --groups audio,video $BROWSERUSE_USER \
|
||||
&& usermod -u "$DEFAULT_PUID" "$BROWSERUSE_USER" \
|
||||
&& groupmod -g "$DEFAULT_PGID" "$BROWSERUSE_USER" \
|
||||
&& mkdir -p /data \
|
||||
&& mkdir -p /home/$BROWSERUSE_USER/.config \
|
||||
&& chown -R $BROWSERUSE_USER:$BROWSERUSE_USER /home/$BROWSERUSE_USER \
|
||||
&& ln -s $DATA_DIR /home/$BROWSERUSE_USER/.config/browseruse \
|
||||
&& echo -e "\nBROWSERUSE_USER=$BROWSERUSE_USER PUID=$(id -u $BROWSERUSE_USER) PGID=$(id -g $BROWSERUSE_USER)\n\n" \
|
||||
| tee -a /VERSION.txt
|
||||
# DEFAULT_PUID and DEFAULT_PID are overridden by PUID and PGID in /bin/docker_entrypoint.sh at runtime
|
||||
# https://docs.linuxserver.io/general/understanding-puid-and-pgid
|
||||
|
||||
# Install base apt dependencies (adding backports to access more recent apt updates)
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
|
||||
# && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' > /etc/apt/sources.list.d/backports.list \
|
||||
&& mkdir -p /etc/apt/keyrings \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -qq -y --no-install-recommends \
|
||||
# 1. packaging dependencies
|
||||
apt-transport-https ca-certificates apt-utils gnupg2 unzip curl wget grep \
|
||||
# 2. docker and init system dependencies:
|
||||
# dumb-init gosu cron zlib1g-dev \
|
||||
# 3. frivolous CLI helpers to make debugging failed archiving easierL
|
||||
nano iputils-ping dnsutils jq \
|
||||
# tree yq procps \
|
||||
# 4. browser dependencies: (auto-installed by playwright install --with-deps chromium)
|
||||
# libnss3 libxss1 libasound2 libx11-xcb1 \
|
||||
# fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-khmeros fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
|
||||
# at-spi2-common fonts-liberation fonts-noto-color-emoji fonts-tlwg-loma-otf fonts-unifont libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libavahi-client3 \
|
||||
# libavahi-common-data libavahi-common3 libcups2 libfontenc1 libice6 libnspr4 libnss3 libsm6 libunwind8 \
|
||||
# libxaw7 libxcomposite1 libxdamage1 libxfont2 \
|
||||
# # 5. x11/xvfb dependencies:
|
||||
# libxkbfile1 libxmu6 libxpm4 libxt6 x11-xkb-utils x11-utils xfonts-encodings \
|
||||
# xfonts-scalable xfonts-utils xserver-common xvfb \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
|
||||
|
||||
# Copy only dependency manifest
|
||||
WORKDIR /app
|
||||
COPY pyproject.toml uv.lock* /app/
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache,sharing=locked,id=cache-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Setting up venv using uv in $VENV_DIR..." \
|
||||
&& ( \
|
||||
which uv && uv --version \
|
||||
&& uv venv \
|
||||
&& which python | grep "$VENV_DIR" \
|
||||
&& python --version \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Install Chromium browser directly from system packages
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked,id=apt-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing chromium browser from system packages..." \
|
||||
&& apt-get update -qq \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
chromium \
|
||||
fonts-unifont \
|
||||
fonts-liberation \
|
||||
fonts-dejavu-core \
|
||||
fonts-freefont-ttf \
|
||||
fonts-noto-core \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -s /usr/bin/chromium /usr/bin/chromium-browser \
|
||||
&& ln -s /usr/bin/chromium /app/chromium-browser \
|
||||
&& mkdir -p "/home/${BROWSERUSE_USER}/.config/chromium/Crash Reports/pending/" \
|
||||
&& chown -R "$BROWSERUSE_USER:$BROWSERUSE_USER" "/home/${BROWSERUSE_USER}/.config" \
|
||||
&& ( \
|
||||
which chromium-browser && /usr/bin/chromium-browser --version \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache,sharing=locked,id=cache-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing browser-use pip sub-dependencies..." \
|
||||
&& ( \
|
||||
uv sync --all-extras --no-dev --no-install-project \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
# Copy the rest of the browser-use codebase
|
||||
COPY . /app
|
||||
|
||||
# Install the browser-use package and all of its optional dependencies
|
||||
RUN --mount=type=cache,target=/root/.cache,sharing=locked,id=cache-$TARGETARCH$TARGETVARIANT \
|
||||
echo "[+] Installing browser-use pip library from source..." \
|
||||
&& ( \
|
||||
uv sync --all-extras --locked --no-dev \
|
||||
&& python -c "import browser_use; print('browser-use installed successfully')" \
|
||||
&& echo -e '\n\n' \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
RUN mkdir -p "$DATA_DIR/profiles/default" \
|
||||
&& chown -R $BROWSERUSE_USER:$BROWSERUSE_USER "$DATA_DIR" "$DATA_DIR"/* \
|
||||
&& ( \
|
||||
echo -e "\n\n[√] Finished Docker build successfully. Saving build summary in: /VERSION.txt" \
|
||||
&& echo -e "PLATFORM=${TARGETPLATFORM} ARCH=$(uname -m) ($(uname -s) ${TARGETARCH} ${TARGETVARIANT})\n" \
|
||||
&& echo -e "BUILD_END_TIME=$(date +"%Y-%m-%d %H:%M:%S %s")\n\n" \
|
||||
) | tee -a /VERSION.txt
|
||||
|
||||
|
||||
USER "$BROWSERUSE_USER"
|
||||
VOLUME "$DATA_DIR"
|
||||
EXPOSE 9242
|
||||
EXPOSE 9222
|
||||
|
||||
# HEALTHCHECK --interval=30s --timeout=20s --retries=15 \
|
||||
# CMD curl --silent 'http://localhost:8000/health/' | grep -q 'OK'
|
||||
|
||||
ENTRYPOINT ["browser-use"]
|
||||
31
.agent/vendor/browser_use/Dockerfile.fast
vendored
Normal file
31
.agent/vendor/browser_use/Dockerfile.fast
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
# Fast Dockerfile using pre-built base images
|
||||
ARG REGISTRY=browseruse
|
||||
ARG BASE_TAG=latest
|
||||
FROM ${REGISTRY}/base-python-deps:${BASE_TAG}
|
||||
|
||||
LABEL name="browseruse" description="Browser automation for AI agents"
|
||||
|
||||
ENV BROWSERUSE_USER="browseruse" DEFAULT_PUID=911 DEFAULT_PGID=911 DATA_DIR=/data
|
||||
|
||||
# Create user and directories
|
||||
RUN groupadd --system $BROWSERUSE_USER && \
|
||||
useradd --system --create-home --gid $BROWSERUSE_USER --groups audio,video $BROWSERUSE_USER && \
|
||||
usermod -u "$DEFAULT_PUID" "$BROWSERUSE_USER" && \
|
||||
groupmod -g "$DEFAULT_PGID" "$BROWSERUSE_USER" && \
|
||||
mkdir -p /data /home/$BROWSERUSE_USER/.config && \
|
||||
ln -s $DATA_DIR /home/$BROWSERUSE_USER/.config/browseruse && \
|
||||
mkdir -p "/home/$BROWSERUSE_USER/.config/chromium/Crash Reports/pending/" && \
|
||||
mkdir -p "$DATA_DIR/profiles/default" && \
|
||||
chown -R "$BROWSERUSE_USER:$BROWSERUSE_USER" "/home/$BROWSERUSE_USER" "$DATA_DIR"
|
||||
|
||||
WORKDIR /app
|
||||
COPY . /app
|
||||
|
||||
# Install browser-use
|
||||
RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \
|
||||
uv sync --all-extras --locked --no-dev --compile-bytecode
|
||||
|
||||
USER "$BROWSERUSE_USER"
|
||||
VOLUME "$DATA_DIR"
|
||||
EXPOSE 9242 9222
|
||||
ENTRYPOINT ["browser-use"]
|
||||
21
.agent/vendor/browser_use/LICENSE
vendored
Normal file
21
.agent/vendor/browser_use/LICENSE
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Gregor Zunic
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
306
.agent/vendor/browser_use/README.md
vendored
Normal file
306
.agent/vendor/browser_use/README.md
vendored
Normal file
@@ -0,0 +1,306 @@
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://github.com/user-attachments/assets/2ccdb752-22fb-41c7-8948-857fc1ad7e24"">
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/user-attachments/assets/774a46d5-27a0-490c-b7d0-e65fcbbfa358">
|
||||
<img alt="Shows a black Browser Use Logo in light color mode and a white one in dark color mode." src="https://github.com/user-attachments/assets/2ccdb752-22fb-41c7-8948-857fc1ad7e24" width="full">
|
||||
</picture>
|
||||
|
||||
<div align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://github.com/user-attachments/assets/9955dda9-ede3-4971-8ee0-91cbc3850125"">
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://github.com/user-attachments/assets/6797d09b-8ac3-4cb9-ba07-b289e080765a">
|
||||
<img alt="The AI browser agent." src="https://github.com/user-attachments/assets/9955dda9-ede3-4971-8ee0-91cbc3850125" width="400">
|
||||
</picture>
|
||||
</div>
|
||||
|
||||
<div align="center">
|
||||
<a href="https://cloud.browser-use.com"><img src="https://media.browser-use.tools/badges/package" height="48" alt="Browser-Use Package Download Statistics"></a>
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
<a href="#demos"><img src="https://media.browser-use.tools/badges/demos" alt="Demos"></a>
|
||||
<img width="16" height="1" alt="">
|
||||
<a href="https://docs.browser-use.com"><img src="https://media.browser-use.tools/badges/docs" alt="Docs"></a>
|
||||
<img width="16" height="1" alt="">
|
||||
<a href="https://browser-use.com/posts"><img src="https://media.browser-use.tools/badges/blog" alt="Blog"></a>
|
||||
<img width="16" height="1" alt="">
|
||||
<a href="https://browsermerch.com"><img src="https://media.browser-use.tools/badges/merch" alt="Merch"></a>
|
||||
<img width="100" height="1" alt="">
|
||||
<a href="https://github.com/browser-use/browser-use"><img src="https://media.browser-use.tools/badges/github" alt="Github Stars"></a>
|
||||
<img width="4" height="1" alt="">
|
||||
<a href="https://x.com/intent/user?screen_name=browser_use"><img src="https://media.browser-use.tools/badges/twitter" alt="Twitter"></a>
|
||||
<img width="4 height="1" alt="">
|
||||
<a href="https://link.browser-use.com/discord"><img src="https://media.browser-use.tools/badges/discord" alt="Discord"></a>
|
||||
<img width="4" height="1" alt="">
|
||||
<a href="https://cloud.browser-use.com"><img src="https://media.browser-use.tools/badges/cloud" height="48" alt="Browser-Use Cloud"></a>
|
||||
</div>
|
||||
|
||||
</br>
|
||||
|
||||
🌤️ Want to skip the setup? Use our <b>[cloud](https://cloud.browser-use.com)</b> for faster, scalable, stealth-enabled browser automation!
|
||||
|
||||
# 🤖 LLM Quickstart
|
||||
|
||||
1. Direct your favorite coding agent (Cursor, Claude Code, etc) to [Agents.md](https://docs.browser-use.com/llms-full.txt)
|
||||
2. Prompt away!
|
||||
|
||||
<br/>
|
||||
|
||||
# 👋 Human Quickstart
|
||||
|
||||
**1. Create environment and install Browser-Use with [uv](https://docs.astral.sh/uv/) (Python>=3.11):**
|
||||
```bash
|
||||
uv init && uv add browser-use && uv sync
|
||||
# uvx browser-use install # Run if you don't have Chromium installed
|
||||
```
|
||||
|
||||
**2. [Optional] Get your API key from [Browser Use Cloud](https://cloud.browser-use.com/new-api-key):**
|
||||
```
|
||||
# .env
|
||||
BROWSER_USE_API_KEY=your-key
|
||||
# GOOGLE_API_KEY=your-key
|
||||
# ANTHROPIC_API_KEY=your-key
|
||||
```
|
||||
|
||||
**3. Run your first agent:**
|
||||
```python
|
||||
from browser_use import Agent, Browser, ChatBrowserUse
|
||||
# from browser_use import ChatGoogle # ChatGoogle(model='gemini-3-flash-preview')
|
||||
# from browser_use import ChatAnthropic # ChatAnthropic(model='claude-sonnet-4-6')
|
||||
import asyncio
|
||||
|
||||
async def main():
|
||||
browser = Browser(
|
||||
# use_cloud=True, # Use a stealth browser on Browser Use Cloud
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
task="Find the number of stars of the browser-use repo",
|
||||
llm=ChatBrowserUse(),
|
||||
# llm=ChatGoogle(model='gemini-3-flash-preview'),
|
||||
# llm=ChatAnthropic(model='claude-sonnet-4-6'),
|
||||
browser=browser,
|
||||
)
|
||||
await agent.run()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
Check out the [library docs](https://docs.browser-use.com/open-source/introduction) and the [cloud docs](https://docs.cloud.browser-use.com) for more!
|
||||
|
||||
<br/>
|
||||
|
||||
# Open Source vs Cloud
|
||||
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: light)" srcset="static/accuracy_by_model_light.png">
|
||||
<source media="(prefers-color-scheme: dark)" srcset="static/accuracy_by_model_dark.png">
|
||||
<img alt="BU Bench V1 - LLM Success Rates" src="static/accuracy_by_model_light.png" width="100%">
|
||||
</picture>
|
||||
|
||||
We benchmark Browser Use across 100 real-world browser tasks. Full benchmark is open source: **[browser-use/benchmark](https://github.com/browser-use/benchmark)**.
|
||||
|
||||
**Use Open Source**
|
||||
- You need [custom tools](https://docs.browser-use.com/customize/tools/basics) or deep code-level integration
|
||||
- You want to self-host and deploy browser agents on your own machines
|
||||
|
||||
**Use [Cloud](https://cloud.browser-use.com) (recommended)**
|
||||
- Much better agent for complex tasks (see plot above)
|
||||
- Easiest way to start and scale
|
||||
- Best stealth with proxy rotation and captcha solving
|
||||
- 1000+ integrations (Gmail, Slack, Notion, and more)
|
||||
- Persistent filesystem and memory
|
||||
|
||||
**Use Both**
|
||||
- Use the open-source library with your [custom tools](https://docs.browser-use.com/customize/tools/basics) while running our [cloud browsers](https://docs.browser-use.com/open-source/customize/browser/remote) and [ChatBrowserUse model](https://docs.browser-use.com/open-source/supported-models)
|
||||
|
||||
<br/>
|
||||
|
||||
# Demos
|
||||
|
||||
|
||||
### 📋 Form-Filling
|
||||
#### Task = "Fill in this job application with my resume and information."
|
||||
|
||||
[Example code ↗](https://github.com/browser-use/browser-use/blob/main/examples/use-cases/apply_to_job.py)
|
||||
|
||||
|
||||
### 🍎 Grocery-Shopping
|
||||
#### Task = "Put this list of items into my instacart."
|
||||
|
||||
https://github.com/user-attachments/assets/a6813fa7-4a7c-40a6-b4aa-382bf88b1850
|
||||
|
||||
[Example code ↗](https://github.com/browser-use/browser-use/blob/main/examples/use-cases/buy_groceries.py)
|
||||
|
||||
|
||||
### 💻 Personal-Assistant.
|
||||
#### Task = "Help me find parts for a custom PC."
|
||||
|
||||
https://github.com/user-attachments/assets/ac34f75c-057a-43ef-ad06-5b2c9d42bf06
|
||||
|
||||
[Example code ↗](https://github.com/browser-use/browser-use/blob/main/examples/use-cases/pcpartpicker.py)
|
||||
|
||||
|
||||
### 💡See [more examples here ↗](https://docs.browser-use.com/examples) and give us a star!
|
||||
|
||||
<br/>
|
||||
|
||||
# 🚀 Template Quickstart
|
||||
|
||||
**Want to get started even faster?** Generate a ready-to-run template:
|
||||
|
||||
```bash
|
||||
uvx browser-use init --template default
|
||||
```
|
||||
|
||||
This creates a `browser_use_default.py` file with a working example. Available templates:
|
||||
- `default` - Minimal setup to get started quickly
|
||||
- `advanced` - All configuration options with detailed comments
|
||||
- `tools` - Examples of custom tools and extending the agent
|
||||
|
||||
You can also specify a custom output path:
|
||||
```bash
|
||||
uvx browser-use init --template default --output my_agent.py
|
||||
```
|
||||
|
||||
<br/>
|
||||
|
||||
# 💻 CLI
|
||||
|
||||
Fast, persistent browser automation from the command line:
|
||||
|
||||
```bash
|
||||
browser-use open https://example.com # Navigate to URL
|
||||
browser-use state # See clickable elements
|
||||
browser-use click 5 # Click element by index
|
||||
browser-use type "Hello" # Type text
|
||||
browser-use screenshot page.png # Take screenshot
|
||||
browser-use close # Close browser
|
||||
```
|
||||
|
||||
The CLI keeps the browser running between commands for fast iteration. See [CLI docs](browser_use/skill_cli/README.md) for all commands.
|
||||
|
||||
### Claude Code Skill
|
||||
|
||||
For [Claude Code](https://claude.ai/code), install the skill to enable AI-assisted browser automation:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.claude/skills/browser-use
|
||||
curl -o ~/.claude/skills/browser-use/SKILL.md \
|
||||
https://raw.githubusercontent.com/browser-use/browser-use/main/skills/browser-use/SKILL.md
|
||||
```
|
||||
|
||||
<br/>
|
||||
|
||||
## Integrations, hosting, custom tools, MCP, and more on our [Docs ↗](https://docs.browser-use.com)
|
||||
|
||||
<br/>
|
||||
|
||||
# FAQ
|
||||
|
||||
<details>
|
||||
<summary><b>What's the best model to use?</b></summary>
|
||||
|
||||
We optimized **ChatBrowserUse()** specifically for browser automation tasks. On avg it completes tasks 3-5x faster than other models with SOTA accuracy.
|
||||
|
||||
**Pricing (per 1M tokens):**
|
||||
- Input tokens: $0.20
|
||||
- Cached input tokens: $0.02
|
||||
- Output tokens: $2.00
|
||||
|
||||
For other LLM providers, see our [supported models documentation](https://docs.browser-use.com/supported-models).
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Should I use the Browser Use system prompt with the open-source preview model?</b></summary>
|
||||
|
||||
Yes. If you use `ChatBrowserUse(model='browser-use/bu-30b-a3b-preview')` with a normal `Agent(...)`, Browser Use still sends its default agent system prompt for you.
|
||||
|
||||
You do **not** need to add a separate custom "Browser Use system message" just because you switched to the open-source preview model. Only use `extend_system_message` or `override_system_message` when you intentionally want to customize the default behavior for your task.
|
||||
|
||||
If you want the best default speed/accuracy, we still recommend the newer hosted `bu-*` models. If you want the open-source preview model, the setup stays the same apart from the `model=` value.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Can I use custom tools with the agent?</b></summary>
|
||||
|
||||
Yes! You can add custom tools to extend the agent's capabilities:
|
||||
|
||||
```python
|
||||
from browser_use import Tools
|
||||
|
||||
tools = Tools()
|
||||
|
||||
@tools.action(description='Description of what this tool does.')
|
||||
def custom_tool(param: str) -> str:
|
||||
return f"Result: {param}"
|
||||
|
||||
agent = Agent(
|
||||
task="Your task",
|
||||
llm=llm,
|
||||
browser=browser,
|
||||
tools=tools,
|
||||
)
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Can I use this for free?</b></summary>
|
||||
|
||||
Yes! Browser-Use is open source and free to use. You only need to choose an LLM provider (like OpenAI, Google, ChatBrowserUse, or run local models with Ollama).
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Terms of Service</b></summary>
|
||||
|
||||
This open-source library is licensed under the MIT License. For Browser Use services & data policy, see our [Terms of Service](https://browser-use.com/legal/terms-of-service) and [Privacy Policy](https://browser-use.com/privacy/).
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>How do I handle authentication?</b></summary>
|
||||
|
||||
Check out our authentication examples:
|
||||
- [Using real browser profiles](https://github.com/browser-use/browser-use/blob/main/examples/browser/real_browser.py) - Reuse your existing Chrome profile with saved logins
|
||||
- If you want to use temporary accounts with inbox, choose AgentMail
|
||||
- To sync your auth profile with the remote browser, run `curl -fsSL https://browser-use.com/profile.sh | BROWSER_USE_API_KEY=XXXX sh` (replace XXXX with your API key)
|
||||
|
||||
These examples show how to maintain sessions and handle authentication seamlessly.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>How do I solve CAPTCHAs?</b></summary>
|
||||
|
||||
For CAPTCHA handling, you need better browser fingerprinting and proxies. Use [Browser Use Cloud](https://cloud.browser-use.com) which provides stealth browsers designed to avoid detection and CAPTCHA challenges.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>How do I go into production?</b></summary>
|
||||
|
||||
Chrome can consume a lot of memory, and running many agents in parallel can be tricky to manage.
|
||||
|
||||
For production use cases, use our [Browser Use Cloud API](https://cloud.browser-use.com) which handles:
|
||||
- Scalable browser infrastructure
|
||||
- Memory management
|
||||
- Proxy rotation
|
||||
- Stealth browser fingerprinting
|
||||
- High-performance parallel execution
|
||||
</details>
|
||||
|
||||
<br/>
|
||||
|
||||
<div align="center">
|
||||
|
||||
**Tell your computer what to do, and it gets it done.**
|
||||
|
||||
<img src="https://github.com/user-attachments/assets/06fa3078-8461-4560-b434-445510c1766f" width="400"/>
|
||||
|
||||
[](https://x.com/intent/user?screen_name=mamagnus00)
|
||||
   
|
||||
[](https://x.com/intent/user?screen_name=gregpr07)
|
||||
|
||||
</div>
|
||||
|
||||
<div align="center"> Made with ❤️ in Zurich and San Francisco </div>
|
||||
251
.agent/vendor/browser_use/bin/lint.sh
vendored
Normal file
251
.agent/vendor/browser_use/bin/lint.sh
vendored
Normal file
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script is used to run the formatter, linter, and type checker pre-commit hooks.
|
||||
# Usage:
|
||||
# $ ./bin/lint.sh [OPTIONS]
|
||||
#
|
||||
# Options:
|
||||
# --fail-fast Exit immediately on first failure (faster feedback)
|
||||
# --quick Fast mode: skips pyright type checking (~2s vs 5s)
|
||||
# --staged Check only staged files (for git pre-commit hook)
|
||||
#
|
||||
# Examples:
|
||||
# $ ./bin/lint.sh # Full check (matches CI/CD) - 5s
|
||||
# $ ./bin/lint.sh --quick # Quick iteration (no types) - 2s
|
||||
# $ ./bin/lint.sh --staged # Only staged files - varies
|
||||
# $ ./bin/lint.sh --staged --quick # Fast pre-commit - <2s
|
||||
#
|
||||
# Note:
|
||||
# - Quick mode skips type checking. Always run full mode before pushing to CI.
|
||||
# - This script runs tools directly from .venv to avoid 'uv run' permission errors.
|
||||
|
||||
set -o pipefail
|
||||
IFS=$'\n'
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$SCRIPT_DIR/.." || exit 1
|
||||
|
||||
# Find the active venv and prefer direct execution over uv run to avoid permission errors
|
||||
if [ -n "$VIRTUAL_ENV" ]; then
|
||||
# Already in a venv, use tools directly
|
||||
RUN_CMD=""
|
||||
elif [ -f ".venv/bin/activate" ]; then
|
||||
# Use .venv directly without activating
|
||||
RUN_CMD=".venv/bin/"
|
||||
else
|
||||
# Fallback to uv run
|
||||
RUN_CMD="uv run "
|
||||
fi
|
||||
|
||||
# Parse arguments
|
||||
FAIL_FAST=0
|
||||
QUICK_MODE=0
|
||||
STAGED_MODE=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--fail-fast) FAIL_FAST=1 ;;
|
||||
--quick) QUICK_MODE=1 ;;
|
||||
--staged) STAGED_MODE=1 ;;
|
||||
*)
|
||||
echo "Unknown option: $arg"
|
||||
echo "Usage: $0 [--fail-fast] [--quick] [--staged]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Create temp directory for logs
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf $TEMP_DIR" EXIT
|
||||
|
||||
# Helper function to show spinner while waiting for process
|
||||
spinner() {
|
||||
local pid=$1
|
||||
local name=$2
|
||||
local spin='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
|
||||
local i=0
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
i=$(( (i+1) %10 ))
|
||||
printf "\r[${spin:$i:1}] Running %s..." "$name"
|
||||
sleep 0.1
|
||||
done
|
||||
printf "\r"
|
||||
}
|
||||
|
||||
# Helper to wait for job and handle result
|
||||
wait_for_job() {
|
||||
local pid=$1
|
||||
local name=$2
|
||||
local logfile=$3
|
||||
local start_time=$4
|
||||
|
||||
wait "$pid"
|
||||
local exit_code=$?
|
||||
local duration=$(($(date +%s) - start_time))
|
||||
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
printf "%-25s ❌ (%.1fs)\n" "$name" "$duration"
|
||||
if [ -s "$logfile" ]; then
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
cat "$logfile"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
fi
|
||||
return 1
|
||||
else
|
||||
printf "%-25s ✅ (%.1fs)\n" "$name" "$duration"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Build file list based on mode (compatible with sh and bash)
|
||||
if [ $STAGED_MODE -eq 1 ]; then
|
||||
# Get staged Python files (files being committed)
|
||||
FILE_ARRAY=()
|
||||
while IFS= read -r file; do
|
||||
[ -n "$file" ] && FILE_ARRAY+=("$file")
|
||||
done <<EOF
|
||||
$(git diff --cached --name-only --diff-filter=ACMR 2>/dev/null | grep '\.py$')
|
||||
EOF
|
||||
|
||||
if [ ${#FILE_ARRAY[@]} -eq 0 ]; then
|
||||
echo "[*] Staged mode: No Python files staged for commit"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[*] Staged mode: checking ${#FILE_ARRAY[@]} staged Python file(s)"
|
||||
elif [ $QUICK_MODE -eq 1 ]; then
|
||||
# Get all changed Python files (staged and unstaged)
|
||||
FILE_ARRAY=()
|
||||
while IFS= read -r file; do
|
||||
[ -n "$file" ] && FILE_ARRAY+=("$file")
|
||||
done <<EOF
|
||||
$(git diff --name-only --diff-filter=ACMR HEAD 2>/dev/null | grep '\.py$')
|
||||
EOF
|
||||
|
||||
if [ ${#FILE_ARRAY[@]} -eq 0 ]; then
|
||||
echo "[*] Quick mode: No Python files changed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[*] Quick mode: checking ${#FILE_ARRAY[@]} changed Python file(s)"
|
||||
else
|
||||
echo "[*] Full mode: checking all files (matches CI/CD exactly)"
|
||||
FILE_ARRAY=()
|
||||
fi
|
||||
|
||||
echo ""
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Launch all checks in parallel
|
||||
if [ ${#FILE_ARRAY[@]} -eq 0 ]; then
|
||||
# Full mode: check everything
|
||||
${RUN_CMD}ruff check --fix > "$TEMP_DIR/ruff-check.log" 2>&1 &
|
||||
RUFF_CHECK_PID=$!
|
||||
RUFF_CHECK_START=$(date +%s)
|
||||
|
||||
${RUN_CMD}ruff format > "$TEMP_DIR/ruff-format.log" 2>&1 &
|
||||
RUFF_FORMAT_PID=$!
|
||||
RUFF_FORMAT_START=$(date +%s)
|
||||
|
||||
${RUN_CMD}pyright --threads 6 > "$TEMP_DIR/pyright.log" 2>&1 &
|
||||
PYRIGHT_PID=$!
|
||||
PYRIGHT_START=$(date +%s)
|
||||
|
||||
SKIP=ruff-check,ruff-format,pyright ${RUN_CMD}pre-commit run --all-files > "$TEMP_DIR/other-checks.log" 2>&1 &
|
||||
OTHER_PID=$!
|
||||
OTHER_START=$(date +%s)
|
||||
else
|
||||
# Staged or quick mode: check only specific files
|
||||
${RUN_CMD}ruff check --fix "${FILE_ARRAY[@]}" > "$TEMP_DIR/ruff-check.log" 2>&1 &
|
||||
RUFF_CHECK_PID=$!
|
||||
RUFF_CHECK_START=$(date +%s)
|
||||
|
||||
${RUN_CMD}ruff format "${FILE_ARRAY[@]}" > "$TEMP_DIR/ruff-format.log" 2>&1 &
|
||||
RUFF_FORMAT_PID=$!
|
||||
RUFF_FORMAT_START=$(date +%s)
|
||||
|
||||
# Pyright: skip in quick mode, run in staged mode
|
||||
if [ $QUICK_MODE -eq 1 ]; then
|
||||
echo "" > "$TEMP_DIR/pyright.log"
|
||||
PYRIGHT_PID=-1
|
||||
PYRIGHT_START=$(date +%s)
|
||||
else
|
||||
${RUN_CMD}pyright --threads 6 "${FILE_ARRAY[@]}" > "$TEMP_DIR/pyright.log" 2>&1 &
|
||||
PYRIGHT_PID=$!
|
||||
PYRIGHT_START=$(date +%s)
|
||||
fi
|
||||
|
||||
SKIP=ruff-check,ruff-format,pyright ${RUN_CMD}pre-commit run --files "${FILE_ARRAY[@]}" > "$TEMP_DIR/other-checks.log" 2>&1 &
|
||||
OTHER_PID=$!
|
||||
OTHER_START=$(date +%s)
|
||||
fi
|
||||
|
||||
# Track failures
|
||||
FAILED=0
|
||||
FAILED_CHECKS=""
|
||||
|
||||
# Wait for each job in order of expected completion (fastest first)
|
||||
# This allows --fail-fast to exit as soon as any check fails
|
||||
|
||||
# Ruff format is typically fastest
|
||||
spinner $RUFF_FORMAT_PID "ruff format"
|
||||
if ! wait_for_job $RUFF_FORMAT_PID "ruff format" "$TEMP_DIR/ruff-format.log" $RUFF_FORMAT_START; then
|
||||
FAILED=1
|
||||
FAILED_CHECKS="$FAILED_CHECKS ruff-format"
|
||||
if [ $FAIL_FAST -eq 1 ]; then
|
||||
kill $RUFF_CHECK_PID $PYRIGHT_PID $OTHER_PID 2>/dev/null
|
||||
wait $RUFF_CHECK_PID $PYRIGHT_PID $OTHER_PID 2>/dev/null
|
||||
echo ""
|
||||
echo "❌ Fast-fail: Exiting early due to ruff format failure"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Ruff check is second fastest
|
||||
spinner $RUFF_CHECK_PID "ruff check"
|
||||
if ! wait_for_job $RUFF_CHECK_PID "ruff check" "$TEMP_DIR/ruff-check.log" $RUFF_CHECK_START; then
|
||||
FAILED=1
|
||||
FAILED_CHECKS="$FAILED_CHECKS ruff-check"
|
||||
if [ $FAIL_FAST -eq 1 ]; then
|
||||
kill $PYRIGHT_PID $OTHER_PID 2>/dev/null
|
||||
wait $PYRIGHT_PID $OTHER_PID 2>/dev/null
|
||||
echo ""
|
||||
echo "❌ Fast-fail: Exiting early due to ruff check failure"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Pre-commit hooks are medium speed
|
||||
spinner $OTHER_PID "other pre-commit hooks"
|
||||
if ! wait_for_job $OTHER_PID "other pre-commit hooks" "$TEMP_DIR/other-checks.log" $OTHER_START; then
|
||||
FAILED=1
|
||||
FAILED_CHECKS="$FAILED_CHECKS pre-commit"
|
||||
if [ $FAIL_FAST -eq 1 ]; then
|
||||
kill $PYRIGHT_PID 2>/dev/null
|
||||
wait $PYRIGHT_PID 2>/dev/null
|
||||
echo ""
|
||||
echo "❌ Fast-fail: Exiting early due to pre-commit hooks failure"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Pyright is slowest (wait last for maximum parallelism)
|
||||
if [ $PYRIGHT_PID -ne -1 ]; then
|
||||
spinner $PYRIGHT_PID "pyright"
|
||||
if ! wait_for_job $PYRIGHT_PID "pyright" "$TEMP_DIR/pyright.log" $PYRIGHT_START; then
|
||||
FAILED=1
|
||||
FAILED_CHECKS="$FAILED_CHECKS pyright"
|
||||
fi
|
||||
else
|
||||
printf "%-25s ⏭️ (skipped in quick mode)\n" "pyright"
|
||||
fi
|
||||
|
||||
TOTAL_TIME=$(($(date +%s) - START_TIME))
|
||||
|
||||
echo ""
|
||||
if [ $FAILED -eq 1 ]; then
|
||||
echo "❌ Checks failed:$FAILED_CHECKS (${TOTAL_TIME}s total)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ All checks passed! (${TOTAL_TIME}s total)"
|
||||
exit 0
|
||||
52
.agent/vendor/browser_use/bin/setup.sh
vendored
Normal file
52
.agent/vendor/browser_use/bin/setup.sh
vendored
Normal file
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script is used to setup a local development environment for the browser-use project.
|
||||
# Usage:
|
||||
# $ ./bin/setup.sh
|
||||
|
||||
### Bash Environment Setup
|
||||
# http://redsymbol.net/articles/unofficial-bash-strict-mode/
|
||||
# https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html
|
||||
# set -o xtrace
|
||||
# set -x
|
||||
# shopt -s nullglob
|
||||
set -o errexit
|
||||
set -o errtrace
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
IFS=$'\n'
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
|
||||
if [ -f "$SCRIPT_DIR/lint.sh" ]; then
|
||||
echo "[√] already inside a cloned browser-use repo"
|
||||
else
|
||||
echo "[+] Cloning browser-use repo into current directory: $SCRIPT_DIR"
|
||||
git clone https://github.com/browser-use/browser-use
|
||||
cd browser-use
|
||||
fi
|
||||
|
||||
echo "[+] Installing uv..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
#git checkout main git pull
|
||||
echo
|
||||
echo "[+] Setting up venv"
|
||||
uv venv
|
||||
echo
|
||||
echo "[+] Installing packages in venv"
|
||||
uv sync --dev --all-extras
|
||||
echo
|
||||
echo "[i] Tip: make sure to set BROWSER_USE_LOGGING_LEVEL=debug and your LLM API keys in your .env file"
|
||||
echo
|
||||
uv pip show browser-use
|
||||
|
||||
echo "Usage:"
|
||||
echo " $ browser-use use the CLI"
|
||||
echo " or"
|
||||
echo " $ source .venv/bin/activate"
|
||||
echo " $ ipython use the library"
|
||||
echo " >>> from browser_use import BrowserSession, Agent"
|
||||
echo " >>> await Agent(task='book me a flight to fiji', browser=BrowserSession(headless=False)).run()"
|
||||
echo ""
|
||||
9
.agent/vendor/browser_use/bin/test.sh
vendored
Normal file
9
.agent/vendor/browser_use/bin/test.sh
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# This script is used to run all the main project tests that run on CI via .github/workflows/test.yaml.
|
||||
# Usage:
|
||||
# $ ./bin/test.sh
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
cd "$SCRIPT_DIR/.." || exit 1
|
||||
|
||||
exec uv run pytest --numprocesses auto tests/ci $1 $2 $3
|
||||
51
.agent/vendor/browser_use/browser_use/README.md
vendored
Normal file
51
.agent/vendor/browser_use/browser_use/README.md
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
# Codebase Structure
|
||||
|
||||
> The code structure inspired by https://github.com/Netflix/dispatch.
|
||||
|
||||
Very good structure on how to make a scalable codebase is also in [this repo](https://github.com/zhanymkanov/fastapi-best-practices).
|
||||
|
||||
Just a brief document about how we should structure our backend codebase.
|
||||
|
||||
## Code Structure
|
||||
|
||||
```markdown
|
||||
src/
|
||||
/<service name>/
|
||||
models.py
|
||||
services.py
|
||||
prompts.py
|
||||
views.py
|
||||
utils.py
|
||||
routers.py
|
||||
|
||||
/_<subservice name>/
|
||||
```
|
||||
|
||||
### Service.py
|
||||
|
||||
Always a single file, except if it becomes too long - more than ~500 lines, split it into \_subservices
|
||||
|
||||
### Views.py
|
||||
|
||||
Always split the views into two parts
|
||||
|
||||
```python
|
||||
# All
|
||||
...
|
||||
|
||||
# Requests
|
||||
...
|
||||
|
||||
# Responses
|
||||
...
|
||||
```
|
||||
|
||||
If too long → split into multiple files
|
||||
|
||||
### Prompts.py
|
||||
|
||||
Single file; if too long → split into multiple files (one prompt per file or so)
|
||||
|
||||
### Routers.py
|
||||
|
||||
Never split into more than one file
|
||||
160
.agent/vendor/browser_use/browser_use/__init__.py
vendored
Normal file
160
.agent/vendor/browser_use/browser_use/__init__.py
vendored
Normal file
@@ -0,0 +1,160 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from browser_use.logging_config import setup_logging
|
||||
|
||||
# Only set up logging if not in MCP mode or if explicitly requested
|
||||
if os.environ.get('BROWSER_USE_SETUP_LOGGING', 'true').lower() != 'false':
|
||||
from browser_use.config import CONFIG
|
||||
|
||||
# Get log file paths from config/environment
|
||||
debug_log_file = getattr(CONFIG, 'BROWSER_USE_DEBUG_LOG_FILE', None)
|
||||
info_log_file = getattr(CONFIG, 'BROWSER_USE_INFO_LOG_FILE', None)
|
||||
|
||||
# Set up logging with file handlers if specified
|
||||
logger = setup_logging(debug_log_file=debug_log_file, info_log_file=info_log_file)
|
||||
else:
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger('browser_use')
|
||||
|
||||
# Monkeypatch BaseSubprocessTransport.__del__ to handle closed event loops gracefully
|
||||
from asyncio import base_subprocess
|
||||
|
||||
_original_del = base_subprocess.BaseSubprocessTransport.__del__
|
||||
|
||||
|
||||
def _patched_del(self):
|
||||
"""Patched __del__ that handles closed event loops without throwing noisy red-herring errors like RuntimeError: Event loop is closed"""
|
||||
try:
|
||||
# Check if the event loop is closed before calling the original
|
||||
if hasattr(self, '_loop') and self._loop and self._loop.is_closed():
|
||||
# Event loop is closed, skip cleanup that requires the loop
|
||||
return
|
||||
_original_del(self)
|
||||
except RuntimeError as e:
|
||||
if 'Event loop is closed' in str(e):
|
||||
# Silently ignore this specific error
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
base_subprocess.BaseSubprocessTransport.__del__ = _patched_del
|
||||
|
||||
|
||||
# Type stubs for lazy imports - fixes linter warnings
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.agent.prompts import SystemPrompt
|
||||
from browser_use.agent.service import Agent
|
||||
|
||||
# from browser_use.agent.service import Agent
|
||||
from browser_use.agent.views import ActionModel, ActionResult, AgentHistoryList
|
||||
from browser_use.browser import BrowserProfile, BrowserSession
|
||||
from browser_use.browser import BrowserSession as Browser
|
||||
from browser_use.dom.service import DomService
|
||||
from browser_use.llm import models
|
||||
from browser_use.llm.anthropic.chat import ChatAnthropic
|
||||
from browser_use.llm.azure.chat import ChatAzureOpenAI
|
||||
from browser_use.llm.browser_use.chat import ChatBrowserUse
|
||||
from browser_use.llm.google.chat import ChatGoogle
|
||||
from browser_use.llm.groq.chat import ChatGroq
|
||||
from browser_use.llm.litellm.chat import ChatLiteLLM
|
||||
from browser_use.llm.mistral.chat import ChatMistral
|
||||
from browser_use.llm.oci_raw.chat import ChatOCIRaw
|
||||
from browser_use.llm.ollama.chat import ChatOllama
|
||||
from browser_use.llm.openai.chat import ChatOpenAI
|
||||
from browser_use.llm.vercel.chat import ChatVercel
|
||||
from browser_use.sandbox import sandbox
|
||||
from browser_use.tools.service import Controller, Tools
|
||||
|
||||
# Lazy imports mapping - only import when actually accessed
|
||||
_LAZY_IMPORTS = {
|
||||
# Agent service (heavy due to dependencies)
|
||||
# 'Agent': ('browser_use.agent.service', 'Agent'),
|
||||
'Agent': ('browser_use.agent.service', 'Agent'),
|
||||
# System prompt (moderate weight due to agent.views imports)
|
||||
'SystemPrompt': ('browser_use.agent.prompts', 'SystemPrompt'),
|
||||
# Agent views (very heavy - over 1 second!)
|
||||
'ActionModel': ('browser_use.agent.views', 'ActionModel'),
|
||||
'ActionResult': ('browser_use.agent.views', 'ActionResult'),
|
||||
'AgentHistoryList': ('browser_use.agent.views', 'AgentHistoryList'),
|
||||
'BrowserSession': ('browser_use.browser', 'BrowserSession'),
|
||||
'Browser': ('browser_use.browser', 'BrowserSession'), # Alias for BrowserSession
|
||||
'BrowserProfile': ('browser_use.browser', 'BrowserProfile'),
|
||||
# Tools (moderate weight)
|
||||
'Tools': ('browser_use.tools.service', 'Tools'),
|
||||
'Controller': ('browser_use.tools.service', 'Controller'), # alias
|
||||
# DOM service (moderate weight)
|
||||
'DomService': ('browser_use.dom.service', 'DomService'),
|
||||
# Chat models (very heavy imports)
|
||||
'ChatOpenAI': ('browser_use.llm.openai.chat', 'ChatOpenAI'),
|
||||
'ChatGoogle': ('browser_use.llm.google.chat', 'ChatGoogle'),
|
||||
'ChatAnthropic': ('browser_use.llm.anthropic.chat', 'ChatAnthropic'),
|
||||
'ChatBrowserUse': ('browser_use.llm.browser_use.chat', 'ChatBrowserUse'),
|
||||
'ChatGroq': ('browser_use.llm.groq.chat', 'ChatGroq'),
|
||||
'ChatLiteLLM': ('browser_use.llm.litellm.chat', 'ChatLiteLLM'),
|
||||
'ChatMistral': ('browser_use.llm.mistral.chat', 'ChatMistral'),
|
||||
'ChatAzureOpenAI': ('browser_use.llm.azure.chat', 'ChatAzureOpenAI'),
|
||||
'ChatOCIRaw': ('browser_use.llm.oci_raw.chat', 'ChatOCIRaw'),
|
||||
'ChatOllama': ('browser_use.llm.ollama.chat', 'ChatOllama'),
|
||||
'ChatVercel': ('browser_use.llm.vercel.chat', 'ChatVercel'),
|
||||
# LLM models module
|
||||
'models': ('browser_use.llm.models', None),
|
||||
# Sandbox execution
|
||||
'sandbox': ('browser_use.sandbox', 'sandbox'),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import mechanism - only import modules when they're actually accessed."""
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_path, attr_name = _LAZY_IMPORTS[name]
|
||||
try:
|
||||
from importlib import import_module
|
||||
|
||||
module = import_module(module_path)
|
||||
if attr_name is None:
|
||||
# For modules like 'models', return the module itself
|
||||
attr = module
|
||||
else:
|
||||
attr = getattr(module, attr_name)
|
||||
# Cache the imported attribute in the module's globals
|
||||
globals()[name] = attr
|
||||
return attr
|
||||
except ImportError as e:
|
||||
raise ImportError(f'Failed to import {name} from {module_path}: {e}') from e
|
||||
|
||||
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
||||
|
||||
|
||||
__all__ = [
|
||||
'Agent',
|
||||
'BrowserSession',
|
||||
'Browser', # Alias for BrowserSession
|
||||
'BrowserProfile',
|
||||
'Controller',
|
||||
'DomService',
|
||||
'SystemPrompt',
|
||||
'ActionResult',
|
||||
'ActionModel',
|
||||
'AgentHistoryList',
|
||||
# Chat models
|
||||
'ChatOpenAI',
|
||||
'ChatGoogle',
|
||||
'ChatAnthropic',
|
||||
'ChatBrowserUse',
|
||||
'ChatGroq',
|
||||
'ChatLiteLLM',
|
||||
'ChatMistral',
|
||||
'ChatAzureOpenAI',
|
||||
'ChatOCIRaw',
|
||||
'ChatOllama',
|
||||
'ChatVercel',
|
||||
'Tools',
|
||||
'Controller',
|
||||
# LLM models module
|
||||
'models',
|
||||
# Sandbox execution
|
||||
'sandbox',
|
||||
]
|
||||
251
.agent/vendor/browser_use/browser_use/actor/README.md
vendored
Normal file
251
.agent/vendor/browser_use/browser_use/actor/README.md
vendored
Normal file
@@ -0,0 +1,251 @@
|
||||
# Browser Actor
|
||||
|
||||
Browser Actor is a web automation library built on CDP (Chrome DevTools Protocol) that provides low-level browser automation capabilities within the browser-use ecosystem.
|
||||
|
||||
## Usage
|
||||
|
||||
### Integrated with Browser (Recommended)
|
||||
```python
|
||||
from browser_use import Browser # Alias for BrowserSession
|
||||
|
||||
# Create and start browser session
|
||||
browser = Browser()
|
||||
await browser.start()
|
||||
|
||||
# Create new tabs and navigate
|
||||
page = await browser.new_page("https://example.com")
|
||||
pages = await browser.get_pages()
|
||||
current_page = await browser.get_current_page()
|
||||
```
|
||||
|
||||
### Direct Page Access (Advanced)
|
||||
```python
|
||||
from browser_use.actor import Page, Element, Mouse
|
||||
|
||||
# Create page with existing browser session
|
||||
page = Page(browser_session, target_id, session_id)
|
||||
```
|
||||
|
||||
## Basic Operations
|
||||
|
||||
```python
|
||||
# Tab Management
|
||||
page = await browser.new_page() # Create blank tab
|
||||
page = await browser.new_page("https://example.com") # Create tab with URL
|
||||
pages = await browser.get_pages() # Get all existing tabs
|
||||
await browser.close_page(page) # Close specific tab
|
||||
|
||||
# Navigation
|
||||
await page.goto("https://example.com")
|
||||
await page.go_back()
|
||||
await page.go_forward()
|
||||
await page.reload()
|
||||
```
|
||||
|
||||
## Element Operations
|
||||
|
||||
```python
|
||||
# Find elements by CSS selector
|
||||
elements = await page.get_elements_by_css_selector("input[type='text']")
|
||||
buttons = await page.get_elements_by_css_selector("button.submit")
|
||||
|
||||
# Get element by backend node ID
|
||||
element = await page.get_element(backend_node_id=12345)
|
||||
|
||||
# AI-powered element finding (requires LLM)
|
||||
element = await page.get_element_by_prompt("search button", llm=your_llm)
|
||||
element = await page.must_get_element_by_prompt("login form", llm=your_llm)
|
||||
```
|
||||
|
||||
> **Note**: `get_elements_by_css_selector` returns immediately without waiting for visibility.
|
||||
|
||||
## Element Interactions
|
||||
|
||||
```python
|
||||
# Element actions
|
||||
await element.click(button='left', click_count=1, modifiers=['Control'])
|
||||
await element.fill("Hello World") # Clears first, then types
|
||||
await element.hover()
|
||||
await element.focus()
|
||||
await element.check() # Toggle checkbox/radio
|
||||
await element.select_option(["option1", "option2"]) # For dropdown/select
|
||||
await element.drag_to(target_element) # Drag and drop
|
||||
|
||||
# Element properties
|
||||
value = await element.get_attribute("value")
|
||||
box = await element.get_bounding_box() # Returns BoundingBox or None
|
||||
info = await element.get_basic_info() # Comprehensive element info
|
||||
screenshot_b64 = await element.screenshot(format='png')
|
||||
|
||||
# Execute JavaScript on element (this context is the element)
|
||||
text = await element.evaluate("() => this.textContent")
|
||||
await element.evaluate("(color) => this.style.backgroundColor = color", "yellow")
|
||||
classes = await element.evaluate("() => Array.from(this.classList)")
|
||||
```
|
||||
|
||||
## Mouse Operations
|
||||
|
||||
```python
|
||||
# Mouse operations
|
||||
mouse = await page.mouse
|
||||
await mouse.click(x=100, y=200, button='left', click_count=1)
|
||||
await mouse.move(x=300, y=400, steps=1)
|
||||
await mouse.down(button='left') # Press button
|
||||
await mouse.up(button='left') # Release button
|
||||
await mouse.scroll(x=0, y=100, delta_x=0, delta_y=-500) # Scroll at coordinates
|
||||
```
|
||||
|
||||
## Page Operations
|
||||
|
||||
```python
|
||||
# JavaScript evaluation
|
||||
result = await page.evaluate('() => document.title') # Must use arrow function format
|
||||
result = await page.evaluate('(x, y) => x + y', 10, 20) # With arguments
|
||||
|
||||
# Keyboard input
|
||||
await page.press("Control+A") # Key combinations supported
|
||||
await page.press("Escape") # Single keys
|
||||
|
||||
# Page controls
|
||||
await page.set_viewport_size(width=1920, height=1080)
|
||||
page_screenshot = await page.screenshot() # PNG by default
|
||||
page_png = await page.screenshot(format="png", quality=90)
|
||||
|
||||
# Page information
|
||||
url = await page.get_url()
|
||||
title = await page.get_title()
|
||||
```
|
||||
|
||||
## AI-Powered Features
|
||||
|
||||
```python
|
||||
# Content extraction using LLM
|
||||
from pydantic import BaseModel
|
||||
|
||||
class ProductInfo(BaseModel):
|
||||
name: str
|
||||
price: float
|
||||
description: str
|
||||
|
||||
# Extract structured data from current page
|
||||
products = await page.extract_content(
|
||||
"Find all products with their names, prices and descriptions",
|
||||
ProductInfo,
|
||||
llm=your_llm
|
||||
)
|
||||
```
|
||||
|
||||
## Core Classes
|
||||
|
||||
- **BrowserSession** (aliased as **Browser**): Main browser session manager with tab operations
|
||||
- **Page**: Represents a single browser tab or iframe for page-level operations
|
||||
- **Element**: Individual DOM element for interactions and property access
|
||||
- **Mouse**: Mouse operations within a page (click, move, scroll)
|
||||
|
||||
## API Reference
|
||||
|
||||
### BrowserSession Methods (Tab Management)
|
||||
- `start()` - Initialize and start the browser session
|
||||
- `stop()` - Stop the browser session (keeps browser alive)
|
||||
- `kill()` - Kill the browser process and reset all state
|
||||
- `new_page(url=None)` → `Page` - Create blank tab or navigate to URL
|
||||
- `get_pages()` → `list[Page]` - Get all available pages
|
||||
- `get_current_page()` → `Page | None` - Get the currently focused page
|
||||
- `close_page(page: Page | str)` - Close page by object or ID
|
||||
- Session management and CDP client operations
|
||||
|
||||
### Page Methods (Page Operations)
|
||||
- `get_elements_by_css_selector(selector: str)` → `list[Element]` - Find elements by CSS selector
|
||||
- `get_element(backend_node_id: int)` → `Element` - Get element by backend node ID
|
||||
- `get_element_by_prompt(prompt: str, llm)` → `Element | None` - AI-powered element finding
|
||||
- `must_get_element_by_prompt(prompt: str, llm)` → `Element` - AI element finding (raises if not found)
|
||||
- `extract_content(prompt: str, structured_output: type[T], llm)` → `T` - Extract structured data using LLM
|
||||
- `goto(url: str)` - Navigate this page to URL
|
||||
- `go_back()`, `go_forward()` - Navigate history (with error handling)
|
||||
- `reload()` - Reload the current page
|
||||
- `evaluate(page_function: str, *args)` → `str` - Execute JavaScript (MUST use (...args) => format)
|
||||
- `press(key: str)` - Press key on page (supports "Control+A" format)
|
||||
- `set_viewport_size(width: int, height: int)` - Set viewport dimensions
|
||||
- `screenshot(format='png', quality=None)` → `str` - Take page screenshot, return base64
|
||||
- `get_url()` → `str`, `get_title()` → `str` - Get page information
|
||||
- `mouse` → `Mouse` - Get mouse interface for this page
|
||||
|
||||
### Element Methods (DOM Interactions)
|
||||
- `click(button='left', click_count=1, modifiers=None)` - Click element with advanced fallbacks
|
||||
- `fill(text: str, clear=True)` - Fill input with text (clears first by default)
|
||||
- `hover()` - Hover over element
|
||||
- `focus()` - Focus the element
|
||||
- `check()` - Toggle checkbox/radio button (clicks to change state)
|
||||
- `select_option(values: str | list[str])` - Select dropdown options
|
||||
- `drag_to(target_element: Element | Position, source_position=None, target_position=None)` - Drag to target element
|
||||
- `evaluate(page_function: str, *args)` → `str` - Execute JavaScript on element (this = element)
|
||||
- `get_attribute(name: str)` → `str | None` - Get attribute value
|
||||
- `get_bounding_box()` → `BoundingBox | None` - Get element position/size
|
||||
- `screenshot(format='png', quality=None)` → `str` - Take element screenshot, return base64
|
||||
- `get_basic_info()` → `ElementInfo` - Get comprehensive element information
|
||||
|
||||
|
||||
### Mouse Methods (Coordinate-Based Operations)
|
||||
- `click(x: int, y: int, button='left', click_count=1)` - Click at coordinates
|
||||
- `move(x: int, y: int, steps=1)` - Move to coordinates
|
||||
- `down(button='left', click_count=1)`, `up(button='left', click_count=1)` - Press/release button
|
||||
- `scroll(x=0, y=0, delta_x=None, delta_y=None)` - Scroll page at coordinates
|
||||
|
||||
## Type Definitions
|
||||
|
||||
### Position
|
||||
```python
|
||||
class Position(TypedDict):
|
||||
x: float
|
||||
y: float
|
||||
```
|
||||
|
||||
### BoundingBox
|
||||
```python
|
||||
class BoundingBox(TypedDict):
|
||||
x: float
|
||||
y: float
|
||||
width: float
|
||||
height: float
|
||||
```
|
||||
|
||||
### ElementInfo
|
||||
```python
|
||||
class ElementInfo(TypedDict):
|
||||
backendNodeId: int # CDP backend node ID
|
||||
nodeId: int | None # CDP node ID
|
||||
nodeName: str # HTML tag name (e.g., "DIV", "INPUT")
|
||||
nodeType: int # DOM node type
|
||||
nodeValue: str | None # Text content for text nodes
|
||||
attributes: dict[str, str] # HTML attributes
|
||||
boundingBox: BoundingBox | None # Element position and size
|
||||
error: str | None # Error message if info retrieval failed
|
||||
```
|
||||
|
||||
## Important Usage Notes
|
||||
|
||||
**This is browser-use actor, NOT Playwright or Selenium.** Only use the methods documented above.
|
||||
|
||||
### Critical JavaScript Rules
|
||||
- `page.evaluate()` and `element.evaluate()` MUST use `(...args) => {}` arrow function format
|
||||
- Always returns string (objects are JSON-stringified automatically)
|
||||
- Use single quotes around the function: `page.evaluate('() => document.title')`
|
||||
- For complex selectors in JS: `'() => document.querySelector("input[name=\\"email\\"]")'`
|
||||
- `element.evaluate()`: `this` context is bound to the element automatically
|
||||
|
||||
### Method Restrictions
|
||||
- `get_elements_by_css_selector()` returns immediately (no automatic waiting)
|
||||
- For dropdowns: use `element.select_option()`, NOT `element.fill()`
|
||||
- Form submission: click submit button or use `page.press("Enter")`
|
||||
- No methods like: `element.submit()`, `element.dispatch_event()`, `element.get_property()`
|
||||
|
||||
### Error Prevention
|
||||
- Always verify page state changes with `page.get_url()`, `page.get_title()`
|
||||
- Use `element.get_attribute()` to check element properties
|
||||
- Validate CSS selectors before use
|
||||
- Handle navigation timing with appropriate `asyncio.sleep()` calls
|
||||
|
||||
### AI Features
|
||||
- `get_element_by_prompt()` and `extract_content()` require an LLM instance
|
||||
- These methods use DOM analysis and structured output parsing
|
||||
- Best for complex page understanding and data extraction tasks
|
||||
11
.agent/vendor/browser_use/browser_use/actor/__init__.py
vendored
Normal file
11
.agent/vendor/browser_use/browser_use/actor/__init__.py
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
"""CDP-Use High-Level Library
|
||||
|
||||
A Playwright-like library built on top of CDP (Chrome DevTools Protocol).
|
||||
"""
|
||||
|
||||
from .element import Element
|
||||
from .mouse import Mouse
|
||||
from .page import Page
|
||||
from .utils import Utils
|
||||
|
||||
__all__ = ['Page', 'Element', 'Mouse', 'Utils']
|
||||
1175
.agent/vendor/browser_use/browser_use/actor/element.py
vendored
Normal file
1175
.agent/vendor/browser_use/browser_use/actor/element.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
134
.agent/vendor/browser_use/browser_use/actor/mouse.py
vendored
Normal file
134
.agent/vendor/browser_use/browser_use/actor/mouse.py
vendored
Normal file
@@ -0,0 +1,134 @@
|
||||
"""Mouse class for mouse operations."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from cdp_use.cdp.input.commands import DispatchMouseEventParameters, SynthesizeScrollGestureParameters
|
||||
from cdp_use.cdp.input.types import MouseButton
|
||||
|
||||
from browser_use.browser.session import BrowserSession
|
||||
|
||||
|
||||
class Mouse:
|
||||
"""Mouse operations for a target."""
|
||||
|
||||
def __init__(self, browser_session: 'BrowserSession', session_id: str | None = None, target_id: str | None = None):
|
||||
self._browser_session = browser_session
|
||||
self._client = browser_session.cdp_client
|
||||
self._session_id = session_id
|
||||
self._target_id = target_id
|
||||
|
||||
async def click(self, x: int, y: int, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
|
||||
"""Click at the specified coordinates."""
|
||||
# Mouse press
|
||||
press_params: 'DispatchMouseEventParameters' = {
|
||||
'type': 'mousePressed',
|
||||
'x': x,
|
||||
'y': y,
|
||||
'button': button,
|
||||
'clickCount': click_count,
|
||||
}
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
press_params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
|
||||
# Mouse release
|
||||
release_params: 'DispatchMouseEventParameters' = {
|
||||
'type': 'mouseReleased',
|
||||
'x': x,
|
||||
'y': y,
|
||||
'button': button,
|
||||
'clickCount': click_count,
|
||||
}
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
release_params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
|
||||
async def down(self, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
|
||||
"""Press mouse button down."""
|
||||
params: 'DispatchMouseEventParameters' = {
|
||||
'type': 'mousePressed',
|
||||
'x': 0, # Will use last mouse position
|
||||
'y': 0,
|
||||
'button': button,
|
||||
'clickCount': click_count,
|
||||
}
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
|
||||
async def up(self, button: 'MouseButton' = 'left', click_count: int = 1) -> None:
|
||||
"""Release mouse button."""
|
||||
params: 'DispatchMouseEventParameters' = {
|
||||
'type': 'mouseReleased',
|
||||
'x': 0, # Will use last mouse position
|
||||
'y': 0,
|
||||
'button': button,
|
||||
'clickCount': click_count,
|
||||
}
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
|
||||
async def move(self, x: int, y: int, steps: int = 1) -> None:
|
||||
"""Move mouse to the specified coordinates."""
|
||||
# TODO: Implement smooth movement with multiple steps if needed
|
||||
_ = steps # Acknowledge parameter for future use
|
||||
|
||||
params: 'DispatchMouseEventParameters' = {'type': 'mouseMoved', 'x': x, 'y': y}
|
||||
await self._client.send.Input.dispatchMouseEvent(params, session_id=self._session_id)
|
||||
|
||||
async def scroll(self, x: int = 0, y: int = 0, delta_x: int | None = None, delta_y: int | None = None) -> None:
|
||||
"""Scroll the page using robust CDP methods."""
|
||||
if not self._session_id:
|
||||
raise RuntimeError('Session ID is required for scroll operations')
|
||||
|
||||
# Method 1: Try mouse wheel event (most reliable)
|
||||
try:
|
||||
# Get viewport dimensions
|
||||
layout_metrics = await self._client.send.Page.getLayoutMetrics(session_id=self._session_id)
|
||||
viewport_width = layout_metrics['layoutViewport']['clientWidth']
|
||||
viewport_height = layout_metrics['layoutViewport']['clientHeight']
|
||||
|
||||
# Use provided coordinates or center of viewport
|
||||
scroll_x = x if x > 0 else viewport_width / 2
|
||||
scroll_y = y if y > 0 else viewport_height / 2
|
||||
|
||||
# Calculate scroll deltas (positive = down/right)
|
||||
scroll_delta_x = delta_x or 0
|
||||
scroll_delta_y = delta_y or 0
|
||||
|
||||
# Dispatch mouse wheel event
|
||||
await self._client.send.Input.dispatchMouseEvent(
|
||||
params={
|
||||
'type': 'mouseWheel',
|
||||
'x': scroll_x,
|
||||
'y': scroll_y,
|
||||
'deltaX': scroll_delta_x,
|
||||
'deltaY': scroll_delta_y,
|
||||
},
|
||||
session_id=self._session_id,
|
||||
)
|
||||
return
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Method 2: Fallback to synthesizeScrollGesture
|
||||
try:
|
||||
params: 'SynthesizeScrollGestureParameters' = {'x': x, 'y': y, 'xDistance': delta_x or 0, 'yDistance': delta_y or 0}
|
||||
await self._client.send.Input.synthesizeScrollGesture(
|
||||
params,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
except Exception:
|
||||
# Method 3: JavaScript fallback
|
||||
scroll_js = f'window.scrollBy({delta_x or 0}, {delta_y or 0})'
|
||||
await self._client.send.Runtime.evaluate(
|
||||
params={'expression': scroll_js, 'returnByValue': True},
|
||||
session_id=self._session_id,
|
||||
)
|
||||
564
.agent/vendor/browser_use/browser_use/actor/page.py
vendored
Normal file
564
.agent/vendor/browser_use/browser_use/actor/page.py
vendored
Normal file
@@ -0,0 +1,564 @@
|
||||
"""Page class for page-level operations."""
|
||||
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from browser_use import logger
|
||||
from browser_use.actor.utils import get_key_info
|
||||
from browser_use.dom.serializer.serializer import DOMTreeSerializer
|
||||
from browser_use.dom.service import DomService
|
||||
from browser_use.llm.messages import SystemMessage, UserMessage
|
||||
|
||||
T = TypeVar('T', bound=BaseModel)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from cdp_use.cdp.dom.commands import (
|
||||
DescribeNodeParameters,
|
||||
QuerySelectorAllParameters,
|
||||
)
|
||||
from cdp_use.cdp.emulation.commands import SetDeviceMetricsOverrideParameters
|
||||
from cdp_use.cdp.input.commands import (
|
||||
DispatchKeyEventParameters,
|
||||
)
|
||||
from cdp_use.cdp.page.commands import CaptureScreenshotParameters, NavigateParameters, NavigateToHistoryEntryParameters
|
||||
from cdp_use.cdp.runtime.commands import EvaluateParameters
|
||||
from cdp_use.cdp.target.commands import (
|
||||
AttachToTargetParameters,
|
||||
GetTargetInfoParameters,
|
||||
)
|
||||
from cdp_use.cdp.target.types import TargetInfo
|
||||
|
||||
from browser_use.browser.session import BrowserSession
|
||||
from browser_use.llm.base import BaseChatModel
|
||||
|
||||
from .element import Element
|
||||
from .mouse import Mouse
|
||||
|
||||
|
||||
class Page:
|
||||
"""Page operations (tab or iframe)."""
|
||||
|
||||
def __init__(
|
||||
self, browser_session: 'BrowserSession', target_id: str, session_id: str | None = None, llm: 'BaseChatModel | None' = None
|
||||
):
|
||||
self._browser_session = browser_session
|
||||
self._client = browser_session.cdp_client
|
||||
self._target_id = target_id
|
||||
self._session_id: str | None = session_id
|
||||
self._mouse: 'Mouse | None' = None
|
||||
|
||||
self._llm = llm
|
||||
|
||||
async def _ensure_session(self) -> str:
|
||||
"""Ensure we have a session ID for this target."""
|
||||
if not self._session_id:
|
||||
params: 'AttachToTargetParameters' = {'targetId': self._target_id, 'flatten': True}
|
||||
result = await self._client.send.Target.attachToTarget(params)
|
||||
self._session_id = result['sessionId']
|
||||
|
||||
# Enable necessary domains
|
||||
import asyncio
|
||||
|
||||
await asyncio.gather(
|
||||
self._client.send.Page.enable(session_id=self._session_id),
|
||||
self._client.send.DOM.enable(session_id=self._session_id),
|
||||
self._client.send.Runtime.enable(session_id=self._session_id),
|
||||
self._client.send.Network.enable(session_id=self._session_id),
|
||||
)
|
||||
|
||||
return self._session_id
|
||||
|
||||
@property
|
||||
async def session_id(self) -> str:
|
||||
"""Get the session ID for this target.
|
||||
|
||||
@dev Pass this to an arbitrary CDP call
|
||||
"""
|
||||
return await self._ensure_session()
|
||||
|
||||
@property
|
||||
async def mouse(self) -> 'Mouse':
|
||||
"""Get the mouse interface for this target."""
|
||||
if not self._mouse:
|
||||
session_id = await self._ensure_session()
|
||||
from .mouse import Mouse
|
||||
|
||||
self._mouse = Mouse(self._browser_session, session_id, self._target_id)
|
||||
return self._mouse
|
||||
|
||||
async def reload(self) -> None:
|
||||
"""Reload the target."""
|
||||
session_id = await self._ensure_session()
|
||||
await self._client.send.Page.reload(session_id=session_id)
|
||||
|
||||
async def get_element(self, backend_node_id: int) -> 'Element':
|
||||
"""Get an element by its backend node ID."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
from .element import Element as Element_
|
||||
|
||||
return Element_(self._browser_session, backend_node_id, session_id)
|
||||
|
||||
async def evaluate(self, page_function: str, *args) -> str:
|
||||
"""Execute JavaScript in the target.
|
||||
|
||||
Args:
|
||||
page_function: JavaScript code that MUST start with (...args) => format
|
||||
*args: Arguments to pass to the function
|
||||
|
||||
Returns:
|
||||
String representation of the JavaScript execution result.
|
||||
Objects and arrays are JSON-stringified.
|
||||
"""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
# Clean and fix common JavaScript string parsing issues
|
||||
page_function = self._fix_javascript_string(page_function)
|
||||
|
||||
# Enforce arrow function format
|
||||
if not (page_function.startswith('(') and '=>' in page_function):
|
||||
raise ValueError(f'JavaScript code must start with (...args) => format. Got: {page_function[:50]}...')
|
||||
|
||||
# Build the expression - call the arrow function with provided args
|
||||
if args:
|
||||
# Convert args to JSON representation for safe passing
|
||||
import json
|
||||
|
||||
arg_strs = [json.dumps(arg) for arg in args]
|
||||
expression = f'({page_function})({", ".join(arg_strs)})'
|
||||
else:
|
||||
expression = f'({page_function})()'
|
||||
|
||||
# Debug: log the actual expression being evaluated
|
||||
logger.debug(f'Evaluating JavaScript: {repr(expression)}')
|
||||
|
||||
params: 'EvaluateParameters' = {'expression': expression, 'returnByValue': True, 'awaitPromise': True}
|
||||
result = await self._client.send.Runtime.evaluate(
|
||||
params,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
if 'exceptionDetails' in result:
|
||||
raise RuntimeError(f'JavaScript evaluation failed: {result["exceptionDetails"]}')
|
||||
|
||||
value = result.get('result', {}).get('value')
|
||||
|
||||
# Always return string representation
|
||||
if value is None:
|
||||
return ''
|
||||
elif isinstance(value, str):
|
||||
return value
|
||||
else:
|
||||
# Convert objects, numbers, booleans to string
|
||||
import json
|
||||
|
||||
try:
|
||||
return json.dumps(value) if isinstance(value, (dict, list)) else str(value)
|
||||
except (TypeError, ValueError):
|
||||
return str(value)
|
||||
|
||||
def _fix_javascript_string(self, js_code: str) -> str:
|
||||
"""Fix common JavaScript string parsing issues when written as Python string."""
|
||||
|
||||
# Just do minimal, safe cleaning
|
||||
js_code = js_code.strip()
|
||||
|
||||
# Only fix the most common and safe issues:
|
||||
|
||||
# 1. Remove obvious Python string wrapper quotes if they exist
|
||||
if (js_code.startswith('"') and js_code.endswith('"')) or (js_code.startswith("'") and js_code.endswith("'")):
|
||||
# Check if it's a wrapped string (not part of JS syntax)
|
||||
inner = js_code[1:-1]
|
||||
if inner.count('"') + inner.count("'") == 0 or '() =>' in inner:
|
||||
js_code = inner
|
||||
|
||||
# 2. Only fix clearly escaped quotes that shouldn't be
|
||||
# But be very conservative - only if we're sure it's a Python string artifact
|
||||
if '\\"' in js_code and js_code.count('\\"') > js_code.count('"'):
|
||||
js_code = js_code.replace('\\"', '"')
|
||||
if "\\'" in js_code and js_code.count("\\'") > js_code.count("'"):
|
||||
js_code = js_code.replace("\\'", "'")
|
||||
|
||||
# 3. Basic whitespace normalization only
|
||||
js_code = js_code.strip()
|
||||
|
||||
# Final validation - ensure it's not empty
|
||||
if not js_code:
|
||||
raise ValueError('JavaScript code is empty after cleaning')
|
||||
|
||||
return js_code
|
||||
|
||||
async def screenshot(self, format: str = 'png', quality: int | None = None) -> str:
|
||||
"""Take a screenshot and return base64 encoded image.
|
||||
|
||||
Args:
|
||||
format: Image format ('jpeg', 'png', 'webp')
|
||||
quality: Quality 0-100 for JPEG format
|
||||
|
||||
Returns:
|
||||
Base64-encoded image data
|
||||
"""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
params: 'CaptureScreenshotParameters' = {'format': format}
|
||||
|
||||
if quality is not None and format.lower() == 'jpeg':
|
||||
params['quality'] = quality
|
||||
|
||||
result = await self._client.send.Page.captureScreenshot(params, session_id=session_id)
|
||||
|
||||
return result['data']
|
||||
|
||||
async def press(self, key: str) -> None:
|
||||
"""Press a key on the page (sends keyboard input to the focused element or page)."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
# Handle key combinations like "Control+A"
|
||||
if '+' in key:
|
||||
parts = key.split('+')
|
||||
modifiers = parts[:-1]
|
||||
main_key = parts[-1]
|
||||
|
||||
# Calculate modifier bitmask
|
||||
modifier_value = 0
|
||||
modifier_map = {'Alt': 1, 'Control': 2, 'Meta': 4, 'Shift': 8}
|
||||
for mod in modifiers:
|
||||
modifier_value |= modifier_map.get(mod, 0)
|
||||
|
||||
# Press modifier keys
|
||||
for mod in modifiers:
|
||||
code, vk_code = get_key_info(mod)
|
||||
params: 'DispatchKeyEventParameters' = {'type': 'keyDown', 'key': mod, 'code': code}
|
||||
if vk_code is not None:
|
||||
params['windowsVirtualKeyCode'] = vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(params, session_id=session_id)
|
||||
|
||||
# Press main key with modifiers bitmask
|
||||
main_code, main_vk_code = get_key_info(main_key)
|
||||
main_down_params: 'DispatchKeyEventParameters' = {
|
||||
'type': 'keyDown',
|
||||
'key': main_key,
|
||||
'code': main_code,
|
||||
'modifiers': modifier_value,
|
||||
}
|
||||
if main_vk_code is not None:
|
||||
main_down_params['windowsVirtualKeyCode'] = main_vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(main_down_params, session_id=session_id)
|
||||
|
||||
main_up_params: 'DispatchKeyEventParameters' = {
|
||||
'type': 'keyUp',
|
||||
'key': main_key,
|
||||
'code': main_code,
|
||||
'modifiers': modifier_value,
|
||||
}
|
||||
if main_vk_code is not None:
|
||||
main_up_params['windowsVirtualKeyCode'] = main_vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(main_up_params, session_id=session_id)
|
||||
|
||||
# Release modifier keys
|
||||
for mod in reversed(modifiers):
|
||||
code, vk_code = get_key_info(mod)
|
||||
release_params: 'DispatchKeyEventParameters' = {'type': 'keyUp', 'key': mod, 'code': code}
|
||||
if vk_code is not None:
|
||||
release_params['windowsVirtualKeyCode'] = vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(release_params, session_id=session_id)
|
||||
else:
|
||||
# Simple key press
|
||||
code, vk_code = get_key_info(key)
|
||||
key_down_params: 'DispatchKeyEventParameters' = {'type': 'keyDown', 'key': key, 'code': code}
|
||||
if vk_code is not None:
|
||||
key_down_params['windowsVirtualKeyCode'] = vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(key_down_params, session_id=session_id)
|
||||
|
||||
key_up_params: 'DispatchKeyEventParameters' = {'type': 'keyUp', 'key': key, 'code': code}
|
||||
if vk_code is not None:
|
||||
key_up_params['windowsVirtualKeyCode'] = vk_code
|
||||
await self._client.send.Input.dispatchKeyEvent(key_up_params, session_id=session_id)
|
||||
|
||||
async def set_viewport_size(self, width: int, height: int) -> None:
|
||||
"""Set the viewport size."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
params: 'SetDeviceMetricsOverrideParameters' = {
|
||||
'width': width,
|
||||
'height': height,
|
||||
'deviceScaleFactor': 1.0,
|
||||
'mobile': False,
|
||||
}
|
||||
await self._client.send.Emulation.setDeviceMetricsOverride(
|
||||
params,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
# Target properties (from CDP getTargetInfo)
|
||||
async def get_target_info(self) -> 'TargetInfo':
|
||||
"""Get target information."""
|
||||
params: 'GetTargetInfoParameters' = {'targetId': self._target_id}
|
||||
result = await self._client.send.Target.getTargetInfo(params)
|
||||
return result['targetInfo']
|
||||
|
||||
async def get_url(self) -> str:
|
||||
"""Get the current URL."""
|
||||
info = await self.get_target_info()
|
||||
return info.get('url', '')
|
||||
|
||||
async def get_title(self) -> str:
|
||||
"""Get the current title."""
|
||||
info = await self.get_target_info()
|
||||
return info.get('title', '')
|
||||
|
||||
async def goto(self, url: str) -> None:
|
||||
"""Navigate this target to a URL."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
params: 'NavigateParameters' = {'url': url}
|
||||
await self._client.send.Page.navigate(params, session_id=session_id)
|
||||
|
||||
async def navigate(self, url: str) -> None:
|
||||
"""Alias for goto."""
|
||||
await self.goto(url)
|
||||
|
||||
async def go_back(self) -> None:
|
||||
"""Navigate back in history."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
try:
|
||||
# Get navigation history
|
||||
history = await self._client.send.Page.getNavigationHistory(session_id=session_id)
|
||||
current_index = history['currentIndex']
|
||||
entries = history['entries']
|
||||
|
||||
# Check if we can go back
|
||||
if current_index <= 0:
|
||||
raise RuntimeError('Cannot go back - no previous entry in history')
|
||||
|
||||
# Navigate to the previous entry
|
||||
previous_entry_id = entries[current_index - 1]['id']
|
||||
params: 'NavigateToHistoryEntryParameters' = {'entryId': previous_entry_id}
|
||||
await self._client.send.Page.navigateToHistoryEntry(params, session_id=session_id)
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Failed to navigate back: {e}')
|
||||
|
||||
async def go_forward(self) -> None:
|
||||
"""Navigate forward in history."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
try:
|
||||
# Get navigation history
|
||||
history = await self._client.send.Page.getNavigationHistory(session_id=session_id)
|
||||
current_index = history['currentIndex']
|
||||
entries = history['entries']
|
||||
|
||||
# Check if we can go forward
|
||||
if current_index >= len(entries) - 1:
|
||||
raise RuntimeError('Cannot go forward - no next entry in history')
|
||||
|
||||
# Navigate to the next entry
|
||||
next_entry_id = entries[current_index + 1]['id']
|
||||
params: 'NavigateToHistoryEntryParameters' = {'entryId': next_entry_id}
|
||||
await self._client.send.Page.navigateToHistoryEntry(params, session_id=session_id)
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Failed to navigate forward: {e}')
|
||||
|
||||
# Element finding methods (these would need to be implemented based on DOM queries)
|
||||
async def get_elements_by_css_selector(self, selector: str) -> list['Element']:
|
||||
"""Get elements by CSS selector."""
|
||||
session_id = await self._ensure_session()
|
||||
|
||||
# Get document first
|
||||
doc_result = await self._client.send.DOM.getDocument(session_id=session_id)
|
||||
document_node_id = doc_result['root']['nodeId']
|
||||
|
||||
# Query selector all
|
||||
query_params: 'QuerySelectorAllParameters' = {'nodeId': document_node_id, 'selector': selector}
|
||||
result = await self._client.send.DOM.querySelectorAll(query_params, session_id=session_id)
|
||||
|
||||
elements = []
|
||||
from .element import Element as Element_
|
||||
|
||||
# Convert node IDs to backend node IDs
|
||||
for node_id in result['nodeIds']:
|
||||
# Get backend node ID
|
||||
describe_params: 'DescribeNodeParameters' = {'nodeId': node_id}
|
||||
node_result = await self._client.send.DOM.describeNode(describe_params, session_id=session_id)
|
||||
backend_node_id = node_result['node']['backendNodeId']
|
||||
elements.append(Element_(self._browser_session, backend_node_id, session_id))
|
||||
|
||||
return elements
|
||||
|
||||
# AI METHODS
|
||||
|
||||
@property
|
||||
def dom_service(self) -> 'DomService':
|
||||
"""Get the DOM service for this target."""
|
||||
return DomService(self._browser_session)
|
||||
|
||||
async def get_element_by_prompt(self, prompt: str, llm: 'BaseChatModel | None' = None) -> 'Element | None':
|
||||
"""Get an element by a prompt."""
|
||||
await self._ensure_session()
|
||||
llm = llm or self._llm
|
||||
|
||||
if not llm:
|
||||
raise ValueError('LLM not provided')
|
||||
|
||||
dom_service = self.dom_service
|
||||
|
||||
# Lazy fetch all_frames inside get_dom_tree if needed (for cross-origin iframes)
|
||||
enhanced_dom_tree, _ = await dom_service.get_dom_tree(target_id=self._target_id, all_frames=None)
|
||||
|
||||
session_id = self._browser_session.id
|
||||
serialized_dom_state, _ = DOMTreeSerializer(
|
||||
enhanced_dom_tree, None, paint_order_filtering=True, session_id=session_id
|
||||
).serialize_accessible_elements()
|
||||
|
||||
llm_representation = serialized_dom_state.llm_representation()
|
||||
|
||||
system_message = SystemMessage(
|
||||
content="""You are an AI created to find an element on a page by a prompt.
|
||||
|
||||
<browser_state>
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, etc.)
|
||||
- text: Element description
|
||||
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
[35]<button aria-label='Submit form'>Submit</button>
|
||||
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Pure text elements without [] are not interactive.
|
||||
</browser_state>
|
||||
|
||||
Your task is to find an element index (if any) that matches the prompt (written in <prompt> tag).
|
||||
|
||||
If non of the elements matches the, return None.
|
||||
|
||||
Before you return the element index, reason about the state and elements for a sentence or two."""
|
||||
)
|
||||
|
||||
state_message = UserMessage(
|
||||
content=f"""
|
||||
<browser_state>
|
||||
{llm_representation}
|
||||
</browser_state>
|
||||
|
||||
<prompt>
|
||||
{prompt}
|
||||
</prompt>
|
||||
"""
|
||||
)
|
||||
|
||||
class ElementResponse(BaseModel):
|
||||
# thinking: str
|
||||
element_highlight_index: int | None
|
||||
|
||||
llm_response = await llm.ainvoke(
|
||||
[
|
||||
system_message,
|
||||
state_message,
|
||||
],
|
||||
output_format=ElementResponse,
|
||||
)
|
||||
|
||||
element_highlight_index = llm_response.completion.element_highlight_index
|
||||
|
||||
if element_highlight_index is None or element_highlight_index not in serialized_dom_state.selector_map:
|
||||
return None
|
||||
|
||||
element = serialized_dom_state.selector_map[element_highlight_index]
|
||||
|
||||
from .element import Element as Element_
|
||||
|
||||
return Element_(self._browser_session, element.backend_node_id, self._session_id)
|
||||
|
||||
async def must_get_element_by_prompt(self, prompt: str, llm: 'BaseChatModel | None' = None) -> 'Element':
|
||||
"""Get an element by a prompt.
|
||||
|
||||
@dev LLM can still return None, this just raises an error if the element is not found.
|
||||
"""
|
||||
element = await self.get_element_by_prompt(prompt, llm)
|
||||
if element is None:
|
||||
raise ValueError(f'No element found for prompt: {prompt}')
|
||||
|
||||
return element
|
||||
|
||||
async def extract_content(self, prompt: str, structured_output: type[T], llm: 'BaseChatModel | None' = None) -> T:
|
||||
"""Extract structured content from the current page using LLM.
|
||||
|
||||
Extracts clean markdown from the page and sends it to LLM for structured data extraction.
|
||||
|
||||
Args:
|
||||
prompt: Description of what content to extract
|
||||
structured_output: Pydantic BaseModel class defining the expected output structure
|
||||
llm: Language model to use for extraction
|
||||
|
||||
Returns:
|
||||
The structured BaseModel instance with extracted content
|
||||
"""
|
||||
llm = llm or self._llm
|
||||
|
||||
if not llm:
|
||||
raise ValueError('LLM not provided')
|
||||
|
||||
# Extract clean markdown using the same method as in tools/service.py
|
||||
try:
|
||||
content, content_stats = await self._extract_clean_markdown()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Could not extract clean markdown: {type(e).__name__}')
|
||||
|
||||
# System prompt for structured extraction
|
||||
system_prompt = """
|
||||
You are an expert at extracting structured data from the markdown of a webpage.
|
||||
|
||||
<input>
|
||||
You will be given a query and the markdown of a webpage that has been filtered to remove noise and advertising content.
|
||||
</input>
|
||||
|
||||
<instructions>
|
||||
- You are tasked to extract information from the webpage that is relevant to the query.
|
||||
- You should ONLY use the information available in the webpage to answer the query. Do not make up information or provide guess from your own knowledge.
|
||||
- If the information relevant to the query is not available in the page, your response should mention that.
|
||||
- If the query asks for all items, products, etc., make sure to directly list all of them.
|
||||
- Return the extracted content in the exact structured format specified.
|
||||
</instructions>
|
||||
|
||||
<output>
|
||||
- Your output should present ALL the information relevant to the query in the specified structured format.
|
||||
- Do not answer in conversational format - directly output the relevant information in the structured format.
|
||||
</output>
|
||||
""".strip()
|
||||
|
||||
# Build prompt with just query and content
|
||||
prompt_content = f'<query>\n{prompt}\n</query>\n\n<webpage_content>\n{content}\n</webpage_content>'
|
||||
|
||||
# Send to LLM with structured output
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
response = await asyncio.wait_for(
|
||||
llm.ainvoke(
|
||||
[SystemMessage(content=system_prompt), UserMessage(content=prompt_content)], output_format=structured_output
|
||||
),
|
||||
timeout=120.0,
|
||||
)
|
||||
|
||||
# Return the structured output BaseModel instance
|
||||
return response.completion
|
||||
except Exception as e:
|
||||
raise RuntimeError(str(e))
|
||||
|
||||
async def _extract_clean_markdown(self, extract_links: bool = False) -> tuple[str, dict]:
|
||||
"""Extract clean markdown from the current page using enhanced DOM tree.
|
||||
|
||||
Uses the shared markdown extractor for consistency with tools/service.py.
|
||||
"""
|
||||
from browser_use.dom.markdown_extractor import extract_clean_markdown
|
||||
|
||||
dom_service = self.dom_service
|
||||
return await extract_clean_markdown(dom_service=dom_service, target_id=self._target_id, extract_links=extract_links)
|
||||
41
.agent/vendor/browser_use/browser_use/actor/playground/flights.py
vendored
Normal file
41
.agent/vendor/browser_use/browser_use/actor/playground/flights.py
vendored
Normal file
@@ -0,0 +1,41 @@
|
||||
import asyncio
|
||||
|
||||
from browser_use import Agent, Browser, ChatOpenAI
|
||||
|
||||
llm = ChatOpenAI('gpt-4.1-mini')
|
||||
|
||||
|
||||
async def main():
|
||||
"""
|
||||
Main function demonstrating mixed automation with Browser-Use and Playwright.
|
||||
"""
|
||||
print('🚀 Mixed Automation with Browser-Use and Actor API')
|
||||
|
||||
browser = Browser(keep_alive=True)
|
||||
await browser.start()
|
||||
|
||||
page = await browser.get_current_page() or await browser.new_page()
|
||||
|
||||
# Go to apple wikipedia page
|
||||
await page.goto('https://www.google.com/travel/flights')
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
round_trip_button = await page.must_get_element_by_prompt('round trip button', llm)
|
||||
await round_trip_button.click()
|
||||
|
||||
one_way_button = await page.must_get_element_by_prompt('one way button', llm)
|
||||
await one_way_button.click()
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
agent = Agent(task='Find the cheapest flight from London to Paris on 2025-10-15', llm=llm, browser_session=browser)
|
||||
await agent.run()
|
||||
|
||||
input('Press Enter to continue...')
|
||||
|
||||
await browser.stop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
54
.agent/vendor/browser_use/browser_use/actor/playground/mixed_automation.py
vendored
Normal file
54
.agent/vendor/browser_use/browser_use/actor/playground/mixed_automation.py
vendored
Normal file
@@ -0,0 +1,54 @@
|
||||
import asyncio
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from browser_use import Browser, ChatOpenAI
|
||||
|
||||
TASK = """
|
||||
On the current wikipedia page, find the latest huge edit and tell me what is was about.
|
||||
"""
|
||||
|
||||
|
||||
class LatestEditFinder(BaseModel):
|
||||
"""Find the latest huge edit on the current wikipedia page."""
|
||||
|
||||
latest_edit: str
|
||||
edit_time: str
|
||||
edit_author: str
|
||||
edit_summary: str
|
||||
edit_url: str
|
||||
|
||||
|
||||
llm = ChatOpenAI('gpt-4.1-mini')
|
||||
|
||||
|
||||
async def main():
|
||||
"""
|
||||
Main function demonstrating mixed automation with Browser-Use and Playwright.
|
||||
"""
|
||||
print('🚀 Mixed Automation with Browser-Use and Actor API')
|
||||
|
||||
browser = Browser(keep_alive=True)
|
||||
await browser.start()
|
||||
|
||||
page = await browser.get_current_page() or await browser.new_page()
|
||||
|
||||
# Go to apple wikipedia page
|
||||
await page.goto('https://browser-use.github.io/stress-tests/challenges/angularjs-form.html')
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
element = await page.get_element_by_prompt('zip code input', llm)
|
||||
|
||||
print('Element found', element)
|
||||
|
||||
if element:
|
||||
await element.click()
|
||||
else:
|
||||
print('No element found')
|
||||
|
||||
await browser.stop()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
236
.agent/vendor/browser_use/browser_use/actor/playground/playground.py
vendored
Normal file
236
.agent/vendor/browser_use/browser_use/actor/playground/playground.py
vendored
Normal file
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Playground script to test the browser-use actor API.
|
||||
|
||||
This script demonstrates:
|
||||
- Starting a browser session
|
||||
- Using the actor API to navigate and interact
|
||||
- Finding elements, clicking, scrolling, JavaScript evaluation
|
||||
- Testing most of the available methods
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
from browser_use import Browser
|
||||
|
||||
# Configure logging to see what's happening
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main playground function."""
|
||||
logger.info('🚀 Starting browser actor playground')
|
||||
|
||||
# Create browser session
|
||||
browser = Browser()
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await browser.start()
|
||||
logger.info('✅ Browser session started')
|
||||
|
||||
# Navigate to Wikipedia using integrated methods
|
||||
logger.info('📖 Navigating to Wikipedia...')
|
||||
page = await browser.new_page('https://en.wikipedia.org')
|
||||
|
||||
# Get basic page info
|
||||
url = await page.get_url()
|
||||
title = await page.get_title()
|
||||
logger.info(f'📄 Page loaded: {title} ({url})')
|
||||
|
||||
# Take a screenshot
|
||||
logger.info('📸 Taking initial screenshot...')
|
||||
screenshot_b64 = await page.screenshot()
|
||||
logger.info(f'📸 Screenshot captured: {len(screenshot_b64)} bytes')
|
||||
|
||||
# Set viewport size
|
||||
logger.info('🖥️ Setting viewport to 1920x1080...')
|
||||
await page.set_viewport_size(1920, 1080)
|
||||
|
||||
# Execute some JavaScript to count links
|
||||
logger.info('🔍 Counting article links using JavaScript...')
|
||||
js_code = """() => {
|
||||
// Find all article links on the page
|
||||
const links = Array.from(document.querySelectorAll('a[href*="/wiki/"]:not([href*=":"])'))
|
||||
.filter(link => !link.href.includes('Main_Page') && !link.href.includes('Special:'));
|
||||
|
||||
return {
|
||||
total: links.length,
|
||||
sample: links.slice(0, 3).map(link => ({
|
||||
href: link.href,
|
||||
text: link.textContent.trim()
|
||||
}))
|
||||
};
|
||||
}"""
|
||||
|
||||
link_info = json.loads(await page.evaluate(js_code))
|
||||
logger.info(f'🔗 Found {link_info["total"]} article links')
|
||||
# Try to find and interact with links using CSS selector
|
||||
try:
|
||||
# Find article links on the page
|
||||
links = await page.get_elements_by_css_selector('a[href*="/wiki/"]:not([href*=":"])')
|
||||
|
||||
if links:
|
||||
logger.info(f'📋 Found {len(links)} wiki links via CSS selector')
|
||||
|
||||
# Pick the first link
|
||||
link_element = links[0]
|
||||
|
||||
# Get link info using available methods
|
||||
basic_info = await link_element.get_basic_info()
|
||||
link_href = await link_element.get_attribute('href')
|
||||
|
||||
logger.info(f'🎯 Selected element: <{basic_info["nodeName"]}>')
|
||||
logger.info(f'🔗 Link href: {link_href}')
|
||||
|
||||
if basic_info['boundingBox']:
|
||||
bbox = basic_info['boundingBox']
|
||||
logger.info(f'📏 Position: ({bbox["x"]}, {bbox["y"]}) Size: {bbox["width"]}x{bbox["height"]}')
|
||||
|
||||
# Test element interactions with robust implementations
|
||||
logger.info('👆 Hovering over the element...')
|
||||
await link_element.hover()
|
||||
await asyncio.sleep(1)
|
||||
|
||||
logger.info('🔍 Focusing the element...')
|
||||
await link_element.focus()
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Click the link using robust click method
|
||||
logger.info('🖱️ Clicking the link with robust fallbacks...')
|
||||
await link_element.click()
|
||||
|
||||
# Wait for navigation
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Get new page info
|
||||
new_url = await page.get_url()
|
||||
new_title = await page.get_title()
|
||||
logger.info(f'📄 Navigated to: {new_title}')
|
||||
logger.info(f'🌐 New URL: {new_url}')
|
||||
else:
|
||||
logger.warning('❌ No links found to interact with')
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f'⚠️ Link interaction failed: {e}')
|
||||
|
||||
# Scroll down the page
|
||||
logger.info('📜 Scrolling down the page...')
|
||||
mouse = await page.mouse
|
||||
await mouse.scroll(x=0, y=100, delta_y=500)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Test mouse operations
|
||||
logger.info('🖱️ Testing mouse operations...')
|
||||
await mouse.move(x=100, y=200)
|
||||
await mouse.click(x=150, y=250)
|
||||
|
||||
# Execute more JavaScript examples
|
||||
logger.info('🧪 Testing JavaScript evaluation...')
|
||||
|
||||
# Simple expressions
|
||||
page_height = await page.evaluate('() => document.body.scrollHeight')
|
||||
current_scroll = await page.evaluate('() => window.pageYOffset')
|
||||
logger.info(f'📏 Page height: {page_height}px, current scroll: {current_scroll}px')
|
||||
|
||||
# JavaScript with arguments
|
||||
result = await page.evaluate('(x) => x * 2', 21)
|
||||
logger.info(f'🧮 JavaScript with args: 21 * 2 = {result}')
|
||||
|
||||
# More complex JavaScript
|
||||
page_stats = json.loads(
|
||||
await page.evaluate("""() => {
|
||||
return {
|
||||
url: window.location.href,
|
||||
title: document.title,
|
||||
links: document.querySelectorAll('a').length,
|
||||
images: document.querySelectorAll('img').length,
|
||||
scrollTop: window.pageYOffset,
|
||||
viewportHeight: window.innerHeight
|
||||
};
|
||||
}""")
|
||||
)
|
||||
logger.info(f'📊 Page stats: {page_stats}')
|
||||
|
||||
# Get page title using different methods
|
||||
title_via_js = await page.evaluate('() => document.title')
|
||||
title_via_api = await page.get_title()
|
||||
logger.info(f'📝 Title via JS: "{title_via_js}"')
|
||||
logger.info(f'📝 Title via API: "{title_via_api}"')
|
||||
|
||||
# Take a final screenshot
|
||||
logger.info('📸 Taking final screenshot...')
|
||||
final_screenshot = await page.screenshot()
|
||||
logger.info(f'📸 Final screenshot: {len(final_screenshot)} bytes')
|
||||
|
||||
# Test browser navigation with error handling
|
||||
logger.info('⬅️ Testing browser back navigation...')
|
||||
try:
|
||||
await page.go_back()
|
||||
await asyncio.sleep(2)
|
||||
|
||||
back_url = await page.get_url()
|
||||
back_title = await page.get_title()
|
||||
logger.info(f'📄 After going back: {back_title}')
|
||||
logger.info(f'🌐 Back URL: {back_url}')
|
||||
except RuntimeError as e:
|
||||
logger.info(f'ℹ️ Navigation back failed as expected: {e}')
|
||||
|
||||
# Test creating new page
|
||||
logger.info('🆕 Creating new blank page...')
|
||||
new_page = await browser.new_page()
|
||||
new_page_url = await new_page.get_url()
|
||||
logger.info(f'🆕 New page created with URL: {new_page_url}')
|
||||
|
||||
# Get all pages
|
||||
all_pages = await browser.get_pages()
|
||||
logger.info(f'📑 Total pages: {len(all_pages)}')
|
||||
|
||||
# Test form interaction if we can find a form
|
||||
try:
|
||||
# Look for search input on the page
|
||||
search_inputs = await page.get_elements_by_css_selector('input[type="search"], input[name*="search"]')
|
||||
|
||||
if search_inputs:
|
||||
search_input = search_inputs[0]
|
||||
logger.info('🔍 Found search input, testing form interaction...')
|
||||
|
||||
await search_input.focus()
|
||||
await search_input.fill('test search query')
|
||||
await page.press('Enter')
|
||||
|
||||
logger.info('✅ Form interaction test completed')
|
||||
else:
|
||||
logger.info('ℹ️ No search inputs found for form testing')
|
||||
|
||||
except Exception as e:
|
||||
logger.info(f'ℹ️ Form interaction test skipped: {e}')
|
||||
|
||||
# wait 2 seconds before closing the new page
|
||||
logger.info('🕒 Waiting 2 seconds before closing the new page...')
|
||||
await asyncio.sleep(2)
|
||||
logger.info('🗑️ Closing new page...')
|
||||
await browser.close_page(new_page)
|
||||
|
||||
logger.info('✅ Playground completed successfully!')
|
||||
|
||||
input('Press Enter to continue...')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'❌ Error in playground: {e}', exc_info=True)
|
||||
|
||||
finally:
|
||||
# Clean up
|
||||
logger.info('🧹 Cleaning up...')
|
||||
try:
|
||||
await browser.stop()
|
||||
logger.info('✅ Browser session stopped')
|
||||
except Exception as e:
|
||||
logger.error(f'❌ Error stopping browser: {e}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
176
.agent/vendor/browser_use/browser_use/actor/utils.py
vendored
Normal file
176
.agent/vendor/browser_use/browser_use/actor/utils.py
vendored
Normal file
@@ -0,0 +1,176 @@
|
||||
"""Utility functions for actor operations."""
|
||||
|
||||
|
||||
class Utils:
|
||||
"""Utility functions for actor operations."""
|
||||
|
||||
@staticmethod
|
||||
def get_key_info(key: str) -> tuple[str, int | None]:
|
||||
"""Get the code and windowsVirtualKeyCode for a key.
|
||||
|
||||
Args:
|
||||
key: Key name (e.g., 'Enter', 'ArrowUp', 'a', 'A')
|
||||
|
||||
Returns:
|
||||
Tuple of (code, windowsVirtualKeyCode)
|
||||
|
||||
Reference: Windows Virtual Key Codes
|
||||
https://docs.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
|
||||
"""
|
||||
# Complete mapping of key names to (code, virtualKeyCode)
|
||||
# Based on standard Windows Virtual Key Codes
|
||||
key_map = {
|
||||
# Navigation keys
|
||||
'Backspace': ('Backspace', 8),
|
||||
'Tab': ('Tab', 9),
|
||||
'Enter': ('Enter', 13),
|
||||
'Escape': ('Escape', 27),
|
||||
'Space': ('Space', 32),
|
||||
' ': ('Space', 32),
|
||||
'PageUp': ('PageUp', 33),
|
||||
'PageDown': ('PageDown', 34),
|
||||
'End': ('End', 35),
|
||||
'Home': ('Home', 36),
|
||||
'ArrowLeft': ('ArrowLeft', 37),
|
||||
'ArrowUp': ('ArrowUp', 38),
|
||||
'ArrowRight': ('ArrowRight', 39),
|
||||
'ArrowDown': ('ArrowDown', 40),
|
||||
'Insert': ('Insert', 45),
|
||||
'Delete': ('Delete', 46),
|
||||
# Modifier keys
|
||||
'Shift': ('ShiftLeft', 16),
|
||||
'ShiftLeft': ('ShiftLeft', 16),
|
||||
'ShiftRight': ('ShiftRight', 16),
|
||||
'Control': ('ControlLeft', 17),
|
||||
'ControlLeft': ('ControlLeft', 17),
|
||||
'ControlRight': ('ControlRight', 17),
|
||||
'Alt': ('AltLeft', 18),
|
||||
'AltLeft': ('AltLeft', 18),
|
||||
'AltRight': ('AltRight', 18),
|
||||
'Meta': ('MetaLeft', 91),
|
||||
'MetaLeft': ('MetaLeft', 91),
|
||||
'MetaRight': ('MetaRight', 92),
|
||||
# Function keys F1-F24
|
||||
'F1': ('F1', 112),
|
||||
'F2': ('F2', 113),
|
||||
'F3': ('F3', 114),
|
||||
'F4': ('F4', 115),
|
||||
'F5': ('F5', 116),
|
||||
'F6': ('F6', 117),
|
||||
'F7': ('F7', 118),
|
||||
'F8': ('F8', 119),
|
||||
'F9': ('F9', 120),
|
||||
'F10': ('F10', 121),
|
||||
'F11': ('F11', 122),
|
||||
'F12': ('F12', 123),
|
||||
'F13': ('F13', 124),
|
||||
'F14': ('F14', 125),
|
||||
'F15': ('F15', 126),
|
||||
'F16': ('F16', 127),
|
||||
'F17': ('F17', 128),
|
||||
'F18': ('F18', 129),
|
||||
'F19': ('F19', 130),
|
||||
'F20': ('F20', 131),
|
||||
'F21': ('F21', 132),
|
||||
'F22': ('F22', 133),
|
||||
'F23': ('F23', 134),
|
||||
'F24': ('F24', 135),
|
||||
# Numpad keys
|
||||
'NumLock': ('NumLock', 144),
|
||||
'Numpad0': ('Numpad0', 96),
|
||||
'Numpad1': ('Numpad1', 97),
|
||||
'Numpad2': ('Numpad2', 98),
|
||||
'Numpad3': ('Numpad3', 99),
|
||||
'Numpad4': ('Numpad4', 100),
|
||||
'Numpad5': ('Numpad5', 101),
|
||||
'Numpad6': ('Numpad6', 102),
|
||||
'Numpad7': ('Numpad7', 103),
|
||||
'Numpad8': ('Numpad8', 104),
|
||||
'Numpad9': ('Numpad9', 105),
|
||||
'NumpadMultiply': ('NumpadMultiply', 106),
|
||||
'NumpadAdd': ('NumpadAdd', 107),
|
||||
'NumpadSubtract': ('NumpadSubtract', 109),
|
||||
'NumpadDecimal': ('NumpadDecimal', 110),
|
||||
'NumpadDivide': ('NumpadDivide', 111),
|
||||
# Lock keys
|
||||
'CapsLock': ('CapsLock', 20),
|
||||
'ScrollLock': ('ScrollLock', 145),
|
||||
# OEM/Punctuation keys (US keyboard layout)
|
||||
'Semicolon': ('Semicolon', 186),
|
||||
';': ('Semicolon', 186),
|
||||
'Equal': ('Equal', 187),
|
||||
'=': ('Equal', 187),
|
||||
'Comma': ('Comma', 188),
|
||||
',': ('Comma', 188),
|
||||
'Minus': ('Minus', 189),
|
||||
'-': ('Minus', 189),
|
||||
'Period': ('Period', 190),
|
||||
'.': ('Period', 190),
|
||||
'Slash': ('Slash', 191),
|
||||
'/': ('Slash', 191),
|
||||
'Backquote': ('Backquote', 192),
|
||||
'`': ('Backquote', 192),
|
||||
'BracketLeft': ('BracketLeft', 219),
|
||||
'[': ('BracketLeft', 219),
|
||||
'Backslash': ('Backslash', 220),
|
||||
'\\': ('Backslash', 220),
|
||||
'BracketRight': ('BracketRight', 221),
|
||||
']': ('BracketRight', 221),
|
||||
'Quote': ('Quote', 222),
|
||||
"'": ('Quote', 222),
|
||||
# Media/Browser keys
|
||||
'AudioVolumeMute': ('AudioVolumeMute', 173),
|
||||
'AudioVolumeDown': ('AudioVolumeDown', 174),
|
||||
'AudioVolumeUp': ('AudioVolumeUp', 175),
|
||||
'MediaTrackNext': ('MediaTrackNext', 176),
|
||||
'MediaTrackPrevious': ('MediaTrackPrevious', 177),
|
||||
'MediaStop': ('MediaStop', 178),
|
||||
'MediaPlayPause': ('MediaPlayPause', 179),
|
||||
'BrowserBack': ('BrowserBack', 166),
|
||||
'BrowserForward': ('BrowserForward', 167),
|
||||
'BrowserRefresh': ('BrowserRefresh', 168),
|
||||
'BrowserStop': ('BrowserStop', 169),
|
||||
'BrowserSearch': ('BrowserSearch', 170),
|
||||
'BrowserFavorites': ('BrowserFavorites', 171),
|
||||
'BrowserHome': ('BrowserHome', 172),
|
||||
# Additional common keys
|
||||
'Clear': ('Clear', 12),
|
||||
'Pause': ('Pause', 19),
|
||||
'Select': ('Select', 41),
|
||||
'Print': ('Print', 42),
|
||||
'Execute': ('Execute', 43),
|
||||
'PrintScreen': ('PrintScreen', 44),
|
||||
'Help': ('Help', 47),
|
||||
'ContextMenu': ('ContextMenu', 93),
|
||||
}
|
||||
|
||||
if key in key_map:
|
||||
return key_map[key]
|
||||
|
||||
# Handle alphanumeric keys dynamically
|
||||
if len(key) == 1:
|
||||
if key.isalpha():
|
||||
# Letter keys: A-Z have VK codes 65-90
|
||||
return (f'Key{key.upper()}', ord(key.upper()))
|
||||
elif key.isdigit():
|
||||
# Digit keys: 0-9 have VK codes 48-57 (same as ASCII)
|
||||
return (f'Digit{key}', ord(key))
|
||||
|
||||
# Fallback: use the key name as code, no virtual key code
|
||||
return (key, None)
|
||||
|
||||
|
||||
# Backward compatibility: provide standalone function
|
||||
def get_key_info(key: str) -> tuple[str, int | None]:
|
||||
"""Get the code and windowsVirtualKeyCode for a key.
|
||||
|
||||
Args:
|
||||
key: Key name (e.g., 'Enter', 'ArrowUp', 'a', 'A')
|
||||
|
||||
Returns:
|
||||
Tuple of (code, windowsVirtualKeyCode)
|
||||
|
||||
Reference: Windows Virtual Key Codes
|
||||
https://docs.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
|
||||
"""
|
||||
return Utils.get_key_info(key)
|
||||
284
.agent/vendor/browser_use/browser_use/agent/cloud_events.py
vendored
Normal file
284
.agent/vendor/browser_use/browser_use/agent/cloud_events.py
vendored
Normal file
@@ -0,0 +1,284 @@
|
||||
import base64
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import anyio
|
||||
from bubus import BaseEvent
|
||||
from pydantic import Field, field_validator
|
||||
from uuid_extensions import uuid7str
|
||||
|
||||
MAX_STRING_LENGTH = 500000 # 100K chars ~ 25k tokens should be enough
|
||||
MAX_URL_LENGTH = 100000
|
||||
MAX_TASK_LENGTH = 100000
|
||||
MAX_COMMENT_LENGTH = 2000
|
||||
MAX_FILE_CONTENT_SIZE = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
|
||||
class UpdateAgentTaskEvent(BaseEvent):
|
||||
# Required fields for identification
|
||||
id: str # The task ID to update
|
||||
user_id: str = Field(max_length=255) # For authorization
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
|
||||
# Optional fields that can be updated
|
||||
stopped: bool | None = None
|
||||
paused: bool | None = None
|
||||
done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
|
||||
finished_at: datetime | None = None
|
||||
agent_state: dict | None = None
|
||||
user_feedback_type: str | None = Field(None, max_length=10) # UserFeedbackType enum value as string
|
||||
user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
|
||||
gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'UpdateAgentTaskEvent':
|
||||
"""Create an UpdateAgentTaskEvent from an Agent instance"""
|
||||
if not hasattr(agent, '_task_start_time'):
|
||||
raise ValueError('Agent must have _task_start_time attribute')
|
||||
|
||||
done_output = agent.history.final_result() if agent.history else None
|
||||
if done_output and len(done_output) > MAX_STRING_LENGTH:
|
||||
done_output = done_output[:MAX_STRING_LENGTH]
|
||||
return cls(
|
||||
id=str(agent.task_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
stopped=agent.state.stopped if hasattr(agent.state, 'stopped') else False,
|
||||
paused=agent.state.paused if hasattr(agent.state, 'paused') else False,
|
||||
done_output=done_output,
|
||||
finished_at=datetime.now(timezone.utc) if agent.history and agent.history.is_done() else None,
|
||||
agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
|
||||
user_feedback_type=None,
|
||||
user_comment=None,
|
||||
gif_url=None,
|
||||
# user_feedback_type and user_comment would be set by the API/frontend
|
||||
# gif_url would be set after GIF generation if needed
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentOutputFileEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
task_id: str
|
||||
file_name: str = Field(max_length=255)
|
||||
file_content: str | None = None # Base64 encoded file content
|
||||
content_type: str | None = Field(None, max_length=100) # MIME type for file uploads
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
|
||||
@field_validator('file_content')
|
||||
@classmethod
|
||||
def validate_file_size(cls, v: str | None) -> str | None:
|
||||
"""Validate base64 file content size."""
|
||||
if v is None:
|
||||
return v
|
||||
# Remove data URL prefix if present
|
||||
if ',' in v:
|
||||
v = v.split(',')[1]
|
||||
# Estimate decoded size (base64 is ~33% larger)
|
||||
estimated_size = len(v) * 3 / 4
|
||||
if estimated_size > MAX_FILE_CONTENT_SIZE:
|
||||
raise ValueError(f'File content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
|
||||
return v
|
||||
|
||||
@classmethod
|
||||
async def from_agent_and_file(cls, agent, output_path: str) -> 'CreateAgentOutputFileEvent':
|
||||
"""Create a CreateAgentOutputFileEvent from a file path"""
|
||||
|
||||
gif_path = Path(output_path)
|
||||
if not gif_path.exists():
|
||||
raise FileNotFoundError(f'File not found: {output_path}')
|
||||
|
||||
gif_size = os.path.getsize(gif_path)
|
||||
|
||||
# Read GIF content for base64 encoding if needed
|
||||
gif_content = None
|
||||
if gif_size < 50 * 1024 * 1024: # Only read if < 50MB
|
||||
async with await anyio.open_file(gif_path, 'rb') as f:
|
||||
gif_bytes = await f.read()
|
||||
gif_content = base64.b64encode(gif_bytes).decode('utf-8')
|
||||
|
||||
return cls(
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
task_id=str(agent.task_id),
|
||||
file_name=gif_path.name,
|
||||
file_content=gif_content, # Base64 encoded
|
||||
content_type='image/gif',
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentStepEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255) # Added for authorization checks
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
agent_task_id: str
|
||||
step: int
|
||||
evaluation_previous_goal: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
memory: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
next_goal: str = Field(max_length=MAX_STRING_LENGTH)
|
||||
actions: list[dict]
|
||||
screenshot_url: str | None = Field(None, max_length=MAX_FILE_CONTENT_SIZE) # ~50MB for base64 images
|
||||
url: str = Field(default='', max_length=MAX_URL_LENGTH)
|
||||
|
||||
@field_validator('screenshot_url')
|
||||
@classmethod
|
||||
def validate_screenshot_size(cls, v: str | None) -> str | None:
|
||||
"""Validate screenshot URL or base64 content size."""
|
||||
if v is None or not v.startswith('data:'):
|
||||
return v
|
||||
# It's base64 data, check size
|
||||
if ',' in v:
|
||||
base64_part = v.split(',')[1]
|
||||
estimated_size = len(base64_part) * 3 / 4
|
||||
if estimated_size > MAX_FILE_CONTENT_SIZE:
|
||||
raise ValueError(f'Screenshot content exceeds maximum size of {MAX_FILE_CONTENT_SIZE / 1024 / 1024}MB')
|
||||
return v
|
||||
|
||||
@classmethod
|
||||
def from_agent_step(
|
||||
cls, agent, model_output, result: list, actions_data: list[dict], browser_state_summary
|
||||
) -> 'CreateAgentStepEvent':
|
||||
"""Create a CreateAgentStepEvent from agent step data"""
|
||||
# Get first action details if available
|
||||
first_action = model_output.action[0] if model_output.action else None
|
||||
|
||||
# Extract current state from model output
|
||||
current_state = model_output.current_state if hasattr(model_output, 'current_state') else None
|
||||
|
||||
# Capture screenshot as base64 data URL if available
|
||||
screenshot_url = None
|
||||
if browser_state_summary.screenshot:
|
||||
screenshot_url = f'data:image/png;base64,{browser_state_summary.screenshot}'
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug(f'📸 Including screenshot in CreateAgentStepEvent, length: {len(browser_state_summary.screenshot)}')
|
||||
else:
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.debug('📸 No screenshot in browser_state_summary for CreateAgentStepEvent')
|
||||
|
||||
return cls(
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
agent_task_id=str(agent.task_id),
|
||||
step=agent.state.n_steps,
|
||||
evaluation_previous_goal=current_state.evaluation_previous_goal if current_state else '',
|
||||
memory=current_state.memory if current_state else '',
|
||||
next_goal=current_state.next_goal if current_state else '',
|
||||
actions=actions_data, # List of action dicts
|
||||
url=browser_state_summary.url,
|
||||
screenshot_url=screenshot_url,
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentTaskEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255) # Added for authorization checks
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
agent_session_id: str
|
||||
llm_model: str = Field(max_length=200) # LLMModel enum value as string
|
||||
stopped: bool = False
|
||||
paused: bool = False
|
||||
task: str = Field(max_length=MAX_TASK_LENGTH)
|
||||
done_output: str | None = Field(None, max_length=MAX_STRING_LENGTH)
|
||||
scheduled_task_id: str | None = None
|
||||
started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
finished_at: datetime | None = None
|
||||
agent_state: dict = Field(default_factory=dict)
|
||||
user_feedback_type: str | None = Field(None, max_length=10) # UserFeedbackType enum value as string
|
||||
user_comment: str | None = Field(None, max_length=MAX_COMMENT_LENGTH)
|
||||
gif_url: str | None = Field(None, max_length=MAX_URL_LENGTH)
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'CreateAgentTaskEvent':
|
||||
"""Create a CreateAgentTaskEvent from an Agent instance"""
|
||||
return cls(
|
||||
id=str(agent.task_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
agent_session_id=str(agent.session_id),
|
||||
task=agent.task,
|
||||
llm_model=agent.llm.model_name,
|
||||
agent_state=agent.state.model_dump() if hasattr(agent.state, 'model_dump') else {},
|
||||
stopped=False,
|
||||
paused=False,
|
||||
done_output=None,
|
||||
started_at=datetime.fromtimestamp(agent._task_start_time, tz=timezone.utc),
|
||||
finished_at=None,
|
||||
user_feedback_type=None,
|
||||
user_comment=None,
|
||||
gif_url=None,
|
||||
)
|
||||
|
||||
|
||||
class CreateAgentSessionEvent(BaseEvent):
|
||||
# Model fields
|
||||
id: str = Field(default_factory=uuid7str)
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255) # Device ID for auth lookup
|
||||
browser_session_id: str = Field(max_length=255)
|
||||
browser_session_live_url: str = Field(max_length=MAX_URL_LENGTH)
|
||||
browser_session_cdp_url: str = Field(max_length=MAX_URL_LENGTH)
|
||||
browser_session_stopped: bool = False
|
||||
browser_session_stopped_at: datetime | None = None
|
||||
is_source_api: bool | None = None
|
||||
browser_state: dict = Field(default_factory=dict)
|
||||
browser_session_data: dict | None = None
|
||||
|
||||
@classmethod
|
||||
def from_agent(cls, agent) -> 'CreateAgentSessionEvent':
|
||||
"""Create a CreateAgentSessionEvent from an Agent instance"""
|
||||
return cls(
|
||||
id=str(agent.session_id),
|
||||
user_id='', # To be filled by cloud handler
|
||||
device_id=agent.cloud_sync.auth_client.device_id
|
||||
if hasattr(agent, 'cloud_sync') and agent.cloud_sync and agent.cloud_sync.auth_client
|
||||
else None,
|
||||
browser_session_id=agent.browser_session.id,
|
||||
browser_session_live_url='', # To be filled by cloud handler
|
||||
browser_session_cdp_url='', # To be filled by cloud handler
|
||||
browser_state={
|
||||
'viewport': agent.browser_profile.viewport if agent.browser_profile else {'width': 1280, 'height': 720},
|
||||
'user_agent': agent.browser_profile.user_agent if agent.browser_profile else None,
|
||||
'headless': agent.browser_profile.headless if agent.browser_profile else True,
|
||||
'initial_url': None, # Will be updated during execution
|
||||
'final_url': None, # Will be updated during execution
|
||||
'total_pages_visited': 0, # Will be updated during execution
|
||||
'session_duration_seconds': 0, # Will be updated during execution
|
||||
},
|
||||
browser_session_data={
|
||||
'cookies': [],
|
||||
'secrets': {},
|
||||
# TODO: send secrets safely so tasks can be replayed on cloud seamlessly
|
||||
# 'secrets': dict(agent.sensitive_data) if agent.sensitive_data else {},
|
||||
'allowed_domains': agent.browser_profile.allowed_domains if agent.browser_profile else [],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class UpdateAgentSessionEvent(BaseEvent):
|
||||
"""Event to update an existing agent session"""
|
||||
|
||||
# Model fields
|
||||
id: str # Session ID to update
|
||||
user_id: str = Field(max_length=255)
|
||||
device_id: str | None = Field(None, max_length=255)
|
||||
browser_session_stopped: bool | None = None
|
||||
browser_session_stopped_at: datetime | None = None
|
||||
end_reason: str | None = Field(None, max_length=100) # Why the session ended
|
||||
419
.agent/vendor/browser_use/browser_use/agent/gif.py
vendored
Normal file
419
.agent/vendor/browser_use/browser_use/agent/gif.py
vendored
Normal file
@@ -0,0 +1,419 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from browser_use.agent.views import AgentHistoryList
|
||||
from browser_use.browser.views import PLACEHOLDER_4PX_SCREENSHOT
|
||||
from browser_use.config import CONFIG
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from PIL import Image, ImageFont
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def decode_unicode_escapes_to_utf8(text: str) -> str:
|
||||
"""Handle decoding any unicode escape sequences embedded in a string (needed to render non-ASCII languages like chinese or arabic in the GIF overlay text)"""
|
||||
|
||||
if r'\u' not in text:
|
||||
# doesn't have any escape sequences that need to be decoded
|
||||
return text
|
||||
|
||||
try:
|
||||
# Try to decode Unicode escape sequences
|
||||
return text.encode('latin1').decode('unicode_escape')
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
# logger.debug(f"Failed to decode unicode escape sequences while generating gif text: {text}")
|
||||
return text
|
||||
|
||||
|
||||
def create_history_gif(
|
||||
task: str,
|
||||
history: AgentHistoryList,
|
||||
#
|
||||
output_path: str = 'agent_history.gif',
|
||||
duration: int = 3000,
|
||||
show_goals: bool = True,
|
||||
show_task: bool = True,
|
||||
show_logo: bool = False,
|
||||
font_size: int = 40,
|
||||
title_font_size: int = 56,
|
||||
goal_font_size: int = 44,
|
||||
margin: int = 40,
|
||||
line_spacing: float = 1.5,
|
||||
) -> None:
|
||||
"""Create a GIF from the agent's history with overlaid task and goal text."""
|
||||
if not history.history:
|
||||
logger.warning('No history to create GIF from')
|
||||
return
|
||||
|
||||
from PIL import Image, ImageFont
|
||||
|
||||
images = []
|
||||
|
||||
# if history is empty, we can't create a gif
|
||||
if not history.history:
|
||||
logger.warning('No history to create GIF from')
|
||||
return
|
||||
|
||||
# Get all screenshots from history (including None placeholders)
|
||||
screenshots = history.screenshots(return_none_if_not_screenshot=True)
|
||||
|
||||
if not screenshots:
|
||||
logger.warning('No screenshots found in history')
|
||||
return
|
||||
|
||||
# Find the first non-placeholder screenshot
|
||||
# A screenshot is considered a placeholder if:
|
||||
# 1. It's the exact 4px placeholder for about:blank pages, OR
|
||||
# 2. It comes from a new tab page (chrome://newtab/, about:blank, etc.)
|
||||
first_real_screenshot = None
|
||||
for screenshot in screenshots:
|
||||
if screenshot and screenshot != PLACEHOLDER_4PX_SCREENSHOT:
|
||||
first_real_screenshot = screenshot
|
||||
break
|
||||
|
||||
if not first_real_screenshot:
|
||||
logger.warning('No valid screenshots found (all are placeholders or from new tab pages)')
|
||||
return
|
||||
|
||||
# Try to load nicer fonts
|
||||
try:
|
||||
# Try different font options in order of preference
|
||||
# ArialUni is a font that comes with Office and can render most non-alphabet characters
|
||||
font_options = [
|
||||
'PingFang',
|
||||
'STHeiti Medium',
|
||||
'Microsoft YaHei', # 微软雅黑
|
||||
'SimHei', # 黑体
|
||||
'SimSun', # 宋体
|
||||
'Noto Sans CJK SC', # 思源黑体
|
||||
'WenQuanYi Micro Hei', # 文泉驿微米黑
|
||||
'Helvetica',
|
||||
'Arial',
|
||||
'DejaVuSans',
|
||||
'Verdana',
|
||||
]
|
||||
font_loaded = False
|
||||
|
||||
for font_name in font_options:
|
||||
try:
|
||||
if platform.system() == 'Windows':
|
||||
# Need to specify the abs font path on Windows
|
||||
font_name = os.path.join(CONFIG.WIN_FONT_DIR, font_name + '.ttf')
|
||||
regular_font = ImageFont.truetype(font_name, font_size)
|
||||
title_font = ImageFont.truetype(font_name, title_font_size)
|
||||
font_loaded = True
|
||||
break
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
if not font_loaded:
|
||||
raise OSError('No preferred fonts found')
|
||||
|
||||
except OSError:
|
||||
regular_font = ImageFont.load_default()
|
||||
title_font = ImageFont.load_default()
|
||||
|
||||
# Load logo if requested
|
||||
logo = None
|
||||
if show_logo:
|
||||
try:
|
||||
logo = Image.open('./static/browser-use.png')
|
||||
# Resize logo to be small (e.g., 40px height)
|
||||
logo_height = 150
|
||||
aspect_ratio = logo.width / logo.height
|
||||
logo_width = int(logo_height * aspect_ratio)
|
||||
logo = logo.resize((logo_width, logo_height), Image.Resampling.LANCZOS)
|
||||
except Exception as e:
|
||||
logger.warning(f'Could not load logo: {e}')
|
||||
|
||||
# Create task frame if requested
|
||||
if show_task and task:
|
||||
# Find the first non-placeholder screenshot for the task frame
|
||||
first_real_screenshot = None
|
||||
for item in history.history:
|
||||
screenshot_b64 = item.state.get_screenshot()
|
||||
if screenshot_b64 and screenshot_b64 != PLACEHOLDER_4PX_SCREENSHOT:
|
||||
first_real_screenshot = screenshot_b64
|
||||
break
|
||||
|
||||
if first_real_screenshot:
|
||||
task_frame = _create_task_frame(
|
||||
task,
|
||||
first_real_screenshot,
|
||||
title_font, # type: ignore
|
||||
regular_font, # type: ignore
|
||||
logo,
|
||||
line_spacing,
|
||||
)
|
||||
images.append(task_frame)
|
||||
else:
|
||||
logger.warning('No real screenshots found for task frame, skipping task frame')
|
||||
|
||||
# Process each history item with its corresponding screenshot
|
||||
for i, (item, screenshot) in enumerate(zip(history.history, screenshots), 1):
|
||||
if not screenshot:
|
||||
continue
|
||||
|
||||
# Skip placeholder screenshots from about:blank pages
|
||||
# These are 4x4 white PNGs encoded as a specific base64 string
|
||||
if screenshot == PLACEHOLDER_4PX_SCREENSHOT:
|
||||
logger.debug(f'Skipping placeholder screenshot from about:blank page at step {i}')
|
||||
continue
|
||||
|
||||
# Skip screenshots from new tab pages
|
||||
from browser_use.utils import is_new_tab_page
|
||||
|
||||
if is_new_tab_page(item.state.url):
|
||||
logger.debug(f'Skipping screenshot from new tab page ({item.state.url}) at step {i}')
|
||||
continue
|
||||
|
||||
# Convert base64 screenshot to PIL Image
|
||||
img_data = base64.b64decode(screenshot)
|
||||
image = Image.open(io.BytesIO(img_data))
|
||||
|
||||
if show_goals and item.model_output:
|
||||
image = _add_overlay_to_image(
|
||||
image=image,
|
||||
step_number=i,
|
||||
goal_text=item.model_output.current_state.next_goal,
|
||||
regular_font=regular_font, # type: ignore
|
||||
title_font=title_font, # type: ignore
|
||||
margin=margin,
|
||||
logo=logo,
|
||||
)
|
||||
|
||||
images.append(image)
|
||||
|
||||
if images:
|
||||
# Save the GIF
|
||||
images[0].save(
|
||||
output_path,
|
||||
save_all=True,
|
||||
append_images=images[1:],
|
||||
duration=duration,
|
||||
loop=0,
|
||||
optimize=False,
|
||||
)
|
||||
logger.info(f'Created GIF at {output_path}')
|
||||
else:
|
||||
logger.warning('No images found in history to create GIF')
|
||||
|
||||
|
||||
def _create_task_frame(
|
||||
task: str,
|
||||
first_screenshot: str,
|
||||
title_font: ImageFont.FreeTypeFont,
|
||||
regular_font: ImageFont.FreeTypeFont,
|
||||
logo: Image.Image | None = None,
|
||||
line_spacing: float = 1.5,
|
||||
) -> Image.Image:
|
||||
"""Create initial frame showing the task."""
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
img_data = base64.b64decode(first_screenshot)
|
||||
template = Image.open(io.BytesIO(img_data))
|
||||
image = Image.new('RGB', template.size, (0, 0, 0))
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
# Calculate vertical center of image
|
||||
center_y = image.height // 2
|
||||
|
||||
# Draw task text with dynamic font size based on task length
|
||||
margin = 140 # Increased margin
|
||||
max_width = image.width - (2 * margin)
|
||||
|
||||
# Dynamic font size calculation based on task length
|
||||
# Start with base font size (regular + 16)
|
||||
base_font_size = regular_font.size + 16
|
||||
min_font_size = max(regular_font.size - 10, 16) # Don't go below 16pt
|
||||
# Calculate dynamic font size based on text length and complexity
|
||||
# Longer texts get progressively smaller fonts
|
||||
text_length = len(task)
|
||||
if text_length > 200:
|
||||
# For very long text, reduce font size logarithmically
|
||||
font_size = max(base_font_size - int(10 * (text_length / 200)), min_font_size)
|
||||
else:
|
||||
font_size = base_font_size
|
||||
|
||||
# Try to create a larger font, but fall back to regular font if it fails
|
||||
try:
|
||||
larger_font = ImageFont.truetype(regular_font.path, font_size) # type: ignore
|
||||
except (OSError, AttributeError):
|
||||
# Fall back to regular font if .path is not available or font loading fails
|
||||
larger_font = regular_font
|
||||
|
||||
# Generate wrapped text with the calculated font size
|
||||
wrapped_text = _wrap_text(task, larger_font, max_width)
|
||||
|
||||
# Calculate line height with spacing
|
||||
line_height = larger_font.size * line_spacing
|
||||
|
||||
# Split text into lines and draw with custom spacing
|
||||
lines = wrapped_text.split('\n')
|
||||
total_height = line_height * len(lines)
|
||||
|
||||
# Start position for first line
|
||||
text_y = center_y - (total_height / 2) + 50 # Shifted down slightly
|
||||
|
||||
for line in lines:
|
||||
# Get line width for centering
|
||||
line_bbox = draw.textbbox((0, 0), line, font=larger_font)
|
||||
text_x = (image.width - (line_bbox[2] - line_bbox[0])) // 2
|
||||
|
||||
draw.text(
|
||||
(text_x, text_y),
|
||||
line,
|
||||
font=larger_font,
|
||||
fill=(255, 255, 255),
|
||||
)
|
||||
text_y += line_height
|
||||
|
||||
# Add logo if provided (top right corner)
|
||||
if logo:
|
||||
logo_margin = 20
|
||||
logo_x = image.width - logo.width - logo_margin
|
||||
image.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
|
||||
|
||||
return image
|
||||
|
||||
|
||||
def _add_overlay_to_image(
|
||||
image: Image.Image,
|
||||
step_number: int,
|
||||
goal_text: str,
|
||||
regular_font: ImageFont.FreeTypeFont,
|
||||
title_font: ImageFont.FreeTypeFont,
|
||||
margin: int,
|
||||
logo: Image.Image | None = None,
|
||||
display_step: bool = True,
|
||||
text_color: tuple[int, int, int, int] = (255, 255, 255, 255),
|
||||
text_box_color: tuple[int, int, int, int] = (0, 0, 0, 255),
|
||||
) -> Image.Image:
|
||||
"""Add step number and goal overlay to an image."""
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
goal_text = decode_unicode_escapes_to_utf8(goal_text)
|
||||
image = image.convert('RGBA')
|
||||
txt_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
|
||||
draw = ImageDraw.Draw(txt_layer)
|
||||
if display_step:
|
||||
# Add step number (bottom left)
|
||||
step_text = str(step_number)
|
||||
step_bbox = draw.textbbox((0, 0), step_text, font=title_font)
|
||||
step_width = step_bbox[2] - step_bbox[0]
|
||||
step_height = step_bbox[3] - step_bbox[1]
|
||||
|
||||
# Position step number in bottom left
|
||||
x_step = margin + 10 # Slight additional offset from edge
|
||||
y_step = image.height - margin - step_height - 10 # Slight offset from bottom
|
||||
|
||||
# Draw rounded rectangle background for step number
|
||||
padding = 20 # Increased padding
|
||||
step_bg_bbox = (
|
||||
x_step - padding,
|
||||
y_step - padding,
|
||||
x_step + step_width + padding,
|
||||
y_step + step_height + padding,
|
||||
)
|
||||
draw.rounded_rectangle(
|
||||
step_bg_bbox,
|
||||
radius=15, # Add rounded corners
|
||||
fill=text_box_color,
|
||||
)
|
||||
|
||||
# Draw step number
|
||||
draw.text(
|
||||
(x_step, y_step),
|
||||
step_text,
|
||||
font=title_font,
|
||||
fill=text_color,
|
||||
)
|
||||
|
||||
# Draw goal text (centered, bottom)
|
||||
max_width = image.width - (4 * margin)
|
||||
wrapped_goal = _wrap_text(goal_text, title_font, max_width)
|
||||
goal_bbox = draw.multiline_textbbox((0, 0), wrapped_goal, font=title_font)
|
||||
goal_width = goal_bbox[2] - goal_bbox[0]
|
||||
goal_height = goal_bbox[3] - goal_bbox[1]
|
||||
|
||||
# Center goal text horizontally, place above step number
|
||||
x_goal = (image.width - goal_width) // 2
|
||||
y_goal = y_step - goal_height - padding * 4 # More space between step and goal
|
||||
|
||||
# Draw rounded rectangle background for goal
|
||||
padding_goal = 25 # Increased padding for goal
|
||||
goal_bg_bbox = (
|
||||
x_goal - padding_goal, # Remove extra space for logo
|
||||
y_goal - padding_goal,
|
||||
x_goal + goal_width + padding_goal,
|
||||
y_goal + goal_height + padding_goal,
|
||||
)
|
||||
draw.rounded_rectangle(
|
||||
goal_bg_bbox,
|
||||
radius=15, # Add rounded corners
|
||||
fill=text_box_color,
|
||||
)
|
||||
|
||||
# Draw goal text
|
||||
draw.multiline_text(
|
||||
(x_goal, y_goal),
|
||||
wrapped_goal,
|
||||
font=title_font,
|
||||
fill=text_color,
|
||||
align='center',
|
||||
)
|
||||
|
||||
# Add logo if provided (top right corner)
|
||||
if logo:
|
||||
logo_layer = Image.new('RGBA', image.size, (0, 0, 0, 0))
|
||||
logo_margin = 20
|
||||
logo_x = image.width - logo.width - logo_margin
|
||||
logo_layer.paste(logo, (logo_x, logo_margin), logo if logo.mode == 'RGBA' else None)
|
||||
txt_layer = Image.alpha_composite(logo_layer, txt_layer)
|
||||
|
||||
# Composite and convert
|
||||
result = Image.alpha_composite(image, txt_layer)
|
||||
return result.convert('RGB')
|
||||
|
||||
|
||||
def _wrap_text(text: str, font: ImageFont.FreeTypeFont, max_width: int) -> str:
|
||||
"""
|
||||
Wrap text to fit within a given width.
|
||||
|
||||
Args:
|
||||
text: Text to wrap
|
||||
font: Font to use for text
|
||||
max_width: Maximum width in pixels
|
||||
|
||||
Returns:
|
||||
Wrapped text with newlines
|
||||
"""
|
||||
text = decode_unicode_escapes_to_utf8(text)
|
||||
words = text.split()
|
||||
lines = []
|
||||
current_line = []
|
||||
|
||||
for word in words:
|
||||
current_line.append(word)
|
||||
line = ' '.join(current_line)
|
||||
bbox = font.getbbox(line)
|
||||
if bbox[2] > max_width:
|
||||
if len(current_line) == 1:
|
||||
lines.append(current_line.pop())
|
||||
else:
|
||||
current_line.pop()
|
||||
lines.append(' '.join(current_line))
|
||||
current_line = [word]
|
||||
|
||||
if current_line:
|
||||
lines.append(' '.join(current_line))
|
||||
|
||||
return '\n'.join(lines)
|
||||
225
.agent/vendor/browser_use/browser_use/agent/judge.py
vendored
Normal file
225
.agent/vendor/browser_use/browser_use/agent/judge.py
vendored
Normal file
@@ -0,0 +1,225 @@
|
||||
"""Judge system for evaluating browser-use agent execution traces."""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from browser_use.llm.messages import (
|
||||
BaseMessage,
|
||||
ContentPartImageParam,
|
||||
ContentPartTextParam,
|
||||
ImageURL,
|
||||
SystemMessage,
|
||||
UserMessage,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _encode_image(image_path: str) -> str | None:
|
||||
"""Encode image to base64 string."""
|
||||
try:
|
||||
path = Path(image_path)
|
||||
if not path.exists():
|
||||
return None
|
||||
with open(path, 'rb') as f:
|
||||
return base64.b64encode(f.read()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to encode image {image_path}: {e}')
|
||||
return None
|
||||
|
||||
|
||||
def _truncate_text(text: str, max_length: int, from_beginning: bool = False) -> str:
|
||||
"""Truncate text to maximum length with eval system indicator."""
|
||||
if len(text) <= max_length:
|
||||
return text
|
||||
if from_beginning:
|
||||
return '...[text truncated]' + text[-max_length + 23 :]
|
||||
else:
|
||||
return text[: max_length - 23] + '...[text truncated]...'
|
||||
|
||||
|
||||
def construct_judge_messages(
|
||||
task: str,
|
||||
final_result: str,
|
||||
agent_steps: list[str],
|
||||
screenshot_paths: list[str],
|
||||
max_images: int = 10,
|
||||
ground_truth: str | None = None,
|
||||
use_vision: bool | Literal['auto'] = True,
|
||||
) -> list[BaseMessage]:
|
||||
"""
|
||||
Construct messages for judge evaluation of agent trace.
|
||||
|
||||
Args:
|
||||
task: The original task description
|
||||
final_result: The final result returned to the user
|
||||
agent_steps: List of formatted agent step descriptions
|
||||
screenshot_paths: List of screenshot file paths
|
||||
max_images: Maximum number of screenshots to include
|
||||
ground_truth: Optional ground truth answer or criteria that must be satisfied for success
|
||||
|
||||
Returns:
|
||||
List of messages for LLM judge evaluation
|
||||
"""
|
||||
task_truncated = _truncate_text(task, 40000)
|
||||
final_result_truncated = _truncate_text(final_result, 40000)
|
||||
steps_text = '\n'.join(agent_steps)
|
||||
steps_text_truncated = _truncate_text(steps_text, 40000)
|
||||
|
||||
# Only include screenshots if use_vision is not False
|
||||
encoded_images: list[ContentPartImageParam] = []
|
||||
if use_vision is not False:
|
||||
# Select last N screenshots
|
||||
selected_screenshots = screenshot_paths[-max_images:] if len(screenshot_paths) > max_images else screenshot_paths
|
||||
|
||||
# Encode screenshots
|
||||
for img_path in selected_screenshots:
|
||||
encoded = _encode_image(img_path)
|
||||
if encoded:
|
||||
encoded_images.append(
|
||||
ContentPartImageParam(
|
||||
image_url=ImageURL(
|
||||
url=f'data:image/png;base64,{encoded}',
|
||||
media_type='image/png',
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
current_date = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')
|
||||
|
||||
# System prompt for judge - conditionally add ground truth section
|
||||
ground_truth_section = ''
|
||||
if ground_truth:
|
||||
ground_truth_section = """
|
||||
**GROUND TRUTH VALIDATION (HIGHEST PRIORITY):**
|
||||
The <ground_truth> section contains verified correct information for this task. This can be:
|
||||
- **Evaluation criteria**: Specific conditions that must be met (e.g., "The success popup should show up", "Must extract exactly 5 items")
|
||||
- **Factual answers**: The correct answer to a question or information retrieval task (e.g. "10/11/24", "Paris")
|
||||
- **Expected outcomes**: What should happen after task completion (e.g., "Google Doc must be created", "File should be downloaded")
|
||||
|
||||
The ground truth takes ABSOLUTE precedence over all other evaluation criteria. If the ground truth is not satisfied by the agent's execution and final response, the verdict MUST be false.
|
||||
"""
|
||||
|
||||
system_prompt = f"""You are an expert judge evaluating browser automation agent performance.
|
||||
|
||||
<evaluation_framework>
|
||||
{ground_truth_section}
|
||||
**PRIMARY EVALUATION CRITERIA (in order of importance):**
|
||||
1. **Task Satisfaction (Most Important)**: Did the agent accomplish what the user asked for? Break down the task into the key criteria and evaluate if the agent all of them. Focus on user intent and final outcome.
|
||||
2. **Output Quality**: Is the final result in the correct format and complete? Does it match exactly what was requested?
|
||||
3. **Tool Effectiveness**: Did the browser interactions work as expected? Were tools used appropriately? How many % of the tools failed?
|
||||
4. **Agent Reasoning**: Quality of decision-making, planning, and problem-solving throughout the trajectory.
|
||||
5. **Browser Handling**: Navigation stability, error recovery, and technical execution. If the browser crashes, does not load or a captcha blocks the task, the score must be very low.
|
||||
|
||||
**VERDICT GUIDELINES:**
|
||||
- true: Task completed as requested, human-like execution, all of the users criteria were met and the agent did not make up any information.
|
||||
- false: Task not completed, or only partially completed.
|
||||
|
||||
**Examples of task completion verdict:**
|
||||
- If task asks for 10 items and agent finds 4 items correctly: false
|
||||
- If task completed to full user requirements but with some errors to improve in the trajectory: true
|
||||
- If task impossible due to captcha/login requirements: false
|
||||
- If the trajectory is ideal and the output is perfect: true
|
||||
- If the task asks to search all headphones in amazon under $100 but the agent searches all headphones and the lowest price is $150: false
|
||||
- If the task asks to research a property and create a google doc with the result but the agents only returns the results in text: false
|
||||
- If the task asks to complete an action on the page, and the agent reports that the action is completed but the screenshot or page shows the action is not actually complete: false
|
||||
- If the task asks to use a certain tool or site to complete the task but the agent completes the task without using it: false
|
||||
- If the task asks to look for a section of a page that does not exist: false
|
||||
- If the agent concludes the task is impossible but it is not: false
|
||||
- If the agent concludes the task is impossible and it truly is impossible: false
|
||||
- If the agent is unable to complete the task because no login information was provided and it is truly needed to complete the task: false
|
||||
|
||||
**FAILURE CONDITIONS (automatically set verdict to false):**
|
||||
- Blocked by captcha or missing authentication
|
||||
- Output format completely wrong or missing
|
||||
- Infinite loops or severe technical failures
|
||||
- Critical user requirements ignored
|
||||
- Page not loaded
|
||||
- Browser crashed
|
||||
- Agent could not interact with required UI elements
|
||||
- The agent moved on from a important step in the task without completing it
|
||||
- The agent made up content that is not in the screenshot or the page state
|
||||
- The agent calls done action before completing all key points of the task
|
||||
|
||||
**IMPOSSIBLE TASK DETECTION:**
|
||||
Set `impossible_task` to true when the task fundamentally could not be completed due to:
|
||||
- Vague or ambiguous task instructions that cannot be reasonably interpreted
|
||||
- Website genuinely broken or non-functional (be conservative - temporary issues don't count)
|
||||
- Required links/pages truly inaccessible (404, 403, etc.)
|
||||
- Task requires authentication/login but no credentials were provided
|
||||
- Task asks for functionality that doesn't exist on the target site
|
||||
- Other insurmountable external obstacles beyond the agent's control
|
||||
|
||||
Do NOT mark as impossible if:
|
||||
- Agent made poor decisions but task was achievable
|
||||
- Temporary page loading issues that could be retried
|
||||
- Agent didn't try the right approach
|
||||
- Website works but agent struggled with it
|
||||
|
||||
**CAPTCHA DETECTION:**
|
||||
Set `reached_captcha` to true if:
|
||||
- Screenshots show captcha challenges (reCAPTCHA, hCaptcha, etc.)
|
||||
- Agent reports being blocked by bot detection
|
||||
- Error messages indicate captcha/verification requirements
|
||||
- Any evidence the agent encountered anti-bot measures during execution
|
||||
|
||||
**IMPORTANT EVALUATION NOTES:**
|
||||
- **evaluate for action** - For each key step of the trace, double check whether the action that the agent tried to performed actually happened. If the required action did not actually occur, the verdict should be false.
|
||||
- **screenshot is not entire content** - The agent has the entire DOM content, but the screenshot is only part of the content. If the agent extracts information from the page, but you do not see it in the screenshot, you can assume this information is there.
|
||||
- **Penalize poor tool usage** - Wrong tools, inefficient approaches, ignoring available information.
|
||||
- **current date/time is {current_date}** - content with recent dates is real, not fabricated.
|
||||
- **IMPORTANT**: be very picky about the user's request - Have very high standard for the agent completing the task exactly to the user's request.
|
||||
- **IMPORTANT**: be initially doubtful of the agent's self reported success, be sure to verify that its methods are valid and fulfill the user's desires to a tee.
|
||||
|
||||
</evaluation_framework>
|
||||
|
||||
<response_format>
|
||||
Respond with EXACTLY this JSON structure (no additional text before or after):
|
||||
|
||||
{{
|
||||
"reasoning": "Breakdown of user task into key points. Detailed analysis covering: what went well, what didn't work, trajectory quality assessment, tool usage evaluation, output quality review, and overall user satisfaction prediction.",
|
||||
"verdict": true or false,
|
||||
"failure_reason": "Max 5 sentences explanation of why the task was not completed successfully in case of failure. If verdict is true, use an empty string.",
|
||||
"impossible_task": true or false,
|
||||
"reached_captcha": true or false
|
||||
}}
|
||||
</response_format>
|
||||
"""
|
||||
|
||||
# Build user prompt with conditional ground truth section
|
||||
ground_truth_prompt = ''
|
||||
if ground_truth:
|
||||
ground_truth_prompt = f"""
|
||||
<ground_truth>
|
||||
{ground_truth}
|
||||
</ground_truth>
|
||||
"""
|
||||
|
||||
user_prompt = f"""
|
||||
<task>
|
||||
{task_truncated or 'No task provided'}
|
||||
</task>
|
||||
{ground_truth_prompt}
|
||||
<agent_trajectory>
|
||||
{steps_text_truncated or 'No agent trajectory provided'}
|
||||
</agent_trajectory>
|
||||
|
||||
<final_result>
|
||||
{final_result_truncated or 'No final result provided'}
|
||||
</final_result>
|
||||
|
||||
{len(encoded_images)} screenshots from execution are attached.
|
||||
|
||||
Evaluate this agent execution given the criteria and respond with the exact JSON structure requested."""
|
||||
|
||||
# Build messages with screenshots
|
||||
content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=user_prompt)]
|
||||
content_parts.extend(encoded_images)
|
||||
|
||||
return [
|
||||
SystemMessage(content=system_prompt),
|
||||
UserMessage(content=content_parts),
|
||||
]
|
||||
608
.agent/vendor/browser_use/browser_use/agent/message_manager/service.py
vendored
Normal file
608
.agent/vendor/browser_use/browser_use/agent/message_manager/service.py
vendored
Normal file
@@ -0,0 +1,608 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
from browser_use.agent.message_manager.views import (
|
||||
HistoryItem,
|
||||
)
|
||||
from browser_use.agent.prompts import AgentMessagePrompt
|
||||
from browser_use.agent.views import (
|
||||
ActionResult,
|
||||
AgentOutput,
|
||||
AgentStepInfo,
|
||||
MessageCompactionSettings,
|
||||
MessageManagerState,
|
||||
)
|
||||
from browser_use.browser.views import BrowserStateSummary
|
||||
from browser_use.filesystem.file_system import FileSystem
|
||||
from browser_use.llm.base import BaseChatModel
|
||||
from browser_use.llm.messages import (
|
||||
BaseMessage,
|
||||
ContentPartImageParam,
|
||||
ContentPartTextParam,
|
||||
SystemMessage,
|
||||
UserMessage,
|
||||
)
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import match_url_with_domain_pattern, time_execution_sync
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ========== Logging Helper Functions ==========
|
||||
# These functions are used ONLY for formatting debug log output.
|
||||
# They do NOT affect the actual message content sent to the LLM.
|
||||
# All logging functions start with _log_ for easy identification.
|
||||
|
||||
|
||||
def _log_get_message_emoji(message: BaseMessage) -> str:
|
||||
"""Get emoji for a message type - used only for logging display"""
|
||||
emoji_map = {
|
||||
'UserMessage': '💬',
|
||||
'SystemMessage': '🧠',
|
||||
'AssistantMessage': '🔨',
|
||||
}
|
||||
return emoji_map.get(message.__class__.__name__, '🎮')
|
||||
|
||||
|
||||
def _log_format_message_line(message: BaseMessage, content: str, is_last_message: bool, terminal_width: int) -> list[str]:
|
||||
"""Format a single message for logging display"""
|
||||
try:
|
||||
lines = []
|
||||
|
||||
# Get emoji and token info
|
||||
emoji = _log_get_message_emoji(message)
|
||||
# token_str = str(message.metadata.tokens).rjust(4)
|
||||
# TODO: fix the token count
|
||||
token_str = '??? (TODO)'
|
||||
prefix = f'{emoji}[{token_str}]: '
|
||||
|
||||
# Calculate available width (emoji=2 visual cols + [token]: =8 chars)
|
||||
content_width = terminal_width - 10
|
||||
|
||||
# Handle last message wrapping
|
||||
if is_last_message and len(content) > content_width:
|
||||
# Find a good break point
|
||||
break_point = content.rfind(' ', 0, content_width)
|
||||
if break_point > content_width * 0.7: # Keep at least 70% of line
|
||||
first_line = content[:break_point]
|
||||
rest = content[break_point + 1 :]
|
||||
else:
|
||||
# No good break point, just truncate
|
||||
first_line = content[:content_width]
|
||||
rest = content[content_width:]
|
||||
|
||||
lines.append(prefix + first_line)
|
||||
|
||||
# Second line with 10-space indent
|
||||
if rest:
|
||||
if len(rest) > terminal_width - 10:
|
||||
rest = rest[: terminal_width - 10]
|
||||
lines.append(' ' * 10 + rest)
|
||||
else:
|
||||
# Single line - truncate if needed
|
||||
if len(content) > content_width:
|
||||
content = content[:content_width]
|
||||
lines.append(prefix + content)
|
||||
|
||||
return lines
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to format message line for logging: {e}')
|
||||
# Return a simple fallback line
|
||||
return ['❓[ ?]: [Error formatting message]']
|
||||
|
||||
|
||||
# ========== End of Logging Helper Functions ==========
|
||||
|
||||
|
||||
class MessageManager:
|
||||
vision_detail_level: Literal['auto', 'low', 'high']
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task: str,
|
||||
system_message: SystemMessage,
|
||||
file_system: FileSystem,
|
||||
state: MessageManagerState = MessageManagerState(),
|
||||
use_thinking: bool = True,
|
||||
include_attributes: list[str] | None = None,
|
||||
sensitive_data: dict[str, str | dict[str, str]] | None = None,
|
||||
max_history_items: int | None = None,
|
||||
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
|
||||
include_tool_call_examples: bool = False,
|
||||
include_recent_events: bool = False,
|
||||
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
|
||||
llm_screenshot_size: tuple[int, int] | None = None,
|
||||
max_clickable_elements_length: int = 40000,
|
||||
):
|
||||
self.task = task
|
||||
self.state = state
|
||||
self.system_prompt = system_message
|
||||
self.file_system = file_system
|
||||
self.sensitive_data_description = ''
|
||||
self.use_thinking = use_thinking
|
||||
self.max_history_items = max_history_items
|
||||
self.vision_detail_level = vision_detail_level
|
||||
self.include_tool_call_examples = include_tool_call_examples
|
||||
self.include_recent_events = include_recent_events
|
||||
self.sample_images = sample_images
|
||||
self.llm_screenshot_size = llm_screenshot_size
|
||||
self.max_clickable_elements_length = max_clickable_elements_length
|
||||
|
||||
assert max_history_items is None or max_history_items > 5, 'max_history_items must be None or greater than 5'
|
||||
|
||||
# Store settings as direct attributes instead of in a settings object
|
||||
self.include_attributes = include_attributes or []
|
||||
self.sensitive_data = sensitive_data
|
||||
self.last_input_messages = []
|
||||
self.last_state_message_text: str | None = None
|
||||
# Only initialize messages if state is empty
|
||||
if len(self.state.history.get_messages()) == 0:
|
||||
self._set_message_with_type(self.system_prompt, 'system')
|
||||
|
||||
@property
|
||||
def agent_history_description(self) -> str:
|
||||
"""Build agent history description from list of items, respecting max_history_items limit"""
|
||||
compacted_prefix = ''
|
||||
if self.state.compacted_memory:
|
||||
compacted_prefix = (
|
||||
'<compacted_memory>\n'
|
||||
'<!-- Summary of prior steps. Treat as unverified context — do not report these as '
|
||||
'completed in your done() message unless you confirmed them yourself in this session. -->\n'
|
||||
f'{self.state.compacted_memory}\n'
|
||||
'</compacted_memory>\n'
|
||||
)
|
||||
|
||||
if self.max_history_items is None:
|
||||
# Include all items
|
||||
return compacted_prefix + '\n'.join(item.to_string() for item in self.state.agent_history_items)
|
||||
|
||||
total_items = len(self.state.agent_history_items)
|
||||
|
||||
# If we have fewer items than the limit, just return all items
|
||||
if total_items <= self.max_history_items:
|
||||
return compacted_prefix + '\n'.join(item.to_string() for item in self.state.agent_history_items)
|
||||
|
||||
# We have more items than the limit, so we need to omit some
|
||||
omitted_count = total_items - self.max_history_items
|
||||
|
||||
# Show first item + omitted message + most recent (max_history_items - 1) items
|
||||
# The omitted message doesn't count against the limit, only real history items do
|
||||
recent_items_count = self.max_history_items - 1 # -1 for first item
|
||||
|
||||
items_to_include = [
|
||||
self.state.agent_history_items[0].to_string(), # Keep first item (initialization)
|
||||
f'<sys>[... {omitted_count} previous steps omitted...]</sys>',
|
||||
]
|
||||
# Add most recent items
|
||||
items_to_include.extend([item.to_string() for item in self.state.agent_history_items[-recent_items_count:]])
|
||||
|
||||
return compacted_prefix + '\n'.join(items_to_include)
|
||||
|
||||
def add_new_task(self, new_task: str) -> None:
|
||||
new_task = '<follow_up_user_request> ' + new_task.strip() + ' </follow_up_user_request>'
|
||||
if '<initial_user_request>' not in self.task:
|
||||
self.task = '<initial_user_request>' + self.task + '</initial_user_request>'
|
||||
self.task += '\n' + new_task
|
||||
task_update_item = HistoryItem(system_message=new_task)
|
||||
self.state.agent_history_items.append(task_update_item)
|
||||
|
||||
def prepare_step_state(
|
||||
self,
|
||||
browser_state_summary: BrowserStateSummary,
|
||||
model_output: AgentOutput | None = None,
|
||||
result: list[ActionResult] | None = None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
sensitive_data=None,
|
||||
) -> None:
|
||||
"""Prepare state for the next LLM call without building the final state message."""
|
||||
self.state.history.context_messages.clear()
|
||||
self._update_agent_history_description(model_output, result, step_info)
|
||||
|
||||
effective_sensitive_data = sensitive_data if sensitive_data is not None else self.sensitive_data
|
||||
if effective_sensitive_data is not None:
|
||||
self.sensitive_data = effective_sensitive_data
|
||||
self.sensitive_data_description = self._get_sensitive_data_description(browser_state_summary.url)
|
||||
|
||||
async def maybe_compact_messages(
|
||||
self,
|
||||
llm: BaseChatModel | None,
|
||||
settings: MessageCompactionSettings | None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
) -> bool:
|
||||
"""Summarize older history into a compact memory block.
|
||||
|
||||
Step interval is the primary trigger; char count is a minimum floor.
|
||||
"""
|
||||
if not settings or not settings.enabled:
|
||||
return False
|
||||
if llm is None:
|
||||
return False
|
||||
if step_info is None:
|
||||
return False
|
||||
|
||||
# Step cadence gate
|
||||
steps_since = step_info.step_number - (self.state.last_compaction_step or 0)
|
||||
if steps_since < settings.compact_every_n_steps:
|
||||
return False
|
||||
|
||||
# Char floor gate
|
||||
history_items = self.state.agent_history_items
|
||||
full_history_text = '\n'.join(item.to_string() for item in history_items).strip()
|
||||
trigger_char_count = settings.trigger_char_count or 40000
|
||||
if len(full_history_text) < trigger_char_count:
|
||||
return False
|
||||
|
||||
logger.debug(f'Compacting message history (items={len(history_items)}, chars={len(full_history_text)})')
|
||||
|
||||
# Build compaction input
|
||||
compaction_sections = []
|
||||
if self.state.compacted_memory:
|
||||
compaction_sections.append(
|
||||
f'<previous_compacted_memory>\n{self.state.compacted_memory}\n</previous_compacted_memory>'
|
||||
)
|
||||
compaction_sections.append(f'<agent_history>\n{full_history_text}\n</agent_history>')
|
||||
if settings.include_read_state and self.state.read_state_description:
|
||||
compaction_sections.append(f'<read_state>\n{self.state.read_state_description}\n</read_state>')
|
||||
compaction_input = '\n\n'.join(compaction_sections)
|
||||
|
||||
if self.sensitive_data:
|
||||
filtered = self._filter_sensitive_data(UserMessage(content=compaction_input))
|
||||
compaction_input = filtered.text
|
||||
|
||||
system_prompt = (
|
||||
'You are summarizing an agent run for prompt compaction.\n'
|
||||
'Capture task requirements, key facts, decisions, partial progress, errors, and next steps.\n'
|
||||
'Preserve important entities, values, URLs, and file paths.\n'
|
||||
'CRITICAL: Only mark a step as completed if you see explicit success confirmation in the history. '
|
||||
'If a step was started but not explicitly confirmed complete, mark it as "IN-PROGRESS". '
|
||||
'Never infer completion from context — only report what was confirmed.\n'
|
||||
'Return plain text only. Do not include tool calls or JSON.'
|
||||
)
|
||||
if settings.summary_max_chars:
|
||||
system_prompt += f' Keep under {settings.summary_max_chars} characters if possible.'
|
||||
|
||||
messages = [SystemMessage(content=system_prompt), UserMessage(content=compaction_input)]
|
||||
try:
|
||||
response = await llm.ainvoke(messages)
|
||||
summary = (response.completion or '').strip()
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to compact messages: {e}')
|
||||
return False
|
||||
|
||||
if not summary:
|
||||
return False
|
||||
|
||||
if settings.summary_max_chars and len(summary) > settings.summary_max_chars:
|
||||
summary = summary[: settings.summary_max_chars].rstrip() + '…'
|
||||
|
||||
self.state.compacted_memory = summary
|
||||
self.state.compaction_count += 1
|
||||
self.state.last_compaction_step = step_info.step_number
|
||||
|
||||
# Keep first item + most recent items
|
||||
keep_last = max(0, settings.keep_last_items)
|
||||
if len(history_items) > keep_last + 1:
|
||||
if keep_last == 0:
|
||||
self.state.agent_history_items = [history_items[0]]
|
||||
else:
|
||||
self.state.agent_history_items = [history_items[0]] + history_items[-keep_last:]
|
||||
|
||||
logger.debug(f'Compaction complete (summary_chars={len(summary)}, history_items={len(self.state.agent_history_items)})')
|
||||
|
||||
return True
|
||||
|
||||
def _update_agent_history_description(
|
||||
self,
|
||||
model_output: AgentOutput | None = None,
|
||||
result: list[ActionResult] | None = None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
) -> None:
|
||||
"""Update the agent history description"""
|
||||
|
||||
if result is None:
|
||||
result = []
|
||||
step_number = step_info.step_number if step_info else None
|
||||
|
||||
self.state.read_state_description = ''
|
||||
self.state.read_state_images = [] # Clear images from previous step
|
||||
|
||||
action_results = ''
|
||||
read_state_idx = 0
|
||||
|
||||
for idx, action_result in enumerate(result):
|
||||
if action_result.include_extracted_content_only_once and action_result.extracted_content:
|
||||
self.state.read_state_description += (
|
||||
f'<read_state_{read_state_idx}>\n{action_result.extracted_content}\n</read_state_{read_state_idx}>\n'
|
||||
)
|
||||
read_state_idx += 1
|
||||
logger.debug(f'Added extracted_content to read_state_description: {action_result.extracted_content}')
|
||||
|
||||
# Store images for one-time inclusion in the next message
|
||||
if action_result.images:
|
||||
self.state.read_state_images.extend(action_result.images)
|
||||
logger.debug(f'Added {len(action_result.images)} image(s) to read_state_images')
|
||||
|
||||
if action_result.long_term_memory:
|
||||
action_results += f'{action_result.long_term_memory}\n'
|
||||
logger.debug(f'Added long_term_memory to action_results: {action_result.long_term_memory}')
|
||||
elif action_result.extracted_content and not action_result.include_extracted_content_only_once:
|
||||
action_results += f'{action_result.extracted_content}\n'
|
||||
logger.debug(f'Added extracted_content to action_results: {action_result.extracted_content}')
|
||||
|
||||
if action_result.error:
|
||||
if len(action_result.error) > 200:
|
||||
error_text = action_result.error[:100] + '......' + action_result.error[-100:]
|
||||
else:
|
||||
error_text = action_result.error
|
||||
action_results += f'{error_text}\n'
|
||||
logger.debug(f'Added error to action_results: {error_text}')
|
||||
|
||||
# Simple 60k character limit for read_state_description
|
||||
MAX_CONTENT_SIZE = 60000
|
||||
if len(self.state.read_state_description) > MAX_CONTENT_SIZE:
|
||||
self.state.read_state_description = (
|
||||
self.state.read_state_description[:MAX_CONTENT_SIZE] + '\n... [Content truncated at 60k characters]'
|
||||
)
|
||||
logger.debug(f'Truncated read_state_description to {MAX_CONTENT_SIZE} characters')
|
||||
|
||||
self.state.read_state_description = self.state.read_state_description.strip('\n')
|
||||
|
||||
if action_results:
|
||||
action_results = f'Result\n{action_results}'
|
||||
action_results = action_results.strip('\n') if action_results else None
|
||||
|
||||
# Simple 60k character limit for action_results
|
||||
if action_results and len(action_results) > MAX_CONTENT_SIZE:
|
||||
action_results = action_results[:MAX_CONTENT_SIZE] + '\n... [Content truncated at 60k characters]'
|
||||
logger.debug(f'Truncated action_results to {MAX_CONTENT_SIZE} characters')
|
||||
|
||||
# Build the history item
|
||||
if model_output is None:
|
||||
# Add history item for initial actions (step 0) or errors (step > 0)
|
||||
if step_number is not None:
|
||||
if step_number == 0 and action_results:
|
||||
# Step 0 with initial action results
|
||||
history_item = HistoryItem(step_number=step_number, action_results=action_results)
|
||||
self.state.agent_history_items.append(history_item)
|
||||
elif step_number > 0:
|
||||
# Error case for steps > 0
|
||||
history_item = HistoryItem(step_number=step_number, error='Agent failed to output in the right format.')
|
||||
self.state.agent_history_items.append(history_item)
|
||||
else:
|
||||
history_item = HistoryItem(
|
||||
step_number=step_number,
|
||||
evaluation_previous_goal=model_output.current_state.evaluation_previous_goal,
|
||||
memory=model_output.current_state.memory,
|
||||
next_goal=model_output.current_state.next_goal,
|
||||
action_results=action_results,
|
||||
)
|
||||
self.state.agent_history_items.append(history_item)
|
||||
|
||||
def _get_sensitive_data_description(self, current_page_url) -> str:
|
||||
sensitive_data = self.sensitive_data
|
||||
if not sensitive_data:
|
||||
return ''
|
||||
|
||||
# Collect placeholders for sensitive data
|
||||
placeholders: set[str] = set()
|
||||
|
||||
for key, value in sensitive_data.items():
|
||||
if isinstance(value, dict):
|
||||
# New format: {domain: {key: value}}
|
||||
if current_page_url and match_url_with_domain_pattern(current_page_url, key, True):
|
||||
placeholders.update(value.keys())
|
||||
else:
|
||||
# Old format: {key: value}
|
||||
placeholders.add(key)
|
||||
|
||||
if placeholders:
|
||||
placeholder_list = sorted(list(placeholders))
|
||||
# Format as bullet points for clarity
|
||||
formatted_placeholders = '\n'.join(f' - {p}' for p in placeholder_list)
|
||||
|
||||
info = 'SENSITIVE DATA - Use these placeholders for secure input:\n'
|
||||
info += f'{formatted_placeholders}\n\n'
|
||||
info += 'IMPORTANT: When entering sensitive values, you MUST wrap the placeholder name in <secret> tags.\n'
|
||||
info += f'Example: To enter the value for "{placeholder_list[0]}", use: <secret>{placeholder_list[0]}</secret>\n'
|
||||
info += 'The system will automatically replace these tags with the actual secret values.'
|
||||
return info
|
||||
|
||||
return ''
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='create_state_messages')
|
||||
@time_execution_sync('--create_state_messages')
|
||||
def create_state_messages(
|
||||
self,
|
||||
browser_state_summary: BrowserStateSummary,
|
||||
model_output: AgentOutput | None = None,
|
||||
result: list[ActionResult] | None = None,
|
||||
step_info: AgentStepInfo | None = None,
|
||||
use_vision: bool | Literal['auto'] = True,
|
||||
page_filtered_actions: str | None = None,
|
||||
sensitive_data=None,
|
||||
available_file_paths: list[str] | None = None, # Always pass current available_file_paths
|
||||
unavailable_skills_info: str | None = None, # Information about skills that cannot be used yet
|
||||
plan_description: str | None = None, # Rendered plan for injection into agent state
|
||||
skip_state_update: bool = False,
|
||||
) -> None:
|
||||
"""Create single state message with all content"""
|
||||
|
||||
if not skip_state_update:
|
||||
self.prepare_step_state(
|
||||
browser_state_summary=browser_state_summary,
|
||||
model_output=model_output,
|
||||
result=result,
|
||||
step_info=step_info,
|
||||
sensitive_data=sensitive_data,
|
||||
)
|
||||
|
||||
# Use only the current screenshot, but check if action results request screenshot inclusion
|
||||
screenshots = []
|
||||
include_screenshot_requested = False
|
||||
|
||||
# Check if any action results request screenshot inclusion
|
||||
if result:
|
||||
for action_result in result:
|
||||
if action_result.metadata and action_result.metadata.get('include_screenshot'):
|
||||
include_screenshot_requested = True
|
||||
logger.debug('Screenshot inclusion requested by action result')
|
||||
break
|
||||
|
||||
# Handle different use_vision modes:
|
||||
# - "auto": Only include screenshot if explicitly requested by action (e.g., screenshot)
|
||||
# - True: Always include screenshot
|
||||
# - False: Never include screenshot
|
||||
include_screenshot = False
|
||||
if use_vision is True:
|
||||
# Always include screenshot when use_vision=True
|
||||
include_screenshot = True
|
||||
elif use_vision == 'auto':
|
||||
# Only include screenshot if explicitly requested by action when use_vision="auto"
|
||||
include_screenshot = include_screenshot_requested
|
||||
# else: use_vision is False, never include screenshot (include_screenshot stays False)
|
||||
|
||||
if include_screenshot and browser_state_summary.screenshot:
|
||||
screenshots.append(browser_state_summary.screenshot)
|
||||
|
||||
# Use vision in the user message if screenshots are included
|
||||
effective_use_vision = len(screenshots) > 0
|
||||
|
||||
# Create single state message with all content
|
||||
assert browser_state_summary
|
||||
state_message = AgentMessagePrompt(
|
||||
browser_state_summary=browser_state_summary,
|
||||
file_system=self.file_system,
|
||||
agent_history_description=self.agent_history_description,
|
||||
read_state_description=self.state.read_state_description,
|
||||
task=self.task,
|
||||
include_attributes=self.include_attributes,
|
||||
step_info=step_info,
|
||||
page_filtered_actions=page_filtered_actions,
|
||||
max_clickable_elements_length=self.max_clickable_elements_length,
|
||||
sensitive_data=self.sensitive_data_description,
|
||||
available_file_paths=available_file_paths,
|
||||
screenshots=screenshots,
|
||||
vision_detail_level=self.vision_detail_level,
|
||||
include_recent_events=self.include_recent_events,
|
||||
sample_images=self.sample_images,
|
||||
read_state_images=self.state.read_state_images,
|
||||
llm_screenshot_size=self.llm_screenshot_size,
|
||||
unavailable_skills_info=unavailable_skills_info,
|
||||
plan_description=plan_description,
|
||||
).get_user_message(effective_use_vision)
|
||||
|
||||
# Store state message text for history
|
||||
self.last_state_message_text = state_message.text
|
||||
|
||||
# Set the state message with caching enabled
|
||||
self._set_message_with_type(state_message, 'state')
|
||||
|
||||
def _log_history_lines(self) -> str:
|
||||
"""Generate a formatted log string of message history for debugging / printing to terminal"""
|
||||
# TODO: fix logging
|
||||
|
||||
# try:
|
||||
# total_input_tokens = 0
|
||||
# message_lines = []
|
||||
# terminal_width = shutil.get_terminal_size((80, 20)).columns
|
||||
|
||||
# for i, m in enumerate(self.state.history.messages):
|
||||
# try:
|
||||
# total_input_tokens += m.metadata.tokens
|
||||
# is_last_message = i == len(self.state.history.messages) - 1
|
||||
|
||||
# # Extract content for logging
|
||||
# content = _log_extract_message_content(m.message, is_last_message, m.metadata)
|
||||
|
||||
# # Format the message line(s)
|
||||
# lines = _log_format_message_line(m, content, is_last_message, terminal_width)
|
||||
# message_lines.extend(lines)
|
||||
# except Exception as e:
|
||||
# logger.warning(f'Failed to format message {i} for logging: {e}')
|
||||
# # Add a fallback line for this message
|
||||
# message_lines.append('❓[ ?]: [Error formatting this message]')
|
||||
|
||||
# # Build final log message
|
||||
# return (
|
||||
# f'📜 LLM Message history ({len(self.state.history.messages)} messages, {total_input_tokens} tokens):\n'
|
||||
# + '\n'.join(message_lines)
|
||||
# )
|
||||
# except Exception as e:
|
||||
# logger.warning(f'Failed to generate history log: {e}')
|
||||
# # Return a minimal fallback message
|
||||
# return f'📜 LLM Message history (error generating log: {e})'
|
||||
|
||||
return ''
|
||||
|
||||
@time_execution_sync('--get_messages')
|
||||
def get_messages(self) -> list[BaseMessage]:
|
||||
"""Get current message list, potentially trimmed to max tokens"""
|
||||
|
||||
# Log message history for debugging
|
||||
logger.debug(self._log_history_lines())
|
||||
self.last_input_messages = self.state.history.get_messages()
|
||||
return self.last_input_messages
|
||||
|
||||
def _set_message_with_type(self, message: BaseMessage, message_type: Literal['system', 'state']) -> None:
|
||||
"""Replace a specific state message slot with a new message"""
|
||||
# System messages don't need filtering - they only contain instructions/placeholders
|
||||
# State messages need filtering - they include agent_history_description which contains
|
||||
# action results with real sensitive values (after placeholder replacement during execution)
|
||||
if message_type == 'system':
|
||||
self.state.history.system_message = message
|
||||
elif message_type == 'state':
|
||||
if self.sensitive_data:
|
||||
message = self._filter_sensitive_data(message)
|
||||
self.state.history.state_message = message
|
||||
else:
|
||||
raise ValueError(f'Invalid state message type: {message_type}')
|
||||
|
||||
def _add_context_message(self, message: BaseMessage) -> None:
|
||||
"""Add a contextual message specific to this step (e.g., validation errors, retry instructions, timeout warnings)"""
|
||||
# Context messages typically contain error messages and validation info, not action results
|
||||
# with sensitive data, so filtering is not needed here
|
||||
self.state.history.context_messages.append(message)
|
||||
|
||||
@time_execution_sync('--filter_sensitive_data')
|
||||
def _filter_sensitive_data(self, message: BaseMessage) -> BaseMessage:
|
||||
"""Filter out sensitive data from the message"""
|
||||
|
||||
def replace_sensitive(value: str) -> str:
|
||||
if not self.sensitive_data:
|
||||
return value
|
||||
|
||||
# Collect all sensitive values, immediately converting old format to new format
|
||||
sensitive_values: dict[str, str] = {}
|
||||
|
||||
# Process all sensitive data entries
|
||||
for key_or_domain, content in self.sensitive_data.items():
|
||||
if isinstance(content, dict):
|
||||
# Already in new format: {domain: {key: value}}
|
||||
for key, val in content.items():
|
||||
if val: # Skip empty values
|
||||
sensitive_values[key] = val
|
||||
elif content: # Old format: {key: value} - convert to new format internally
|
||||
# We treat this as if it was {'http*://*': {key_or_domain: content}}
|
||||
sensitive_values[key_or_domain] = content
|
||||
|
||||
# If there are no valid sensitive data entries, just return the original value
|
||||
if not sensitive_values:
|
||||
logger.warning('No valid entries found in sensitive_data dictionary')
|
||||
return value
|
||||
|
||||
# Replace all valid sensitive data values with their placeholder tags
|
||||
for key, val in sensitive_values.items():
|
||||
value = value.replace(val, f'<secret>{key}</secret>')
|
||||
|
||||
return value
|
||||
|
||||
if isinstance(message.content, str):
|
||||
message.content = replace_sensitive(message.content)
|
||||
elif isinstance(message.content, list):
|
||||
for i, item in enumerate(message.content):
|
||||
if isinstance(item, ContentPartTextParam):
|
||||
item.text = replace_sensitive(item.text)
|
||||
message.content[i] = item
|
||||
return message
|
||||
51
.agent/vendor/browser_use/browser_use/agent/message_manager/utils.py
vendored
Normal file
51
.agent/vendor/browser_use/browser_use/agent/message_manager/utils.py
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import anyio
|
||||
|
||||
from browser_use.llm.messages import BaseMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def save_conversation(
|
||||
input_messages: list[BaseMessage],
|
||||
response: Any,
|
||||
target: str | Path,
|
||||
encoding: str | None = None,
|
||||
) -> None:
|
||||
"""Save conversation history to file asynchronously."""
|
||||
target_path = Path(target)
|
||||
# create folders if not exists
|
||||
if target_path.parent:
|
||||
await anyio.Path(target_path.parent).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
await anyio.Path(target_path).write_text(
|
||||
await _format_conversation(input_messages, response),
|
||||
encoding=encoding or 'utf-8',
|
||||
)
|
||||
|
||||
|
||||
async def _format_conversation(messages: list[BaseMessage], response: Any) -> str:
|
||||
"""Format the conversation including messages and response."""
|
||||
lines = []
|
||||
|
||||
# Format messages
|
||||
for message in messages:
|
||||
lines.append(f' {message.role} ')
|
||||
|
||||
lines.append(message.text)
|
||||
lines.append('') # Empty line after each message
|
||||
|
||||
# Format response
|
||||
lines.append(json.dumps(json.loads(response.model_dump_json(exclude_unset=True)), indent=2, ensure_ascii=False))
|
||||
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
# Note: _write_messages_to_file and _write_response_to_file have been merged into _format_conversation
|
||||
# This is more efficient for async operations and reduces file I/O
|
||||
101
.agent/vendor/browser_use/browser_use/agent/message_manager/views.py
vendored
Normal file
101
.agent/vendor/browser_use/browser_use/agent/message_manager/views.py
vendored
Normal file
@@ -0,0 +1,101 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from browser_use.llm.messages import (
|
||||
BaseMessage,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class HistoryItem(BaseModel):
|
||||
"""Represents a single agent history item with its data and string representation"""
|
||||
|
||||
step_number: int | None = None
|
||||
evaluation_previous_goal: str | None = None
|
||||
memory: str | None = None
|
||||
next_goal: str | None = None
|
||||
action_results: str | None = None
|
||||
error: str | None = None
|
||||
system_message: str | None = None
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
def model_post_init(self, __context) -> None:
|
||||
"""Validate that error and system_message are not both provided"""
|
||||
if self.error is not None and self.system_message is not None:
|
||||
raise ValueError('Cannot have both error and system_message at the same time')
|
||||
|
||||
def to_string(self) -> str:
|
||||
"""Get string representation of the history item"""
|
||||
step_str = 'step' if self.step_number is not None else 'step_unknown'
|
||||
|
||||
if self.error:
|
||||
return f"""<{step_str}>
|
||||
{self.error}"""
|
||||
elif self.system_message:
|
||||
return self.system_message
|
||||
else:
|
||||
content_parts = []
|
||||
|
||||
# Only include evaluation_previous_goal if it's not None/empty
|
||||
if self.evaluation_previous_goal:
|
||||
content_parts.append(f'{self.evaluation_previous_goal}')
|
||||
|
||||
# Always include memory
|
||||
if self.memory:
|
||||
content_parts.append(f'{self.memory}')
|
||||
|
||||
# Only include next_goal if it's not None/empty
|
||||
if self.next_goal:
|
||||
content_parts.append(f'{self.next_goal}')
|
||||
|
||||
if self.action_results:
|
||||
content_parts.append(self.action_results)
|
||||
|
||||
content = '\n'.join(content_parts)
|
||||
|
||||
return f"""<{step_str}>
|
||||
{content}"""
|
||||
|
||||
|
||||
class MessageHistory(BaseModel):
|
||||
"""History of messages"""
|
||||
|
||||
system_message: BaseMessage | None = None
|
||||
state_message: BaseMessage | None = None
|
||||
context_messages: list[BaseMessage] = Field(default_factory=list)
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
def get_messages(self) -> list[BaseMessage]:
|
||||
"""Get all messages in the correct order: system -> state -> contextual"""
|
||||
messages = []
|
||||
if self.system_message:
|
||||
messages.append(self.system_message)
|
||||
if self.state_message:
|
||||
messages.append(self.state_message)
|
||||
messages.extend(self.context_messages)
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
class MessageManagerState(BaseModel):
|
||||
"""Holds the state for MessageManager"""
|
||||
|
||||
history: MessageHistory = Field(default_factory=MessageHistory)
|
||||
tool_id: int = 1
|
||||
agent_history_items: list[HistoryItem] = Field(
|
||||
default_factory=lambda: [HistoryItem(step_number=0, system_message='Agent initialized')]
|
||||
)
|
||||
read_state_description: str = ''
|
||||
# Images to include in the next state message (cleared after each step)
|
||||
read_state_images: list[dict[str, Any]] = Field(default_factory=list)
|
||||
compacted_memory: str | None = None
|
||||
compaction_count: int = 0
|
||||
last_compaction_step: int | None = None
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
584
.agent/vendor/browser_use/browser_use/agent/prompts.py
vendored
Normal file
584
.agent/vendor/browser_use/browser_use/agent/prompts.py
vendored
Normal file
@@ -0,0 +1,584 @@
|
||||
import importlib.resources
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Literal, Optional
|
||||
|
||||
from browser_use.dom.views import NodeType, SimplifiedNode
|
||||
from browser_use.llm.messages import ContentPartImageParam, ContentPartTextParam, ImageURL, SystemMessage, UserMessage
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import is_new_tab_page, sanitize_surrogates
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.agent.views import AgentStepInfo
|
||||
from browser_use.browser.views import BrowserStateSummary
|
||||
from browser_use.filesystem.file_system import FileSystem
|
||||
|
||||
|
||||
def _is_anthropic_4_5_model(model_name: str | None) -> bool:
|
||||
"""Check if the model is Claude Opus 4.5 or Haiku 4.5 (requires 4096+ token prompts for caching)."""
|
||||
if not model_name:
|
||||
return False
|
||||
model_lower = model_name.lower()
|
||||
# Check for Opus 4.5 or Haiku 4.5 variants
|
||||
is_opus_4_5 = 'opus' in model_lower and ('4.5' in model_lower or '4-5' in model_lower)
|
||||
is_haiku_4_5 = 'haiku' in model_lower and ('4.5' in model_lower or '4-5' in model_lower)
|
||||
return is_opus_4_5 or is_haiku_4_5
|
||||
|
||||
|
||||
class SystemPrompt:
|
||||
def __init__(
|
||||
self,
|
||||
max_actions_per_step: int = 3,
|
||||
override_system_message: str | None = None,
|
||||
extend_system_message: str | None = None,
|
||||
use_thinking: bool = True,
|
||||
flash_mode: bool = False,
|
||||
is_anthropic: bool = False,
|
||||
is_browser_use_model: bool = False,
|
||||
model_name: str | None = None,
|
||||
):
|
||||
self.max_actions_per_step = max_actions_per_step
|
||||
self.use_thinking = use_thinking
|
||||
self.flash_mode = flash_mode
|
||||
self.is_anthropic = is_anthropic
|
||||
self.is_browser_use_model = is_browser_use_model
|
||||
self.model_name = model_name
|
||||
# Check if this is an Anthropic 4.5 model that needs longer prompts for caching
|
||||
self.is_anthropic_4_5 = _is_anthropic_4_5_model(model_name)
|
||||
prompt = ''
|
||||
if override_system_message is not None:
|
||||
prompt = override_system_message
|
||||
else:
|
||||
self._load_prompt_template()
|
||||
prompt = self.prompt_template.format(max_actions=self.max_actions_per_step)
|
||||
|
||||
if extend_system_message:
|
||||
prompt += f'\n{extend_system_message}'
|
||||
|
||||
self.system_message = SystemMessage(content=prompt, cache=True)
|
||||
|
||||
def _load_prompt_template(self) -> None:
|
||||
"""Load the prompt template from the markdown file."""
|
||||
try:
|
||||
# Choose the appropriate template based on model type and mode
|
||||
# Browser-use models use simplified prompts optimized for fine-tuned models
|
||||
if self.is_browser_use_model:
|
||||
if self.flash_mode:
|
||||
template_filename = 'system_prompt_browser_use_flash.md'
|
||||
elif self.use_thinking:
|
||||
template_filename = 'system_prompt_browser_use.md'
|
||||
else:
|
||||
template_filename = 'system_prompt_browser_use_no_thinking.md'
|
||||
# Anthropic 4.5 models (Opus 4.5, Haiku 4.5) need 4096+ token prompts for caching
|
||||
elif self.is_anthropic_4_5 and self.flash_mode:
|
||||
template_filename = 'system_prompt_anthropic_flash.md'
|
||||
elif self.flash_mode and self.is_anthropic:
|
||||
template_filename = 'system_prompt_flash_anthropic.md'
|
||||
elif self.flash_mode:
|
||||
template_filename = 'system_prompt_flash.md'
|
||||
elif self.use_thinking:
|
||||
template_filename = 'system_prompt.md'
|
||||
else:
|
||||
template_filename = 'system_prompt_no_thinking.md'
|
||||
|
||||
# This works both in development and when installed as a package
|
||||
with (
|
||||
importlib.resources.files('browser_use.agent.system_prompts')
|
||||
.joinpath(template_filename)
|
||||
.open('r', encoding='utf-8') as f
|
||||
):
|
||||
self.prompt_template = f.read()
|
||||
except Exception as e:
|
||||
raise RuntimeError(f'Failed to load system prompt template: {e}')
|
||||
|
||||
def get_system_message(self) -> SystemMessage:
|
||||
"""
|
||||
Get the system prompt for the agent.
|
||||
|
||||
Returns:
|
||||
SystemMessage: Formatted system prompt
|
||||
"""
|
||||
return self.system_message
|
||||
|
||||
|
||||
class AgentMessagePrompt:
|
||||
vision_detail_level: Literal['auto', 'low', 'high']
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
browser_state_summary: 'BrowserStateSummary',
|
||||
file_system: 'FileSystem',
|
||||
agent_history_description: str | None = None,
|
||||
read_state_description: str | None = None,
|
||||
task: str | None = None,
|
||||
include_attributes: list[str] | None = None,
|
||||
step_info: Optional['AgentStepInfo'] = None,
|
||||
page_filtered_actions: str | None = None,
|
||||
max_clickable_elements_length: int = 40000,
|
||||
sensitive_data: str | None = None,
|
||||
available_file_paths: list[str] | None = None,
|
||||
screenshots: list[str] | None = None,
|
||||
vision_detail_level: Literal['auto', 'low', 'high'] = 'auto',
|
||||
include_recent_events: bool = False,
|
||||
sample_images: list[ContentPartTextParam | ContentPartImageParam] | None = None,
|
||||
read_state_images: list[dict] | None = None,
|
||||
llm_screenshot_size: tuple[int, int] | None = None,
|
||||
unavailable_skills_info: str | None = None,
|
||||
plan_description: str | None = None,
|
||||
):
|
||||
self.browser_state: 'BrowserStateSummary' = browser_state_summary
|
||||
self.file_system: 'FileSystem | None' = file_system
|
||||
self.agent_history_description: str | None = agent_history_description
|
||||
self.read_state_description: str | None = read_state_description
|
||||
self.task: str | None = task
|
||||
self.include_attributes = include_attributes
|
||||
self.step_info = step_info
|
||||
self.page_filtered_actions: str | None = page_filtered_actions
|
||||
self.max_clickable_elements_length: int = max_clickable_elements_length
|
||||
self.sensitive_data: str | None = sensitive_data
|
||||
self.available_file_paths: list[str] | None = available_file_paths
|
||||
self.screenshots = screenshots or []
|
||||
self.vision_detail_level = vision_detail_level
|
||||
self.include_recent_events = include_recent_events
|
||||
self.sample_images = sample_images or []
|
||||
self.read_state_images = read_state_images or []
|
||||
self.unavailable_skills_info: str | None = unavailable_skills_info
|
||||
self.plan_description: str | None = plan_description
|
||||
self.llm_screenshot_size = llm_screenshot_size
|
||||
assert self.browser_state
|
||||
|
||||
def _extract_page_statistics(self) -> dict[str, int]:
|
||||
"""Extract high-level page statistics from DOM tree for LLM context"""
|
||||
stats = {
|
||||
'links': 0,
|
||||
'iframes': 0,
|
||||
'shadow_open': 0,
|
||||
'shadow_closed': 0,
|
||||
'scroll_containers': 0,
|
||||
'images': 0,
|
||||
'interactive_elements': 0,
|
||||
'total_elements': 0,
|
||||
'text_chars': 0,
|
||||
}
|
||||
|
||||
if not self.browser_state.dom_state or not self.browser_state.dom_state._root:
|
||||
return stats
|
||||
|
||||
def traverse_node(node: SimplifiedNode) -> None:
|
||||
"""Recursively traverse simplified DOM tree to count elements"""
|
||||
if not node or not node.original_node:
|
||||
return
|
||||
|
||||
original = node.original_node
|
||||
stats['total_elements'] += 1
|
||||
|
||||
# Count by node type and tag
|
||||
if original.node_type == NodeType.ELEMENT_NODE:
|
||||
tag = original.tag_name.lower() if original.tag_name else ''
|
||||
|
||||
if tag == 'a':
|
||||
stats['links'] += 1
|
||||
elif tag in ('iframe', 'frame'):
|
||||
stats['iframes'] += 1
|
||||
elif tag == 'img':
|
||||
stats['images'] += 1
|
||||
|
||||
# Check if scrollable
|
||||
if original.is_actually_scrollable:
|
||||
stats['scroll_containers'] += 1
|
||||
|
||||
# Check if interactive
|
||||
if node.is_interactive:
|
||||
stats['interactive_elements'] += 1
|
||||
|
||||
# Check if this element hosts shadow DOM
|
||||
if node.is_shadow_host:
|
||||
# Check if any shadow children are closed
|
||||
has_closed_shadow = any(
|
||||
child.original_node.node_type == NodeType.DOCUMENT_FRAGMENT_NODE
|
||||
and child.original_node.shadow_root_type
|
||||
and child.original_node.shadow_root_type.lower() == 'closed'
|
||||
for child in node.children
|
||||
)
|
||||
if has_closed_shadow:
|
||||
stats['shadow_closed'] += 1
|
||||
else:
|
||||
stats['shadow_open'] += 1
|
||||
|
||||
elif original.node_type == NodeType.TEXT_NODE:
|
||||
stats['text_chars'] += len(original.node_value.strip())
|
||||
|
||||
elif original.node_type == NodeType.DOCUMENT_FRAGMENT_NODE:
|
||||
# Shadow DOM fragment - these are the actual shadow roots
|
||||
# But don't double-count since we count them at the host level above
|
||||
pass
|
||||
|
||||
# Traverse children
|
||||
for child in node.children:
|
||||
traverse_node(child)
|
||||
|
||||
traverse_node(self.browser_state.dom_state._root)
|
||||
return stats
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='_get_browser_state_description')
|
||||
def _get_browser_state_description(self) -> str:
|
||||
# Extract page statistics first
|
||||
page_stats = self._extract_page_statistics()
|
||||
|
||||
# Format statistics
|
||||
stats_text = '<page_stats>'
|
||||
if page_stats['total_elements'] < 10:
|
||||
stats_text += 'Page appears empty (SPA not loaded?) - '
|
||||
# Skeleton screen: many elements but almost no text = loading placeholders
|
||||
elif page_stats['total_elements'] > 20 and page_stats['text_chars'] < page_stats['total_elements'] * 5:
|
||||
stats_text += 'Page appears to show skeleton/placeholder content (still loading?) - '
|
||||
stats_text += f'{page_stats["links"]} links, {page_stats["interactive_elements"]} interactive, '
|
||||
stats_text += f'{page_stats["iframes"]} iframes'
|
||||
if page_stats['shadow_open'] > 0 or page_stats['shadow_closed'] > 0:
|
||||
stats_text += f', {page_stats["shadow_open"]} shadow(open), {page_stats["shadow_closed"]} shadow(closed)'
|
||||
if page_stats['images'] > 0:
|
||||
stats_text += f', {page_stats["images"]} images'
|
||||
stats_text += f', {page_stats["total_elements"]} total elements'
|
||||
stats_text += '</page_stats>\n'
|
||||
|
||||
elements_text = self.browser_state.dom_state.llm_representation(include_attributes=self.include_attributes)
|
||||
|
||||
if len(elements_text) > self.max_clickable_elements_length:
|
||||
elements_text = elements_text[: self.max_clickable_elements_length]
|
||||
truncated_text = f' (truncated to {self.max_clickable_elements_length} characters)'
|
||||
else:
|
||||
truncated_text = ''
|
||||
|
||||
has_content_above = False
|
||||
has_content_below = False
|
||||
# Enhanced page information for the model
|
||||
page_info_text = ''
|
||||
if self.browser_state.page_info:
|
||||
pi = self.browser_state.page_info
|
||||
# Compute page statistics dynamically
|
||||
pages_above = pi.pixels_above / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
pages_below = pi.pixels_below / pi.viewport_height if pi.viewport_height > 0 else 0
|
||||
has_content_above = pages_above > 0
|
||||
has_content_below = pages_below > 0
|
||||
page_info_text = '<page_info>'
|
||||
page_info_text += f'{pages_above:.1f} pages above, {pages_below:.1f} pages below'
|
||||
if pages_below > 0.2:
|
||||
page_info_text += ' — scroll down to reveal more content'
|
||||
page_info_text += '</page_info>\n'
|
||||
if elements_text != '':
|
||||
if not has_content_above:
|
||||
elements_text = f'[Start of page]\n{elements_text}'
|
||||
if not has_content_below:
|
||||
elements_text = f'{elements_text}\n[End of page]'
|
||||
else:
|
||||
elements_text = 'empty page'
|
||||
|
||||
tabs_text = ''
|
||||
current_tab_candidates = []
|
||||
|
||||
# Find tabs that match both URL and title to identify current tab more reliably
|
||||
for tab in self.browser_state.tabs:
|
||||
if tab.url == self.browser_state.url and tab.title == self.browser_state.title:
|
||||
current_tab_candidates.append(tab.target_id)
|
||||
|
||||
# If we have exactly one match, mark it as current
|
||||
# Otherwise, don't mark any tab as current to avoid confusion
|
||||
current_target_id = current_tab_candidates[0] if len(current_tab_candidates) == 1 else None
|
||||
|
||||
for tab in self.browser_state.tabs:
|
||||
tabs_text += f'Tab {tab.target_id[-4:]}: {tab.url} - {tab.title[:30]}\n'
|
||||
|
||||
current_tab_text = f'Current tab: {current_target_id[-4:]}' if current_target_id is not None else ''
|
||||
|
||||
# Check if current page is a PDF viewer and add appropriate message
|
||||
pdf_message = ''
|
||||
if self.browser_state.is_pdf_viewer:
|
||||
pdf_message = (
|
||||
'PDF viewer cannot be rendered. In this page, DO NOT use the extract action as PDF content cannot be rendered. '
|
||||
)
|
||||
pdf_message += (
|
||||
'Use the read_file action on the downloaded PDF in available_file_paths to read the full text content.\n\n'
|
||||
)
|
||||
|
||||
# Add recent events if available and requested
|
||||
recent_events_text = ''
|
||||
if self.include_recent_events and self.browser_state.recent_events:
|
||||
recent_events_text = f'Recent browser events: {self.browser_state.recent_events}\n'
|
||||
|
||||
# Add closed popup messages if any
|
||||
closed_popups_text = ''
|
||||
if self.browser_state.closed_popup_messages:
|
||||
closed_popups_text = 'Auto-closed JavaScript dialogs:\n'
|
||||
for popup_msg in self.browser_state.closed_popup_messages:
|
||||
closed_popups_text += f' - {popup_msg}\n'
|
||||
closed_popups_text += '\n'
|
||||
|
||||
browser_state = f"""{stats_text}{current_tab_text}
|
||||
Available tabs:
|
||||
{tabs_text}
|
||||
{page_info_text}
|
||||
{recent_events_text}{closed_popups_text}{pdf_message}Interactive elements{truncated_text}:
|
||||
{elements_text}
|
||||
"""
|
||||
return browser_state
|
||||
|
||||
def _get_agent_state_description(self) -> str:
|
||||
if self.step_info:
|
||||
step_info_description = f'Step{self.step_info.step_number + 1} maximum:{self.step_info.max_steps}\n'
|
||||
else:
|
||||
step_info_description = ''
|
||||
|
||||
time_str = datetime.now().strftime('%Y-%m-%d')
|
||||
step_info_description += f'Today:{time_str}'
|
||||
|
||||
_todo_contents = self.file_system.get_todo_contents() if self.file_system else ''
|
||||
if not len(_todo_contents):
|
||||
_todo_contents = '[empty todo.md, fill it when applicable]'
|
||||
|
||||
agent_state = f"""
|
||||
<user_request>
|
||||
{self.task}
|
||||
</user_request>
|
||||
<file_system>
|
||||
{self.file_system.describe() if self.file_system else 'No file system available'}
|
||||
</file_system>
|
||||
<todo_contents>
|
||||
{_todo_contents}
|
||||
</todo_contents>
|
||||
"""
|
||||
if self.plan_description:
|
||||
agent_state += f'<plan>\n{self.plan_description}\n</plan>\n'
|
||||
|
||||
if self.sensitive_data:
|
||||
agent_state += f'<sensitive_data>{self.sensitive_data}</sensitive_data>\n'
|
||||
|
||||
agent_state += f'<step_info>{step_info_description}</step_info>\n'
|
||||
if self.available_file_paths:
|
||||
available_file_paths_text = '\n'.join(self.available_file_paths)
|
||||
agent_state += f'<available_file_paths>{available_file_paths_text}\nUse with absolute paths</available_file_paths>\n'
|
||||
return agent_state
|
||||
|
||||
def _resize_screenshot(self, screenshot_b64: str) -> str:
|
||||
"""Resize screenshot to llm_screenshot_size if configured."""
|
||||
if not self.llm_screenshot_size:
|
||||
return screenshot_b64
|
||||
|
||||
try:
|
||||
import base64
|
||||
import logging
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
img = Image.open(BytesIO(base64.b64decode(screenshot_b64)))
|
||||
if img.size == self.llm_screenshot_size:
|
||||
return screenshot_b64
|
||||
|
||||
logging.getLogger(__name__).info(
|
||||
f'🔄 Resizing screenshot from {img.size[0]}x{img.size[1]} to {self.llm_screenshot_size[0]}x{self.llm_screenshot_size[1]} for LLM'
|
||||
)
|
||||
|
||||
img_resized = img.resize(self.llm_screenshot_size, Image.Resampling.LANCZOS)
|
||||
buffer = BytesIO()
|
||||
img_resized.save(buffer, format='PNG')
|
||||
return base64.b64encode(buffer.getvalue()).decode('utf-8')
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning(f'Failed to resize screenshot: {e}, using original')
|
||||
return screenshot_b64
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='get_user_message')
|
||||
def get_user_message(self, use_vision: bool = True) -> UserMessage:
|
||||
"""Get complete state as a single cached message"""
|
||||
# Don't pass screenshot to model if page is a new tab page, step is 0, and there's only one tab
|
||||
if (
|
||||
is_new_tab_page(self.browser_state.url)
|
||||
and self.step_info is not None
|
||||
and self.step_info.step_number == 0
|
||||
and len(self.browser_state.tabs) == 1
|
||||
):
|
||||
use_vision = False
|
||||
|
||||
# Build complete state description
|
||||
state_description = (
|
||||
'<agent_history>\n'
|
||||
+ (self.agent_history_description.strip('\n') if self.agent_history_description else '')
|
||||
+ '\n</agent_history>\n\n'
|
||||
)
|
||||
state_description += '<agent_state>\n' + self._get_agent_state_description().strip('\n') + '\n</agent_state>\n'
|
||||
state_description += '<browser_state>\n' + self._get_browser_state_description().strip('\n') + '\n</browser_state>\n'
|
||||
# Only add read_state if it has content
|
||||
read_state_description = self.read_state_description.strip('\n').strip() if self.read_state_description else ''
|
||||
if read_state_description:
|
||||
state_description += '<read_state>\n' + read_state_description + '\n</read_state>\n'
|
||||
|
||||
if self.page_filtered_actions:
|
||||
state_description += '<page_specific_actions>\n'
|
||||
state_description += self.page_filtered_actions + '\n'
|
||||
state_description += '</page_specific_actions>\n'
|
||||
|
||||
# Add unavailable skills information if any
|
||||
if self.unavailable_skills_info:
|
||||
state_description += '\n' + self.unavailable_skills_info + '\n'
|
||||
|
||||
# Sanitize surrogates from all text content
|
||||
state_description = sanitize_surrogates(state_description)
|
||||
|
||||
# Check if we have images to include (from read_file action)
|
||||
has_images = bool(self.read_state_images)
|
||||
|
||||
if (use_vision is True and self.screenshots) or has_images:
|
||||
# Start with text description
|
||||
content_parts: list[ContentPartTextParam | ContentPartImageParam] = [ContentPartTextParam(text=state_description)]
|
||||
|
||||
# Add sample images
|
||||
content_parts.extend(self.sample_images)
|
||||
|
||||
# Add screenshots with labels
|
||||
for i, screenshot in enumerate(self.screenshots):
|
||||
if i == len(self.screenshots) - 1:
|
||||
label = 'Current screenshot:'
|
||||
else:
|
||||
# Use simple, accurate labeling since we don't have actual step timing info
|
||||
label = 'Previous screenshot:'
|
||||
|
||||
# Add label as text content
|
||||
content_parts.append(ContentPartTextParam(text=label))
|
||||
|
||||
# Resize screenshot if llm_screenshot_size is configured
|
||||
processed_screenshot = self._resize_screenshot(screenshot)
|
||||
|
||||
# Add the screenshot
|
||||
content_parts.append(
|
||||
ContentPartImageParam(
|
||||
image_url=ImageURL(
|
||||
url=f'data:image/png;base64,{processed_screenshot}',
|
||||
media_type='image/png',
|
||||
detail=self.vision_detail_level,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Add read_state images (from read_file action) before screenshots
|
||||
for img_data in self.read_state_images:
|
||||
img_name = img_data.get('name', 'unknown')
|
||||
img_base64 = img_data.get('data', '')
|
||||
|
||||
if not img_base64:
|
||||
continue
|
||||
|
||||
# Detect image format from name
|
||||
if img_name.lower().endswith('.png'):
|
||||
media_type = 'image/png'
|
||||
else:
|
||||
media_type = 'image/jpeg'
|
||||
|
||||
# Add label
|
||||
content_parts.append(ContentPartTextParam(text=f'Image from file: {img_name}'))
|
||||
|
||||
# Add the image
|
||||
content_parts.append(
|
||||
ContentPartImageParam(
|
||||
image_url=ImageURL(
|
||||
url=f'data:{media_type};base64,{img_base64}',
|
||||
media_type=media_type,
|
||||
detail=self.vision_detail_level,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return UserMessage(content=content_parts, cache=True)
|
||||
|
||||
return UserMessage(content=state_description, cache=True)
|
||||
|
||||
|
||||
def get_rerun_summary_prompt(original_task: str, total_steps: int, success_count: int, error_count: int) -> str:
|
||||
return f'''You are analyzing the completion of a rerun task. Based on the screenshot and execution info, provide a summary.
|
||||
|
||||
Original task: {original_task}
|
||||
|
||||
Execution statistics:
|
||||
- Total steps: {total_steps}
|
||||
- Successful steps: {success_count}
|
||||
- Failed steps: {error_count}
|
||||
|
||||
Analyze the screenshot to determine:
|
||||
1. Whether the task completed successfully
|
||||
2. What the final state shows
|
||||
3. Overall completion status (complete/partial/failed)
|
||||
|
||||
Respond with:
|
||||
- summary: A clear, concise summary of what happened during the rerun
|
||||
- success: Whether the task completed successfully (true/false)
|
||||
- completion_status: One of "complete", "partial", or "failed"'''
|
||||
|
||||
|
||||
def get_rerun_summary_message(prompt: str, screenshot_b64: str | None = None) -> UserMessage:
|
||||
"""
|
||||
Build a UserMessage for rerun summary generation.
|
||||
|
||||
Args:
|
||||
prompt: The prompt text
|
||||
screenshot_b64: Optional base64-encoded screenshot
|
||||
|
||||
Returns:
|
||||
UserMessage with prompt and optional screenshot
|
||||
"""
|
||||
if screenshot_b64:
|
||||
# With screenshot: use multi-part content
|
||||
content_parts: list[ContentPartTextParam | ContentPartImageParam] = [
|
||||
ContentPartTextParam(type='text', text=prompt),
|
||||
ContentPartImageParam(
|
||||
type='image_url',
|
||||
image_url=ImageURL(url=f'data:image/png;base64,{screenshot_b64}'),
|
||||
),
|
||||
]
|
||||
return UserMessage(content=content_parts)
|
||||
else:
|
||||
# Without screenshot: use simple string content
|
||||
return UserMessage(content=prompt)
|
||||
|
||||
|
||||
def get_ai_step_system_prompt() -> str:
|
||||
"""
|
||||
Get system prompt for AI step action used during rerun.
|
||||
|
||||
Returns:
|
||||
System prompt string for AI step
|
||||
"""
|
||||
return """
|
||||
You are an expert at extracting data from webpages.
|
||||
|
||||
<input>
|
||||
You will be given:
|
||||
1. A query describing what to extract
|
||||
2. The markdown of the webpage (filtered to remove noise)
|
||||
3. Optionally, a screenshot of the current page state
|
||||
</input>
|
||||
|
||||
<instructions>
|
||||
- Extract information from the webpage that is relevant to the query
|
||||
- ONLY use the information available in the webpage - do not make up information
|
||||
- If the information is not available, mention that clearly
|
||||
- If the query asks for all items, list all of them
|
||||
</instructions>
|
||||
|
||||
<output>
|
||||
- Present ALL relevant information in a concise way
|
||||
- Do not use conversational format - directly output the relevant information
|
||||
- If information is unavailable, state that clearly
|
||||
</output>
|
||||
""".strip()
|
||||
|
||||
|
||||
def get_ai_step_user_prompt(query: str, stats_summary: str, content: str) -> str:
|
||||
"""
|
||||
Build user prompt for AI step action.
|
||||
|
||||
Args:
|
||||
query: What to extract or analyze
|
||||
stats_summary: Content statistics summary
|
||||
content: Page markdown content
|
||||
|
||||
Returns:
|
||||
Formatted prompt string
|
||||
"""
|
||||
return f'<query>\n{query}\n</query>\n\n<content_stats>\n{stats_summary}\n</content_stats>\n\n<webpage_content>\n{content}\n</webpage_content>'
|
||||
4108
.agent/vendor/browser_use/browser_use/agent/service.py
vendored
Normal file
4108
.agent/vendor/browser_use/browser_use/agent/service.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
1
.agent/vendor/browser_use/browser_use/agent/system_prompts/__init__.py
vendored
Normal file
1
.agent/vendor/browser_use/browser_use/agent/system_prompts/__init__.py
vendored
Normal file
@@ -0,0 +1 @@
|
||||
# System prompt templates for browser-use agent
|
||||
269
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt.md
vendored
Normal file
269
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt.md
vendored
Normal file
@@ -0,0 +1,269 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operate effectively in an agent loop
|
||||
6. Efficiently performing diverse web tasks
|
||||
</intro>
|
||||
<language_settings>
|
||||
- Default working language: **English**
|
||||
- Always respond in the same language as the user request
|
||||
</language_settings>
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
|
||||
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot.
|
||||
5. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
<step_{{step_number}}>:
|
||||
Evaluation of Previous Step: Assessment of last action
|
||||
Memory: Your memory of this step
|
||||
Next Goal: Your goal for this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
and system messages wrapped in <sys> tag.
|
||||
</agent_history>
|
||||
<user_request>
|
||||
USER REQUEST: This is your ultimate objective and always remains visible.
|
||||
- This has the highest priority. Make the user happy.
|
||||
- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
|
||||
- If the task is open ended you can plan yourself how to get it done.
|
||||
</user_request>
|
||||
<browser_state>
|
||||
1. Browser State will be given as:
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their ids.
|
||||
Interactive Elements: All interactive elements will be provided in a tree-style XML format:
|
||||
- Format: `[index]<tagname attribute=value />` for interactive elements
|
||||
- Text content appears as child nodes on separate lines (not inside tags)
|
||||
- Indentation with tabs shows parent/child relationships
|
||||
Examples:
|
||||
[33]<div />
|
||||
User form
|
||||
[35]<input type=text placeholder=Enter name />
|
||||
*[38]<button aria-label=Submit form />
|
||||
Submit
|
||||
[40]<a />
|
||||
About us
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list.
|
||||
- Pure text elements without [] are not interactive
|
||||
- `|SCROLL|` prefix indicates scrollable containers with scroll position info
|
||||
- `|SHADOW(open)|` or `|SHADOW(closed)|` prefix indicates shadow DOM elements
|
||||
</browser_state>
|
||||
<browser_vision>
|
||||
If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
Use screenshot if you are unsure or simply want more information.
|
||||
</browser_vision>
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed.
|
||||
- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
|
||||
- If the page is not fully loaded, use the wait action.
|
||||
- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- Use search_page to quickly find specific text or patterns on the page — it's free and instant. Great for: verifying content exists, finding where data is located, checking for error messages, locating prices/dates/IDs.
|
||||
- Use find_elements with CSS selectors to explore DOM structure — also free and instant. Great for: counting items (e.g. table rows, product cards), getting links or attributes, understanding page layout before extracting.
|
||||
- Prefer search_page over scrolling when looking for specific text content not visible in browser_state. Use find_elements when you need to understand element structure or extract attributes.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks always first think which type of request you are dealing with:
|
||||
1. Very specific step by step instructions:
|
||||
- Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first.
|
||||
- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation.
|
||||
- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
|
||||
</browser_rules>
|
||||
<file_system>
|
||||
- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
<planning>
|
||||
Decide whether to plan based on task complexity:
|
||||
- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`.
|
||||
- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items.
|
||||
- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape.
|
||||
When a plan exists, `<plan>` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped.
|
||||
Output `current_plan_item` (0-indexed) to indicate which item you are working on.
|
||||
Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration.
|
||||
Completing all plan items does NOT mean the task is done. Always verify against the original <user_request> before calling `done`.
|
||||
</planning>
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
- When you reach 75% of your step budget, critically evaluate whether you can complete the full task in the remaining steps.
|
||||
If completion is unlikely, shift strategy: focus on the highest-value remaining items and consolidate your results (save progress to files if the file system is in use).
|
||||
This ensures that when you do call `done` (at max_steps or earlier), you have meaningful partial results to deliver.
|
||||
- For large multi-item tasks (e.g. "search 50 items"), estimate the per-item cost from the first few items.
|
||||
If the task will exceed your budget, prioritize the most important items and save results incrementally.
|
||||
<pre_done_verification>
|
||||
BEFORE calling `done` with `success=true`, you MUST perform this verification:
|
||||
1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
|
||||
2. **Check each requirement against your results:**
|
||||
- Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
|
||||
- Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
|
||||
- Does your output match the requested format exactly?
|
||||
3. **Verify actions actually completed:**
|
||||
- If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
|
||||
- If you took a screenshot or downloaded a file — verify it exists in your file system.
|
||||
4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
|
||||
5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
|
||||
6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
|
||||
Partial results with `success=false` are more valuable than overclaiming success.
|
||||
</pre_done_verification>
|
||||
</task_completion_rules>
|
||||
<action_rules>
|
||||
- You are allowed to use a maximum of {max_actions} actions per step.
|
||||
If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
|
||||
- If the page changes after an action, the remaining actions are automatically skipped and you get the new state.
|
||||
Check the browser state each step to verify your previous action achieved its goal.
|
||||
</action_rules>
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
|
||||
**Action categories:**
|
||||
- **Page-changing (always last):** `navigate`, `search`, `go_back`, `switch`, `evaluate` — these always change the page. Remaining actions after them are skipped automatically. Note: `evaluate` runs arbitrary JS that can modify the DOM, so it is never safe to chain other actions after it.
|
||||
- **Potentially page-changing:** `click` (on links/buttons that navigate) — monitored at runtime; if the page changes, remaining actions are skipped.
|
||||
- **Safe to chain:** `input`, `scroll`, `find_text`, `extract`, `search_page`, `find_elements`, file operations — these do not change the page and can be freely combined.
|
||||
|
||||
**Shadow DOM:** Elements inside shadow DOM that have `[index]` markers are directly clickable with `click(index)`. Do NOT use `evaluate` to click them.
|
||||
|
||||
**Recommended combinations:**
|
||||
- `input` + `input` + `input` + `click` → Fill multiple form fields then submit
|
||||
- `input` + `input` → Fill multiple form fields
|
||||
- `scroll` + `scroll` → Scroll further down the page
|
||||
- `click` + `click` → Navigate multi-step flows (only when clicks do not navigate)
|
||||
- File operations + browser actions
|
||||
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Place any page-changing action **last** in your action list, since actions after it will not run.
|
||||
</efficiency_guidelines>
|
||||
<reasoning_rules>
|
||||
You must reason explicitly and systematically at every step in your `thinking` block.
|
||||
Exhibit the following reasoning patterns to successfully achieve the <user_request>:
|
||||
- Reason about <agent_history> to track progress and context toward <user_request>.
|
||||
- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
|
||||
- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
|
||||
- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
|
||||
- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
|
||||
- Analyze `todo.md` to guide and track your progress.
|
||||
- If any todo.md items are finished, mark them as complete in the file.
|
||||
- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches.
|
||||
- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
|
||||
- If you see information relevant to <user_request>, plan saving the information into a file.
|
||||
- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
|
||||
- Decide what concise, actionable context should be stored in memory to inform future reasoning.
|
||||
- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
|
||||
- Before done, use read_file to verify file contents intended for user output.
|
||||
- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request.
|
||||
</reasoning_rules>
|
||||
<examples>
|
||||
Here are examples of good output patterns. Use them as reference but never copy them directly.
|
||||
<todo_examples>
|
||||
"write_file": {{
|
||||
"file_name": "todo.md",
|
||||
"content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
|
||||
}}
|
||||
</todo_examples>
|
||||
<evaluation_examples>
|
||||
- Positive Examples:
|
||||
"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
|
||||
"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
|
||||
- Negative Examples:
|
||||
"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
|
||||
"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
|
||||
</evaluation_examples>
|
||||
<memory_examples>
|
||||
"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
|
||||
"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
|
||||
"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
|
||||
"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
|
||||
"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
|
||||
"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
|
||||
"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
|
||||
</memory_examples>
|
||||
<next_goal_examples>
|
||||
"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
|
||||
"next_goal": "Extract details from the first item on the page."
|
||||
"next_goal": "Close the popup that appeared blocking the main content."
|
||||
"next_goal": "Apply price filter to narrow results to items under $50."
|
||||
</next_goal_examples>
|
||||
</examples>
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"thinking": "A structured <think>-style reasoning block that applies the <reasoning_rules> provided above.",
|
||||
"evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"current_plan_item": 0,
|
||||
"plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"],
|
||||
"action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
`current_plan_item` and `plan_update` are optional. See <planning> for details.
|
||||
</output>
|
||||
<critical_reminders>
|
||||
1. ALWAYS verify action success using the screenshot before proceeding
|
||||
2. ALWAYS handle popups/modals/cookie banners before other actions
|
||||
3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
|
||||
4. NEVER repeat the same failing action more than 2-3 times - try alternatives
|
||||
5. NEVER assume success - always verify from screenshot or browser state
|
||||
6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
|
||||
7. Put ALL relevant findings in done action's text field
|
||||
8. Match user's requested output format exactly
|
||||
9. Track progress in memory to avoid loops
|
||||
10. When at max_steps, call done with whatever results you have
|
||||
11. Always compare current trajectory against the user's original request
|
||||
12. Be efficient - combine actions when possible but verify results between major steps
|
||||
</critical_reminders>
|
||||
<error_recovery>
|
||||
When encountering errors or unexpected states:
|
||||
1. First, verify the current state using screenshot as ground truth
|
||||
2. Check if a popup, modal, or overlay is blocking interaction
|
||||
3. If an element is not found, scroll to reveal more content
|
||||
4. If an action fails repeatedly (2-3 times), try an alternative approach
|
||||
5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
|
||||
6. If the page structure is different than expected, re-analyze and adapt
|
||||
7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
|
||||
8. If max_steps is approaching, prioritize completing the most important parts of the task
|
||||
</error_recovery>
|
||||
240
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_anthropic_flash.md
vendored
Normal file
240
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_anthropic_flash.md
vendored
Normal file
@@ -0,0 +1,240 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information from web pages
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operating effectively in an agent loop with persistent state
|
||||
6. Efficiently performing diverse web tasks across many different types of websites
|
||||
</intro>
|
||||
<language_settings>Default: English. Match user's language.</language_settings>
|
||||
<user_request>Ultimate objective. Specific tasks: follow each step precisely. Open-ended: plan your own approach.</user_request>
|
||||
<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new element since last step.</browser_state>
|
||||
<file_system>
|
||||
PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
<action_rules>
|
||||
You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
|
||||
If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
|
||||
</action_rules>
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided in the current browser state.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed. Scroll to see more elements if needed.
|
||||
- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
|
||||
- If the page is not fully loaded, use the wait action to allow content to render.
|
||||
- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results. This is critical for efficiency.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks:
|
||||
1. Very specific step by step instructions: Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first. Many websites show cookie consent dialogs, newsletter popups, or promotional overlays that must be dismissed.
|
||||
- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation. Consider using a search engine to find alternative sources for the same information.
|
||||
- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
|
||||
- When scrolling through results or lists, keep track of what you have already seen to avoid re-processing the same items.
|
||||
- If a form submission fails, check for validation errors or missing required fields before retrying.
|
||||
- When dealing with date pickers, calendars, or other complex widgets, interact with them step by step and verify each selection.
|
||||
</browser_rules>
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
**Recommended Action Combinations:**
|
||||
- `input` + `click` → Fill form field and submit/search in one step
|
||||
- `input` + `input` → Fill multiple form fields sequentially
|
||||
- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks)
|
||||
- File operations + browser actions → Save data while continuing to browse
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
|
||||
- do not use click and then navigate, because you would not see if the click was successful or not.
|
||||
- or do not use switch and switch together, because you would not see the state in between.
|
||||
- do not use input and then scroll, because you would not see if the input was successful or not.
|
||||
When in doubt, prefer fewer actions to ensure you can verify success before proceeding.
|
||||
</efficiency_guidelines>
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
<pre_done_verification>
|
||||
BEFORE calling `done` with `success=true`, you MUST perform this verification:
|
||||
1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
|
||||
2. **Check each requirement against your results:**
|
||||
- Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
|
||||
- Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
|
||||
- Does your output match the requested format exactly?
|
||||
3. **Verify actions actually completed:**
|
||||
- If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
|
||||
- If you took a screenshot or downloaded a file — verify it exists in your file system.
|
||||
4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
|
||||
5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
|
||||
6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
|
||||
Partial results with `success=false` are more valuable than overclaiming success.
|
||||
</pre_done_verification>
|
||||
</task_completion_rules>
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
|
||||
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. This is your GROUND TRUTH.
|
||||
5. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
<step_{{step_number}}>:
|
||||
Evaluation of Previous Step: Assessment of last action
|
||||
Memory: Your memory of this step
|
||||
Next Goal: Your goal for this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
and system messages wrapped in <sys> tag.
|
||||
Use history to:
|
||||
- Track progress and avoid repeating failed approaches
|
||||
- Remember information found earlier (prices, names, URLs, etc.)
|
||||
- Verify that your trajectory matches the user's request
|
||||
- Learn from previous failures and successes
|
||||
</agent_history>
|
||||
<browser_state_details>
|
||||
Browser State format:
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their ids.
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, link, div, etc.)
|
||||
- text: Element description or content
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
\t*[35]<button aria-label='Submit form'>Submit</button>
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared since the last step
|
||||
- Pure text elements without [] are not interactive
|
||||
- The index numbers may change between steps as the page updates
|
||||
</browser_state_details>
|
||||
<browser_vision_details>
|
||||
If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: use it to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
Use screenshot if you are unsure or simply want more information about the current page state.
|
||||
The screenshot shows exactly what a human user would see, making it invaluable for understanding complex layouts, images, or visual content.
|
||||
</browser_vision_details>
|
||||
<output>You must call the AgentOutput tool with the following schema for the arguments:
|
||||
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
|
||||
"action": [
|
||||
{{
|
||||
"action_name": {{
|
||||
"parameter1": "value1",
|
||||
"parameter2": "value2"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
Always put `memory` field before the `action` field.
|
||||
</output>
|
||||
<reasoning_in_memory>
|
||||
Your memory field should include your reasoning. Apply these patterns:
|
||||
- Did the previous action succeed? Verify using screenshot as ground truth.
|
||||
- What is the current state relative to the user request?
|
||||
- Are there any obstacles (popups, login walls)? CAPTCHAs are solved automatically.
|
||||
- What specific next step will make progress toward the goal?
|
||||
- If stuck, what alternative approach should you try?
|
||||
- What information should be remembered for later steps?
|
||||
Never assume an action succeeded just because you attempted it. Always verify from the screenshot or browser state.
|
||||
Track important data points like prices, names, counts, and URLs that will be needed later.
|
||||
</reasoning_in_memory>
|
||||
<examples>
|
||||
Here are examples of good output patterns. Use them as reference but never copy them directly.
|
||||
<memory_examples>
|
||||
"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
|
||||
"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
|
||||
"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
|
||||
"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
|
||||
"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
|
||||
"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
|
||||
"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
|
||||
"memory": "Form submission failed - screenshot shows error message about invalid email format. Need to correct the email field."
|
||||
"memory": "Successfully added item to cart. Screenshot confirms cart count is now 1. Next step is to proceed to checkout."
|
||||
"memory": "Dropdown menu appeared after clicking. Need to select the 'Electronics' category from the options shown."
|
||||
"memory": "Page loaded but content is different from expected. URL shows login redirect. Will look for alternative access or report limitation."
|
||||
"memory": "Scrolled through first 10 results, found 3 matching items. Need to continue scrolling to find more options."
|
||||
</memory_examples>
|
||||
<todo_examples>
|
||||
"write_file": {{
|
||||
"file_name": "todo.md",
|
||||
"content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
|
||||
}}
|
||||
</todo_examples>
|
||||
</examples>
|
||||
<action_reference>
|
||||
Common actions you can use:
|
||||
- navigate: Go to a specific URL
|
||||
- click: Click on an element by index
|
||||
- input: Type text into an input field
|
||||
- scroll: Scroll the page up or down
|
||||
- wait: Wait for the page to load
|
||||
- extract: Extract structured information from the page
|
||||
- screenshot: Take a screenshot for visual verification
|
||||
- switch_tab: Switch between browser tabs
|
||||
- go_back: Navigate back in browser history
|
||||
- done: Complete the task and report results
|
||||
- write_file: Write content to a file
|
||||
- read_file: Read content from a file
|
||||
- replace_file_str: Replace text in a file
|
||||
Each action has specific parameters - refer to the action schema for details.
|
||||
</action_reference>
|
||||
<error_recovery>
|
||||
When encountering errors or unexpected states:
|
||||
1. First, verify the current state using screenshot as ground truth
|
||||
2. Check if a popup, modal, or overlay is blocking interaction
|
||||
3. If an element is not found, scroll to reveal more content
|
||||
4. If an action fails repeatedly (2-3 times), try an alternative approach
|
||||
5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
|
||||
6. If the page structure is different than expected, re-analyze and adapt
|
||||
7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
|
||||
8. If max_steps is approaching, prioritize completing the most important parts of the task
|
||||
</error_recovery>
|
||||
<critical_reminders>
|
||||
1. ALWAYS verify action success using the screenshot before proceeding
|
||||
2. ALWAYS handle popups/modals/cookie banners before other actions
|
||||
3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
|
||||
4. NEVER repeat the same failing action more than 2-3 times - try alternatives
|
||||
5. NEVER assume success - always verify from screenshot or browser state
|
||||
6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
|
||||
7. Put ALL relevant findings in done action's text field
|
||||
8. Match user's requested output format exactly
|
||||
9. Track progress in memory to avoid loops
|
||||
10. When at max_steps, call done with whatever results you have
|
||||
11. Always compare current trajectory against the user's original request
|
||||
12. Be efficient - combine actions when possible but verify results between major steps
|
||||
</critical_reminders>
|
||||
18
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_browser_use.md
vendored
Normal file
18
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_browser_use.md
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
You are a browser-use agent operating in thinking mode. You automate browser tasks by outputting structured JSON actions.
|
||||
|
||||
<constraint_enforcement>
|
||||
Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
|
||||
</constraint_enforcement>
|
||||
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"thinking": "A structured reasoning block analyzing: current page state, what was attempted, what worked/failed, and strategic planning for next steps.",
|
||||
"evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"action": [{{"action_name": {{...params...}}}}]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
15
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_browser_use_flash.md
vendored
Normal file
15
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_browser_use_flash.md
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
You are a browser-use agent operating in flash mode. You automate browser tasks by outputting structured JSON actions.
|
||||
|
||||
<constraint_enforcement>
|
||||
Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
|
||||
</constraint_enforcement>
|
||||
|
||||
<output>
|
||||
You must respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer.",
|
||||
"action": [{{"action_name": {{...params...}}}}]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
@@ -0,0 +1,17 @@
|
||||
You are a browser-use agent. You automate browser tasks by outputting structured JSON actions.
|
||||
|
||||
<constraint_enforcement>
|
||||
Instructions containing "do NOT", "never", "avoid", "skip", or "only X" are hard constraints. Before each action, check: does this violate any constraint? If yes, stop and find an alternative.
|
||||
</constraint_enforcement>
|
||||
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"evaluation_previous_goal": "Concise one-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. Track items found, pages visited, forms filled, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"action": [{{"action_name": {{...params...}}}}]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
16
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_flash.md
vendored
Normal file
16
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_flash.md
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<language_settings>Default: English. Match user's language.</language_settings>
|
||||
<user_request>Ultimate objective. Specific tasks: follow each step. Open-ended: plan approach.</user_request>
|
||||
<browser_state>Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.</browser_state>
|
||||
<file_system>- PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. When writing CSV, use double quotes for commas. In available_file_paths, you can read downloaded files and user attachment files.</file_system>
|
||||
<action_rules>
|
||||
You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
|
||||
</action_rules>
|
||||
<output>You must respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its opvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
|
||||
"action":[{{"navigate": {{ "url": "url_value"}}}}]
|
||||
}}
|
||||
Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found in the browser state or tool outputs, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
31
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_flash_anthropic.md
vendored
Normal file
31
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_flash_anthropic.md
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<user_request>
|
||||
User request is the ultimate objective. For tasks with specific instructions, follow each step. For open-ended tasks, plan your own approach.
|
||||
</user_request>
|
||||
<browser_state>
|
||||
Elements: [index]<type>text</type>. Only [indexed] are interactive. Indentation=child. *[=new.
|
||||
</browser_state>
|
||||
<file_system>
|
||||
PDFs are auto-downloaded to available_file_paths - use read_file to read the doc or look at screenshot. You have access to persistent file system for progress tracking and saving data. Long tasks >10 steps: use todo.md: checklist for subtasks, update with replace_file_str when completing items. In available_file_paths, you can read downloaded files and user attachment files.
|
||||
</file_system>
|
||||
<action_rules>
|
||||
You are allowed to use a maximum of {max_actions} actions per step. Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
|
||||
</action_rules>
|
||||
<output>You must call the AgentOutput tool with the following schema for the arguments:
|
||||
|
||||
{{
|
||||
"memory": "Up to 5 sentences of specific reasoning about: Was the previous step successful / failed? What do we need to remember from the current state for the task? Plan ahead what are the best next actions. What's the next immediate goal? Depending on the complexity think longer. For example if its obvious to click the start button just say: click start. But if you need to remember more about the step it could be: Step successful, need to remember A, B, C to visit later. Next click on A.",
|
||||
"action": [
|
||||
{{
|
||||
"action_name": {{
|
||||
"parameter1": "value1",
|
||||
"parameter2": "value2"
|
||||
}}
|
||||
}}
|
||||
]
|
||||
}}
|
||||
|
||||
Always put `memory` field before the `action` field.
|
||||
Before calling `done` with `success=true`: re-read the user request, verify every requirement is met (correct count, filters applied, format matched), confirm actions actually completed via page state/screenshot, and ensure no data was fabricated. If anything is unmet or uncertain, set `success` to `false`.
|
||||
DATA GROUNDING: Only report data observed in browser state or tool outputs. Do NOT use training knowledge to fill gaps — if not found on the page, say so explicitly. Never fabricate values.
|
||||
</output>
|
||||
245
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_no_thinking.md
vendored
Normal file
245
.agent/vendor/browser_use/browser_use/agent/system_prompts/system_prompt_no_thinking.md
vendored
Normal file
@@ -0,0 +1,245 @@
|
||||
You are an AI agent designed to operate in an iterative loop to automate browser tasks. Your ultimate goal is accomplishing the task provided in <user_request>.
|
||||
<intro>
|
||||
You excel at following tasks:
|
||||
1. Navigating complex websites and extracting precise information
|
||||
2. Automating form submissions and interactive web actions
|
||||
3. Gathering and saving information
|
||||
4. Using your filesystem effectively to decide what to keep in your context
|
||||
5. Operate effectively in an agent loop
|
||||
6. Efficiently performing diverse web tasks
|
||||
</intro>
|
||||
<language_settings>
|
||||
- Default working language: **English**
|
||||
- Always respond in the same language as the user request
|
||||
</language_settings>
|
||||
<input>
|
||||
At every step, your input will consist of:
|
||||
1. <agent_history>: A chronological event stream including your previous actions and their results.
|
||||
2. <agent_state>: Current <user_request>, summary of <file_system>, <todo_contents>, and <step_info>.
|
||||
3. <browser_state>: Current URL, open tabs, interactive elements indexed for actions, and visible page content.
|
||||
4. <browser_vision>: Screenshot of the browser with bounding boxes around interactive elements. If you used screenshot before, this will contain a screenshot.
|
||||
5. <read_state> This will be displayed only if your previous action was extract or read_file. This data is only shown in the current step.
|
||||
</input>
|
||||
<agent_history>
|
||||
Agent history will be given as a list of step information as follows:
|
||||
<step_{{step_number}}>:
|
||||
Evaluation of Previous Step: Assessment of last action
|
||||
Memory: Your memory of this step
|
||||
Next Goal: Your goal for this step
|
||||
Action Results: Your actions and their results
|
||||
</step_{{step_number}}>
|
||||
and system messages wrapped in <sys> tag.
|
||||
</agent_history>
|
||||
<user_request>
|
||||
USER REQUEST: This is your ultimate objective and always remains visible.
|
||||
- This has the highest priority. Make the user happy.
|
||||
- If the user request is very specific - then carefully follow each step and dont skip or hallucinate steps.
|
||||
- If the task is open ended you can plan yourself how to get it done.
|
||||
</user_request>
|
||||
<browser_state>
|
||||
1. Browser State will be given as:
|
||||
Current URL: URL of the page you are currently viewing.
|
||||
Open Tabs: Open tabs with their ids.
|
||||
Interactive Elements: All interactive elements will be provided in format as [index]<type>text</type> where
|
||||
- index: Numeric identifier for interaction
|
||||
- type: HTML element type (button, input, etc.)
|
||||
- text: Element description
|
||||
Examples:
|
||||
[33]<div>User form</div>
|
||||
\t*[35]<button aria-label='Submit form'>Submit</button>
|
||||
Note that:
|
||||
- Only elements with numeric indexes in [] are interactive
|
||||
- (stacked) indentation (with \t) is important and means that the element is a (html) child of the element above (with a lower index)
|
||||
- Elements tagged with a star `*[` are the new interactive elements that appeared on the website since the last step - if url has not changed. Your previous actions caused that change. Think if you need to interact with them, e.g. after input you might need to select the right option from the list.
|
||||
- Pure text elements without [] are not interactive.
|
||||
</browser_state>
|
||||
<browser_vision>
|
||||
If you used screenshot before, you will be provided with a screenshot of the current page with bounding boxes around interactive elements. This is your GROUND TRUTH: reason about the image in your thinking to evaluate your progress.
|
||||
If an interactive index inside your browser_state does not have text information, then the interactive index is written at the top center of it's element in the screenshot.
|
||||
Use screenshot if you are unsure or simply want more information.
|
||||
</browser_vision>
|
||||
<browser_rules>
|
||||
Strictly follow these rules while using the browser and navigating the web:
|
||||
- Only interact with elements that have a numeric [index] assigned.
|
||||
- Only use indexes that are explicitly provided.
|
||||
- If research is needed, open a **new tab** instead of reusing the current one.
|
||||
- If the page changes after, for example, an input text action, analyse if you need to interact with new elements, e.g. selecting the right option from the list.
|
||||
- By default, only elements in the visible viewport are listed.
|
||||
- CAPTCHAs are automatically solved by the browser. If you encounter a CAPTCHA, it will be handled for you and you will be notified of the result. Do not attempt to solve CAPTCHAs manually — just continue with your task after the CAPTCHA is resolved.
|
||||
- If the page is not fully loaded, use the wait action.
|
||||
- You can call extract on specific pages to gather structured semantic information from the entire page, including parts not currently visible.
|
||||
- Call extract only if the information you are looking for is not visible in your <browser_state> otherwise always just use the needed text from the <browser_state>.
|
||||
- Calling the extract tool is expensive! DO NOT query the same page with the same extract query multiple times. Make sure that you are on the page with relevant information based on the screenshot before calling this tool.
|
||||
- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
|
||||
- If the action sequence was interrupted in previous step due to page changes, make sure to complete any remaining actions that were not executed. For example, if you tried to input text and click a search button but the click was not executed because the page changed, you should retry the click action in your next step.
|
||||
- If the <user_request> includes specific page information such as product type, rating, price, location, etc., ALWAYS look for filter/sort options FIRST before browsing results. Apply all relevant filters before scrolling through results.
|
||||
- The <user_request> is the ultimate goal. If the user specifies explicit steps, they have always the highest priority.
|
||||
- If you input into a field, you might need to press enter, click the search button, or select from dropdown for completion.
|
||||
- For autocomplete/combobox fields (e.g. search boxes with suggestions, fields with role="combobox"): type your search text, then WAIT for the suggestions dropdown to appear in the next step. If suggestions appear (new elements marked with *[), click the correct one instead of pressing Enter. If no suggestions appear after one step, you may press Enter or submit normally.
|
||||
- Don't login into a page if you don't have to. Don't login if you don't have the credentials.
|
||||
- There are 2 types of tasks always first think which type of request you are dealing with:
|
||||
1. Very specific step by step instructions:
|
||||
- Follow them as very precise and don't skip steps. Try to complete everything as requested.
|
||||
2. Open ended tasks. Plan yourself, be creative in achieving them.
|
||||
- If you get stuck e.g. with logins in open-ended tasks you can re-evaluate the task and try alternative ways, e.g. sometimes accidentally login pops up, even though there some part of the page is accessible or you get some information via web search. CAPTCHAs are handled automatically.
|
||||
- If you reach a PDF viewer, the file is automatically downloaded and you can see its path in <available_file_paths>. You can either read the file or scroll in the page to see more.
|
||||
- Handle popups, modals, cookie banners, and overlays immediately before attempting other actions. Look for close buttons (X, Close, Dismiss, No thanks, Skip) or accept/reject options. If a popup blocks interaction with the main page, handle it first.
|
||||
- If you encounter access denied (403), bot detection, or rate limiting, do NOT repeatedly retry the same URL. Try alternative approaches or report the limitation.
|
||||
- Detect and break out of unproductive loops: if you are on the same URL for 3+ steps without meaningful progress, or the same action fails 2-3 times, try a different approach. Track what you have tried in memory to avoid repeating failed approaches.
|
||||
</browser_rules>
|
||||
<file_system>
|
||||
- You have access to a persistent file system which you can use to track progress, store results, and manage long tasks.
|
||||
- Your file system is initialized with a `todo.md`: Use this to keep a checklist for known subtasks. Use `replace_file` tool to update markers in `todo.md` as first action whenever you complete an item. This file should guide your step-by-step execution when you have a long running task.
|
||||
- If you are writing a `csv` file, make sure to use double quotes if cell elements contain commas.
|
||||
- If the file is too large, you are only given a preview of your file. Use `read_file` to see the full content if necessary.
|
||||
- If exists, <available_file_paths> includes files you have downloaded or uploaded by the user. You can only read or upload these files but you don't have write access.
|
||||
- If the task is really long, initialize a `results.md` file to accumulate your results.
|
||||
- DO NOT use the file system if the task is less than 10 steps!
|
||||
</file_system>
|
||||
<planning>
|
||||
Decide whether to plan based on task complexity:
|
||||
- Simple task (1-3 actions, e.g. "go to X and click Y"): Act directly. Do NOT output `plan_update`.
|
||||
- Complex but clear task (multi-step, known approach): Output `plan_update` immediately with 3-10 todo items.
|
||||
- Complex and unclear task (unfamiliar site, vague goal): Explore for a few steps first, then output `plan_update` once you understand the landscape.
|
||||
When a plan exists, `<plan>` in your input shows status markers: [x]=done, [>]=current, [ ]=pending, [-]=skipped.
|
||||
Output `current_plan_item` (0-indexed) to indicate which item you are working on.
|
||||
Output `plan_update` again only to revise the plan after unexpected obstacles or after exploration.
|
||||
Completing all plan items does NOT mean the task is done. Always verify against the original <user_request> before calling `done`.
|
||||
</planning>
|
||||
<task_completion_rules>
|
||||
You must call the `done` action in one of two cases:
|
||||
- When you have fully completed the USER REQUEST.
|
||||
- When you reach the final allowed step (`max_steps`), even if the task is incomplete.
|
||||
- If it is ABSOLUTELY IMPOSSIBLE to continue.
|
||||
The `done` action is your opportunity to terminate and share your findings with the user.
|
||||
- Set `success` to `true` only if the full USER REQUEST has been completed with no missing components.
|
||||
- If any part of the request is missing, incomplete, or uncertain, set `success` to `false`.
|
||||
- You can use the `text` field of the `done` action to communicate your findings and `files_to_display` to send file attachments to the user, e.g. `["results.md"]`.
|
||||
- Put ALL the relevant information you found so far in the `text` field when you call `done` action.
|
||||
- Combine `text` and `files_to_display` to provide a coherent reply to the user and fulfill the USER REQUEST.
|
||||
- You are ONLY ALLOWED to call `done` as a single action. Don't call it together with other actions.
|
||||
- If the user asks for specified format, such as "return JSON with following structure", "return a list of format...", MAKE sure to use the right format in your answer.
|
||||
- If the user asks for a structured output, your `done` action's schema will be modified. Take this schema into account when solving the task!
|
||||
<pre_done_verification>
|
||||
BEFORE calling `done` with `success=true`, you MUST perform this verification:
|
||||
1. **Re-read the USER REQUEST** — list every concrete requirement (items to find, actions to perform, format to use, filters to apply).
|
||||
2. **Check each requirement against your results:**
|
||||
- Did you extract the CORRECT number of items? (e.g., "list 5 items" → count them)
|
||||
- Did you apply ALL specified filters/criteria? (e.g., price range, date, location)
|
||||
- Does your output match the requested format exactly?
|
||||
3. **Verify actions actually completed:**
|
||||
- If you submitted a form, posted a comment, or saved a file — check the page state or screenshot to confirm it happened.
|
||||
- If you took a screenshot or downloaded a file — verify it exists in your file system.
|
||||
4. **Verify data grounding:** Every URL, price, name, and value must appear verbatim in your tool outputs or browser_state. Do NOT use your training knowledge to fill gaps — if information was not found on the page during this session, say so explicitly. Never fabricate or invent values.
|
||||
5. **Blocking error check:** If you hit an unresolved blocker (payment declined, login failed without credentials, email/verification wall, required paywall, access denied not bypassed) → set `success=false`. Temporary obstacles you overcame (auto-solved CAPTCHAs, dismissed popups, retried errors) do NOT count.
|
||||
6. **If ANY requirement is unmet, uncertain, or unverifiable — set `success` to `false`.**
|
||||
Partial results with `success=false` are more valuable than overclaiming success.
|
||||
</pre_done_verification>
|
||||
</task_completion_rules>
|
||||
<action_rules>
|
||||
- You are allowed to use a maximum of {max_actions} actions per step.
|
||||
If you are allowed multiple actions, you can specify multiple actions in the list to be executed sequentially (one after another).
|
||||
- If the page changes after an action, the sequence is interrupted and you get the new state. You can see this in your agent history when this happens.
|
||||
Check the browser state each step to verify your previous action achieved its goal. When chaining multiple actions, never take consequential actions (submitting forms, clicking consequential buttons) without confirming necessary changes occurred.
|
||||
</action_rules>
|
||||
<efficiency_guidelines>
|
||||
You can output multiple actions in one step. Try to be efficient where it makes sense. Do not predict actions which do not make sense for the current page.
|
||||
**Recommended Action Combinations:**
|
||||
- `input` + `click` → Fill form field and submit/search in one step
|
||||
- `input` + `input` → Fill multiple form fields
|
||||
- `click` + `click` → Navigate through multi-step flows (when the page does not navigate between clicks)
|
||||
- File operations + browser actions
|
||||
Do not try multiple different paths in one step. Always have one clear goal per step.
|
||||
Its important that you see in the next step if your action was successful, so do not chain actions which change the browser state multiple times, e.g.
|
||||
- do not use click and then navigate, because you would not see if the click was successful or not.
|
||||
- or do not use switch and switch together, because you would not see the state in between.
|
||||
- do not use input and then scroll, because you would not see if the input was successful or not.
|
||||
</efficiency_guidelines>
|
||||
<reasoning_rules>
|
||||
Be clear and concise in your decision-making. Exhibit the following reasoning patterns to successfully achieve the <user_request>:
|
||||
- Reason about <agent_history> to track progress and context toward <user_request>.
|
||||
- Analyze the most recent "Next Goal" and "Action Result" in <agent_history> and clearly state what you previously tried to achieve.
|
||||
- Analyze all relevant items in <agent_history>, <browser_state>, <read_state>, <file_system>, <read_state> and the screenshot to understand your state.
|
||||
- Explicitly judge success/failure/uncertainty of the last action. Never assume an action succeeded just because it appears to be executed in your last step in <agent_history>. For example, you might have "Action 1/1: Input '2025-05-05' into element 3." in your history even though inputting text failed. Always verify using <browser_vision> (screenshot) as the primary ground truth. If a screenshot is unavailable, fall back to <browser_state>. If the expected change is missing, mark the last action as failed (or uncertain) and plan a recovery.
|
||||
- If todo.md is empty and the task is multi-step, generate a stepwise plan in todo.md using file tools.
|
||||
- Analyze `todo.md` to guide and track your progress.
|
||||
- If any todo.md items are finished, mark them as complete in the file.
|
||||
- Analyze whether you are stuck, e.g. when you repeat the same actions multiple times without any progress. Then consider alternative approaches.
|
||||
- Analyze the <read_state> where one-time information are displayed due to your previous action. Reason about whether you want to keep this information in memory and plan writing them into a file if applicable using the file tools.
|
||||
- If you see information relevant to <user_request>, plan saving the information into a file.
|
||||
- Before writing data into a file, analyze the <file_system> and check if the file already has some content to avoid overwriting.
|
||||
- Decide what concise, actionable context should be stored in memory to inform future reasoning.
|
||||
- When ready to finish, state you are preparing to call done and communicate completion/results to the user.
|
||||
- Before done, use read_file to verify file contents intended for user output.
|
||||
- Always reason about the <user_request>. Make sure to carefully analyze the specific steps and information required. E.g. specific filters, specific form fields, specific information to search. Make sure to always compare the current trajectory with the user request.
|
||||
</reasoning_rules>
|
||||
<examples>
|
||||
Here are examples of good output patterns. Use them as reference but never copy them directly.
|
||||
<todo_examples>
|
||||
"write_file": {{
|
||||
"file_name": "todo.md",
|
||||
"content": "# ArXiv CS.AI Recent Papers Collection Task\n\n## Goal: Collect metadata for 20 most recent papers\n\n## Tasks:\n- [ ] Navigate to https://arxiv.org/list/cs.AI/recent\n- [ ] Initialize papers.md file for storing paper data\n- [ ] Collect paper 1/20: The Automated LLM Speedrunning Benchmark\n- [x] Collect paper 2/20: AI Model Passport\n- [ ] Collect paper 3/20: Embodied AI Agents\n- [ ] Collect paper 4/20: Conceptual Topic Aggregation\n- [ ] Collect paper 5/20: Artificial Intelligent Disobedience\n- [ ] Continue collecting remaining papers from current page\n- [ ] Navigate through subsequent pages if needed\n- [ ] Continue until 20 papers are collected\n- [ ] Verify all 20 papers have complete metadata\n- [ ] Final review and completion"
|
||||
}}
|
||||
</todo_examples>
|
||||
<evaluation_examples>
|
||||
- Positive Examples:
|
||||
"evaluation_previous_goal": "Successfully navigated to the product page and found the target information. Verdict: Success"
|
||||
"evaluation_previous_goal": "Clicked the login button and user authentication form appeared. Verdict: Success"
|
||||
- Negative Examples:
|
||||
"evaluation_previous_goal": "Failed to input text into the search bar as I cannot see it in the image. Verdict: Failure"
|
||||
"evaluation_previous_goal": "Clicked the submit button with index 15 but the form was not submitted successfully. Verdict: Failure"
|
||||
</evaluation_examples>
|
||||
<memory_examples>
|
||||
"memory": "Visited 2 of 5 target websites. Collected pricing data from Amazon ($39.99) and eBay ($42.00). Still need to check Walmart, Target, and Best Buy for the laptop comparison."
|
||||
"memory": "Found many pending reports that need to be analyzed in the main page. Successfully processed the first 2 reports on quarterly sales data and moving on to inventory analysis and customer feedback reports."
|
||||
"memory": "Search returned results but no filter applied yet. User wants items under $50 with 4+ stars. Will apply price filter first, then rating filter."
|
||||
"memory": "Popup appeared blocking the page. Need to close it first before continuing with search."
|
||||
"memory": "Previous click on search button failed - page did not change. Will try pressing Enter in the search field instead."
|
||||
"memory": "Captcha appeared twice on this site. Will try alternative approach via search engine instead of direct navigation."
|
||||
"memory": "403 error on main product page. Will try searching for the product on a different site instead of retrying."
|
||||
</memory_examples>
|
||||
<next_goal_examples>
|
||||
"next_goal": "Click on the 'Add to Cart' button to proceed with the purchase flow."
|
||||
"next_goal": "Extract details from the first item on the page."
|
||||
"next_goal": "Close the popup that appeared blocking the main content."
|
||||
"next_goal": "Apply price filter to narrow results to items under $50."
|
||||
</next_goal_examples>
|
||||
</examples>
|
||||
<output>
|
||||
You must ALWAYS respond with a valid JSON in this exact format:
|
||||
{{
|
||||
"evaluation_previous_goal": "One-sentence analysis of your last action. Clearly state success, failure, or uncertain.",
|
||||
"memory": "1-3 sentences of specific memory of this step and overall progress. You should put here everything that will help you track progress in future steps. Like counting pages visited, items found, etc.",
|
||||
"next_goal": "State the next immediate goal and action to achieve it, in one clear sentence.",
|
||||
"current_plan_item": 0,
|
||||
"plan_update": ["Todo item 1", "Todo item 2", "Todo item 3"],
|
||||
"action":[{{"navigate": {{ "url": "url_value"}}}}, // ... more actions in sequence]
|
||||
}}
|
||||
Action list should NEVER be empty.
|
||||
`current_plan_item` and `plan_update` are optional. See <planning> for details.
|
||||
</output>
|
||||
<critical_reminders>
|
||||
1. ALWAYS verify action success using the screenshot before proceeding
|
||||
2. ALWAYS handle popups/modals/cookie banners before other actions
|
||||
3. ALWAYS apply filters when user specifies criteria (price, rating, location, etc.)
|
||||
4. NEVER repeat the same failing action more than 2-3 times - try alternatives
|
||||
5. NEVER assume success - always verify from screenshot or browser state
|
||||
6. CAPTCHAs are solved automatically. If blocked by login/403, try alternative approaches rather than retrying
|
||||
7. Put ALL relevant findings in done action's text field
|
||||
8. Match user's requested output format exactly
|
||||
9. Track progress in memory to avoid loops
|
||||
10. When at max_steps, call done with whatever results you have
|
||||
11. Always compare current trajectory against the user's original request
|
||||
12. Be efficient - combine actions when possible but verify results between major steps
|
||||
</critical_reminders>
|
||||
<error_recovery>
|
||||
When encountering errors or unexpected states:
|
||||
1. First, verify the current state using screenshot as ground truth
|
||||
2. Check if a popup, modal, or overlay is blocking interaction
|
||||
3. If an element is not found, scroll to reveal more content
|
||||
4. If an action fails repeatedly (2-3 times), try an alternative approach
|
||||
5. If blocked by login/403, consider alternative sites or search engines. CAPTCHAs are solved automatically.
|
||||
6. If the page structure is different than expected, re-analyze and adapt
|
||||
7. If stuck in a loop, explicitly acknowledge it in memory and change strategy
|
||||
8. If max_steps is approaching, prioritize completing the most important parts of the task
|
||||
</error_recovery>
|
||||
276
.agent/vendor/browser_use/browser_use/agent/variable_detector.py
vendored
Normal file
276
.agent/vendor/browser_use/browser_use/agent/variable_detector.py
vendored
Normal file
@@ -0,0 +1,276 @@
|
||||
"""Detect variables in agent history for reuse"""
|
||||
|
||||
import re
|
||||
|
||||
from browser_use.agent.views import AgentHistoryList, DetectedVariable
|
||||
from browser_use.dom.views import DOMInteractedElement
|
||||
|
||||
|
||||
def detect_variables_in_history(history: AgentHistoryList) -> dict[str, DetectedVariable]:
|
||||
"""
|
||||
Analyze agent history and detect reusable variables.
|
||||
|
||||
Uses two strategies:
|
||||
1. Element attributes (id, name, type, placeholder, aria-label) - most reliable
|
||||
2. Value pattern matching (email, phone, date formats) - fallback
|
||||
|
||||
Returns:
|
||||
Dictionary mapping variable names to DetectedVariable objects
|
||||
"""
|
||||
detected: dict[str, DetectedVariable] = {}
|
||||
detected_values: set[str] = set() # Track which values we've already detected
|
||||
|
||||
for step_idx, history_item in enumerate(history.history):
|
||||
if not history_item.model_output:
|
||||
continue
|
||||
|
||||
for action_idx, action in enumerate(history_item.model_output.action):
|
||||
# Convert action to dict - handle both Pydantic models and dict-like objects
|
||||
if hasattr(action, 'model_dump'):
|
||||
action_dict = action.model_dump()
|
||||
elif isinstance(action, dict):
|
||||
action_dict = action
|
||||
else:
|
||||
# For SimpleNamespace or similar objects
|
||||
action_dict = vars(action)
|
||||
|
||||
# Get the interacted element for this action (if available)
|
||||
element = None
|
||||
if history_item.state and history_item.state.interacted_element:
|
||||
if len(history_item.state.interacted_element) > action_idx:
|
||||
element = history_item.state.interacted_element[action_idx]
|
||||
|
||||
# Detect variables in this action
|
||||
_detect_in_action(action_dict, element, detected, detected_values)
|
||||
|
||||
return detected
|
||||
|
||||
|
||||
def _detect_in_action(
|
||||
action_dict: dict,
|
||||
element: DOMInteractedElement | None,
|
||||
detected: dict[str, DetectedVariable],
|
||||
detected_values: set[str],
|
||||
) -> None:
|
||||
"""Detect variables in a single action using element context"""
|
||||
|
||||
# Extract action type and parameters
|
||||
for action_type, params in action_dict.items():
|
||||
if not isinstance(params, dict):
|
||||
continue
|
||||
|
||||
# Check fields that commonly contain variables
|
||||
fields_to_check = ['text', 'query']
|
||||
|
||||
for field in fields_to_check:
|
||||
if field not in params:
|
||||
continue
|
||||
|
||||
value = params[field]
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
continue
|
||||
|
||||
# Skip if we already detected this exact value
|
||||
if value in detected_values:
|
||||
continue
|
||||
|
||||
# Try to detect variable type (with element context)
|
||||
var_info = _detect_variable_type(value, element)
|
||||
if not var_info:
|
||||
continue
|
||||
|
||||
var_name, var_format = var_info
|
||||
|
||||
# Ensure unique variable name
|
||||
var_name = _ensure_unique_name(var_name, detected)
|
||||
|
||||
# Add detected variable
|
||||
detected[var_name] = DetectedVariable(
|
||||
name=var_name,
|
||||
original_value=value,
|
||||
type='string',
|
||||
format=var_format,
|
||||
)
|
||||
|
||||
detected_values.add(value)
|
||||
|
||||
|
||||
def _detect_variable_type(
|
||||
value: str,
|
||||
element: DOMInteractedElement | None = None,
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""
|
||||
Detect if a value looks like a variable, using element context when available.
|
||||
|
||||
Priority:
|
||||
1. Element attributes (id, name, type, placeholder, aria-label) - most reliable
|
||||
2. Value pattern matching (email, phone, date formats) - fallback
|
||||
|
||||
Returns:
|
||||
(variable_name, format) or None if not detected
|
||||
"""
|
||||
|
||||
# STRATEGY 1: Use element attributes (most reliable)
|
||||
if element and element.attributes:
|
||||
attr_detection = _detect_from_attributes(element.attributes)
|
||||
if attr_detection:
|
||||
return attr_detection
|
||||
|
||||
# STRATEGY 2: Pattern matching on value (fallback)
|
||||
return _detect_from_value_pattern(value)
|
||||
|
||||
|
||||
def _detect_from_attributes(attributes: dict[str, str]) -> tuple[str, str | None] | None:
|
||||
"""
|
||||
Detect variable from element attributes.
|
||||
|
||||
Check attributes in priority order:
|
||||
1. type attribute (HTML5 input types - most specific)
|
||||
2. id, name, placeholder, aria-label (semantic hints)
|
||||
"""
|
||||
|
||||
# Check 'type' attribute first (HTML5 input types)
|
||||
input_type = attributes.get('type', '').lower()
|
||||
if input_type == 'email':
|
||||
return ('email', 'email')
|
||||
elif input_type == 'tel':
|
||||
return ('phone', 'phone')
|
||||
elif input_type == 'date':
|
||||
return ('date', 'date')
|
||||
elif input_type == 'number':
|
||||
return ('number', 'number')
|
||||
elif input_type == 'url':
|
||||
return ('url', 'url')
|
||||
|
||||
# Combine semantic attributes for keyword matching
|
||||
semantic_attrs = [
|
||||
attributes.get('id', ''),
|
||||
attributes.get('name', ''),
|
||||
attributes.get('placeholder', ''),
|
||||
attributes.get('aria-label', ''),
|
||||
]
|
||||
|
||||
combined_text = ' '.join(semantic_attrs).lower()
|
||||
|
||||
# Address detection
|
||||
if any(keyword in combined_text for keyword in ['address', 'street', 'addr']):
|
||||
if 'billing' in combined_text:
|
||||
return ('billing_address', None)
|
||||
elif 'shipping' in combined_text:
|
||||
return ('shipping_address', None)
|
||||
else:
|
||||
return ('address', None)
|
||||
|
||||
# Comment/Note detection
|
||||
if any(keyword in combined_text for keyword in ['comment', 'note', 'message', 'description']):
|
||||
return ('comment', None)
|
||||
|
||||
# Email detection
|
||||
if 'email' in combined_text or 'e-mail' in combined_text:
|
||||
return ('email', 'email')
|
||||
|
||||
# Phone detection
|
||||
if any(keyword in combined_text for keyword in ['phone', 'tel', 'mobile', 'cell']):
|
||||
return ('phone', 'phone')
|
||||
|
||||
# Name detection (order matters - check specific before general)
|
||||
if 'first' in combined_text and 'name' in combined_text:
|
||||
return ('first_name', None)
|
||||
elif 'last' in combined_text and 'name' in combined_text:
|
||||
return ('last_name', None)
|
||||
elif 'full' in combined_text and 'name' in combined_text:
|
||||
return ('full_name', None)
|
||||
elif 'name' in combined_text:
|
||||
return ('name', None)
|
||||
|
||||
# Date detection
|
||||
if any(keyword in combined_text for keyword in ['date', 'dob', 'birth']):
|
||||
return ('date', 'date')
|
||||
|
||||
# City detection
|
||||
if 'city' in combined_text:
|
||||
return ('city', None)
|
||||
|
||||
# State/Province detection
|
||||
if 'state' in combined_text or 'province' in combined_text:
|
||||
return ('state', None)
|
||||
|
||||
# Country detection
|
||||
if 'country' in combined_text:
|
||||
return ('country', None)
|
||||
|
||||
# Zip code detection
|
||||
if any(keyword in combined_text for keyword in ['zip', 'postal', 'postcode']):
|
||||
return ('zip_code', 'postal_code')
|
||||
|
||||
# Company detection
|
||||
if 'company' in combined_text or 'organization' in combined_text:
|
||||
return ('company', None)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _detect_from_value_pattern(value: str) -> tuple[str, str | None] | None:
|
||||
"""
|
||||
Detect variable type from value pattern (fallback when no element context).
|
||||
|
||||
Patterns:
|
||||
- Email: contains @ and . with valid format
|
||||
- Phone: digits with separators, 10+ chars
|
||||
- Date: YYYY-MM-DD format
|
||||
- Name: Capitalized word(s), 2-30 chars, letters only
|
||||
- Number: Pure digits, 1-9 chars
|
||||
"""
|
||||
|
||||
# Email detection - most specific first
|
||||
if '@' in value and '.' in value:
|
||||
# Basic email validation
|
||||
if re.match(r'^[\w\.-]+@[\w\.-]+\.\w+$', value):
|
||||
return ('email', 'email')
|
||||
|
||||
# Phone detection (digits with separators, 10+ chars)
|
||||
if re.match(r'^[\d\s\-\(\)\+]+$', value):
|
||||
# Remove separators and check length
|
||||
digits_only = re.sub(r'[\s\-\(\)\+]', '', value)
|
||||
if len(digits_only) >= 10:
|
||||
return ('phone', 'phone')
|
||||
|
||||
# Date detection (YYYY-MM-DD or similar)
|
||||
if re.match(r'^\d{4}-\d{2}-\d{2}$', value):
|
||||
return ('date', 'date')
|
||||
|
||||
# Name detection (capitalized, only letters/spaces, 2-30 chars)
|
||||
if value and value[0].isupper() and value.replace(' ', '').replace('-', '').isalpha() and 2 <= len(value) <= 30:
|
||||
words = value.split()
|
||||
if len(words) == 1:
|
||||
return ('first_name', None)
|
||||
elif len(words) == 2:
|
||||
return ('full_name', None)
|
||||
else:
|
||||
return ('name', None)
|
||||
|
||||
# Number detection (pure digits, not phone length)
|
||||
if value.isdigit() and 1 <= len(value) <= 9:
|
||||
return ('number', 'number')
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_unique_name(base_name: str, existing: dict[str, DetectedVariable]) -> str:
|
||||
"""
|
||||
Ensure variable name is unique by adding suffix if needed.
|
||||
|
||||
Examples:
|
||||
first_name → first_name
|
||||
first_name (exists) → first_name_2
|
||||
first_name_2 (exists) → first_name_3
|
||||
"""
|
||||
if base_name not in existing:
|
||||
return base_name
|
||||
|
||||
# Add numeric suffix
|
||||
counter = 2
|
||||
while f'{base_name}_{counter}' in existing:
|
||||
counter += 1
|
||||
|
||||
return f'{base_name}_{counter}'
|
||||
1009
.agent/vendor/browser_use/browser_use/agent/views.py
vendored
Normal file
1009
.agent/vendor/browser_use/browser_use/agent/views.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
41
.agent/vendor/browser_use/browser_use/browser/__init__.py
vendored
Normal file
41
.agent/vendor/browser_use/browser_use/browser/__init__.py
vendored
Normal file
@@ -0,0 +1,41 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
# Type stubs for lazy imports
|
||||
if TYPE_CHECKING:
|
||||
from .profile import BrowserProfile, ProxySettings
|
||||
from .session import BrowserSession
|
||||
|
||||
|
||||
# Lazy imports mapping for heavy browser components
|
||||
_LAZY_IMPORTS = {
|
||||
'ProxySettings': ('.profile', 'ProxySettings'),
|
||||
'BrowserProfile': ('.profile', 'BrowserProfile'),
|
||||
'BrowserSession': ('.session', 'BrowserSession'),
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Lazy import mechanism for heavy browser components."""
|
||||
if name in _LAZY_IMPORTS:
|
||||
module_path, attr_name = _LAZY_IMPORTS[name]
|
||||
try:
|
||||
from importlib import import_module
|
||||
|
||||
# Use relative import for current package
|
||||
full_module_path = f'browser_use.browser{module_path}'
|
||||
module = import_module(full_module_path)
|
||||
attr = getattr(module, attr_name)
|
||||
# Cache the imported attribute in the module's globals
|
||||
globals()[name] = attr
|
||||
return attr
|
||||
except ImportError as e:
|
||||
raise ImportError(f'Failed to import {name} from {full_module_path}: {e}') from e
|
||||
|
||||
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
||||
|
||||
|
||||
__all__ = [
|
||||
'BrowserSession',
|
||||
'BrowserProfile',
|
||||
'ProxySettings',
|
||||
]
|
||||
203
.agent/vendor/browser_use/browser_use/browser/cloud/cloud.py
vendored
Normal file
203
.agent/vendor/browser_use/browser_use/browser/cloud/cloud.py
vendored
Normal file
@@ -0,0 +1,203 @@
|
||||
"""Cloud browser service integration for browser-use.
|
||||
|
||||
This module provides integration with the browser-use cloud browser service.
|
||||
When cloud_browser=True, it automatically creates a cloud browser instance
|
||||
and returns the CDP URL for connection.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
from browser_use.browser.cloud.views import CloudBrowserAuthError, CloudBrowserError, CloudBrowserResponse, CreateBrowserRequest
|
||||
from browser_use.sync.auth import CloudAuthConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CloudBrowserClient:
|
||||
"""Client for browser-use cloud browser service."""
|
||||
|
||||
def __init__(self, api_base_url: str = 'https://api.browser-use.com'):
|
||||
self.api_base_url = api_base_url
|
||||
self.client = httpx.AsyncClient(timeout=30.0)
|
||||
self.current_session_id: str | None = None
|
||||
|
||||
async def create_browser(
|
||||
self, request: CreateBrowserRequest, extra_headers: dict[str, str] | None = None
|
||||
) -> CloudBrowserResponse:
|
||||
"""Create a new cloud browser instance. For full docs refer to https://docs.cloud.browser-use.com/api-reference/v-2-api-current/browsers/create-browser-session-browsers-post
|
||||
|
||||
Args:
|
||||
request: CreateBrowserRequest object containing browser creation parameters
|
||||
|
||||
Returns:
|
||||
CloudBrowserResponse: Contains CDP URL and other browser info
|
||||
"""
|
||||
url = f'{self.api_base_url}/api/v2/browsers'
|
||||
|
||||
# Try to get API key from environment variable first, then auth config
|
||||
api_token = os.getenv('BROWSER_USE_API_KEY')
|
||||
|
||||
if not api_token:
|
||||
# Fallback to auth config file
|
||||
try:
|
||||
auth_config = CloudAuthConfig.load_from_file()
|
||||
api_token = auth_config.api_token
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not api_token:
|
||||
raise CloudBrowserAuthError(
|
||||
'No authentication token found. Please set BROWSER_USE_API_KEY environment variable to authenticate with the cloud service. You can also create an API key at https://cloud.browser-use.com/new-api-key'
|
||||
)
|
||||
|
||||
headers = {'X-Browser-Use-API-Key': api_token, 'Content-Type': 'application/json', **(extra_headers or {})}
|
||||
|
||||
# Convert request to dictionary and exclude unset fields
|
||||
request_body = request.model_dump(exclude_unset=True)
|
||||
|
||||
try:
|
||||
logger.info('🌤️ Creating cloud browser instance...')
|
||||
|
||||
response = await self.client.post(url, headers=headers, json=request_body)
|
||||
|
||||
if response.status_code == 401:
|
||||
raise CloudBrowserAuthError(
|
||||
'Authentication failed. Please make sure you have set BROWSER_USE_API_KEY environment variable to authenticate with the cloud service. You can also create an API key at https://cloud.browser-use.com/new-api-key'
|
||||
)
|
||||
elif response.status_code == 403:
|
||||
raise CloudBrowserAuthError('Access forbidden. Please check your browser-use cloud subscription status.')
|
||||
elif not response.is_success:
|
||||
error_msg = f'Failed to create cloud browser: HTTP {response.status_code}'
|
||||
try:
|
||||
error_data = response.json()
|
||||
if 'detail' in error_data:
|
||||
error_msg += f' - {error_data["detail"]}'
|
||||
except Exception:
|
||||
pass
|
||||
raise CloudBrowserError(error_msg)
|
||||
|
||||
browser_data = response.json()
|
||||
browser_response = CloudBrowserResponse(**browser_data)
|
||||
|
||||
# Store session ID for cleanup
|
||||
self.current_session_id = browser_response.id
|
||||
|
||||
logger.info(f'🌤️ Cloud browser created successfully: {browser_response.id}')
|
||||
logger.debug(f'🌤️ CDP URL: {browser_response.cdpUrl}')
|
||||
# Cyan color for live URL
|
||||
logger.info(f'\033[36m🔗 Live URL: {browser_response.liveUrl}\033[0m')
|
||||
|
||||
return browser_response
|
||||
|
||||
except httpx.TimeoutException:
|
||||
raise CloudBrowserError('Timeout while creating cloud browser. Please try again.')
|
||||
except httpx.ConnectError:
|
||||
raise CloudBrowserError('Failed to connect to cloud browser service. Please check your internet connection.')
|
||||
except Exception as e:
|
||||
if isinstance(e, (CloudBrowserError, CloudBrowserAuthError)):
|
||||
raise
|
||||
raise CloudBrowserError(f'Unexpected error creating cloud browser: {e}')
|
||||
|
||||
async def stop_browser(
|
||||
self, session_id: str | None = None, extra_headers: dict[str, str] | None = None
|
||||
) -> CloudBrowserResponse:
|
||||
"""Stop a cloud browser session.
|
||||
|
||||
Args:
|
||||
session_id: Session ID to stop. If None, uses current session.
|
||||
|
||||
Returns:
|
||||
CloudBrowserResponse: Updated browser info with stopped status
|
||||
|
||||
Raises:
|
||||
CloudBrowserAuthError: If authentication fails
|
||||
CloudBrowserError: If stopping fails
|
||||
"""
|
||||
if session_id is None:
|
||||
session_id = self.current_session_id
|
||||
|
||||
if not session_id:
|
||||
raise CloudBrowserError('No session ID provided and no current session available')
|
||||
|
||||
url = f'{self.api_base_url}/api/v2/browsers/{session_id}'
|
||||
|
||||
# Try to get API key from environment variable first, then auth config
|
||||
api_token = os.getenv('BROWSER_USE_API_KEY')
|
||||
|
||||
if not api_token:
|
||||
# Fallback to auth config file
|
||||
try:
|
||||
auth_config = CloudAuthConfig.load_from_file()
|
||||
api_token = auth_config.api_token
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not api_token:
|
||||
raise CloudBrowserAuthError(
|
||||
'No authentication token found. Please set BROWSER_USE_API_KEY environment variable to authenticate with the cloud service. You can also create an API key at https://cloud.browser-use.com/new-api-key'
|
||||
)
|
||||
|
||||
headers = {'X-Browser-Use-API-Key': api_token, 'Content-Type': 'application/json', **(extra_headers or {})}
|
||||
|
||||
request_body = {'action': 'stop'}
|
||||
|
||||
try:
|
||||
logger.info(f'🌤️ Stopping cloud browser session: {session_id}')
|
||||
|
||||
response = await self.client.patch(url, headers=headers, json=request_body)
|
||||
|
||||
if response.status_code == 401:
|
||||
raise CloudBrowserAuthError(
|
||||
'Authentication failed. Please make sure you have set the BROWSER_USE_API_KEY environment variable to authenticate with the cloud service.'
|
||||
)
|
||||
elif response.status_code == 404:
|
||||
# Session already stopped or doesn't exist - treating as error and clearing session
|
||||
logger.debug(f'🌤️ Cloud browser session {session_id} not found (already stopped)')
|
||||
# Clear current session if it was this one
|
||||
if session_id == self.current_session_id:
|
||||
self.current_session_id = None
|
||||
raise CloudBrowserError(f'Cloud browser session {session_id} not found')
|
||||
elif not response.is_success:
|
||||
error_msg = f'Failed to stop cloud browser: HTTP {response.status_code}'
|
||||
try:
|
||||
error_data = response.json()
|
||||
if 'detail' in error_data:
|
||||
error_msg += f' - {error_data["detail"]}'
|
||||
except Exception:
|
||||
pass
|
||||
raise CloudBrowserError(error_msg)
|
||||
|
||||
browser_data = response.json()
|
||||
browser_response = CloudBrowserResponse(**browser_data)
|
||||
|
||||
# Clear current session if it was this one
|
||||
if session_id == self.current_session_id:
|
||||
self.current_session_id = None
|
||||
|
||||
logger.info(f'🌤️ Cloud browser session stopped: {browser_response.id}')
|
||||
logger.debug(f'🌤️ Status: {browser_response.status}')
|
||||
|
||||
return browser_response
|
||||
|
||||
except httpx.TimeoutException:
|
||||
raise CloudBrowserError('Timeout while stopping cloud browser. Please try again.')
|
||||
except httpx.ConnectError:
|
||||
raise CloudBrowserError('Failed to connect to cloud browser service. Please check your internet connection.')
|
||||
except Exception as e:
|
||||
if isinstance(e, (CloudBrowserError, CloudBrowserAuthError)):
|
||||
raise
|
||||
raise CloudBrowserError(f'Unexpected error stopping cloud browser: {e}')
|
||||
|
||||
async def close(self):
|
||||
"""Close the HTTP client and cleanup any active sessions."""
|
||||
# Try to stop current session if active
|
||||
if self.current_session_id:
|
||||
try:
|
||||
await self.stop_browser()
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to stop cloud browser session during cleanup: {e}')
|
||||
|
||||
await self.client.aclose()
|
||||
89
.agent/vendor/browser_use/browser_use/browser/cloud/views.py
vendored
Normal file
89
.agent/vendor/browser_use/browser_use/browser/cloud/views.py
vendored
Normal file
@@ -0,0 +1,89 @@
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
ProxyCountryCode = (
|
||||
Literal[
|
||||
'us', # United States
|
||||
'uk', # United Kingdom
|
||||
'fr', # France
|
||||
'it', # Italy
|
||||
'jp', # Japan
|
||||
'au', # Australia
|
||||
'de', # Germany
|
||||
'fi', # Finland
|
||||
'ca', # Canada
|
||||
'in', # India
|
||||
]
|
||||
| str
|
||||
)
|
||||
|
||||
# Browser session timeout limits (in minutes)
|
||||
MAX_FREE_USER_SESSION_TIMEOUT = 15 # Free users limited to 15 minutes
|
||||
MAX_PAID_USER_SESSION_TIMEOUT = 240 # Paid users can go up to 4 hours
|
||||
|
||||
|
||||
# Requests
|
||||
class CreateBrowserRequest(BaseModel):
|
||||
"""Request to create a cloud browser instance.
|
||||
|
||||
Args:
|
||||
cloud_profile_id: The ID of the profile to use for the session
|
||||
cloud_proxy_country_code: Country code for proxy location
|
||||
cloud_timeout: The timeout for the session in minutes
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra='forbid', populate_by_name=True)
|
||||
|
||||
profile_id: UUID | str | None = Field(
|
||||
default=None,
|
||||
alias='cloud_profile_id',
|
||||
description='The ID of the profile to use for the session. Can be a UUID or a string of UUID.',
|
||||
title='Cloud Profile ID',
|
||||
)
|
||||
|
||||
proxy_country_code: ProxyCountryCode | None = Field(
|
||||
default=None,
|
||||
alias='cloud_proxy_country_code',
|
||||
description='Country code for proxy location.',
|
||||
title='Cloud Proxy Country Code',
|
||||
)
|
||||
|
||||
timeout: int | None = Field(
|
||||
ge=1,
|
||||
le=MAX_PAID_USER_SESSION_TIMEOUT,
|
||||
default=None,
|
||||
alias='cloud_timeout',
|
||||
description=f'The timeout for the session in minutes. Free users are limited to {MAX_FREE_USER_SESSION_TIMEOUT} minutes, paid users can use up to {MAX_PAID_USER_SESSION_TIMEOUT} minutes ({MAX_PAID_USER_SESSION_TIMEOUT // 60} hours).',
|
||||
title='Cloud Timeout',
|
||||
)
|
||||
|
||||
|
||||
CloudBrowserParams = CreateBrowserRequest # alias for easier readability
|
||||
|
||||
|
||||
# Responses
|
||||
class CloudBrowserResponse(BaseModel):
|
||||
"""Response from cloud browser API."""
|
||||
|
||||
id: str
|
||||
status: str
|
||||
liveUrl: str = Field(alias='liveUrl')
|
||||
cdpUrl: str = Field(alias='cdpUrl')
|
||||
timeoutAt: str = Field(alias='timeoutAt')
|
||||
startedAt: str = Field(alias='startedAt')
|
||||
finishedAt: str | None = Field(alias='finishedAt', default=None)
|
||||
|
||||
|
||||
# Errors
|
||||
class CloudBrowserError(Exception):
|
||||
"""Exception raised when cloud browser operations fail."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class CloudBrowserAuthError(CloudBrowserError):
|
||||
"""Exception raised when cloud browser authentication fails."""
|
||||
|
||||
pass
|
||||
922
.agent/vendor/browser_use/browser_use/browser/demo_mode.py
vendored
Normal file
922
.agent/vendor/browser_use/browser_use/browser/demo_mode.py
vendored
Normal file
File diff suppressed because one or more lines are too long
667
.agent/vendor/browser_use/browser_use/browser/events.py
vendored
Normal file
667
.agent/vendor/browser_use/browser_use/browser/events.py
vendored
Normal file
@@ -0,0 +1,667 @@
|
||||
"""Event definitions for browser communication."""
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from typing import Any, Literal
|
||||
|
||||
from bubus import BaseEvent
|
||||
from bubus.models import T_EventResultType
|
||||
from cdp_use.cdp.target import TargetID
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from browser_use.browser.views import BrowserStateSummary
|
||||
from browser_use.dom.views import EnhancedDOMTreeNode
|
||||
|
||||
|
||||
def _get_timeout(env_var: str, default: float) -> float | None:
|
||||
"""
|
||||
Safely parse environment variable timeout values with robust error handling.
|
||||
|
||||
Args:
|
||||
env_var: Environment variable name (e.g. 'TIMEOUT_NavigateToUrlEvent')
|
||||
default: Default timeout value as float (e.g. 15.0)
|
||||
|
||||
Returns:
|
||||
Parsed float value or the default if parsing fails
|
||||
|
||||
Raises:
|
||||
ValueError: Only if both env_var and default are invalid (should not happen with valid defaults)
|
||||
"""
|
||||
# Try environment variable first
|
||||
env_value = os.getenv(env_var)
|
||||
if env_value:
|
||||
try:
|
||||
parsed = float(env_value)
|
||||
if parsed < 0:
|
||||
print(f'Warning: {env_var}={env_value} is negative, using default {default}')
|
||||
return default
|
||||
return parsed
|
||||
except (ValueError, TypeError):
|
||||
print(f'Warning: {env_var}={env_value} is not a valid number, using default {default}')
|
||||
|
||||
# Fall back to default
|
||||
return default
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Agent/Tools -> BrowserSession Events (High-level browser actions)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class ElementSelectedEvent(BaseEvent[T_EventResultType]):
|
||||
"""An element was selected."""
|
||||
|
||||
node: EnhancedDOMTreeNode
|
||||
|
||||
@field_validator('node', mode='before')
|
||||
@classmethod
|
||||
def serialize_node(cls, data: EnhancedDOMTreeNode | None) -> EnhancedDOMTreeNode | None:
|
||||
if data is None:
|
||||
return None
|
||||
return EnhancedDOMTreeNode(
|
||||
node_id=data.node_id,
|
||||
backend_node_id=data.backend_node_id,
|
||||
session_id=data.session_id,
|
||||
frame_id=data.frame_id,
|
||||
target_id=data.target_id,
|
||||
node_type=data.node_type,
|
||||
node_name=data.node_name,
|
||||
node_value=data.node_value,
|
||||
attributes=data.attributes,
|
||||
is_scrollable=data.is_scrollable,
|
||||
is_visible=data.is_visible,
|
||||
absolute_position=data.absolute_position,
|
||||
# override the circular reference fields in EnhancedDOMTreeNode as they cant be serialized and aren't needed by event handlers
|
||||
# only used internally by the DOM service during DOM tree building process, not intended public API use
|
||||
content_document=None,
|
||||
shadow_root_type=None,
|
||||
shadow_roots=[],
|
||||
parent_node=None,
|
||||
children_nodes=[],
|
||||
ax_node=None,
|
||||
snapshot_node=None,
|
||||
)
|
||||
|
||||
|
||||
# TODO: add page handle to events
|
||||
# class PageHandle(share a base with browser.session.CDPSession?):
|
||||
# url: str
|
||||
# target_id: TargetID
|
||||
# @classmethod
|
||||
# def from_target_id(cls, target_id: TargetID) -> Self:
|
||||
# return cls(target_id=target_id)
|
||||
# @classmethod
|
||||
# def from_target_id(cls, target_id: TargetID) -> Self:
|
||||
# return cls(target_id=target_id)
|
||||
# @classmethod
|
||||
# def from_url(cls, url: str) -> Self:
|
||||
# @property
|
||||
# def root_frame_id(self) -> str:
|
||||
# return self.target_id
|
||||
# @property
|
||||
# def session_id(self) -> str:
|
||||
# return browser_session.get_or_create_cdp_session(self.target_id).session_id
|
||||
|
||||
# class PageSelectedEvent(BaseEvent[T_EventResultType]):
|
||||
# """An event like SwitchToTabEvent(page=PageHandle) or CloseTabEvent(page=PageHandle)"""
|
||||
# page: PageHandle
|
||||
|
||||
|
||||
class NavigateToUrlEvent(BaseEvent[None]):
|
||||
"""Navigate to a specific URL."""
|
||||
|
||||
url: str
|
||||
wait_until: Literal['load', 'domcontentloaded', 'networkidle', 'commit'] = 'load'
|
||||
timeout_ms: int | None = None
|
||||
new_tab: bool = Field(
|
||||
default=False, description='Set True to leave the current tab alone and open a new tab in the foreground for the new URL'
|
||||
)
|
||||
# existing_tab: PageHandle | None = None # TODO
|
||||
|
||||
# time limits enforced by bubus, not exposed to LLM:
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigateToUrlEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class ClickElementEvent(ElementSelectedEvent[dict[str, Any] | None]):
|
||||
"""Click an element."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
button: Literal['left', 'right', 'middle'] = 'left'
|
||||
# click_count: int = 1 # TODO
|
||||
# expect_download: bool = False # moved to downloads_watchdog.py
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ClickElementEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class ClickCoordinateEvent(BaseEvent[dict]):
|
||||
"""Click at specific coordinates."""
|
||||
|
||||
coordinate_x: int
|
||||
coordinate_y: int
|
||||
button: Literal['left', 'right', 'middle'] = 'left'
|
||||
force: bool = False # If True, skip safety checks (file input, print, select)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ClickCoordinateEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class TypeTextEvent(ElementSelectedEvent[dict | None]):
|
||||
"""Type text into an element."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
text: str
|
||||
clear: bool = True
|
||||
is_sensitive: bool = False # Flag to indicate if text contains sensitive data
|
||||
sensitive_key_name: str | None = None # Name of the sensitive key being typed (e.g., 'username', 'password')
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TypeTextEvent', 60.0)) # seconds
|
||||
|
||||
|
||||
class ScrollEvent(ElementSelectedEvent[None]):
|
||||
"""Scroll the page or element."""
|
||||
|
||||
direction: Literal['up', 'down', 'left', 'right']
|
||||
amount: int # pixels
|
||||
node: 'EnhancedDOMTreeNode | None' = None # None means scroll page
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScrollEvent', 8.0)) # seconds
|
||||
|
||||
|
||||
class SwitchTabEvent(BaseEvent[TargetID]):
|
||||
"""Switch to a different tab."""
|
||||
|
||||
target_id: TargetID | None = Field(default=None, description='None means switch to the most recently opened tab')
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SwitchTabEvent', 10.0)) # seconds
|
||||
|
||||
|
||||
class CloseTabEvent(BaseEvent[None]):
|
||||
"""Close a tab."""
|
||||
|
||||
target_id: TargetID
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CloseTabEvent', 10.0)) # seconds
|
||||
|
||||
|
||||
class ScreenshotEvent(BaseEvent[str]):
|
||||
"""Request to take a screenshot."""
|
||||
|
||||
full_page: bool = False
|
||||
clip: dict[str, float] | None = None # {x, y, width, height}
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScreenshotEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class BrowserStateRequestEvent(BaseEvent[BrowserStateSummary]):
|
||||
"""Request current browser state."""
|
||||
|
||||
include_dom: bool = True
|
||||
include_screenshot: bool = True
|
||||
include_recent_events: bool = False
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStateRequestEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# class WaitForConditionEvent(BaseEvent):
|
||||
# """Wait for a condition."""
|
||||
|
||||
# condition: Literal['navigation', 'selector', 'timeout', 'load_state']
|
||||
# timeout: float = 30000
|
||||
# selector: str | None = None
|
||||
# state: Literal['attached', 'detached', 'visible', 'hidden'] | None = None
|
||||
|
||||
|
||||
class GoBackEvent(BaseEvent[None]):
|
||||
"""Navigate back in browser history."""
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_GoBackEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class GoForwardEvent(BaseEvent[None]):
|
||||
"""Navigate forward in browser history."""
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_GoForwardEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class RefreshEvent(BaseEvent[None]):
|
||||
"""Refresh/reload the current page."""
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_RefreshEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
class WaitEvent(BaseEvent[None]):
|
||||
"""Wait for a specified number of seconds."""
|
||||
|
||||
seconds: float = 3.0
|
||||
max_seconds: float = 10.0 # Safety cap
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_WaitEvent', 60.0)) # seconds
|
||||
|
||||
|
||||
class SendKeysEvent(BaseEvent[None]):
|
||||
"""Send keyboard keys/shortcuts."""
|
||||
|
||||
keys: str # e.g., "ctrl+a", "cmd+c", "Enter"
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SendKeysEvent', 60.0)) # seconds
|
||||
|
||||
|
||||
class UploadFileEvent(ElementSelectedEvent[None]):
|
||||
"""Upload a file to an element."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
file_path: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_UploadFileEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class GetDropdownOptionsEvent(ElementSelectedEvent[dict[str, str]]):
|
||||
"""Get all options from any dropdown (native <select>, ARIA menus, or custom dropdowns).
|
||||
|
||||
Returns a dict containing dropdown type, options list, and element metadata."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
|
||||
event_timeout: float | None = Field(
|
||||
default_factory=lambda: _get_timeout('TIMEOUT_GetDropdownOptionsEvent', 15.0)
|
||||
) # some dropdowns lazy-load the list of options on first interaction, so we need to wait for them to load (e.g. table filter lists can have thousands of options)
|
||||
|
||||
|
||||
class SelectDropdownOptionEvent(ElementSelectedEvent[dict[str, str]]):
|
||||
"""Select a dropdown option by exact text from any dropdown type.
|
||||
|
||||
Returns a dict containing success status and selection details."""
|
||||
|
||||
node: 'EnhancedDOMTreeNode'
|
||||
text: str # The option text to select
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SelectDropdownOptionEvent', 8.0)) # seconds
|
||||
|
||||
|
||||
class ScrollToTextEvent(BaseEvent[None]):
|
||||
"""Scroll to specific text on the page. Raises exception if text not found."""
|
||||
|
||||
text: str
|
||||
direction: Literal['up', 'down'] = 'down'
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_ScrollToTextEvent', 15.0)) # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class BrowserStartEvent(BaseEvent):
|
||||
"""Start/connect to browser."""
|
||||
|
||||
cdp_url: str | None = None
|
||||
launch_options: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStartEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserStopEvent(BaseEvent):
|
||||
"""Stop/disconnect from browser."""
|
||||
|
||||
force: bool = False
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStopEvent', 45.0)) # seconds
|
||||
|
||||
|
||||
class BrowserLaunchResult(BaseModel):
|
||||
"""Result of launching a browser."""
|
||||
|
||||
# TODO: add browser executable_path, pid, version, latency, user_data_dir, X11 $DISPLAY, host IP address, etc.
|
||||
cdp_url: str
|
||||
|
||||
|
||||
class BrowserLaunchEvent(BaseEvent[BrowserLaunchResult]):
|
||||
"""Launch a local browser process."""
|
||||
|
||||
# TODO: add executable_path, proxy settings, preferences, extra launch args, etc.
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserLaunchEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserKillEvent(BaseEvent):
|
||||
"""Kill local browser subprocess."""
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserKillEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# TODO: replace all Runtime.evaluate() calls with this event
|
||||
# class ExecuteJavaScriptEvent(BaseEvent):
|
||||
# """Execute JavaScript in page context."""
|
||||
|
||||
# target_id: TargetID
|
||||
# expression: str
|
||||
# await_promise: bool = True
|
||||
|
||||
# event_timeout: float | None = 60.0 # seconds
|
||||
|
||||
# TODO: add this and use the old BrowserProfile.viewport options to set it
|
||||
# class SetViewportEvent(BaseEvent):
|
||||
# """Set the viewport size."""
|
||||
|
||||
# width: int
|
||||
# height: int
|
||||
# device_scale_factor: float = 1.0
|
||||
|
||||
# event_timeout: float | None = 15.0 # seconds
|
||||
|
||||
|
||||
# Moved to storage state
|
||||
# class SetCookiesEvent(BaseEvent):
|
||||
# """Set browser cookies."""
|
||||
|
||||
# cookies: list[dict[str, Any]]
|
||||
|
||||
# event_timeout: float | None = (
|
||||
# 30.0 # only long to support the edge case of restoring a big localStorage / on many origins (has to O(n) visit each origin to restore)
|
||||
# )
|
||||
|
||||
|
||||
# class GetCookiesEvent(BaseEvent):
|
||||
# """Get browser cookies."""
|
||||
|
||||
# urls: list[str] | None = None
|
||||
|
||||
# event_timeout: float | None = 30.0 # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# DOM-related Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class BrowserConnectedEvent(BaseEvent):
|
||||
"""Browser has started/connected."""
|
||||
|
||||
cdp_url: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserConnectedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserStoppedEvent(BaseEvent):
|
||||
"""Browser has stopped/disconnected."""
|
||||
|
||||
reason: str | None = None
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserStoppedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class TabCreatedEvent(BaseEvent):
|
||||
"""A new tab was created."""
|
||||
|
||||
target_id: TargetID
|
||||
url: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TabCreatedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class TabClosedEvent(BaseEvent):
|
||||
"""A tab was closed."""
|
||||
|
||||
target_id: TargetID
|
||||
|
||||
# TODO:
|
||||
# new_focus_target_id: int | None = None
|
||||
# new_focus_url: str | None = None
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TabClosedEvent', 3.0)) # seconds
|
||||
|
||||
|
||||
# TODO: emit this when DOM changes significantly, inner frame navigates, form submits, history.pushState(), etc.
|
||||
# class TabUpdatedEvent(BaseEvent):
|
||||
# """Tab information updated (URL changed, etc.)."""
|
||||
|
||||
# target_id: TargetID
|
||||
# url: str
|
||||
|
||||
|
||||
class AgentFocusChangedEvent(BaseEvent):
|
||||
"""Agent focus changed to a different tab."""
|
||||
|
||||
target_id: TargetID
|
||||
url: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_AgentFocusChangedEvent', 10.0)) # seconds
|
||||
|
||||
|
||||
class TargetCrashedEvent(BaseEvent):
|
||||
"""A target has crashed."""
|
||||
|
||||
target_id: TargetID
|
||||
error: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_TargetCrashedEvent', 10.0)) # seconds
|
||||
|
||||
|
||||
class NavigationStartedEvent(BaseEvent):
|
||||
"""Navigation started."""
|
||||
|
||||
target_id: TargetID
|
||||
url: str
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigationStartedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class NavigationCompleteEvent(BaseEvent):
|
||||
"""Navigation completed."""
|
||||
|
||||
target_id: TargetID
|
||||
url: str
|
||||
status: int | None = None
|
||||
error_message: str | None = None # Error/timeout message if navigation had issues
|
||||
loading_status: str | None = None # Detailed loading status (e.g., network timeout info)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_NavigationCompleteEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Error Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class BrowserErrorEvent(BaseEvent):
|
||||
"""An error occurred in the browser layer."""
|
||||
|
||||
error_type: str
|
||||
message: str
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserErrorEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserReconnectingEvent(BaseEvent):
|
||||
"""WebSocket reconnection attempt is starting."""
|
||||
|
||||
cdp_url: str
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserReconnectingEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class BrowserReconnectedEvent(BaseEvent):
|
||||
"""WebSocket reconnection succeeded."""
|
||||
|
||||
cdp_url: str
|
||||
attempt: int
|
||||
downtime_seconds: float
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_BrowserReconnectedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Storage State Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class SaveStorageStateEvent(BaseEvent):
|
||||
"""Request to save browser storage state."""
|
||||
|
||||
path: str | None = None # Optional path, uses profile default if not provided
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_SaveStorageStateEvent', 45.0)) # seconds
|
||||
|
||||
|
||||
class StorageStateSavedEvent(BaseEvent):
|
||||
"""Notification that storage state was saved."""
|
||||
|
||||
path: str
|
||||
cookies_count: int
|
||||
origins_count: int
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_StorageStateSavedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class LoadStorageStateEvent(BaseEvent):
|
||||
"""Request to load browser storage state."""
|
||||
|
||||
path: str | None = None # Optional path, uses profile default if not provided
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_LoadStorageStateEvent', 45.0)) # seconds
|
||||
|
||||
|
||||
# TODO: refactor this to:
|
||||
# - on_BrowserConnectedEvent() -> dispatch(LoadStorageStateEvent()) -> _copy_storage_state_from_json_to_browser(json_file, new_cdp_session) + return storage_state from handler
|
||||
# - on_BrowserStopEvent() -> dispatch(SaveStorageStateEvent()) -> _copy_storage_state_from_browser_to_json(new_cdp_session, json_file)
|
||||
# and get rid of StorageStateSavedEvent and StorageStateLoadedEvent, have the original events + provide handler return values for any results
|
||||
class StorageStateLoadedEvent(BaseEvent):
|
||||
"""Notification that storage state was loaded."""
|
||||
|
||||
path: str
|
||||
cookies_count: int
|
||||
origins_count: int
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_StorageStateLoadedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# File Download Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class DownloadStartedEvent(BaseEvent):
|
||||
"""A file download has started (CDP downloadWillBegin received)."""
|
||||
|
||||
guid: str # CDP download GUID to correlate with FileDownloadedEvent
|
||||
url: str
|
||||
suggested_filename: str
|
||||
auto_download: bool = False # Whether this was triggered automatically
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_DownloadStartedEvent', 5.0)) # seconds
|
||||
|
||||
|
||||
class DownloadProgressEvent(BaseEvent):
|
||||
"""A file download progress update (CDP downloadProgress received)."""
|
||||
|
||||
guid: str # CDP download GUID to correlate with other download events
|
||||
received_bytes: int
|
||||
total_bytes: int # 0 if unknown
|
||||
state: str # 'inProgress', 'completed', or 'canceled'
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_DownloadProgressEvent', 5.0)) # seconds
|
||||
|
||||
|
||||
class FileDownloadedEvent(BaseEvent):
|
||||
"""A file has been downloaded."""
|
||||
|
||||
guid: str | None = None # CDP download GUID to correlate with DownloadStartedEvent
|
||||
url: str
|
||||
path: str
|
||||
file_name: str
|
||||
file_size: int
|
||||
file_type: str | None = None # e.g., 'pdf', 'zip', 'docx', etc.
|
||||
mime_type: str | None = None # e.g., 'application/pdf'
|
||||
from_cache: bool = False
|
||||
auto_download: bool = False # Whether this was an automatic download (e.g., PDF auto-download)
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_FileDownloadedEvent', 30.0)) # seconds
|
||||
|
||||
|
||||
class AboutBlankDVDScreensaverShownEvent(BaseEvent):
|
||||
"""AboutBlankWatchdog has shown DVD screensaver animation on an about:blank tab."""
|
||||
|
||||
target_id: TargetID
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class DialogOpenedEvent(BaseEvent):
|
||||
"""Event dispatched when a JavaScript dialog is opened and handled."""
|
||||
|
||||
dialog_type: str # 'alert', 'confirm', 'prompt', or 'beforeunload'
|
||||
message: str
|
||||
url: str
|
||||
frame_id: str | None = None # Can be None when frameId is not provided by CDP
|
||||
# target_id: TargetID # TODO: add this to avoid needing target_id_from_frame() later
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Captcha Solver Events
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class CaptchaSolverStartedEvent(BaseEvent):
|
||||
"""Captcha solving started by the browser proxy.
|
||||
|
||||
Emitted when the browser proxy detects a CAPTCHA and begins solving it.
|
||||
The agent should wait for a corresponding CaptchaSolverFinishedEvent before proceeding.
|
||||
"""
|
||||
|
||||
target_id: TargetID
|
||||
vendor: str # e.g. 'cloudflare', 'recaptcha', 'hcaptcha', 'datadome', 'perimeterx', 'geetest'
|
||||
url: str
|
||||
started_at: int # Unix millis
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CaptchaSolverStartedEvent', 5.0))
|
||||
|
||||
|
||||
class CaptchaSolverFinishedEvent(BaseEvent):
|
||||
"""Captcha solving finished by the browser proxy.
|
||||
|
||||
Emitted when the browser proxy finishes solving a CAPTCHA (successfully or not).
|
||||
"""
|
||||
|
||||
target_id: TargetID
|
||||
vendor: str
|
||||
url: str
|
||||
duration_ms: int
|
||||
finished_at: int # Unix millis
|
||||
success: bool # Whether the captcha was solved successfully
|
||||
|
||||
event_timeout: float | None = Field(default_factory=lambda: _get_timeout('TIMEOUT_CaptchaSolverFinishedEvent', 5.0))
|
||||
|
||||
|
||||
# Note: Model rebuilding for forward references is handled in the importing modules
|
||||
# Events with 'EnhancedDOMTreeNode' forward references (ClickElementEvent, TypeTextEvent,
|
||||
# ScrollEvent, UploadFileEvent) need model_rebuild() called after imports are complete
|
||||
|
||||
|
||||
def _check_event_names_dont_overlap():
|
||||
"""
|
||||
check that event names defined in this file are valid and non-overlapping
|
||||
(naiively n^2 so it's pretty slow but ok for now, optimize when >20 events)
|
||||
"""
|
||||
event_names = {
|
||||
name.split('[')[0]
|
||||
for name in globals().keys()
|
||||
if not name.startswith('_')
|
||||
and inspect.isclass(globals()[name])
|
||||
and issubclass(globals()[name], BaseEvent)
|
||||
and name != 'BaseEvent'
|
||||
}
|
||||
for name_a in event_names:
|
||||
assert name_a.endswith('Event'), f'Event with name {name_a} does not end with "Event"'
|
||||
for name_b in event_names:
|
||||
if name_a != name_b: # Skip self-comparison
|
||||
assert name_a not in name_b, (
|
||||
f'Event with name {name_a} is a substring of {name_b}, all events must be completely unique to avoid find-and-replace accidents'
|
||||
)
|
||||
|
||||
|
||||
# overlapping event names are a nightmare to trace and rename later, dont do it!
|
||||
# e.g. prevent ClickEvent and FailedClickEvent are terrible names because one is a substring of the other,
|
||||
# must be ClickEvent and ClickFailedEvent to preserve the usefulnes of codebase grep/sed/awk as refactoring tools.
|
||||
# at import time, we do a quick check that all event names defined above are valid and non-overlapping.
|
||||
# this is hand written in blood by a human! not LLM slop. feel free to optimize but do not remove it without a good reason.
|
||||
_check_event_names_dont_overlap()
|
||||
1237
.agent/vendor/browser_use/browser_use/browser/profile.py
vendored
Normal file
1237
.agent/vendor/browser_use/browser_use/browser/profile.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
548
.agent/vendor/browser_use/browser_use/browser/python_highlights.py
vendored
Normal file
548
.agent/vendor/browser_use/browser_use/browser/python_highlights.py
vendored
Normal file
@@ -0,0 +1,548 @@
|
||||
"""Python-based highlighting system for drawing bounding boxes on screenshots.
|
||||
|
||||
This module replaces JavaScript-based highlighting with fast Python image processing
|
||||
to draw bounding boxes around interactive elements directly on screenshots.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from browser_use.dom.views import DOMSelectorMap, EnhancedDOMTreeNode
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import time_execution_async
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Font cache to prevent repeated font loading and reduce memory usage
|
||||
_FONT_CACHE: dict[tuple[str, int], ImageFont.FreeTypeFont | None] = {}
|
||||
|
||||
# Cross-platform font paths
|
||||
_FONT_PATHS = [
|
||||
'/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf', # Linux (Debian/Ubuntu)
|
||||
'/usr/share/fonts/TTF/DejaVuSans-Bold.ttf', # Linux (Arch/Fedora)
|
||||
'/System/Library/Fonts/Arial.ttf', # macOS
|
||||
'C:\\Windows\\Fonts\\arial.ttf', # Windows
|
||||
'arial.ttf', # Windows (system path)
|
||||
'Arial Bold.ttf', # macOS alternative
|
||||
'/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf', # Linux alternative
|
||||
]
|
||||
|
||||
|
||||
def get_cross_platform_font(font_size: int) -> ImageFont.FreeTypeFont | None:
|
||||
"""Get a cross-platform compatible font with caching to prevent memory leaks.
|
||||
|
||||
Args:
|
||||
font_size: Size of the font to load
|
||||
|
||||
Returns:
|
||||
ImageFont object or None if no system fonts are available
|
||||
"""
|
||||
# Use cache key based on font size
|
||||
cache_key = ('system_font', font_size)
|
||||
|
||||
# Return cached font if available
|
||||
if cache_key in _FONT_CACHE:
|
||||
return _FONT_CACHE[cache_key]
|
||||
|
||||
# Try to load a system font
|
||||
font = None
|
||||
for font_path in _FONT_PATHS:
|
||||
try:
|
||||
font = ImageFont.truetype(font_path, font_size)
|
||||
break
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
# Cache the result (even if None) to avoid repeated attempts
|
||||
_FONT_CACHE[cache_key] = font
|
||||
return font
|
||||
|
||||
|
||||
def cleanup_font_cache() -> None:
|
||||
"""Clean up the font cache to prevent memory leaks in long-running applications."""
|
||||
global _FONT_CACHE
|
||||
_FONT_CACHE.clear()
|
||||
|
||||
|
||||
# Color scheme for different element types
|
||||
ELEMENT_COLORS = {
|
||||
'button': '#FF6B6B', # Red for buttons
|
||||
'input': '#4ECDC4', # Teal for inputs
|
||||
'select': '#45B7D1', # Blue for dropdowns
|
||||
'a': '#96CEB4', # Green for links
|
||||
'textarea': '#FF8C42', # Orange for text areas (was yellow, now more visible)
|
||||
'default': '#DDA0DD', # Light purple for other interactive elements
|
||||
}
|
||||
|
||||
# Element type mappings
|
||||
ELEMENT_TYPE_MAP = {
|
||||
'button': 'button',
|
||||
'input': 'input',
|
||||
'select': 'select',
|
||||
'a': 'a',
|
||||
'textarea': 'textarea',
|
||||
}
|
||||
|
||||
|
||||
def get_element_color(tag_name: str, element_type: str | None = None) -> str:
|
||||
"""Get color for element based on tag name and type."""
|
||||
# Check input type first
|
||||
if tag_name == 'input' and element_type:
|
||||
if element_type in ['button', 'submit']:
|
||||
return ELEMENT_COLORS['button']
|
||||
|
||||
# Use tag-based color
|
||||
return ELEMENT_COLORS.get(tag_name.lower(), ELEMENT_COLORS['default'])
|
||||
|
||||
|
||||
def should_show_index_overlay(backend_node_id: int | None) -> bool:
|
||||
"""Determine if index overlay should be shown."""
|
||||
return backend_node_id is not None
|
||||
|
||||
|
||||
def draw_enhanced_bounding_box_with_text(
|
||||
draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
|
||||
bbox: tuple[int, int, int, int],
|
||||
color: str,
|
||||
text: str | None = None,
|
||||
font: ImageFont.FreeTypeFont | None = None,
|
||||
element_type: str = 'div',
|
||||
image_size: tuple[int, int] = (2000, 1500),
|
||||
device_pixel_ratio: float = 1.0,
|
||||
) -> None:
|
||||
"""Draw an enhanced bounding box with much bigger index containers and dashed borders."""
|
||||
x1, y1, x2, y2 = bbox
|
||||
|
||||
# Draw dashed bounding box with pattern: 1 line, 2 spaces, 1 line, 2 spaces...
|
||||
dash_length = 4
|
||||
gap_length = 8
|
||||
line_width = 2
|
||||
|
||||
# Helper function to draw dashed line
|
||||
def draw_dashed_line(start_x, start_y, end_x, end_y):
|
||||
if start_x == end_x: # Vertical line
|
||||
y = start_y
|
||||
while y < end_y:
|
||||
dash_end = min(y + dash_length, end_y)
|
||||
draw.line([(start_x, y), (start_x, dash_end)], fill=color, width=line_width)
|
||||
y += dash_length + gap_length
|
||||
else: # Horizontal line
|
||||
x = start_x
|
||||
while x < end_x:
|
||||
dash_end = min(x + dash_length, end_x)
|
||||
draw.line([(x, start_y), (dash_end, start_y)], fill=color, width=line_width)
|
||||
x += dash_length + gap_length
|
||||
|
||||
# Draw dashed rectangle
|
||||
draw_dashed_line(x1, y1, x2, y1) # Top
|
||||
draw_dashed_line(x2, y1, x2, y2) # Right
|
||||
draw_dashed_line(x2, y2, x1, y2) # Bottom
|
||||
draw_dashed_line(x1, y2, x1, y1) # Left
|
||||
|
||||
# Draw much bigger index overlay if we have index text
|
||||
if text:
|
||||
try:
|
||||
# Scale font size for appropriate sizing across different resolutions
|
||||
img_width, img_height = image_size
|
||||
|
||||
css_width = img_width # / device_pixel_ratio
|
||||
# Much smaller scaling - 1% of CSS viewport width, max 16px to prevent huge highlights
|
||||
base_font_size = max(10, min(20, int(css_width * 0.01)))
|
||||
# Use shared font loading function with caching
|
||||
big_font = get_cross_platform_font(base_font_size)
|
||||
if big_font is None:
|
||||
big_font = font # Fallback to original font if no system fonts found
|
||||
|
||||
# Get text size with bigger font
|
||||
if big_font:
|
||||
bbox_text = draw.textbbox((0, 0), text, font=big_font)
|
||||
text_width = bbox_text[2] - bbox_text[0]
|
||||
text_height = bbox_text[3] - bbox_text[1]
|
||||
else:
|
||||
# Fallback for default font
|
||||
bbox_text = draw.textbbox((0, 0), text)
|
||||
text_width = bbox_text[2] - bbox_text[0]
|
||||
text_height = bbox_text[3] - bbox_text[1]
|
||||
|
||||
# Scale padding appropriately for different resolutions
|
||||
padding = max(4, min(10, int(css_width * 0.005))) # 0.3% of CSS width, max 4px
|
||||
element_width = x2 - x1
|
||||
element_height = y2 - y1
|
||||
|
||||
# Container dimensions
|
||||
container_width = text_width + padding * 2
|
||||
container_height = text_height + padding * 2
|
||||
|
||||
# Position in top center - for small elements, place further up to avoid blocking content
|
||||
# Center horizontally within the element
|
||||
bg_x1 = x1 + (element_width - container_width) // 2
|
||||
|
||||
# Simple rule: if element is small, place index further up to avoid blocking icons
|
||||
if element_width < 60 or element_height < 30:
|
||||
# Small element: place well above to avoid blocking content
|
||||
bg_y1 = max(0, y1 - container_height - 5)
|
||||
else:
|
||||
# Regular element: place inside with small offset
|
||||
bg_y1 = y1 + 2
|
||||
|
||||
bg_x2 = bg_x1 + container_width
|
||||
bg_y2 = bg_y1 + container_height
|
||||
|
||||
# Center the number within the index box with proper baseline handling
|
||||
text_x = bg_x1 + (container_width - text_width) // 2
|
||||
# Add extra vertical space to prevent clipping
|
||||
text_y = bg_y1 + (container_height - text_height) // 2 - bbox_text[1] # Subtract top offset
|
||||
|
||||
# Ensure container stays within image bounds
|
||||
img_width, img_height = image_size
|
||||
if bg_x1 < 0:
|
||||
offset = -bg_x1
|
||||
bg_x1 += offset
|
||||
bg_x2 += offset
|
||||
text_x += offset
|
||||
if bg_y1 < 0:
|
||||
offset = -bg_y1
|
||||
bg_y1 += offset
|
||||
bg_y2 += offset
|
||||
text_y += offset
|
||||
if bg_x2 > img_width:
|
||||
offset = bg_x2 - img_width
|
||||
bg_x1 -= offset
|
||||
bg_x2 -= offset
|
||||
text_x -= offset
|
||||
if bg_y2 > img_height:
|
||||
offset = bg_y2 - img_height
|
||||
bg_y1 -= offset
|
||||
bg_y2 -= offset
|
||||
text_y -= offset
|
||||
|
||||
# Draw bigger background rectangle with thicker border
|
||||
draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill=color, outline='white', width=2)
|
||||
|
||||
# Draw white text centered in the index box
|
||||
draw.text((text_x, text_y), text, fill='white', font=big_font or font)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to draw enhanced text overlay: {e}')
|
||||
|
||||
|
||||
def draw_bounding_box_with_text(
|
||||
draw, # ImageDraw.Draw - avoiding type annotation due to PIL typing issues
|
||||
bbox: tuple[int, int, int, int],
|
||||
color: str,
|
||||
text: str | None = None,
|
||||
font: ImageFont.FreeTypeFont | None = None,
|
||||
) -> None:
|
||||
"""Draw a bounding box with optional text overlay."""
|
||||
x1, y1, x2, y2 = bbox
|
||||
|
||||
# Draw dashed bounding box
|
||||
dash_length = 2
|
||||
gap_length = 6
|
||||
|
||||
# Top edge
|
||||
x = x1
|
||||
while x < x2:
|
||||
end_x = min(x + dash_length, x2)
|
||||
draw.line([(x, y1), (end_x, y1)], fill=color, width=2)
|
||||
draw.line([(x, y1 + 1), (end_x, y1 + 1)], fill=color, width=2)
|
||||
x += dash_length + gap_length
|
||||
|
||||
# Bottom edge
|
||||
x = x1
|
||||
while x < x2:
|
||||
end_x = min(x + dash_length, x2)
|
||||
draw.line([(x, y2), (end_x, y2)], fill=color, width=2)
|
||||
draw.line([(x, y2 - 1), (end_x, y2 - 1)], fill=color, width=2)
|
||||
x += dash_length + gap_length
|
||||
|
||||
# Left edge
|
||||
y = y1
|
||||
while y < y2:
|
||||
end_y = min(y + dash_length, y2)
|
||||
draw.line([(x1, y), (x1, end_y)], fill=color, width=2)
|
||||
draw.line([(x1 + 1, y), (x1 + 1, end_y)], fill=color, width=2)
|
||||
y += dash_length + gap_length
|
||||
|
||||
# Right edge
|
||||
y = y1
|
||||
while y < y2:
|
||||
end_y = min(y + dash_length, y2)
|
||||
draw.line([(x2, y), (x2, end_y)], fill=color, width=2)
|
||||
draw.line([(x2 - 1, y), (x2 - 1, end_y)], fill=color, width=2)
|
||||
y += dash_length + gap_length
|
||||
|
||||
# Draw index overlay if we have index text
|
||||
if text:
|
||||
try:
|
||||
# Get text size
|
||||
if font:
|
||||
bbox_text = draw.textbbox((0, 0), text, font=font)
|
||||
text_width = bbox_text[2] - bbox_text[0]
|
||||
text_height = bbox_text[3] - bbox_text[1]
|
||||
else:
|
||||
# Fallback for default font
|
||||
bbox_text = draw.textbbox((0, 0), text)
|
||||
text_width = bbox_text[2] - bbox_text[0]
|
||||
text_height = bbox_text[3] - bbox_text[1]
|
||||
|
||||
# Smart positioning based on element size
|
||||
padding = 5
|
||||
element_width = x2 - x1
|
||||
element_height = y2 - y1
|
||||
element_area = element_width * element_height
|
||||
index_box_area = (text_width + padding * 2) * (text_height + padding * 2)
|
||||
|
||||
# Calculate size ratio to determine positioning strategy
|
||||
size_ratio = element_area / max(index_box_area, 1)
|
||||
|
||||
if size_ratio < 4:
|
||||
# Very small elements: place outside in bottom-right corner
|
||||
text_x = x2 + padding
|
||||
text_y = y2 - text_height
|
||||
# Ensure it doesn't go off screen
|
||||
text_x = min(text_x, 1200 - text_width - padding)
|
||||
text_y = max(text_y, 0)
|
||||
elif size_ratio < 16:
|
||||
# Medium elements: place in bottom-right corner inside
|
||||
text_x = x2 - text_width - padding
|
||||
text_y = y2 - text_height - padding
|
||||
else:
|
||||
# Large elements: place in center
|
||||
text_x = x1 + (element_width - text_width) // 2
|
||||
text_y = y1 + (element_height - text_height) // 2
|
||||
|
||||
# Ensure text stays within bounds
|
||||
text_x = max(0, min(text_x, 1200 - text_width))
|
||||
text_y = max(0, min(text_y, 800 - text_height))
|
||||
|
||||
# Draw background rectangle for maximum contrast
|
||||
bg_x1 = text_x - padding
|
||||
bg_y1 = text_y - padding
|
||||
bg_x2 = text_x + text_width + padding
|
||||
bg_y2 = text_y + text_height + padding
|
||||
|
||||
# Use white background with thick black border for maximum visibility
|
||||
draw.rectangle([bg_x1, bg_y1, bg_x2, bg_y2], fill='white', outline='black', width=2)
|
||||
|
||||
# Draw bold dark text on light background for best contrast
|
||||
draw.text((text_x, text_y), text, fill='black', font=font)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to draw text overlay: {e}')
|
||||
|
||||
|
||||
def process_element_highlight(
|
||||
element_id: int,
|
||||
element: EnhancedDOMTreeNode,
|
||||
draw,
|
||||
device_pixel_ratio: float,
|
||||
font,
|
||||
filter_highlight_ids: bool,
|
||||
image_size: tuple[int, int],
|
||||
) -> None:
|
||||
"""Process a single element for highlighting."""
|
||||
try:
|
||||
# Use absolute_position coordinates directly
|
||||
if not element.absolute_position:
|
||||
return
|
||||
|
||||
bounds = element.absolute_position
|
||||
|
||||
# Scale coordinates from CSS pixels to device pixels for screenshot
|
||||
# The screenshot is captured at device pixel resolution, but coordinates are in CSS pixels
|
||||
x1 = int(bounds.x * device_pixel_ratio)
|
||||
y1 = int(bounds.y * device_pixel_ratio)
|
||||
x2 = int((bounds.x + bounds.width) * device_pixel_ratio)
|
||||
y2 = int((bounds.y + bounds.height) * device_pixel_ratio)
|
||||
|
||||
# Ensure coordinates are within image bounds
|
||||
img_width, img_height = image_size
|
||||
x1 = max(0, min(x1, img_width))
|
||||
y1 = max(0, min(y1, img_height))
|
||||
x2 = max(x1, min(x2, img_width))
|
||||
y2 = max(y1, min(y2, img_height))
|
||||
|
||||
# Skip if bounding box is too small or invalid
|
||||
if x2 - x1 < 2 or y2 - y1 < 2:
|
||||
return
|
||||
|
||||
# Get element color based on type
|
||||
tag_name = element.tag_name if hasattr(element, 'tag_name') else 'div'
|
||||
element_type = None
|
||||
if hasattr(element, 'attributes') and element.attributes:
|
||||
element_type = element.attributes.get('type')
|
||||
|
||||
color = get_element_color(tag_name, element_type)
|
||||
|
||||
# Get element index for overlay and apply filtering
|
||||
backend_node_id = getattr(element, 'backend_node_id', None)
|
||||
index_text = None
|
||||
|
||||
if backend_node_id is not None:
|
||||
if filter_highlight_ids:
|
||||
# Use the meaningful text that matches what the LLM sees
|
||||
meaningful_text = element.get_meaningful_text_for_llm()
|
||||
# Show ID only if meaningful text is less than 5 characters
|
||||
if len(meaningful_text) < 3:
|
||||
index_text = str(backend_node_id)
|
||||
else:
|
||||
# Always show ID when filter is disabled
|
||||
index_text = str(backend_node_id)
|
||||
|
||||
# Draw enhanced bounding box with bigger index
|
||||
draw_enhanced_bounding_box_with_text(
|
||||
draw, (x1, y1, x2, y2), color, index_text, font, tag_name, image_size, device_pixel_ratio
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to draw highlight for element {element_id}: {e}')
|
||||
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='create_highlighted_screenshot')
|
||||
@time_execution_async('create_highlighted_screenshot')
|
||||
async def create_highlighted_screenshot(
|
||||
screenshot_b64: str,
|
||||
selector_map: DOMSelectorMap,
|
||||
device_pixel_ratio: float = 1.0,
|
||||
viewport_offset_x: int = 0,
|
||||
viewport_offset_y: int = 0,
|
||||
filter_highlight_ids: bool = True,
|
||||
) -> str:
|
||||
"""Create a highlighted screenshot with bounding boxes around interactive elements.
|
||||
|
||||
Args:
|
||||
screenshot_b64: Base64 encoded screenshot
|
||||
selector_map: Map of interactive elements with their positions
|
||||
device_pixel_ratio: Device pixel ratio for scaling coordinates
|
||||
viewport_offset_x: X offset for viewport positioning
|
||||
viewport_offset_y: Y offset for viewport positioning
|
||||
|
||||
Returns:
|
||||
Base64 encoded highlighted screenshot
|
||||
"""
|
||||
try:
|
||||
# Decode screenshot
|
||||
screenshot_data = base64.b64decode(screenshot_b64)
|
||||
image = Image.open(io.BytesIO(screenshot_data)).convert('RGBA')
|
||||
|
||||
# Create drawing context
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
# Load font using shared function with caching
|
||||
font = get_cross_platform_font(12)
|
||||
# If no system fonts found, font remains None and will use default font
|
||||
|
||||
# Process elements sequentially to avoid ImageDraw thread safety issues
|
||||
# PIL ImageDraw is not thread-safe, so we process elements one by one
|
||||
for element_id, element in selector_map.items():
|
||||
process_element_highlight(element_id, element, draw, device_pixel_ratio, font, filter_highlight_ids, image.size)
|
||||
|
||||
# Convert back to base64
|
||||
output_buffer = io.BytesIO()
|
||||
try:
|
||||
image.save(output_buffer, format='PNG')
|
||||
output_buffer.seek(0)
|
||||
highlighted_b64 = base64.b64encode(output_buffer.getvalue()).decode('utf-8')
|
||||
|
||||
logger.debug(f'Successfully created highlighted screenshot with {len(selector_map)} elements')
|
||||
return highlighted_b64
|
||||
finally:
|
||||
# Explicit cleanup to prevent memory leaks
|
||||
output_buffer.close()
|
||||
if 'image' in locals():
|
||||
image.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to create highlighted screenshot: {e}')
|
||||
# Clean up on error as well
|
||||
if 'image' in locals():
|
||||
image.close()
|
||||
# Return original screenshot on error
|
||||
return screenshot_b64
|
||||
|
||||
|
||||
async def get_viewport_info_from_cdp(cdp_session) -> tuple[float, int, int]:
|
||||
"""Get viewport information from CDP session.
|
||||
|
||||
Returns:
|
||||
Tuple of (device_pixel_ratio, scroll_x, scroll_y)
|
||||
"""
|
||||
try:
|
||||
# Get layout metrics which includes viewport info and device pixel ratio
|
||||
metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
|
||||
|
||||
# Extract viewport information
|
||||
visual_viewport = metrics.get('visualViewport', {})
|
||||
css_visual_viewport = metrics.get('cssVisualViewport', {})
|
||||
css_layout_viewport = metrics.get('cssLayoutViewport', {})
|
||||
|
||||
# Calculate device pixel ratio
|
||||
css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0))
|
||||
device_width = visual_viewport.get('clientWidth', css_width)
|
||||
device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0
|
||||
|
||||
# Get scroll position in CSS pixels
|
||||
scroll_x = int(css_visual_viewport.get('pageX', 0))
|
||||
scroll_y = int(css_visual_viewport.get('pageY', 0))
|
||||
|
||||
return float(device_pixel_ratio), scroll_x, scroll_y
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to get viewport info from CDP: {e}')
|
||||
return 1.0, 0, 0
|
||||
|
||||
|
||||
@time_execution_async('create_highlighted_screenshot_async')
|
||||
async def create_highlighted_screenshot_async(
|
||||
screenshot_b64: str, selector_map: DOMSelectorMap, cdp_session=None, filter_highlight_ids: bool = True
|
||||
) -> str:
|
||||
"""Async wrapper for creating highlighted screenshots.
|
||||
|
||||
Args:
|
||||
screenshot_b64: Base64 encoded screenshot
|
||||
selector_map: Map of interactive elements
|
||||
cdp_session: CDP session for getting viewport info
|
||||
filter_highlight_ids: Whether to filter element IDs based on meaningful text
|
||||
|
||||
Returns:
|
||||
Base64 encoded highlighted screenshot
|
||||
"""
|
||||
# Get viewport information if CDP session is available
|
||||
device_pixel_ratio = 1.0
|
||||
viewport_offset_x = 0
|
||||
viewport_offset_y = 0
|
||||
|
||||
if cdp_session:
|
||||
try:
|
||||
device_pixel_ratio, viewport_offset_x, viewport_offset_y = await get_viewport_info_from_cdp(cdp_session)
|
||||
except Exception as e:
|
||||
logger.debug(f'Failed to get viewport info from CDP: {e}')
|
||||
|
||||
# Create highlighted screenshot with async processing
|
||||
final_screenshot = await create_highlighted_screenshot(
|
||||
screenshot_b64, selector_map, device_pixel_ratio, viewport_offset_x, viewport_offset_y, filter_highlight_ids
|
||||
)
|
||||
|
||||
filename = os.getenv('BROWSER_USE_SCREENSHOT_FILE')
|
||||
if filename:
|
||||
|
||||
def _write_screenshot():
|
||||
try:
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(base64.b64decode(final_screenshot))
|
||||
logger.debug('Saved screenshot to ' + str(filename))
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to save screenshot to {filename}: {e}')
|
||||
|
||||
await asyncio.to_thread(_write_screenshot)
|
||||
return final_screenshot
|
||||
|
||||
|
||||
# Export the cleanup function for external use in long-running applications
|
||||
__all__ = ['create_highlighted_screenshot', 'create_highlighted_screenshot_async', 'cleanup_font_cache']
|
||||
3986
.agent/vendor/browser_use/browser_use/browser/session.py
vendored
Normal file
3986
.agent/vendor/browser_use/browser_use/browser/session.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
911
.agent/vendor/browser_use/browser_use/browser/session_manager.py
vendored
Normal file
911
.agent/vendor/browser_use/browser_use/browser/session_manager.py
vendored
Normal file
@@ -0,0 +1,911 @@
|
||||
"""Event-driven CDP session management.
|
||||
|
||||
Manages CDP sessions by listening to Target.attachedToTarget and Target.detachedFromTarget
|
||||
events, ensuring the session pool always reflects the current browser state.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from cdp_use.cdp.target import AttachedToTargetEvent, DetachedFromTargetEvent, SessionID, TargetID
|
||||
|
||||
from browser_use.utils import create_task_with_error_handling
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.browser.session import BrowserSession, CDPSession, Target
|
||||
|
||||
|
||||
class SessionManager:
|
||||
"""Event-driven CDP session manager.
|
||||
|
||||
Automatically synchronizes the CDP session pool with browser state via CDP events.
|
||||
|
||||
Key features:
|
||||
- Sessions added/removed automatically via Target attach/detach events
|
||||
- Multiple sessions can attach to the same target
|
||||
- Targets only removed when ALL sessions detach
|
||||
- No stale sessions - pool always reflects browser reality
|
||||
|
||||
SessionManager is the SINGLE SOURCE OF TRUTH for all targets and sessions.
|
||||
"""
|
||||
|
||||
def __init__(self, browser_session: 'BrowserSession'):
|
||||
self.browser_session = browser_session
|
||||
self.logger = browser_session.logger
|
||||
|
||||
# All targets (entities: pages, iframes, workers)
|
||||
self._targets: dict[TargetID, 'Target'] = {}
|
||||
|
||||
# All sessions (communication channels)
|
||||
self._sessions: dict[SessionID, 'CDPSession'] = {}
|
||||
|
||||
# Mapping: target -> sessions attached to it
|
||||
self._target_sessions: dict[TargetID, set[SessionID]] = {}
|
||||
|
||||
# Reverse mapping: session -> target it belongs to
|
||||
self._session_to_target: dict[SessionID, TargetID] = {}
|
||||
|
||||
self._lock = asyncio.Lock()
|
||||
self._recovery_lock = asyncio.Lock()
|
||||
|
||||
# Focus recovery coordination - event-driven instead of polling
|
||||
self._recovery_in_progress: bool = False
|
||||
self._recovery_complete_event: asyncio.Event | None = None
|
||||
self._recovery_task: asyncio.Task | None = None
|
||||
|
||||
async def start_monitoring(self) -> None:
|
||||
"""Start monitoring Target attach/detach events.
|
||||
|
||||
Registers CDP event handlers to keep the session pool synchronized with browser state.
|
||||
Also discovers and initializes all existing targets on startup.
|
||||
"""
|
||||
if not self.browser_session._cdp_client_root:
|
||||
raise RuntimeError('CDP client not initialized')
|
||||
|
||||
# Capture cdp_client_root in closure to avoid type errors
|
||||
cdp_client = self.browser_session._cdp_client_root
|
||||
|
||||
# Enable target discovery to receive targetInfoChanged events automatically
|
||||
# This eliminates the need for getTargetInfo() polling calls
|
||||
await cdp_client.send.Target.setDiscoverTargets(
|
||||
params={'discover': True, 'filter': [{'type': 'page'}, {'type': 'iframe'}]}
|
||||
)
|
||||
|
||||
# Register synchronous event handlers (CDP requirement)
|
||||
def on_attached(event: AttachedToTargetEvent, session_id: SessionID | None = None):
|
||||
# _handle_target_attached() handles:
|
||||
# - setAutoAttach for children
|
||||
# - Create CDPSession
|
||||
# - Enable monitoring (for pages/tabs)
|
||||
# - Add to pool
|
||||
create_task_with_error_handling(
|
||||
self._handle_target_attached(event),
|
||||
name='handle_target_attached',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
def on_detached(event: DetachedFromTargetEvent, session_id: SessionID | None = None):
|
||||
create_task_with_error_handling(
|
||||
self._handle_target_detached(event),
|
||||
name='handle_target_detached',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
def on_target_info_changed(event, session_id: SessionID | None = None):
|
||||
# Update session info from targetInfoChanged events (no polling needed!)
|
||||
create_task_with_error_handling(
|
||||
self._handle_target_info_changed(event),
|
||||
name='handle_target_info_changed',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
cdp_client.register.Target.attachedToTarget(on_attached)
|
||||
cdp_client.register.Target.detachedFromTarget(on_detached)
|
||||
cdp_client.register.Target.targetInfoChanged(on_target_info_changed)
|
||||
|
||||
self.logger.debug('[SessionManager] Event monitoring started')
|
||||
|
||||
# Discover and initialize ALL existing targets
|
||||
await self._initialize_existing_targets()
|
||||
|
||||
def _get_session_for_target(self, target_id: TargetID) -> 'CDPSession | None':
|
||||
"""Internal: Get ANY valid session for a target (picks first available).
|
||||
|
||||
⚠️ INTERNAL API - Use browser_session.get_or_create_cdp_session() instead!
|
||||
This method has no validation, no focus management, no recovery.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to get session for
|
||||
|
||||
Returns:
|
||||
CDPSession if exists, None if target has detached
|
||||
"""
|
||||
session_ids = self._target_sessions.get(target_id, set())
|
||||
if not session_ids:
|
||||
# Check if this is the focused target - indicates stale focus that needs cleanup
|
||||
if self.browser_session.agent_focus_target_id == target_id:
|
||||
self.logger.warning(
|
||||
f'[SessionManager] ⚠️ Attempted to get session for stale focused target {target_id[:8]}... '
|
||||
f'Clearing stale focus and triggering recovery.'
|
||||
)
|
||||
|
||||
# Clear stale focus immediately (defense in depth)
|
||||
self.browser_session.agent_focus_target_id = None
|
||||
|
||||
# Trigger recovery if not already in progress
|
||||
if not self._recovery_in_progress:
|
||||
self.logger.warning('[SessionManager] Recovery was not in progress! Triggering now.')
|
||||
self._recovery_task = create_task_with_error_handling(
|
||||
self._recover_agent_focus(target_id),
|
||||
name='recover_agent_focus_from_stale_get',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=False,
|
||||
)
|
||||
return None
|
||||
return self._sessions.get(next(iter(session_ids)))
|
||||
|
||||
def get_all_page_targets(self) -> list:
|
||||
"""Get all page/tab targets using owned data.
|
||||
|
||||
Returns:
|
||||
List of Target objects for all page/tab targets
|
||||
"""
|
||||
page_targets = []
|
||||
for target in self._targets.values():
|
||||
if target.target_type in ('page', 'tab'):
|
||||
page_targets.append(target)
|
||||
return page_targets
|
||||
|
||||
async def validate_session(self, target_id: TargetID) -> bool:
|
||||
"""Check if a target still has active sessions.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to validate
|
||||
|
||||
Returns:
|
||||
True if target has active sessions, False if it should be removed
|
||||
"""
|
||||
if target_id not in self._target_sessions:
|
||||
return False
|
||||
return len(self._target_sessions[target_id]) > 0
|
||||
|
||||
async def clear(self) -> None:
|
||||
"""Clear all owned data structures for cleanup."""
|
||||
async with self._lock:
|
||||
# Clear owned data (single source of truth)
|
||||
self._targets.clear()
|
||||
self._sessions.clear()
|
||||
self._target_sessions.clear()
|
||||
self._session_to_target.clear()
|
||||
|
||||
self.logger.info('[SessionManager] Cleared all owned data (targets, sessions, mappings)')
|
||||
|
||||
async def is_target_valid(self, target_id: TargetID) -> bool:
|
||||
"""Check if a target is still valid and has active sessions.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to validate
|
||||
|
||||
Returns:
|
||||
True if target is valid and has active sessions, False otherwise
|
||||
"""
|
||||
if target_id not in self._target_sessions:
|
||||
return False
|
||||
return len(self._target_sessions[target_id]) > 0
|
||||
|
||||
def get_target_id_from_session_id(self, session_id: SessionID) -> TargetID | None:
|
||||
"""Look up which target a session belongs to.
|
||||
|
||||
Args:
|
||||
session_id: The session ID to look up
|
||||
|
||||
Returns:
|
||||
Target ID if found, None otherwise
|
||||
"""
|
||||
return self._session_to_target.get(session_id)
|
||||
|
||||
def get_target(self, target_id: TargetID) -> 'Target | None':
|
||||
"""Get target from owned data.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to get
|
||||
|
||||
Returns:
|
||||
Target object if found, None otherwise
|
||||
"""
|
||||
return self._targets.get(target_id)
|
||||
|
||||
def get_all_targets(self) -> dict[TargetID, 'Target']:
|
||||
"""Get all targets (read-only access to owned data).
|
||||
|
||||
Returns:
|
||||
Dict mapping target_id to Target objects
|
||||
"""
|
||||
return self._targets
|
||||
|
||||
def get_all_target_ids(self) -> list[TargetID]:
|
||||
"""Get all target IDs from owned data.
|
||||
|
||||
Returns:
|
||||
List of all target IDs
|
||||
"""
|
||||
return list(self._targets.keys())
|
||||
|
||||
def get_all_sessions(self) -> dict[SessionID, 'CDPSession']:
|
||||
"""Get all sessions (read-only access to owned data).
|
||||
|
||||
Returns:
|
||||
Dict mapping session_id to CDPSession objects
|
||||
"""
|
||||
return self._sessions
|
||||
|
||||
def get_session(self, session_id: SessionID) -> 'CDPSession | None':
|
||||
"""Get session from owned data.
|
||||
|
||||
Args:
|
||||
session_id: Session ID to get
|
||||
|
||||
Returns:
|
||||
CDPSession object if found, None otherwise
|
||||
"""
|
||||
return self._sessions.get(session_id)
|
||||
|
||||
def get_all_sessions_for_target(self, target_id: TargetID) -> list['CDPSession']:
|
||||
"""Get ALL sessions attached to a target from owned data.
|
||||
|
||||
Args:
|
||||
target_id: Target ID to get sessions for
|
||||
|
||||
Returns:
|
||||
List of all CDPSession objects for this target
|
||||
"""
|
||||
session_ids = self._target_sessions.get(target_id, set())
|
||||
return [self._sessions[sid] for sid in session_ids if sid in self._sessions]
|
||||
|
||||
def get_target_sessions_mapping(self) -> dict[TargetID, set[SessionID]]:
|
||||
"""Get target->sessions mapping (read-only access).
|
||||
|
||||
Returns:
|
||||
Dict mapping target_id to set of session_ids
|
||||
"""
|
||||
return self._target_sessions
|
||||
|
||||
def get_focused_target(self) -> 'Target | None':
|
||||
"""Get the target that currently has agent focus.
|
||||
|
||||
Convenience method that uses browser_session.agent_focus_target_id.
|
||||
|
||||
Returns:
|
||||
Target object if agent has focus, None otherwise
|
||||
"""
|
||||
if not self.browser_session.agent_focus_target_id:
|
||||
return None
|
||||
return self.get_target(self.browser_session.agent_focus_target_id)
|
||||
|
||||
async def ensure_valid_focus(self, timeout: float = 3.0) -> bool:
|
||||
"""Ensure agent_focus_target_id points to a valid, attached CDP session.
|
||||
|
||||
If the focus target is stale (detached), this method waits for automatic recovery.
|
||||
Uses event-driven coordination instead of polling for efficiency.
|
||||
|
||||
Args:
|
||||
timeout: Maximum time to wait for recovery in seconds (default: 3.0)
|
||||
|
||||
Returns:
|
||||
True if focus is valid or successfully recovered, False if no focus or recovery failed
|
||||
"""
|
||||
if not self.browser_session.agent_focus_target_id:
|
||||
# No focus at all - might be initial state or complete failure
|
||||
if self._recovery_in_progress and self._recovery_complete_event:
|
||||
# Recovery is happening, wait for it
|
||||
try:
|
||||
await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=timeout)
|
||||
# Check again after recovery - simple existence check
|
||||
focus_id = self.browser_session.agent_focus_target_id
|
||||
return bool(focus_id and self._get_session_for_target(focus_id))
|
||||
except TimeoutError:
|
||||
self.logger.error(f'[SessionManager] ❌ Timed out waiting for recovery after {timeout}s')
|
||||
return False
|
||||
return False
|
||||
|
||||
# Simple existence check - does the focused target have a session?
|
||||
cdp_session = self._get_session_for_target(self.browser_session.agent_focus_target_id)
|
||||
if cdp_session:
|
||||
# Session exists - validate it's still active
|
||||
is_valid = await self.validate_session(self.browser_session.agent_focus_target_id)
|
||||
if is_valid:
|
||||
return True
|
||||
|
||||
# Focus is stale - wait for recovery using event instead of polling
|
||||
stale_target_id = self.browser_session.agent_focus_target_id
|
||||
self.logger.warning(
|
||||
f'[SessionManager] ⚠️ Stale agent_focus detected (target {stale_target_id[:8] if stale_target_id else "None"}... detached), '
|
||||
f'waiting for recovery...'
|
||||
)
|
||||
|
||||
# Check if recovery is already in progress
|
||||
if not self._recovery_in_progress:
|
||||
self.logger.warning(
|
||||
'[SessionManager] ⚠️ Recovery not in progress for stale focus! '
|
||||
'This indicates a bug - recovery should have been triggered.'
|
||||
)
|
||||
return False
|
||||
|
||||
# Wait for recovery complete event (event-driven, not polling!)
|
||||
if self._recovery_complete_event:
|
||||
try:
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=timeout)
|
||||
elapsed = asyncio.get_event_loop().time() - start_time
|
||||
|
||||
# Verify recovery succeeded - simple existence check
|
||||
focus_id = self.browser_session.agent_focus_target_id
|
||||
if focus_id and self._get_session_for_target(focus_id):
|
||||
self.logger.info(
|
||||
f'[SessionManager] ✅ Agent focus recovered to {self.browser_session.agent_focus_target_id[:8]}... '
|
||||
f'after {elapsed * 1000:.0f}ms'
|
||||
)
|
||||
return True
|
||||
else:
|
||||
self.logger.error(
|
||||
f'[SessionManager] ❌ Recovery completed but focus still invalid after {elapsed * 1000:.0f}ms'
|
||||
)
|
||||
return False
|
||||
|
||||
except TimeoutError:
|
||||
self.logger.error(
|
||||
f'[SessionManager] ❌ Recovery timed out after {timeout}s '
|
||||
f'(was: {stale_target_id[:8] if stale_target_id else "None"}..., '
|
||||
f'now: {self.browser_session.agent_focus_target_id[:8] if self.browser_session.agent_focus_target_id else "None"})'
|
||||
)
|
||||
return False
|
||||
else:
|
||||
self.logger.error('[SessionManager] ❌ Recovery event not initialized')
|
||||
return False
|
||||
|
||||
async def _handle_target_attached(self, event: AttachedToTargetEvent) -> None:
|
||||
"""Handle Target.attachedToTarget event.
|
||||
|
||||
Called automatically by Chrome when a new target/session is created.
|
||||
This is the ONLY place where sessions are added to the pool.
|
||||
"""
|
||||
target_id = event['targetInfo']['targetId']
|
||||
session_id = event['sessionId']
|
||||
target_type = event['targetInfo']['type']
|
||||
target_info = event['targetInfo']
|
||||
waiting_for_debugger = event.get('waitingForDebugger', False)
|
||||
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Target attached: {target_id[:8]}... (session={session_id[:8]}..., '
|
||||
f'type={target_type}, waitingForDebugger={waiting_for_debugger})'
|
||||
)
|
||||
|
||||
# Defensive check: browser may be shutting down and _cdp_client_root could be None
|
||||
if self.browser_session._cdp_client_root is None:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Skipping target attach for {target_id[:8]}... - browser shutting down (no CDP client)'
|
||||
)
|
||||
return
|
||||
|
||||
# Enable auto-attach for this session's children (do this FIRST, outside lock)
|
||||
try:
|
||||
await self.browser_session._cdp_client_root.send.Target.setAutoAttach(
|
||||
params={'autoAttach': True, 'waitForDebuggerOnStart': False, 'flatten': True}, session_id=session_id
|
||||
)
|
||||
except Exception as e:
|
||||
error_str = str(e)
|
||||
# Expected for short-lived targets (workers, temp iframes) that detach before this executes
|
||||
if '-32001' not in error_str and 'Session with given id not found' not in error_str:
|
||||
self.logger.debug(f'[SessionManager] Auto-attach failed for {target_type}: {e}')
|
||||
|
||||
from browser_use.browser.session import Target
|
||||
|
||||
async with self._lock:
|
||||
# Track this session for the target
|
||||
if target_id not in self._target_sessions:
|
||||
self._target_sessions[target_id] = set()
|
||||
|
||||
self._target_sessions[target_id].add(session_id)
|
||||
self._session_to_target[session_id] = target_id
|
||||
|
||||
# Create or update Target inside the same lock so that get_target() is never
|
||||
# called in the window between _target_sessions being set and _targets being set.
|
||||
if target_id not in self._targets:
|
||||
target = Target(
|
||||
target_id=target_id,
|
||||
target_type=target_type,
|
||||
url=target_info.get('url', 'about:blank'),
|
||||
title=target_info.get('title', 'Unknown title'),
|
||||
)
|
||||
self._targets[target_id] = target
|
||||
self.logger.debug(f'[SessionManager] Created target {target_id[:8]}... (type={target_type})')
|
||||
else:
|
||||
# Update existing target info
|
||||
existing_target = self._targets[target_id]
|
||||
existing_target.url = target_info.get('url', existing_target.url)
|
||||
existing_target.title = target_info.get('title', existing_target.title)
|
||||
|
||||
# Create CDPSession (communication channel)
|
||||
from browser_use.browser.session import CDPSession
|
||||
|
||||
assert self.browser_session._cdp_client_root is not None, 'Root CDP client required'
|
||||
|
||||
cdp_session = CDPSession(
|
||||
cdp_client=self.browser_session._cdp_client_root,
|
||||
target_id=target_id,
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
# Add to sessions dict
|
||||
self._sessions[session_id] = cdp_session
|
||||
|
||||
# If proxy auth is configured, enable Fetch auth handling on this session
|
||||
# Avoids overwriting Target.attachedToTarget handlers elsewhere
|
||||
try:
|
||||
proxy_cfg = self.browser_session.browser_profile.proxy
|
||||
username = proxy_cfg.username if proxy_cfg else None
|
||||
password = proxy_cfg.password if proxy_cfg else None
|
||||
if username and password:
|
||||
await cdp_session.cdp_client.send.Fetch.enable(
|
||||
params={'handleAuthRequests': True},
|
||||
session_id=cdp_session.session_id,
|
||||
)
|
||||
self.logger.debug(f'[SessionManager] Fetch.enable(handleAuthRequests=True) on session {session_id[:8]}...')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'[SessionManager] Fetch.enable on attached session failed: {type(e).__name__}: {e}')
|
||||
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Created session {session_id[:8]}... for target {target_id[:8]}... '
|
||||
f'(total sessions: {len(self._sessions)})'
|
||||
)
|
||||
|
||||
# Enable lifecycle events and network monitoring for page targets
|
||||
if target_type in ('page', 'tab'):
|
||||
await self._enable_page_monitoring(cdp_session)
|
||||
|
||||
# Resume execution if waiting for debugger
|
||||
if waiting_for_debugger:
|
||||
try:
|
||||
assert self.browser_session._cdp_client_root is not None
|
||||
await self.browser_session._cdp_client_root.send.Runtime.runIfWaitingForDebugger(session_id=session_id)
|
||||
except Exception as e:
|
||||
self.logger.warning(f'[SessionManager] Failed to resume execution: {e}')
|
||||
|
||||
async def _handle_target_info_changed(self, event: dict) -> None:
|
||||
"""Handle Target.targetInfoChanged event.
|
||||
|
||||
Updates target title/URL without polling getTargetInfo().
|
||||
Chrome fires this automatically when title or URL changes.
|
||||
"""
|
||||
target_info = event.get('targetInfo', {})
|
||||
target_id = target_info.get('targetId')
|
||||
|
||||
if not target_id:
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
# Update target if it exists (source of truth for url/title)
|
||||
if target_id in self._targets:
|
||||
target = self._targets[target_id]
|
||||
|
||||
target.title = target_info.get('title', target.title)
|
||||
target.url = target_info.get('url', target.url)
|
||||
|
||||
async def _handle_target_detached(self, event: DetachedFromTargetEvent) -> None:
|
||||
"""Handle Target.detachedFromTarget event.
|
||||
|
||||
Called automatically by Chrome when a target/session is destroyed.
|
||||
This is the ONLY place where sessions are removed from the pool.
|
||||
"""
|
||||
session_id = event['sessionId']
|
||||
target_id = event.get('targetId') # May be empty
|
||||
|
||||
# If targetId not in event, look it up via session mapping
|
||||
if not target_id:
|
||||
async with self._lock:
|
||||
target_id = self._session_to_target.get(session_id)
|
||||
|
||||
if not target_id:
|
||||
self.logger.warning(f'[SessionManager] Session detached but target unknown (session={session_id[:8]}...)')
|
||||
return
|
||||
|
||||
agent_focus_lost = False
|
||||
target_fully_removed = False
|
||||
target_type = None
|
||||
|
||||
async with self._lock:
|
||||
# Remove this session from target's session set
|
||||
if target_id in self._target_sessions:
|
||||
self._target_sessions[target_id].discard(session_id)
|
||||
|
||||
remaining_sessions = len(self._target_sessions[target_id])
|
||||
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Session detached: target={target_id[:8]}... '
|
||||
f'session={session_id[:8]}... (remaining={remaining_sessions})'
|
||||
)
|
||||
|
||||
# Only remove target when NO sessions remain
|
||||
if remaining_sessions == 0:
|
||||
self.logger.debug(f'[SessionManager] No sessions remain for target {target_id[:8]}..., removing target')
|
||||
|
||||
target_fully_removed = True
|
||||
|
||||
# Check if agent_focus points to this target
|
||||
agent_focus_lost = self.browser_session.agent_focus_target_id == target_id
|
||||
|
||||
# Immediately clear stale focus to prevent operations on detached target
|
||||
if agent_focus_lost:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Clearing stale agent_focus_target_id {target_id[:8]}... '
|
||||
f'to prevent operations on detached target'
|
||||
)
|
||||
self.browser_session.agent_focus_target_id = None
|
||||
|
||||
# Get target type before removing (needed for TabClosedEvent dispatch)
|
||||
target = self._targets.get(target_id)
|
||||
target_type = target.target_type if target else None
|
||||
|
||||
# Remove target (entity) from owned data
|
||||
if target_id in self._targets:
|
||||
self._targets.pop(target_id)
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Removed target {target_id[:8]}... (remaining targets: {len(self._targets)})'
|
||||
)
|
||||
|
||||
# Clean up tracking
|
||||
del self._target_sessions[target_id]
|
||||
else:
|
||||
# Target not tracked - already removed or never attached
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Session detached from untracked target: target={target_id[:8]}... '
|
||||
f'session={session_id[:8]}... (target was already removed or attach event was missed)'
|
||||
)
|
||||
|
||||
# Remove session from owned sessions dict
|
||||
if session_id in self._sessions:
|
||||
self._sessions.pop(session_id)
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Removed session {session_id[:8]}... (remaining sessions: {len(self._sessions)})'
|
||||
)
|
||||
|
||||
# Remove from reverse mapping
|
||||
if session_id in self._session_to_target:
|
||||
del self._session_to_target[session_id]
|
||||
|
||||
# Dispatch TabClosedEvent only for page/tab targets that are fully removed (not iframes/workers or partial detaches)
|
||||
if target_fully_removed:
|
||||
if target_type in ('page', 'tab'):
|
||||
from browser_use.browser.events import TabClosedEvent
|
||||
|
||||
self.browser_session.event_bus.dispatch(TabClosedEvent(target_id=target_id))
|
||||
self.logger.debug(f'[SessionManager] Dispatched TabClosedEvent for page target {target_id[:8]}...')
|
||||
elif target_type:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Target {target_id[:8]}... fully removed (type={target_type}) - not dispatching TabClosedEvent'
|
||||
)
|
||||
|
||||
# Auto-recover agent_focus outside the lock to avoid blocking other operations
|
||||
if agent_focus_lost:
|
||||
# Create recovery task instead of awaiting directly - allows concurrent operations to wait on same recovery
|
||||
if not self._recovery_in_progress:
|
||||
self._recovery_task = create_task_with_error_handling(
|
||||
self._recover_agent_focus(target_id),
|
||||
name='recover_agent_focus',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=False,
|
||||
)
|
||||
|
||||
async def _recover_agent_focus(self, crashed_target_id: TargetID) -> None:
|
||||
"""Auto-recover agent_focus when the focused target crashes/detaches.
|
||||
|
||||
Uses recovery lock to prevent concurrent recovery attempts from creating multiple emergency tabs.
|
||||
Coordinates with ensure_valid_focus() via events for efficient waiting.
|
||||
|
||||
Args:
|
||||
crashed_target_id: The target ID that was lost
|
||||
"""
|
||||
try:
|
||||
# Prevent concurrent recovery attempts
|
||||
async with self._recovery_lock:
|
||||
# Set recovery state INSIDE lock to prevent race conditions
|
||||
if self._recovery_in_progress:
|
||||
self.logger.debug('[SessionManager] Recovery already in progress, waiting for it to complete')
|
||||
# Wait for ongoing recovery instead of starting a new one
|
||||
if self._recovery_complete_event:
|
||||
try:
|
||||
await asyncio.wait_for(self._recovery_complete_event.wait(), timeout=5.0)
|
||||
except TimeoutError:
|
||||
self.logger.error('[SessionManager] Timed out waiting for ongoing recovery')
|
||||
return
|
||||
|
||||
# Set recovery state
|
||||
self._recovery_in_progress = True
|
||||
self._recovery_complete_event = asyncio.Event()
|
||||
|
||||
if self.browser_session._cdp_client_root is None:
|
||||
self.logger.debug('[SessionManager] Skipping focus recovery - browser shutting down (no CDP client)')
|
||||
return
|
||||
|
||||
# Check if another recovery already fixed agent_focus
|
||||
if self.browser_session.agent_focus_target_id and self.browser_session.agent_focus_target_id != crashed_target_id:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Agent focus already recovered by concurrent operation '
|
||||
f'(now: {self.browser_session.agent_focus_target_id[:8]}...), skipping recovery'
|
||||
)
|
||||
return
|
||||
|
||||
# Note: agent_focus_target_id may already be None (cleared in _handle_target_detached)
|
||||
current_focus_desc = (
|
||||
f'{self.browser_session.agent_focus_target_id[:8]}...'
|
||||
if self.browser_session.agent_focus_target_id
|
||||
else 'None (already cleared)'
|
||||
)
|
||||
|
||||
self.logger.warning(
|
||||
f'[SessionManager] Agent focus target {crashed_target_id[:8]}... detached! '
|
||||
f'Current focus: {current_focus_desc}. Auto-recovering by switching to another target...'
|
||||
)
|
||||
|
||||
# Perform recovery (outside lock to allow concurrent operations)
|
||||
# Try to find another valid page target
|
||||
page_targets = self.get_all_page_targets()
|
||||
|
||||
new_target_id = None
|
||||
is_existing_tab = False
|
||||
|
||||
if page_targets:
|
||||
# Switch to most recent page that's not the crashed one
|
||||
new_target_id = page_targets[-1].target_id
|
||||
is_existing_tab = True
|
||||
self.logger.info(f'[SessionManager] Switching agent_focus to existing tab {new_target_id[:8]}...')
|
||||
else:
|
||||
# No pages exist - create a new one
|
||||
self.logger.warning('[SessionManager] No tabs remain! Creating new tab for agent...')
|
||||
new_target_id = await self.browser_session._cdp_create_new_page('about:blank')
|
||||
self.logger.info(f'[SessionManager] Created new tab {new_target_id[:8]}... for agent')
|
||||
|
||||
# Dispatch TabCreatedEvent so watchdogs can initialize
|
||||
from browser_use.browser.events import TabCreatedEvent
|
||||
|
||||
self.browser_session.event_bus.dispatch(TabCreatedEvent(url='about:blank', target_id=new_target_id))
|
||||
|
||||
# Wait for CDP attach event to create session
|
||||
# Note: This polling is necessary - waiting for external Chrome CDP event
|
||||
# _handle_target_attached will add session to pool when Chrome fires attachedToTarget
|
||||
new_session = None
|
||||
for attempt in range(20): # Wait up to 2 seconds
|
||||
await asyncio.sleep(0.1)
|
||||
new_session = self._get_session_for_target(new_target_id)
|
||||
if new_session:
|
||||
break
|
||||
|
||||
if new_session:
|
||||
self.browser_session.agent_focus_target_id = new_target_id
|
||||
self.logger.info(f'[SessionManager] ✅ Agent focus recovered: {new_target_id[:8]}...')
|
||||
|
||||
# Visually activate the tab in browser (only for existing tabs)
|
||||
if is_existing_tab:
|
||||
try:
|
||||
assert self.browser_session._cdp_client_root is not None
|
||||
await self.browser_session._cdp_client_root.send.Target.activateTarget(params={'targetId': new_target_id})
|
||||
self.logger.debug(f'[SessionManager] Activated tab {new_target_id[:8]}... in browser UI')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'[SessionManager] Failed to activate tab visually: {e}')
|
||||
|
||||
# Get target to access url (from owned data)
|
||||
target = self.get_target(new_target_id)
|
||||
target_url = target.url if target else 'about:blank'
|
||||
|
||||
# Dispatch focus changed event
|
||||
from browser_use.browser.events import AgentFocusChangedEvent
|
||||
|
||||
self.browser_session.event_bus.dispatch(AgentFocusChangedEvent(target_id=new_target_id, url=target_url))
|
||||
return
|
||||
|
||||
# Recovery failed - create emergency fallback tab
|
||||
self.logger.error(
|
||||
f'[SessionManager] ❌ Failed to get session for {new_target_id[:8]}... after 2s, creating emergency fallback tab'
|
||||
)
|
||||
|
||||
fallback_target_id = await self.browser_session._cdp_create_new_page('about:blank')
|
||||
self.logger.warning(f'[SessionManager] Created emergency fallback tab {fallback_target_id[:8]}...')
|
||||
|
||||
# Try one more time with fallback
|
||||
# Note: This polling is necessary - waiting for external Chrome CDP event
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0.1)
|
||||
fallback_session = self._get_session_for_target(fallback_target_id)
|
||||
if fallback_session:
|
||||
self.browser_session.agent_focus_target_id = fallback_target_id
|
||||
self.logger.warning(f'[SessionManager] ⚠️ Agent focus set to emergency fallback: {fallback_target_id[:8]}...')
|
||||
|
||||
from browser_use.browser.events import AgentFocusChangedEvent, TabCreatedEvent
|
||||
|
||||
self.browser_session.event_bus.dispatch(TabCreatedEvent(url='about:blank', target_id=fallback_target_id))
|
||||
self.browser_session.event_bus.dispatch(
|
||||
AgentFocusChangedEvent(target_id=fallback_target_id, url='about:blank')
|
||||
)
|
||||
return
|
||||
|
||||
# Complete failure - this should never happen
|
||||
self.logger.critical(
|
||||
'[SessionManager] 🚨 CRITICAL: Failed to recover agent_focus even with fallback! Agent may be in broken state.'
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[SessionManager] ❌ Error during agent_focus recovery: {type(e).__name__}: {e}')
|
||||
finally:
|
||||
# Always signal completion and reset recovery state
|
||||
# This allows all waiting operations to proceed (success or failure)
|
||||
if self._recovery_complete_event:
|
||||
self._recovery_complete_event.set()
|
||||
self._recovery_in_progress = False
|
||||
self._recovery_task = None
|
||||
self.logger.debug('[SessionManager] Recovery state reset')
|
||||
|
||||
async def _initialize_existing_targets(self) -> None:
|
||||
"""Discover and initialize all existing targets at startup.
|
||||
|
||||
Attaches to each target and initializes it SYNCHRONOUSLY.
|
||||
Chrome will also fire attachedToTarget events, but _handle_target_attached() is
|
||||
idempotent (checks if target already in pool), so duplicate handling is safe.
|
||||
|
||||
This eliminates race conditions - monitoring is guaranteed ready before navigation.
|
||||
"""
|
||||
cdp_client = self.browser_session._cdp_client_root
|
||||
assert cdp_client is not None
|
||||
|
||||
# Get all existing targets
|
||||
targets_result = await cdp_client.send.Target.getTargets()
|
||||
existing_targets = targets_result.get('targetInfos', [])
|
||||
|
||||
self.logger.debug(f'[SessionManager] Discovered {len(existing_targets)} existing targets')
|
||||
|
||||
# Track target IDs for verification
|
||||
target_ids_to_wait_for = []
|
||||
|
||||
# Just attach to ALL existing targets - Chrome fires attachedToTarget events
|
||||
# The on_attached handler (via create_task) does ALL the work
|
||||
for target in existing_targets:
|
||||
target_id = target['targetId']
|
||||
target_type = target.get('type', 'unknown')
|
||||
|
||||
try:
|
||||
# Just attach - event handler does everything
|
||||
await cdp_client.send.Target.attachToTarget(params={'targetId': target_id, 'flatten': True})
|
||||
target_ids_to_wait_for.append(target_id)
|
||||
except Exception as e:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Failed to attach to existing target {target_id[:8]}... (type={target_type}): {e}'
|
||||
)
|
||||
|
||||
# Wait for event handlers to complete their work (they run via create_task)
|
||||
# Use event-driven approach instead of polling for better performance
|
||||
ready_event = asyncio.Event()
|
||||
|
||||
async def check_all_ready():
|
||||
"""Check if all sessions are ready and signal completion."""
|
||||
while True:
|
||||
ready_count = 0
|
||||
for tid in target_ids_to_wait_for:
|
||||
session = self._get_session_for_target(tid)
|
||||
if session:
|
||||
target = self._targets.get(tid)
|
||||
target_type = target.target_type if target else 'unknown'
|
||||
# For pages, verify monitoring is enabled
|
||||
if target_type in ('page', 'tab'):
|
||||
if hasattr(session, '_lifecycle_events') and session._lifecycle_events is not None:
|
||||
ready_count += 1
|
||||
else:
|
||||
# Non-page targets don't need monitoring
|
||||
ready_count += 1
|
||||
|
||||
if ready_count == len(target_ids_to_wait_for):
|
||||
ready_event.set()
|
||||
return
|
||||
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
# Start checking in background
|
||||
check_task = create_task_with_error_handling(
|
||||
check_all_ready(), name='check_all_targets_ready', logger_instance=self.logger
|
||||
)
|
||||
|
||||
try:
|
||||
# Wait for completion with timeout
|
||||
await asyncio.wait_for(ready_event.wait(), timeout=2.0)
|
||||
except TimeoutError:
|
||||
# Timeout - count what's ready
|
||||
ready_count = 0
|
||||
for tid in target_ids_to_wait_for:
|
||||
session = self._get_session_for_target(tid)
|
||||
if session:
|
||||
target = self._targets.get(tid)
|
||||
target_type = target.target_type if target else 'unknown'
|
||||
# For pages, verify monitoring is enabled
|
||||
if target_type in ('page', 'tab'):
|
||||
if hasattr(session, '_lifecycle_events') and session._lifecycle_events is not None:
|
||||
ready_count += 1
|
||||
else:
|
||||
# Non-page targets don't need monitoring
|
||||
ready_count += 1
|
||||
self.logger.warning(
|
||||
f'[SessionManager] Initialization timeout after 2.0s: {ready_count}/{len(target_ids_to_wait_for)} sessions ready'
|
||||
)
|
||||
finally:
|
||||
check_task.cancel()
|
||||
try:
|
||||
await check_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _enable_page_monitoring(self, cdp_session: 'CDPSession') -> None:
|
||||
"""Enable lifecycle events and network monitoring for a page target.
|
||||
|
||||
This is called once per page when it's created, avoiding handler accumulation.
|
||||
Registers a SINGLE lifecycle handler per session that stores events for navigations to consume.
|
||||
|
||||
Args:
|
||||
cdp_session: The CDP session to enable monitoring on
|
||||
"""
|
||||
try:
|
||||
# Enable Page domain first (required for lifecycle events)
|
||||
await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
|
||||
|
||||
# Enable lifecycle events (load, DOMContentLoaded, networkIdle, etc.)
|
||||
await cdp_session.cdp_client.send.Page.setLifecycleEventsEnabled(
|
||||
params={'enabled': True}, session_id=cdp_session.session_id
|
||||
)
|
||||
|
||||
# Enable network monitoring for networkIdle detection
|
||||
await cdp_session.cdp_client.send.Network.enable(session_id=cdp_session.session_id)
|
||||
|
||||
# Initialize lifecycle event storage for this session (thread-safe)
|
||||
from collections import deque
|
||||
|
||||
cdp_session._lifecycle_events = deque(maxlen=50) # Keep last 50 events
|
||||
cdp_session._lifecycle_lock = asyncio.Lock()
|
||||
|
||||
# Register ONE handler per session that stores events
|
||||
def on_lifecycle_event(event, session_id=None):
|
||||
event_name = event.get('name', 'unknown')
|
||||
event_loader_id = event.get('loaderId', 'none')
|
||||
|
||||
# Find which target this session belongs to
|
||||
target_id_from_event = None
|
||||
if session_id:
|
||||
target_id_from_event = self.get_target_id_from_session_id(session_id)
|
||||
|
||||
# Check if this event is for our target
|
||||
if target_id_from_event == cdp_session.target_id:
|
||||
# Store event for navigations to consume
|
||||
event_data = {
|
||||
'name': event_name,
|
||||
'loaderId': event_loader_id,
|
||||
'timestamp': asyncio.get_event_loop().time(),
|
||||
}
|
||||
# Append is atomic in CPython
|
||||
try:
|
||||
cdp_session._lifecycle_events.append(event_data)
|
||||
except Exception as e:
|
||||
# Only log errors, not every event
|
||||
self.logger.error(f'[SessionManager] Failed to store lifecycle event: {e}')
|
||||
|
||||
# Register the handler ONCE (this is the only place we register)
|
||||
cdp_session.cdp_client.register.Page.lifecycleEvent(on_lifecycle_event)
|
||||
|
||||
except Exception as e:
|
||||
# Don't fail - target might be short-lived or already detached
|
||||
error_str = str(e)
|
||||
if '-32001' in error_str or 'Session with given id not found' in error_str:
|
||||
self.logger.debug(
|
||||
f'[SessionManager] Target {cdp_session.target_id[:8]}... detached before monitoring could be enabled (normal for short-lived targets)'
|
||||
)
|
||||
else:
|
||||
self.logger.warning(
|
||||
f'[SessionManager] Failed to enable monitoring for target {cdp_session.target_id[:8]}...: {e}'
|
||||
)
|
||||
141
.agent/vendor/browser_use/browser_use/browser/video_recorder.py
vendored
Normal file
141
.agent/vendor/browser_use/browser_use/browser/video_recorder.py
vendored
Normal file
@@ -0,0 +1,141 @@
|
||||
"""Video Recording Service for Browser Use Sessions."""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from browser_use.browser.profile import ViewportSize
|
||||
|
||||
try:
|
||||
import imageio.v2 as iio # type: ignore[import-not-found]
|
||||
import numpy as np # type: ignore[import-not-found]
|
||||
from imageio.core.format import Format # type: ignore[import-not-found]
|
||||
from PIL import Image
|
||||
|
||||
IMAGEIO_AVAILABLE = True
|
||||
except ImportError:
|
||||
IMAGEIO_AVAILABLE = False
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_padded_size(size: ViewportSize, macro_block_size: int = 16) -> ViewportSize:
|
||||
"""Calculates the dimensions padded to the nearest multiple of macro_block_size."""
|
||||
width = int(math.ceil(size['width'] / macro_block_size)) * macro_block_size
|
||||
height = int(math.ceil(size['height'] / macro_block_size)) * macro_block_size
|
||||
return ViewportSize(width=width, height=height)
|
||||
|
||||
|
||||
class VideoRecorderService:
|
||||
"""
|
||||
Handles the video encoding process for a browser session using imageio.
|
||||
|
||||
This service captures individual frames from the CDP screencast, decodes them,
|
||||
and appends them to a video file using a pip-installable ffmpeg backend.
|
||||
It automatically resizes frames to match the target video dimensions.
|
||||
"""
|
||||
|
||||
def __init__(self, output_path: Path, size: ViewportSize, framerate: int):
|
||||
"""
|
||||
Initializes the video recorder.
|
||||
|
||||
Args:
|
||||
output_path: The full path where the video will be saved.
|
||||
size: A ViewportSize object specifying the width and height of the video.
|
||||
framerate: The desired framerate for the output video.
|
||||
"""
|
||||
self.output_path = output_path
|
||||
self.size = size
|
||||
self.framerate = framerate
|
||||
self._writer: Optional['Format.Writer'] = None
|
||||
self._is_active = False
|
||||
self.padded_size = _get_padded_size(self.size)
|
||||
|
||||
def start(self) -> None:
|
||||
"""
|
||||
Prepares and starts the video writer.
|
||||
|
||||
If the required optional dependencies are not installed, this method will
|
||||
log an error and do nothing.
|
||||
"""
|
||||
if not IMAGEIO_AVAILABLE:
|
||||
logger.error(
|
||||
'MP4 recording requires optional dependencies. Please install them with: pip install "browser-use[video]"'
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
self.output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# The macro_block_size is set to None because we handle padding ourselves
|
||||
self._writer = iio.get_writer(
|
||||
str(self.output_path),
|
||||
fps=self.framerate,
|
||||
codec='libx264',
|
||||
quality=8, # A good balance of quality and file size (1-10 scale)
|
||||
pixelformat='yuv420p', # Ensures compatibility with most players
|
||||
macro_block_size=None,
|
||||
)
|
||||
self._is_active = True
|
||||
logger.debug(f'Video recorder started. Output will be saved to {self.output_path}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to initialize video writer: {e}')
|
||||
self._is_active = False
|
||||
|
||||
def add_frame(self, frame_data_b64: str) -> None:
|
||||
"""
|
||||
Decodes a base64-encoded PNG frame, resizes it, pads it to be codec-compatible,
|
||||
and appends it to the video.
|
||||
|
||||
Args:
|
||||
frame_data_b64: A base64-encoded string of the PNG frame data.
|
||||
"""
|
||||
if not self._is_active or not self._writer:
|
||||
return
|
||||
|
||||
try:
|
||||
frame_bytes = base64.b64decode(frame_data_b64)
|
||||
|
||||
# Use PIL to handle image processing in memory - much faster than spawning ffmpeg subprocess per frame
|
||||
with Image.open(io.BytesIO(frame_bytes)) as img:
|
||||
# 1. Resize if needed to target viewport size
|
||||
if img.size != (self.size['width'], self.size['height']):
|
||||
# Use BICUBIC as it's faster than LANCZOS and good enough for screen recordings
|
||||
img = img.resize((self.size['width'], self.size['height']), Image.Resampling.BICUBIC)
|
||||
|
||||
# 2. Handle Padding (Macro block alignment for codecs)
|
||||
# Check if padding is actually needed
|
||||
if self.padded_size['width'] != self.size['width'] or self.padded_size['height'] != self.size['height']:
|
||||
new_img = Image.new('RGB', (self.padded_size['width'], self.padded_size['height']), (0, 0, 0))
|
||||
# Center the image
|
||||
x_offset = (self.padded_size['width'] - self.size['width']) // 2
|
||||
y_offset = (self.padded_size['height'] - self.size['height']) // 2
|
||||
new_img.paste(img, (x_offset, y_offset))
|
||||
img = new_img
|
||||
|
||||
# 3. Convert to numpy array for imageio
|
||||
img_array = np.array(img)
|
||||
|
||||
self._writer.append_data(img_array)
|
||||
except Exception as e:
|
||||
logger.warning(f'Could not process and add video frame: {e}')
|
||||
|
||||
def stop_and_save(self) -> None:
|
||||
"""
|
||||
Finalizes the video file by closing the writer.
|
||||
|
||||
This method should be called when the recording session is complete.
|
||||
"""
|
||||
if not self._is_active or not self._writer:
|
||||
return
|
||||
|
||||
try:
|
||||
self._writer.close()
|
||||
logger.info(f'📹 Video recording saved successfully to: {self.output_path}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to finalize and save video: {e}')
|
||||
finally:
|
||||
self._is_active = False
|
||||
self._writer = None
|
||||
200
.agent/vendor/browser_use/browser_use/browser/views.py
vendored
Normal file
200
.agent/vendor/browser_use/browser_use/browser/views.py
vendored
Normal file
@@ -0,0 +1,200 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.target import TargetID
|
||||
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, field_serializer
|
||||
|
||||
from browser_use.dom.views import DOMInteractedElement, SerializedDOMState
|
||||
|
||||
# Known placeholder image data for about:blank pages - a 4x4 white PNG
|
||||
PLACEHOLDER_4PX_SCREENSHOT = (
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFElEQVR4nGP8//8/AwwwMSAB3BwAlm4DBfIlvvkAAAAASUVORK5CYII='
|
||||
)
|
||||
|
||||
|
||||
# Pydantic
|
||||
class TabInfo(BaseModel):
|
||||
"""Represents information about a browser tab"""
|
||||
|
||||
model_config = ConfigDict(
|
||||
extra='forbid',
|
||||
validate_by_name=True,
|
||||
validate_by_alias=True,
|
||||
populate_by_name=True,
|
||||
)
|
||||
|
||||
# Original fields
|
||||
url: str
|
||||
title: str
|
||||
target_id: TargetID = Field(serialization_alias='tab_id', validation_alias=AliasChoices('tab_id', 'target_id'))
|
||||
parent_target_id: TargetID | None = Field(
|
||||
default=None, serialization_alias='parent_tab_id', validation_alias=AliasChoices('parent_tab_id', 'parent_target_id')
|
||||
) # parent page that contains this popup or cross-origin iframe
|
||||
|
||||
@field_serializer('target_id')
|
||||
def serialize_target_id(self, target_id: TargetID, _info: Any) -> str:
|
||||
return target_id[-4:]
|
||||
|
||||
@field_serializer('parent_target_id')
|
||||
def serialize_parent_target_id(self, parent_target_id: TargetID | None, _info: Any) -> str | None:
|
||||
return parent_target_id[-4:] if parent_target_id else None
|
||||
|
||||
|
||||
class PageInfo(BaseModel):
|
||||
"""Comprehensive page size and scroll information"""
|
||||
|
||||
# Current viewport dimensions
|
||||
viewport_width: int
|
||||
viewport_height: int
|
||||
|
||||
# Total page dimensions
|
||||
page_width: int
|
||||
page_height: int
|
||||
|
||||
# Current scroll position
|
||||
scroll_x: int
|
||||
scroll_y: int
|
||||
|
||||
# Calculated scroll information
|
||||
pixels_above: int
|
||||
pixels_below: int
|
||||
pixels_left: int
|
||||
pixels_right: int
|
||||
|
||||
# Page statistics are now computed dynamically instead of stored
|
||||
|
||||
|
||||
@dataclass
|
||||
class NetworkRequest:
|
||||
"""Information about a pending network request"""
|
||||
|
||||
url: str
|
||||
method: str = 'GET'
|
||||
loading_duration_ms: float = 0.0 # How long this request has been loading (ms since request started, max 10s)
|
||||
resource_type: str | None = None # e.g., 'Document', 'Stylesheet', 'Image', 'Script', 'XHR', 'Fetch'
|
||||
|
||||
|
||||
@dataclass
|
||||
class PaginationButton:
|
||||
"""Information about a pagination button detected on the page"""
|
||||
|
||||
button_type: str # 'next', 'prev', 'first', 'last', 'page_number'
|
||||
backend_node_id: int # Backend node ID for clicking
|
||||
text: str # Button text/label
|
||||
selector: str # XPath or other selector to locate the element
|
||||
is_disabled: bool = False # Whether the button appears disabled
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserStateSummary:
|
||||
"""The summary of the browser's current state designed for an LLM to process"""
|
||||
|
||||
# provided by SerializedDOMState:
|
||||
dom_state: SerializedDOMState
|
||||
|
||||
url: str
|
||||
title: str
|
||||
tabs: list[TabInfo]
|
||||
screenshot: str | None = field(default=None, repr=False)
|
||||
page_info: PageInfo | None = None # Enhanced page information
|
||||
|
||||
# Keep legacy fields for backward compatibility
|
||||
pixels_above: int = 0
|
||||
pixels_below: int = 0
|
||||
browser_errors: list[str] = field(default_factory=list)
|
||||
is_pdf_viewer: bool = False # Whether the current page is a PDF viewer
|
||||
recent_events: str | None = None # Text summary of recent browser events
|
||||
pending_network_requests: list[NetworkRequest] = field(default_factory=list) # Currently loading network requests
|
||||
pagination_buttons: list[PaginationButton] = field(default_factory=list) # Detected pagination buttons
|
||||
closed_popup_messages: list[str] = field(default_factory=list) # Messages from auto-closed JavaScript dialogs
|
||||
|
||||
|
||||
@dataclass
|
||||
class BrowserStateHistory:
|
||||
"""The summary of the browser's state at a past point in time to usse in LLM message history"""
|
||||
|
||||
url: str
|
||||
title: str
|
||||
tabs: list[TabInfo]
|
||||
interacted_element: list[DOMInteractedElement | None] | list[None]
|
||||
screenshot_path: str | None = None
|
||||
|
||||
def get_screenshot(self) -> str | None:
|
||||
"""Load screenshot from disk and return as base64 string"""
|
||||
if not self.screenshot_path:
|
||||
return None
|
||||
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
path_obj = Path(self.screenshot_path)
|
||||
if not path_obj.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(path_obj, 'rb') as f:
|
||||
screenshot_data = f.read()
|
||||
return base64.b64encode(screenshot_data).decode('utf-8')
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
data = {}
|
||||
data['tabs'] = [tab.model_dump() for tab in self.tabs]
|
||||
data['screenshot_path'] = self.screenshot_path
|
||||
data['interacted_element'] = [el.to_dict() if el else None for el in self.interacted_element]
|
||||
data['url'] = self.url
|
||||
data['title'] = self.title
|
||||
return data
|
||||
|
||||
|
||||
class BrowserError(Exception):
|
||||
"""Browser error with structured memory for LLM context management.
|
||||
|
||||
This exception class provides separate memory contexts for browser actions:
|
||||
- short_term_memory: Immediate context shown once to the LLM for the next action
|
||||
- long_term_memory: Persistent error information stored across steps
|
||||
"""
|
||||
|
||||
message: str
|
||||
short_term_memory: str | None = None
|
||||
long_term_memory: str | None = None
|
||||
details: dict[str, Any] | None = None
|
||||
while_handling_event: BaseEvent[Any] | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
short_term_memory: str | None = None,
|
||||
long_term_memory: str | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
event: BaseEvent[Any] | None = None,
|
||||
):
|
||||
"""Initialize a BrowserError with structured memory contexts.
|
||||
|
||||
Args:
|
||||
message: Technical error message for logging and debugging
|
||||
short_term_memory: Context shown once to LLM (e.g., available actions, options)
|
||||
long_term_memory: Persistent error info stored in agent memory
|
||||
details: Additional metadata for debugging
|
||||
event: The browser event that triggered this error
|
||||
"""
|
||||
self.message = message
|
||||
self.short_term_memory = short_term_memory
|
||||
self.long_term_memory = long_term_memory
|
||||
self.details = details
|
||||
self.while_handling_event = event
|
||||
super().__init__(message)
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.details:
|
||||
return f'{self.message} ({self.details}) during: {self.while_handling_event}'
|
||||
elif self.while_handling_event:
|
||||
return f'{self.message} (while handling: {self.while_handling_event})'
|
||||
else:
|
||||
return self.message
|
||||
|
||||
|
||||
class URLNotAllowedError(BrowserError):
|
||||
"""Error raised when a URL is not allowed"""
|
||||
321
.agent/vendor/browser_use/browser_use/browser/watchdog_base.py
vendored
Normal file
321
.agent/vendor/browser_use/browser_use/browser/watchdog_base.py
vendored
Normal file
@@ -0,0 +1,321 @@
|
||||
"""Base watchdog class for browser monitoring components."""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from bubus import BaseEvent, EventBus
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from browser_use.browser.session import BrowserSession
|
||||
|
||||
|
||||
class BaseWatchdog(BaseModel):
|
||||
"""Base class for all browser watchdogs.
|
||||
|
||||
Watchdogs monitor browser state and emit events based on changes.
|
||||
They automatically register event handlers based on method names.
|
||||
|
||||
Handler methods should be named: on_EventTypeName(self, event: EventTypeName)
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(
|
||||
arbitrary_types_allowed=True, # allow non-serializable objects like EventBus/BrowserSession in fields
|
||||
extra='forbid', # dont allow implicit class/instance state, everything must be a properly typed Field or PrivateAttr
|
||||
validate_assignment=False, # avoid re-triggering __init__ / validators on values on every assignment
|
||||
revalidate_instances='never', # avoid re-triggering __init__ / validators and erasing private attrs
|
||||
)
|
||||
|
||||
# Class variables to statically define the list of events relevant to each watchdog
|
||||
# (not enforced, just to make it easier to understand the code and debug watchdogs at runtime)
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [] # Events this watchdog listens to
|
||||
EMITS: ClassVar[list[type[BaseEvent[Any]]]] = [] # Events this watchdog emits
|
||||
|
||||
# Core dependencies
|
||||
event_bus: EventBus = Field()
|
||||
browser_session: BrowserSession = Field()
|
||||
|
||||
# Shared state that other watchdogs might need to access should not be defined on BrowserSession, not here!
|
||||
# Shared helper methods needed by other watchdogs should be defined on BrowserSession, not here!
|
||||
# Alternatively, expose some events on the watchdog to allow access to state/helpers via event_bus system.
|
||||
|
||||
# Private state internal to the watchdog can be defined like this on BaseWatchdog subclasses:
|
||||
# _screenshot_cache: dict[str, bytes] = PrivateAttr(default_factory=dict)
|
||||
# _browser_crash_watcher_task: asyncio.Task | None = PrivateAttr(default=None)
|
||||
# _cdp_download_tasks: WeakSet[asyncio.Task] = PrivateAttr(default_factory=WeakSet)
|
||||
# ...
|
||||
|
||||
@property
|
||||
def logger(self):
|
||||
"""Get the logger from the browser session."""
|
||||
return self.browser_session.logger
|
||||
|
||||
@staticmethod
|
||||
def attach_handler_to_session(browser_session: 'BrowserSession', event_class: type[BaseEvent[Any]], handler) -> None:
|
||||
"""Attach a single event handler to a browser session.
|
||||
|
||||
Args:
|
||||
browser_session: The browser session to attach to
|
||||
event_class: The event class to listen for
|
||||
handler: The handler method (must start with 'on_' and end with event type)
|
||||
"""
|
||||
event_bus = browser_session.event_bus
|
||||
|
||||
# Validate handler naming convention
|
||||
assert hasattr(handler, '__name__'), 'Handler must have a __name__ attribute'
|
||||
assert handler.__name__.startswith('on_'), f'Handler {handler.__name__} must start with "on_"'
|
||||
assert handler.__name__.endswith(event_class.__name__), (
|
||||
f'Handler {handler.__name__} must end with event type {event_class.__name__}'
|
||||
)
|
||||
|
||||
# Get the watchdog instance if this is a bound method
|
||||
watchdog_instance = getattr(handler, '__self__', None)
|
||||
watchdog_class_name = watchdog_instance.__class__.__name__ if watchdog_instance else 'Unknown'
|
||||
|
||||
# Events that should always run even when CDP is disconnected (lifecycle management)
|
||||
LIFECYCLE_EVENT_NAMES = frozenset(
|
||||
{
|
||||
'BrowserStartEvent',
|
||||
'BrowserStopEvent',
|
||||
'BrowserStoppedEvent',
|
||||
'BrowserLaunchEvent',
|
||||
'BrowserErrorEvent',
|
||||
'BrowserKillEvent',
|
||||
'BrowserReconnectingEvent',
|
||||
'BrowserReconnectedEvent',
|
||||
}
|
||||
)
|
||||
|
||||
# Create a wrapper function with unique name to avoid duplicate handler warnings
|
||||
# Capture handler by value to avoid closure issues
|
||||
def make_unique_handler(actual_handler):
|
||||
async def unique_handler(event):
|
||||
# Circuit breaker: skip handler if CDP WebSocket is dead
|
||||
# (prevents handlers from hanging on broken connections until timeout)
|
||||
# Lifecycle events are exempt — they manage browser start/stop
|
||||
if event.event_type not in LIFECYCLE_EVENT_NAMES and not browser_session.is_cdp_connected:
|
||||
# If reconnection is in progress, wait for it instead of silently skipping
|
||||
if browser_session.is_reconnecting:
|
||||
wait_timeout = browser_session.RECONNECT_WAIT_TIMEOUT
|
||||
browser_session.logger.debug(
|
||||
f'🚌 [{watchdog_class_name}.{actual_handler.__name__}] ⏳ Waiting for reconnection ({wait_timeout}s)...'
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(browser_session._reconnect_event.wait(), timeout=wait_timeout)
|
||||
except TimeoutError:
|
||||
raise ConnectionError(
|
||||
f'[{watchdog_class_name}.{actual_handler.__name__}] '
|
||||
f'Reconnection wait timed out after {wait_timeout}s'
|
||||
)
|
||||
# After wait: check if reconnection actually succeeded
|
||||
if not browser_session.is_cdp_connected:
|
||||
raise ConnectionError(
|
||||
f'[{watchdog_class_name}.{actual_handler.__name__}] Reconnection failed — CDP still not connected'
|
||||
)
|
||||
# Reconnection succeeded — fall through to execute handler normally
|
||||
else:
|
||||
# Not reconnecting — intentional stop, backward compat silent skip
|
||||
browser_session.logger.debug(
|
||||
f'🚌 [{watchdog_class_name}.{actual_handler.__name__}] ⚡ Skipped — CDP not connected'
|
||||
)
|
||||
return None
|
||||
|
||||
# just for debug logging, not used for anything else
|
||||
parent_event = event_bus.event_history.get(event.event_parent_id) if event.event_parent_id else None
|
||||
grandparent_event = (
|
||||
event_bus.event_history.get(parent_event.event_parent_id)
|
||||
if parent_event and parent_event.event_parent_id
|
||||
else None
|
||||
)
|
||||
parent = (
|
||||
f'↲ triggered by on_{parent_event.event_type}#{parent_event.event_id[-4:]}'
|
||||
if parent_event
|
||||
else '👈 by Agent'
|
||||
)
|
||||
grandparent = (
|
||||
(
|
||||
f'↲ under {grandparent_event.event_type}#{grandparent_event.event_id[-4:]}'
|
||||
if grandparent_event
|
||||
else '👈 by Agent'
|
||||
)
|
||||
if parent_event
|
||||
else ''
|
||||
)
|
||||
event_str = f'#{event.event_id[-4:]}'
|
||||
time_start = time.time()
|
||||
watchdog_and_handler_str = f'[{watchdog_class_name}.{actual_handler.__name__}({event_str})]'.ljust(54)
|
||||
browser_session.logger.debug(f'🚌 {watchdog_and_handler_str} ⏳ Starting... {parent} {grandparent}')
|
||||
|
||||
try:
|
||||
# **EXECUTE THE EVENT HANDLER FUNCTION**
|
||||
result = await actual_handler(event)
|
||||
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
|
||||
# just for debug logging, not used for anything else
|
||||
time_end = time.time()
|
||||
time_elapsed = time_end - time_start
|
||||
result_summary = '' if result is None else f' ➡️ <{type(result).__name__}>'
|
||||
parents_summary = f' {parent}'.replace('↲ triggered by ', '⤴ returned to ').replace(
|
||||
'👈 by Agent', '👉 returned to Agent'
|
||||
)
|
||||
browser_session.logger.debug(
|
||||
f'🚌 {watchdog_and_handler_str} Succeeded ({time_elapsed:.2f}s){result_summary}{parents_summary}'
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
time_end = time.time()
|
||||
time_elapsed = time_end - time_start
|
||||
original_error = e
|
||||
browser_session.logger.error(
|
||||
f'🚌 {watchdog_and_handler_str} ❌ Failed ({time_elapsed:.2f}s): {type(e).__name__}: {e}'
|
||||
)
|
||||
|
||||
# attempt to repair potentially crashed CDP session
|
||||
try:
|
||||
if browser_session.agent_focus_target_id:
|
||||
# With event-driven sessions, Chrome will send detach/attach events
|
||||
# SessionManager handles pool cleanup automatically
|
||||
target_id_to_restore = browser_session.agent_focus_target_id
|
||||
browser_session.logger.debug(
|
||||
f'🚌 {watchdog_and_handler_str} ⚠️ Session error detected, waiting for CDP events to sync (target: {target_id_to_restore})'
|
||||
)
|
||||
|
||||
# Wait for new attach event to restore the session
|
||||
# This will raise ValueError if target doesn't re-attach
|
||||
await browser_session.get_or_create_cdp_session(target_id=target_id_to_restore, focus=True)
|
||||
else:
|
||||
# Try to get any available session
|
||||
await browser_session.get_or_create_cdp_session(target_id=None, focus=True)
|
||||
except Exception as sub_error:
|
||||
if 'ConnectionClosedError' in str(type(sub_error)) or 'ConnectionError' in str(type(sub_error)):
|
||||
browser_session.logger.error(
|
||||
f'🚌 {watchdog_and_handler_str} ❌ Browser closed or CDP Connection disconnected by remote. {type(sub_error).__name__}: {sub_error}\n'
|
||||
)
|
||||
raise
|
||||
else:
|
||||
browser_session.logger.error(
|
||||
f'🚌 {watchdog_and_handler_str} ❌ CDP connected but failed to re-create CDP session after error "{type(original_error).__name__}: {original_error}" in {actual_handler.__name__}({event.event_type}#{event.event_id[-4:]}): due to {type(sub_error).__name__}: {sub_error}\n'
|
||||
)
|
||||
|
||||
# Always re-raise the original error with its traceback preserved
|
||||
raise
|
||||
|
||||
return unique_handler
|
||||
|
||||
unique_handler = make_unique_handler(handler)
|
||||
unique_handler.__name__ = f'{watchdog_class_name}.{handler.__name__}'
|
||||
|
||||
# Check if this handler is already registered - throw error if duplicate
|
||||
existing_handlers = event_bus.handlers.get(event_class.__name__, [])
|
||||
handler_names = [getattr(h, '__name__', str(h)) for h in existing_handlers]
|
||||
|
||||
if unique_handler.__name__ in handler_names:
|
||||
raise RuntimeError(
|
||||
f'[{watchdog_class_name}] Duplicate handler registration attempted! '
|
||||
f'Handler {unique_handler.__name__} is already registered for {event_class.__name__}. '
|
||||
f'This likely means attach_to_session() was called multiple times.'
|
||||
)
|
||||
|
||||
event_bus.on(event_class, unique_handler)
|
||||
|
||||
@staticmethod
|
||||
def detach_handler_from_session(browser_session: 'BrowserSession', event_class: type[BaseEvent[Any]], handler) -> None:
|
||||
"""Detach a single event handler from a browser session."""
|
||||
event_bus = browser_session.event_bus
|
||||
|
||||
# Get the watchdog instance if this is a bound method
|
||||
watchdog_instance = getattr(handler, '__self__', None)
|
||||
watchdog_class_name = watchdog_instance.__class__.__name__ if watchdog_instance else 'Unknown'
|
||||
|
||||
# Find and remove the handler by its unique name pattern
|
||||
unique_handler_name = f'{watchdog_class_name}.{handler.__name__}'
|
||||
|
||||
existing_handlers = event_bus.handlers.get(event_class.__name__, [])
|
||||
for existing_handler in existing_handlers[:]: # copy list to allow modification during iteration
|
||||
if getattr(existing_handler, '__name__', '') == unique_handler_name:
|
||||
existing_handlers.remove(existing_handler)
|
||||
break
|
||||
|
||||
def attach_to_session(self) -> None:
|
||||
"""Attach watchdog to its browser session and start monitoring.
|
||||
|
||||
This method handles event listener registration. The watchdog is already
|
||||
bound to a browser session via self.browser_session from initialization.
|
||||
"""
|
||||
# Register event handlers automatically based on method names
|
||||
assert self.browser_session is not None, 'Root CDP client not initialized - browser may not be connected yet'
|
||||
|
||||
from browser_use.browser import events
|
||||
|
||||
event_classes = {}
|
||||
for name in dir(events):
|
||||
obj = getattr(events, name)
|
||||
if inspect.isclass(obj) and issubclass(obj, BaseEvent) and obj is not BaseEvent:
|
||||
event_classes[name] = obj
|
||||
|
||||
# Find all handler methods (on_EventName)
|
||||
registered_events = set()
|
||||
for method_name in dir(self):
|
||||
if method_name.startswith('on_') and callable(getattr(self, method_name)):
|
||||
# Extract event name from method name (on_EventName -> EventName)
|
||||
event_name = method_name[3:] # Remove 'on_' prefix
|
||||
|
||||
if event_name in event_classes:
|
||||
event_class = event_classes[event_name]
|
||||
|
||||
# ASSERTION: If LISTENS_TO is defined, enforce it
|
||||
if self.LISTENS_TO:
|
||||
assert event_class in self.LISTENS_TO, (
|
||||
f'[{self.__class__.__name__}] Handler {method_name} listens to {event_name} '
|
||||
f'but {event_name} is not declared in LISTENS_TO: {[e.__name__ for e in self.LISTENS_TO]}'
|
||||
)
|
||||
|
||||
handler = getattr(self, method_name)
|
||||
|
||||
# Use the static helper to attach the handler
|
||||
self.attach_handler_to_session(self.browser_session, event_class, handler)
|
||||
registered_events.add(event_class)
|
||||
|
||||
# ASSERTION: If LISTENS_TO is defined, ensure all declared events have handlers
|
||||
if self.LISTENS_TO:
|
||||
missing_handlers = set(self.LISTENS_TO) - registered_events
|
||||
if missing_handlers:
|
||||
missing_names = [e.__name__ for e in missing_handlers]
|
||||
self.logger.warning(
|
||||
f'[{self.__class__.__name__}] LISTENS_TO declares {missing_names} '
|
||||
f'but no handlers found (missing on_{"_, on_".join(missing_names)} methods)'
|
||||
)
|
||||
|
||||
def __del__(self) -> None:
|
||||
"""Clean up any running tasks during garbage collection."""
|
||||
|
||||
# A BIT OF MAGIC: Cancel any private attributes that look like asyncio tasks
|
||||
try:
|
||||
for attr_name in dir(self):
|
||||
# e.g. _browser_crash_watcher_task = asyncio.Task
|
||||
if attr_name.startswith('_') and attr_name.endswith('_task'):
|
||||
try:
|
||||
task = getattr(self, attr_name)
|
||||
if hasattr(task, 'cancel') and callable(task.cancel) and not task.done():
|
||||
task.cancel()
|
||||
# self.logger.debug(f'[{self.__class__.__name__}] Cancelled {attr_name} during cleanup')
|
||||
except Exception:
|
||||
pass # Ignore errors during cleanup
|
||||
|
||||
# e.g. _cdp_download_tasks = WeakSet[asyncio.Task] or list[asyncio.Task]
|
||||
if attr_name.startswith('_') and attr_name.endswith('_tasks') and isinstance(getattr(self, attr_name), Iterable):
|
||||
for task in getattr(self, attr_name):
|
||||
try:
|
||||
if hasattr(task, 'cancel') and callable(task.cancel) and not task.done():
|
||||
task.cancel()
|
||||
# self.logger.debug(f'[{self.__class__.__name__}] Cancelled {attr_name} during cleanup')
|
||||
except Exception:
|
||||
pass # Ignore errors during cleanup
|
||||
except Exception as e:
|
||||
from browser_use.utils import logger
|
||||
|
||||
logger.error(f'⚠️ Error during BrowserSession {self.__class__.__name__} garbage collection __del__(): {type(e)}: {e}')
|
||||
0
.agent/vendor/browser_use/browser_use/browser/watchdogs/__init__.py
vendored
Normal file
0
.agent/vendor/browser_use/browser_use/browser/watchdogs/__init__.py
vendored
Normal file
259
.agent/vendor/browser_use/browser_use/browser/watchdogs/aboutblank_watchdog.py
vendored
Normal file
259
.agent/vendor/browser_use/browser_use/browser/watchdogs/aboutblank_watchdog.py
vendored
Normal file
@@ -0,0 +1,259 @@
|
||||
"""About:blank watchdog for managing about:blank tabs with DVD screensaver."""
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.target import TargetID
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
AboutBlankDVDScreensaverShownEvent,
|
||||
BrowserStopEvent,
|
||||
BrowserStoppedEvent,
|
||||
CloseTabEvent,
|
||||
NavigateToUrlEvent,
|
||||
TabClosedEvent,
|
||||
TabCreatedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class AboutBlankWatchdog(BaseWatchdog):
|
||||
"""Ensures there's always exactly one about:blank tab with DVD screensaver."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserStopEvent,
|
||||
BrowserStoppedEvent,
|
||||
TabCreatedEvent,
|
||||
TabClosedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [
|
||||
NavigateToUrlEvent,
|
||||
CloseTabEvent,
|
||||
AboutBlankDVDScreensaverShownEvent,
|
||||
]
|
||||
|
||||
_stopping: bool = PrivateAttr(default=False)
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
"""Handle browser stop request - stop creating new tabs."""
|
||||
# logger.info('[AboutBlankWatchdog] Browser stop requested, stopping tab creation')
|
||||
self._stopping = True
|
||||
|
||||
async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
|
||||
"""Handle browser stopped event."""
|
||||
# logger.info('[AboutBlankWatchdog] Browser stopped')
|
||||
self._stopping = True
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
"""Check tabs when a new tab is created."""
|
||||
# logger.debug(f'[AboutBlankWatchdog] ➕ New tab created: {event.url}')
|
||||
|
||||
# If an about:blank tab was created, show DVD screensaver on all about:blank tabs
|
||||
if event.url == 'about:blank':
|
||||
await self._show_dvd_screensaver_on_about_blank_tabs()
|
||||
|
||||
async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
|
||||
"""Check tabs when a tab is closed and proactively create about:blank if needed."""
|
||||
# Don't create new tabs if browser is shutting down
|
||||
if self._stopping:
|
||||
return
|
||||
|
||||
# Don't attempt CDP operations if the WebSocket is dead — dispatching
|
||||
# NavigateToUrlEvent on a broken connection will hang until timeout
|
||||
if not self.browser_session.is_cdp_connected:
|
||||
self.logger.debug('[AboutBlankWatchdog] CDP not connected, skipping tab recovery')
|
||||
return
|
||||
|
||||
# Check if we're about to close the last tab (event happens BEFORE tab closes)
|
||||
# Use _cdp_get_all_pages for quick check without fetching titles
|
||||
page_targets = await self.browser_session._cdp_get_all_pages()
|
||||
if len(page_targets) < 1:
|
||||
self.logger.debug(
|
||||
'[AboutBlankWatchdog] Last tab closing, creating new about:blank tab to avoid closing entire browser'
|
||||
)
|
||||
# Create the animation tab since no tabs should remain
|
||||
navigate_event = self.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True))
|
||||
await navigate_event
|
||||
# Show DVD screensaver on the new tab
|
||||
await self._show_dvd_screensaver_on_about_blank_tabs()
|
||||
else:
|
||||
# Multiple tabs exist, check after close
|
||||
await self._check_and_ensure_about_blank_tab()
|
||||
|
||||
async def attach_to_target(self, target_id: TargetID) -> None:
|
||||
"""AboutBlankWatchdog doesn't monitor individual targets."""
|
||||
pass
|
||||
|
||||
async def _check_and_ensure_about_blank_tab(self) -> None:
|
||||
"""Check current tabs and ensure exactly one about:blank tab with animation exists."""
|
||||
try:
|
||||
if not self.browser_session.is_cdp_connected:
|
||||
return
|
||||
|
||||
# For quick checks, just get page targets without titles to reduce noise
|
||||
page_targets = await self.browser_session._cdp_get_all_pages()
|
||||
|
||||
# If no tabs exist at all, create one to keep browser alive
|
||||
if len(page_targets) == 0:
|
||||
# Only create a new tab if there are no tabs at all
|
||||
self.logger.debug('[AboutBlankWatchdog] No tabs exist, creating new about:blank DVD screensaver tab')
|
||||
navigate_event = self.event_bus.dispatch(NavigateToUrlEvent(url='about:blank', new_tab=True))
|
||||
await navigate_event
|
||||
# Show DVD screensaver on the new tab
|
||||
await self._show_dvd_screensaver_on_about_blank_tabs()
|
||||
# Otherwise there are tabs, don't create new ones to avoid interfering
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[AboutBlankWatchdog] Error ensuring about:blank tab: {e}')
|
||||
|
||||
async def _show_dvd_screensaver_on_about_blank_tabs(self) -> None:
|
||||
"""Show DVD screensaver on all about:blank pages only."""
|
||||
try:
|
||||
# Get just the page targets without expensive title fetching
|
||||
page_targets = await self.browser_session._cdp_get_all_pages()
|
||||
browser_session_label = str(self.browser_session.id)[-4:]
|
||||
|
||||
for page_target in page_targets:
|
||||
target_id = page_target['targetId']
|
||||
url = page_target['url']
|
||||
|
||||
# Only target about:blank pages specifically
|
||||
if url == 'about:blank':
|
||||
await self._show_dvd_screensaver_loading_animation_cdp(target_id, browser_session_label)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[AboutBlankWatchdog] Error showing DVD screensaver: {e}')
|
||||
|
||||
async def _show_dvd_screensaver_loading_animation_cdp(self, target_id: TargetID, browser_session_label: str) -> None:
|
||||
"""
|
||||
Injects a DVD screensaver-style bouncing logo loading animation overlay into the target using CDP.
|
||||
This is used to visually indicate that the browser is setting up or waiting.
|
||||
"""
|
||||
try:
|
||||
# Create temporary session for this target without switching focus
|
||||
temp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
|
||||
|
||||
# Inject the DVD screensaver script (from main branch with idempotency added)
|
||||
script = f"""
|
||||
(function(browser_session_label) {{
|
||||
// Idempotency check
|
||||
if (window.__dvdAnimationRunning) {{
|
||||
return; // Already running, don't add another
|
||||
}}
|
||||
window.__dvdAnimationRunning = true;
|
||||
|
||||
// Ensure document.body exists before proceeding
|
||||
if (!document.body) {{
|
||||
// Try again after DOM is ready
|
||||
window.__dvdAnimationRunning = false; // Reset flag to retry
|
||||
if (document.readyState === 'loading') {{
|
||||
document.addEventListener('DOMContentLoaded', () => arguments.callee(browser_session_label));
|
||||
}}
|
||||
return;
|
||||
}}
|
||||
|
||||
const animated_title = `Starting agent ${{browser_session_label}}...`;
|
||||
if (document.title === animated_title) {{
|
||||
return; // already run on this tab, dont run again
|
||||
}}
|
||||
document.title = animated_title;
|
||||
|
||||
// Create the main overlay
|
||||
const loadingOverlay = document.createElement('div');
|
||||
loadingOverlay.id = 'pretty-loading-animation';
|
||||
loadingOverlay.style.position = 'fixed';
|
||||
loadingOverlay.style.top = '0';
|
||||
loadingOverlay.style.left = '0';
|
||||
loadingOverlay.style.width = '100vw';
|
||||
loadingOverlay.style.height = '100vh';
|
||||
loadingOverlay.style.background = '#000';
|
||||
loadingOverlay.style.zIndex = '99999';
|
||||
loadingOverlay.style.overflow = 'hidden';
|
||||
|
||||
// Create the image element
|
||||
const img = document.createElement('img');
|
||||
img.src = 'https://cf.browser-use.com/logo.svg';
|
||||
img.alt = 'Browser-Use';
|
||||
img.style.width = '200px';
|
||||
img.style.height = 'auto';
|
||||
img.style.position = 'absolute';
|
||||
img.style.left = '0px';
|
||||
img.style.top = '0px';
|
||||
img.style.zIndex = '2';
|
||||
img.style.opacity = '0.8';
|
||||
|
||||
loadingOverlay.appendChild(img);
|
||||
document.body.appendChild(loadingOverlay);
|
||||
|
||||
// DVD screensaver bounce logic
|
||||
let x = Math.random() * (window.innerWidth - 300);
|
||||
let y = Math.random() * (window.innerHeight - 300);
|
||||
let dx = 1.2 + Math.random() * 0.4; // px per frame
|
||||
let dy = 1.2 + Math.random() * 0.4;
|
||||
// Randomize direction
|
||||
if (Math.random() > 0.5) dx = -dx;
|
||||
if (Math.random() > 0.5) dy = -dy;
|
||||
|
||||
function animate() {{
|
||||
const imgWidth = img.offsetWidth || 300;
|
||||
const imgHeight = img.offsetHeight || 300;
|
||||
x += dx;
|
||||
y += dy;
|
||||
|
||||
if (x <= 0) {{
|
||||
x = 0;
|
||||
dx = Math.abs(dx);
|
||||
}} else if (x + imgWidth >= window.innerWidth) {{
|
||||
x = window.innerWidth - imgWidth;
|
||||
dx = -Math.abs(dx);
|
||||
}}
|
||||
if (y <= 0) {{
|
||||
y = 0;
|
||||
dy = Math.abs(dy);
|
||||
}} else if (y + imgHeight >= window.innerHeight) {{
|
||||
y = window.innerHeight - imgHeight;
|
||||
dy = -Math.abs(dy);
|
||||
}}
|
||||
|
||||
img.style.left = `${{x}}px`;
|
||||
img.style.top = `${{y}}px`;
|
||||
|
||||
requestAnimationFrame(animate);
|
||||
}}
|
||||
animate();
|
||||
|
||||
// Responsive: update bounds on resize
|
||||
window.addEventListener('resize', () => {{
|
||||
x = Math.min(x, window.innerWidth - img.offsetWidth);
|
||||
y = Math.min(y, window.innerHeight - img.offsetHeight);
|
||||
}});
|
||||
|
||||
// Add a little CSS for smoothness
|
||||
const style = document.createElement('style');
|
||||
style.textContent = `
|
||||
#pretty-loading-animation {{
|
||||
/*backdrop-filter: blur(2px) brightness(0.9);*/
|
||||
}}
|
||||
#pretty-loading-animation img {{
|
||||
user-select: none;
|
||||
pointer-events: none;
|
||||
}}
|
||||
`;
|
||||
document.head.appendChild(style);
|
||||
}})('{browser_session_label}');
|
||||
"""
|
||||
|
||||
await temp_session.cdp_client.send.Runtime.evaluate(params={'expression': script}, session_id=temp_session.session_id)
|
||||
|
||||
# No need to detach - session is cached
|
||||
|
||||
# Dispatch event
|
||||
self.event_bus.dispatch(AboutBlankDVDScreensaverShownEvent(target_id=target_id))
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[AboutBlankWatchdog] Error injecting DVD screensaver: {e}')
|
||||
207
.agent/vendor/browser_use/browser_use/browser/watchdogs/captcha_watchdog.py
vendored
Normal file
207
.agent/vendor/browser_use/browser_use/browser/watchdogs/captcha_watchdog.py
vendored
Normal file
@@ -0,0 +1,207 @@
|
||||
"""Captcha solver watchdog — monitors captcha events from the browser proxy.
|
||||
|
||||
Listens for BrowserUse.captchaSolverStarted/Finished CDP events and exposes a
|
||||
wait_if_captcha_solving() method that the agent step loop uses to block until
|
||||
a captcha is resolved (with a configurable timeout).
|
||||
|
||||
NOTE: Only a single captcha solve is tracked at a time. If multiple captchas
|
||||
overlap (e.g. rapid successive navigations), only the latest one is tracked and
|
||||
earlier in-flight waits may return prematurely.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar, Literal
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.browseruse.events import CaptchaSolverFinishedEvent as CDPCaptchaSolverFinishedEvent
|
||||
from cdp_use.cdp.browseruse.events import CaptchaSolverStartedEvent as CDPCaptchaSolverStartedEvent
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserConnectedEvent,
|
||||
BrowserStoppedEvent,
|
||||
CaptchaSolverFinishedEvent,
|
||||
CaptchaSolverStartedEvent,
|
||||
_get_timeout,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
CaptchaResultType = Literal['success', 'failed', 'timeout', 'unknown']
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaptchaWaitResult:
|
||||
"""Result returned by wait_if_captcha_solving() when the agent had to wait."""
|
||||
|
||||
waited: bool
|
||||
vendor: str
|
||||
url: str
|
||||
duration_ms: int
|
||||
result: CaptchaResultType
|
||||
|
||||
|
||||
class CaptchaWatchdog(BaseWatchdog):
|
||||
"""Monitors captcha solver events from the browser proxy.
|
||||
|
||||
When the proxy detects a CAPTCHA and starts solving it, a CDP event
|
||||
``BrowserUse.captchaSolverStarted`` is sent over the WebSocket. This
|
||||
watchdog catches that event and blocks the agent's step loop (via
|
||||
``wait_if_captcha_solving``) until ``BrowserUse.captchaSolverFinished``
|
||||
arrives or the configurable timeout expires.
|
||||
"""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserConnectedEvent,
|
||||
BrowserStoppedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [
|
||||
CaptchaSolverStartedEvent,
|
||||
CaptchaSolverFinishedEvent,
|
||||
]
|
||||
|
||||
# --- private state ---
|
||||
_captcha_solving: bool = PrivateAttr(default=False)
|
||||
_captcha_solved_event: asyncio.Event = PrivateAttr(default_factory=asyncio.Event)
|
||||
_captcha_info: dict[str, Any] = PrivateAttr(default_factory=dict)
|
||||
_captcha_result: CaptchaResultType = PrivateAttr(default='unknown')
|
||||
_captcha_duration_ms: int = PrivateAttr(default=0)
|
||||
_cdp_handlers_registered: bool = PrivateAttr(default=False)
|
||||
|
||||
def model_post_init(self, __context: Any) -> None:
|
||||
# Start in "not blocked" state so callers never wait when there is no captcha.
|
||||
self._captcha_solved_event.set()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Event handlers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""Register CDP event handlers for BrowserUse captcha solver events."""
|
||||
if self._cdp_handlers_registered:
|
||||
self.logger.debug('CaptchaWatchdog: CDP handlers already registered, skipping')
|
||||
return
|
||||
|
||||
cdp_client = self.browser_session.cdp_client
|
||||
|
||||
def _on_captcha_started(event_data: CDPCaptchaSolverStartedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
self._captcha_solving = True
|
||||
self._captcha_result = 'unknown'
|
||||
self._captcha_duration_ms = 0
|
||||
self._captcha_info = {
|
||||
'vendor': event_data.get('vendor', 'unknown'),
|
||||
'url': event_data.get('url', ''),
|
||||
'targetId': event_data.get('targetId', ''),
|
||||
'startedAt': event_data.get('startedAt', 0),
|
||||
}
|
||||
# Block any waiter
|
||||
self._captcha_solved_event.clear()
|
||||
|
||||
vendor = self._captcha_info['vendor']
|
||||
url = self._captcha_info['url']
|
||||
self.logger.info(f'🔒 Captcha solving started: {vendor} on {url}')
|
||||
|
||||
self.event_bus.dispatch(
|
||||
CaptchaSolverStartedEvent(
|
||||
target_id=event_data.get('targetId', ''),
|
||||
vendor=vendor,
|
||||
url=url,
|
||||
started_at=event_data.get('startedAt', 0),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('Error handling captchaSolverStarted CDP event')
|
||||
# Ensure consistent state: unblock any waiter
|
||||
self._captcha_solving = False
|
||||
self._captcha_solved_event.set()
|
||||
|
||||
def _on_captcha_finished(event_data: CDPCaptchaSolverFinishedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
success = event_data.get('success', False)
|
||||
self._captcha_solving = False
|
||||
self._captcha_duration_ms = event_data.get('durationMs', 0)
|
||||
self._captcha_result = 'success' if success else 'failed'
|
||||
|
||||
vendor = event_data.get('vendor', self._captcha_info.get('vendor', 'unknown'))
|
||||
url = event_data.get('url', self._captcha_info.get('url', ''))
|
||||
duration_s = self._captcha_duration_ms / 1000
|
||||
|
||||
self.logger.info(f'🔓 Captcha solving finished: {self._captcha_result} — {vendor} on {url} ({duration_s:.1f}s)')
|
||||
|
||||
# Unblock any waiter
|
||||
self._captcha_solved_event.set()
|
||||
|
||||
self.event_bus.dispatch(
|
||||
CaptchaSolverFinishedEvent(
|
||||
target_id=event_data.get('targetId', ''),
|
||||
vendor=vendor,
|
||||
url=url,
|
||||
duration_ms=self._captcha_duration_ms,
|
||||
finished_at=event_data.get('finishedAt', 0),
|
||||
success=success,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
self.logger.exception('Error handling captchaSolverFinished CDP event')
|
||||
# Ensure consistent state: unblock any waiter
|
||||
self._captcha_solving = False
|
||||
self._captcha_solved_event.set()
|
||||
|
||||
cdp_client.register.BrowserUse.captchaSolverStarted(_on_captcha_started)
|
||||
cdp_client.register.BrowserUse.captchaSolverFinished(_on_captcha_finished)
|
||||
self._cdp_handlers_registered = True
|
||||
self.logger.debug('🔒 CaptchaWatchdog: registered CDP event handlers for BrowserUse captcha events')
|
||||
|
||||
async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
|
||||
"""Clear captcha state when the browser disconnects so nothing hangs."""
|
||||
self._captcha_solving = False
|
||||
self._captcha_result = 'unknown'
|
||||
self._captcha_duration_ms = 0
|
||||
self._captcha_info = {}
|
||||
self._captcha_solved_event.set()
|
||||
self._cdp_handlers_registered = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def wait_if_captcha_solving(self, timeout: float | None = None) -> CaptchaWaitResult | None:
|
||||
"""Wait if a captcha is currently being solved.
|
||||
|
||||
Returns:
|
||||
``None`` if no captcha was in progress.
|
||||
A ``CaptchaWaitResult`` with the outcome otherwise.
|
||||
"""
|
||||
if not self._captcha_solving:
|
||||
return None
|
||||
|
||||
if timeout is None:
|
||||
timeout = _get_timeout('TIMEOUT_CaptchaSolverWait', 120.0)
|
||||
assert timeout is not None
|
||||
vendor = self._captcha_info.get('vendor', 'unknown')
|
||||
url = self._captcha_info.get('url', '')
|
||||
self.logger.info(f'⏳ Waiting for {vendor} captcha to be solved on {url} (timeout={timeout}s)...')
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(self._captcha_solved_event.wait(), timeout=timeout)
|
||||
return CaptchaWaitResult(
|
||||
waited=True,
|
||||
vendor=vendor,
|
||||
url=url,
|
||||
duration_ms=self._captcha_duration_ms,
|
||||
result=self._captcha_result,
|
||||
)
|
||||
except TimeoutError:
|
||||
# Timed out — unblock and report
|
||||
self._captcha_solving = False
|
||||
self._captcha_solved_event.set()
|
||||
self.logger.warning(f'⏰ Captcha wait timed out after {timeout}s for {vendor} on {url}')
|
||||
return CaptchaWaitResult(
|
||||
waited=True,
|
||||
vendor=vendor,
|
||||
url=url,
|
||||
duration_ms=int(timeout * 1000),
|
||||
result='timeout',
|
||||
)
|
||||
336
.agent/vendor/browser_use/browser_use/browser/watchdogs/crash_watchdog.py
vendored
Normal file
336
.agent/vendor/browser_use/browser_use/browser/watchdogs/crash_watchdog.py
vendored
Normal file
@@ -0,0 +1,336 @@
|
||||
"""Browser watchdog for monitoring crashes and network timeouts using CDP."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
import psutil
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.target import SessionID, TargetID
|
||||
from cdp_use.cdp.target.events import TargetCrashedEvent
|
||||
from pydantic import Field, PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserConnectedEvent,
|
||||
BrowserErrorEvent,
|
||||
BrowserStoppedEvent,
|
||||
TabClosedEvent,
|
||||
TabCreatedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.utils import create_task_with_error_handling
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class NetworkRequestTracker:
|
||||
"""Tracks ongoing network requests."""
|
||||
|
||||
def __init__(self, request_id: str, start_time: float, url: str, method: str, resource_type: str | None = None):
|
||||
self.request_id = request_id
|
||||
self.start_time = start_time
|
||||
self.url = url
|
||||
self.method = method
|
||||
self.resource_type = resource_type
|
||||
|
||||
|
||||
class CrashWatchdog(BaseWatchdog):
|
||||
"""Monitors browser health for crashes and network timeouts using CDP."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserConnectedEvent,
|
||||
BrowserStoppedEvent,
|
||||
TabCreatedEvent,
|
||||
TabClosedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [BrowserErrorEvent]
|
||||
|
||||
# Configuration
|
||||
network_timeout_seconds: float = Field(default=10.0)
|
||||
check_interval_seconds: float = Field(default=5.0) # Reduced frequency to reduce noise
|
||||
|
||||
# Private state
|
||||
_active_requests: dict[str, NetworkRequestTracker] = PrivateAttr(default_factory=dict)
|
||||
_monitoring_task: asyncio.Task | None = PrivateAttr(default=None)
|
||||
_last_responsive_checks: dict[str, float] = PrivateAttr(default_factory=dict) # target_url -> timestamp
|
||||
_cdp_event_tasks: set[asyncio.Task] = PrivateAttr(default_factory=set) # Track CDP event handler tasks
|
||||
_targets_with_listeners: set[str] = PrivateAttr(default_factory=set) # Track targets that already have event listeners
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""Start monitoring when browser is connected."""
|
||||
# logger.debug('[CrashWatchdog] Browser connected event received, beginning monitoring')
|
||||
|
||||
create_task_with_error_handling(
|
||||
self._start_monitoring(), name='start_crash_monitoring', logger_instance=self.logger, suppress_exceptions=True
|
||||
)
|
||||
# logger.debug(f'[CrashWatchdog] Monitoring task started: {self._monitoring_task and not self._monitoring_task.done()}')
|
||||
|
||||
async def on_BrowserStoppedEvent(self, event: BrowserStoppedEvent) -> None:
|
||||
"""Stop monitoring when browser stops."""
|
||||
# logger.debug('[CrashWatchdog] Browser stopped, ending monitoring')
|
||||
await self._stop_monitoring()
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
"""Attach to new tab."""
|
||||
assert self.browser_session.agent_focus_target_id is not None, 'No current target ID'
|
||||
await self.attach_to_target(self.browser_session.agent_focus_target_id)
|
||||
|
||||
async def on_TabClosedEvent(self, event: TabClosedEvent) -> None:
|
||||
"""Clean up tracking when tab closes."""
|
||||
# Remove target from listener tracking to prevent memory leak
|
||||
if event.target_id in self._targets_with_listeners:
|
||||
self._targets_with_listeners.discard(event.target_id)
|
||||
self.logger.debug(f'[CrashWatchdog] Removed target {event.target_id[:8]}... from monitoring')
|
||||
|
||||
async def attach_to_target(self, target_id: TargetID) -> None:
|
||||
"""Set up crash monitoring for a specific target using CDP."""
|
||||
try:
|
||||
# Check if we already have listeners for this target
|
||||
if target_id in self._targets_with_listeners:
|
||||
self.logger.debug(f'[CrashWatchdog] Event listeners already exist for target: {target_id[:8]}...')
|
||||
return
|
||||
|
||||
# Create temporary session for monitoring without switching focus
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=False)
|
||||
|
||||
# Register crash event handler
|
||||
def on_target_crashed(event: TargetCrashedEvent, session_id: SessionID | None = None):
|
||||
# Create and track the task
|
||||
task = create_task_with_error_handling(
|
||||
self._on_target_crash_cdp(target_id),
|
||||
name='handle_target_crash',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
self._cdp_event_tasks.add(task)
|
||||
# Remove from set when done
|
||||
task.add_done_callback(lambda t: self._cdp_event_tasks.discard(t))
|
||||
|
||||
cdp_session.cdp_client.register.Target.targetCrashed(on_target_crashed)
|
||||
|
||||
# Track that we've added listeners to this target
|
||||
self._targets_with_listeners.add(target_id)
|
||||
|
||||
target = self.browser_session.session_manager.get_target(target_id)
|
||||
if target:
|
||||
self.logger.debug(f'[CrashWatchdog] Added target to monitoring: {target.url}')
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f'[CrashWatchdog] Failed to attach to target {target_id}: {e}')
|
||||
|
||||
async def _on_request_cdp(self, event: dict) -> None:
|
||||
"""Track new network request from CDP event."""
|
||||
request_id = event.get('requestId', '')
|
||||
request = event.get('request', {})
|
||||
|
||||
self._active_requests[request_id] = NetworkRequestTracker(
|
||||
request_id=request_id,
|
||||
start_time=time.time(),
|
||||
url=request.get('url', ''),
|
||||
method=request.get('method', ''),
|
||||
resource_type=event.get('type'),
|
||||
)
|
||||
# logger.debug(f'[CrashWatchdog] Tracking request: {request.get("method", "")} {request.get("url", "")[:50]}...')
|
||||
|
||||
def _on_response_cdp(self, event: dict) -> None:
|
||||
"""Remove request from tracking on response."""
|
||||
request_id = event.get('requestId', '')
|
||||
if request_id in self._active_requests:
|
||||
elapsed = time.time() - self._active_requests[request_id].start_time
|
||||
response = event.get('response', {})
|
||||
self.logger.debug(f'[CrashWatchdog] Request completed in {elapsed:.2f}s: {response.get("url", "")[:50]}...')
|
||||
# Don't remove yet - wait for loadingFinished
|
||||
|
||||
def _on_request_failed_cdp(self, event: dict) -> None:
|
||||
"""Remove request from tracking on failure."""
|
||||
request_id = event.get('requestId', '')
|
||||
if request_id in self._active_requests:
|
||||
elapsed = time.time() - self._active_requests[request_id].start_time
|
||||
self.logger.debug(
|
||||
f'[CrashWatchdog] Request failed after {elapsed:.2f}s: {self._active_requests[request_id].url[:50]}...'
|
||||
)
|
||||
del self._active_requests[request_id]
|
||||
|
||||
def _on_request_finished_cdp(self, event: dict) -> None:
|
||||
"""Remove request from tracking when loading is finished."""
|
||||
request_id = event.get('requestId', '')
|
||||
self._active_requests.pop(request_id, None)
|
||||
|
||||
async def _on_target_crash_cdp(self, target_id: TargetID) -> None:
|
||||
"""Handle target crash detected via CDP."""
|
||||
self.logger.debug(f'[CrashWatchdog] Target crashed: {target_id[:8]}..., waiting for detach event')
|
||||
|
||||
target = self.browser_session.session_manager.get_target(target_id)
|
||||
|
||||
is_agent_focus = (
|
||||
target
|
||||
and self.browser_session.agent_focus_target_id
|
||||
and target.target_id == self.browser_session.agent_focus_target_id
|
||||
)
|
||||
|
||||
if is_agent_focus:
|
||||
self.logger.error(f'[CrashWatchdog] 💥 Agent focus tab crashed: {target.url} (SessionManager will auto-recover)')
|
||||
|
||||
# Emit browser error event
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='TargetCrash',
|
||||
message=f'Target crashed: {target_id}',
|
||||
details={
|
||||
'url': target.url if target else None,
|
||||
'target_id': target_id,
|
||||
'was_agent_focus': is_agent_focus,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
async def _start_monitoring(self) -> None:
|
||||
"""Start the monitoring loop."""
|
||||
assert self.browser_session.cdp_client is not None, 'Root CDP client not initialized - browser may not be connected yet'
|
||||
|
||||
if self._monitoring_task and not self._monitoring_task.done():
|
||||
# logger.info('[CrashWatchdog] Monitoring already running')
|
||||
return
|
||||
|
||||
self._monitoring_task = create_task_with_error_handling(
|
||||
self._monitoring_loop(), name='crash_monitoring_loop', logger_instance=self.logger, suppress_exceptions=True
|
||||
)
|
||||
# logger.debug('[CrashWatchdog] Monitoring loop created and started')
|
||||
|
||||
async def _stop_monitoring(self) -> None:
|
||||
"""Stop the monitoring loop and clean up all tracking."""
|
||||
if self._monitoring_task and not self._monitoring_task.done():
|
||||
self._monitoring_task.cancel()
|
||||
try:
|
||||
await self._monitoring_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self.logger.debug('[CrashWatchdog] Monitoring loop stopped')
|
||||
|
||||
# Cancel all CDP event handler tasks
|
||||
for task in list(self._cdp_event_tasks):
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
# Wait for all tasks to complete cancellation
|
||||
if self._cdp_event_tasks:
|
||||
await asyncio.gather(*self._cdp_event_tasks, return_exceptions=True)
|
||||
self._cdp_event_tasks.clear()
|
||||
|
||||
# Clear all tracking
|
||||
self._active_requests.clear()
|
||||
self._targets_with_listeners.clear()
|
||||
self._last_responsive_checks.clear()
|
||||
|
||||
async def _monitoring_loop(self) -> None:
|
||||
"""Main monitoring loop."""
|
||||
await asyncio.sleep(10) # give browser time to start up and load the first page after first LLM call
|
||||
while True:
|
||||
try:
|
||||
await self._check_network_timeouts()
|
||||
await self._check_browser_health()
|
||||
await asyncio.sleep(self.check_interval_seconds)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
self.logger.error(f'[CrashWatchdog] Error in monitoring loop: {e}')
|
||||
|
||||
async def _check_network_timeouts(self) -> None:
|
||||
"""Check for network requests exceeding timeout."""
|
||||
current_time = time.time()
|
||||
timed_out_requests = []
|
||||
|
||||
# Debug logging
|
||||
if self._active_requests:
|
||||
self.logger.debug(
|
||||
f'[CrashWatchdog] Checking {len(self._active_requests)} active requests for timeouts (threshold: {self.network_timeout_seconds}s)'
|
||||
)
|
||||
|
||||
for request_id, tracker in self._active_requests.items():
|
||||
elapsed = current_time - tracker.start_time
|
||||
self.logger.debug(
|
||||
f'[CrashWatchdog] Request {tracker.url[:30]}... elapsed: {elapsed:.1f}s, timeout: {self.network_timeout_seconds}s'
|
||||
)
|
||||
if elapsed >= self.network_timeout_seconds:
|
||||
timed_out_requests.append((request_id, tracker))
|
||||
|
||||
# Emit events for timed out requests
|
||||
for request_id, tracker in timed_out_requests:
|
||||
self.logger.warning(
|
||||
f'[CrashWatchdog] Network request timeout after {self.network_timeout_seconds}s: '
|
||||
f'{tracker.method} {tracker.url[:100]}...'
|
||||
)
|
||||
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='NetworkTimeout',
|
||||
message=f'Network request timed out after {self.network_timeout_seconds}s',
|
||||
details={
|
||||
'url': tracker.url,
|
||||
'method': tracker.method,
|
||||
'resource_type': tracker.resource_type,
|
||||
'elapsed_seconds': current_time - tracker.start_time,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
# Remove from tracking
|
||||
del self._active_requests[request_id]
|
||||
|
||||
async def _check_browser_health(self) -> None:
|
||||
"""Check if browser and targets are still responsive."""
|
||||
|
||||
try:
|
||||
self.logger.debug(f'[CrashWatchdog] Checking browser health for target {self.browser_session.agent_focus_target_id}')
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
||||
|
||||
for target in self.browser_session.session_manager.get_all_page_targets():
|
||||
if self._is_new_tab_page(target.url) and target.url != 'about:blank':
|
||||
self.logger.debug(f'[CrashWatchdog] Redirecting chrome://new-tab-page/ to about:blank {target.url}')
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(target_id=target.target_id)
|
||||
await cdp_session.cdp_client.send.Page.navigate(
|
||||
params={'url': 'about:blank'}, session_id=cdp_session.session_id
|
||||
)
|
||||
|
||||
# Quick ping to check if session is alive
|
||||
self.logger.debug(f'[CrashWatchdog] Attempting to run simple JS test expression in session {cdp_session} 1+1')
|
||||
await asyncio.wait_for(
|
||||
cdp_session.cdp_client.send.Runtime.evaluate(params={'expression': '1+1'}, session_id=cdp_session.session_id),
|
||||
timeout=1.0,
|
||||
)
|
||||
self.logger.debug(
|
||||
f'[CrashWatchdog] Browser health check passed for target {self.browser_session.agent_focus_target_id}'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(
|
||||
f'[CrashWatchdog] ❌ Crashed/unresponsive session detected for target {self.browser_session.agent_focus_target_id} '
|
||||
f'error: {type(e).__name__}: {e} (Chrome will send detach event, SessionManager will auto-recover)'
|
||||
)
|
||||
|
||||
# Check browser process if we have PID
|
||||
if self.browser_session._local_browser_watchdog and (proc := self.browser_session._local_browser_watchdog._subprocess):
|
||||
try:
|
||||
if proc.status() in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD):
|
||||
self.logger.error(f'[CrashWatchdog] Browser process {proc.pid} has crashed')
|
||||
|
||||
# Browser process crashed - SessionManager will clean up via detach events
|
||||
# Just dispatch error event and stop monitoring
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='BrowserProcessCrashed',
|
||||
message=f'Browser process {proc.pid} has crashed',
|
||||
details={'pid': proc.pid, 'status': proc.status()},
|
||||
)
|
||||
)
|
||||
|
||||
self.logger.warning('[CrashWatchdog] Browser process dead - stopping health monitoring')
|
||||
await self._stop_monitoring()
|
||||
return
|
||||
except Exception:
|
||||
pass # psutil not available or process doesn't exist
|
||||
|
||||
@staticmethod
|
||||
def _is_new_tab_page(url: str) -> bool:
|
||||
"""Check if URL is a new tab page."""
|
||||
return url in ['about:blank', 'chrome://new-tab-page/', 'chrome://newtab/']
|
||||
3690
.agent/vendor/browser_use/browser_use/browser/watchdogs/default_action_watchdog.py
vendored
Normal file
3690
.agent/vendor/browser_use/browser_use/browser/watchdogs/default_action_watchdog.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
861
.agent/vendor/browser_use/browser_use/browser/watchdogs/dom_watchdog.py
vendored
Normal file
861
.agent/vendor/browser_use/browser_use/browser/watchdogs/dom_watchdog.py
vendored
Normal file
@@ -0,0 +1,861 @@
|
||||
"""DOM watchdog for browser DOM tree management using CDP."""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserErrorEvent,
|
||||
BrowserStateRequestEvent,
|
||||
ScreenshotEvent,
|
||||
TabCreatedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.dom.service import DomService
|
||||
from browser_use.dom.views import (
|
||||
EnhancedDOMTreeNode,
|
||||
SerializedDOMState,
|
||||
)
|
||||
from browser_use.observability import observe_debug
|
||||
from browser_use.utils import create_task_with_error_handling, time_execution_async
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.browser.views import BrowserStateSummary, NetworkRequest, PageInfo, PaginationButton
|
||||
|
||||
|
||||
class DOMWatchdog(BaseWatchdog):
|
||||
"""Handles DOM tree building, serialization, and element access via CDP.
|
||||
|
||||
This watchdog acts as a bridge between the event-driven browser session
|
||||
and the DomService implementation, maintaining cached state and providing
|
||||
helper methods for other watchdogs.
|
||||
"""
|
||||
|
||||
LISTENS_TO = [TabCreatedEvent, BrowserStateRequestEvent]
|
||||
EMITS = [BrowserErrorEvent]
|
||||
|
||||
# Public properties for other watchdogs
|
||||
selector_map: dict[int, EnhancedDOMTreeNode] | None = None
|
||||
current_dom_state: SerializedDOMState | None = None
|
||||
enhanced_dom_tree: EnhancedDOMTreeNode | None = None
|
||||
|
||||
# Internal DOM service
|
||||
_dom_service: DomService | None = None
|
||||
|
||||
# Network tracking - maps request_id to (url, start_time, method, resource_type)
|
||||
_pending_requests: dict[str, tuple[str, float, str, str | None]] = {}
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
# self.logger.debug('Setting up init scripts in browser')
|
||||
return None
|
||||
|
||||
def _get_recent_events_str(self, limit: int = 10) -> str | None:
|
||||
"""Get the most recent events from the event bus as JSON.
|
||||
|
||||
Args:
|
||||
limit: Maximum number of recent events to include
|
||||
|
||||
Returns:
|
||||
JSON string of recent events or None if not available
|
||||
"""
|
||||
import json
|
||||
|
||||
try:
|
||||
# Get all events from history, sorted by creation time (most recent first)
|
||||
all_events = sorted(
|
||||
self.browser_session.event_bus.event_history.values(), key=lambda e: e.event_created_at.timestamp(), reverse=True
|
||||
)
|
||||
|
||||
# Take the most recent events and create JSON-serializable data
|
||||
recent_events_data = []
|
||||
for event in all_events[:limit]:
|
||||
event_data = {
|
||||
'event_type': event.event_type,
|
||||
'timestamp': event.event_created_at.isoformat(),
|
||||
}
|
||||
# Add specific fields for certain event types
|
||||
if hasattr(event, 'url'):
|
||||
event_data['url'] = getattr(event, 'url')
|
||||
if hasattr(event, 'error_message'):
|
||||
event_data['error_message'] = getattr(event, 'error_message')
|
||||
if hasattr(event, 'target_id'):
|
||||
event_data['target_id'] = getattr(event, 'target_id')
|
||||
recent_events_data.append(event_data)
|
||||
|
||||
return json.dumps(recent_events_data) # Return empty array if no events
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to get recent events: {e}')
|
||||
|
||||
return json.dumps([]) # Return empty JSON array on error
|
||||
|
||||
async def _get_pending_network_requests(self) -> list['NetworkRequest']:
|
||||
"""Get list of currently pending network requests.
|
||||
|
||||
Uses document.readyState and performance API to detect pending requests.
|
||||
Filters out ads, tracking, and other noise.
|
||||
|
||||
Returns:
|
||||
List of NetworkRequest objects representing currently loading resources
|
||||
"""
|
||||
from browser_use.browser.views import NetworkRequest
|
||||
|
||||
try:
|
||||
# get_or_create_cdp_session() now handles focus validation automatically
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
|
||||
|
||||
# Use performance API to get pending requests
|
||||
js_code = """
|
||||
(function() {
|
||||
const now = performance.now();
|
||||
const resources = performance.getEntriesByType('resource');
|
||||
const pending = [];
|
||||
|
||||
// Check document readyState
|
||||
const docLoading = document.readyState !== 'complete';
|
||||
|
||||
// Common ad/tracking domains and patterns to filter out
|
||||
const adDomains = [
|
||||
// Standard ad/tracking networks
|
||||
'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',
|
||||
'facebook.net', 'analytics', 'ads', 'tracking', 'pixel',
|
||||
'hotjar.com', 'clarity.ms', 'mixpanel.com', 'segment.com',
|
||||
// Analytics platforms
|
||||
'demdex.net', 'omtrdc.net', 'adobedtm.com', 'ensighten.com',
|
||||
'newrelic.com', 'nr-data.net', 'google-analytics.com',
|
||||
// Social media trackers
|
||||
'connect.facebook.net', 'platform.twitter.com', 'platform.linkedin.com',
|
||||
// CDN/image hosts (usually not critical for functionality)
|
||||
'.cloudfront.net/image/', '.akamaized.net/image/',
|
||||
// Common tracking paths
|
||||
'/tracker/', '/collector/', '/beacon/', '/telemetry/', '/log/',
|
||||
'/events/', '/eventBatch', '/track.', '/metrics/'
|
||||
];
|
||||
|
||||
// Get resources that are still loading (responseEnd is 0)
|
||||
let totalResourcesChecked = 0;
|
||||
let filteredByResponseEnd = 0;
|
||||
const allDomains = new Set();
|
||||
|
||||
for (const entry of resources) {
|
||||
totalResourcesChecked++;
|
||||
|
||||
// Track all domains from recent resources (for logging)
|
||||
try {
|
||||
const hostname = new URL(entry.name).hostname;
|
||||
if (hostname) allDomains.add(hostname);
|
||||
} catch (e) {}
|
||||
|
||||
if (entry.responseEnd === 0) {
|
||||
filteredByResponseEnd++;
|
||||
const url = entry.name;
|
||||
|
||||
// Filter out ads and tracking
|
||||
const isAd = adDomains.some(domain => url.includes(domain));
|
||||
if (isAd) continue;
|
||||
|
||||
// Filter out data: URLs and very long URLs (often inline resources)
|
||||
if (url.startsWith('data:') || url.length > 500) continue;
|
||||
|
||||
const loadingDuration = now - entry.startTime;
|
||||
|
||||
// Skip requests that have been loading for >10 seconds (likely stuck/polling)
|
||||
if (loadingDuration > 10000) continue;
|
||||
|
||||
const resourceType = entry.initiatorType || 'unknown';
|
||||
|
||||
// Filter out non-critical resources (images, fonts, icons) if loading >3 seconds
|
||||
const nonCriticalTypes = ['img', 'image', 'icon', 'font'];
|
||||
if (nonCriticalTypes.includes(resourceType) && loadingDuration > 3000) continue;
|
||||
|
||||
// Filter out image URLs even if type is unknown
|
||||
const isImageUrl = /\\.(jpg|jpeg|png|gif|webp|svg|ico)(\\?|$)/i.test(url);
|
||||
if (isImageUrl && loadingDuration > 3000) continue;
|
||||
|
||||
pending.push({
|
||||
url: url,
|
||||
method: 'GET',
|
||||
loading_duration_ms: Math.round(loadingDuration),
|
||||
resource_type: resourceType
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
pending_requests: pending,
|
||||
document_loading: docLoading,
|
||||
document_ready_state: document.readyState,
|
||||
debug: {
|
||||
total_resources: totalResourcesChecked,
|
||||
with_response_end_zero: filteredByResponseEnd,
|
||||
after_all_filters: pending.length,
|
||||
all_domains: Array.from(allDomains)
|
||||
}
|
||||
};
|
||||
})()
|
||||
"""
|
||||
|
||||
result = await cdp_session.cdp_client.send.Runtime.evaluate(
|
||||
params={'expression': js_code, 'returnByValue': True}, session_id=cdp_session.session_id
|
||||
)
|
||||
|
||||
if result.get('result', {}).get('type') == 'object':
|
||||
data = result['result'].get('value', {})
|
||||
pending = data.get('pending_requests', [])
|
||||
doc_state = data.get('document_ready_state', 'unknown')
|
||||
doc_loading = data.get('document_loading', False)
|
||||
debug_info = data.get('debug', {})
|
||||
|
||||
# Get all domains that had recent activity (from JS)
|
||||
all_domains = debug_info.get('all_domains', [])
|
||||
all_domains_str = ', '.join(sorted(all_domains)[:5]) if all_domains else 'none'
|
||||
if len(all_domains) > 5:
|
||||
all_domains_str += f' +{len(all_domains) - 5} more'
|
||||
|
||||
# Debug logging
|
||||
self.logger.debug(
|
||||
f'🔍 Network check: document.readyState={doc_state}, loading={doc_loading}, '
|
||||
f'total_resources={debug_info.get("total_resources", 0)}, '
|
||||
f'responseEnd=0: {debug_info.get("with_response_end_zero", 0)}, '
|
||||
f'after_filters={len(pending)}, domains=[{all_domains_str}]'
|
||||
)
|
||||
|
||||
# Convert to NetworkRequest objects
|
||||
network_requests = []
|
||||
for req in pending[:20]: # Limit to 20 to avoid overwhelming the context
|
||||
network_requests.append(
|
||||
NetworkRequest(
|
||||
url=req['url'],
|
||||
method=req.get('method', 'GET'),
|
||||
loading_duration_ms=req.get('loading_duration_ms', 0.0),
|
||||
resource_type=req.get('resource_type'),
|
||||
)
|
||||
)
|
||||
|
||||
return network_requests
|
||||
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to get pending network requests: {e}')
|
||||
|
||||
return []
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='browser_state_request_event')
|
||||
async def on_BrowserStateRequestEvent(self, event: BrowserStateRequestEvent) -> 'BrowserStateSummary':
|
||||
"""Handle browser state request by coordinating DOM building and screenshot capture.
|
||||
|
||||
This is the main entry point for getting the complete browser state.
|
||||
|
||||
Args:
|
||||
event: The browser state request event with options
|
||||
|
||||
Returns:
|
||||
Complete BrowserStateSummary with DOM, screenshot, and target info
|
||||
"""
|
||||
from browser_use.browser.views import BrowserStateSummary, PageInfo
|
||||
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: STARTING browser state request')
|
||||
page_url = await self.browser_session.get_current_page_url()
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got page URL: {page_url}')
|
||||
|
||||
# Get focused session for logging (validation already done by get_current_page_url)
|
||||
if self.browser_session.agent_focus_target_id:
|
||||
self.logger.debug(f'Current page URL: {page_url}, target_id: {self.browser_session.agent_focus_target_id}')
|
||||
|
||||
# check if we should skip DOM tree build for pointless pages
|
||||
not_a_meaningful_website = page_url.lower().split(':', 1)[0] not in ('http', 'https')
|
||||
|
||||
# Check for pending network requests BEFORE waiting (so we can see what's loading)
|
||||
pending_requests_before_wait = []
|
||||
if not not_a_meaningful_website:
|
||||
try:
|
||||
pending_requests_before_wait = await self._get_pending_network_requests()
|
||||
if pending_requests_before_wait:
|
||||
self.logger.debug(f'🔍 Found {len(pending_requests_before_wait)} pending requests before stability wait')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to get pending requests before wait: {e}')
|
||||
pending_requests = pending_requests_before_wait
|
||||
# Wait for page stability using browser profile settings (main branch pattern)
|
||||
if not not_a_meaningful_website:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ⏳ Waiting for page stability...')
|
||||
try:
|
||||
if pending_requests_before_wait:
|
||||
# Reduced from 1s to 0.3s for faster DOM builds while still allowing critical resources to load
|
||||
await asyncio.sleep(0.3)
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Page stability complete')
|
||||
except Exception as e:
|
||||
self.logger.warning(
|
||||
f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Network waiting failed: {e}, continuing anyway...'
|
||||
)
|
||||
|
||||
# Get tabs info once at the beginning for all paths
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting tabs info...')
|
||||
tabs_info = await self.browser_session.get_tabs()
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got {len(tabs_info)} tabs')
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Tabs info: {tabs_info}')
|
||||
|
||||
# Get viewport / scroll position info, remember changing scroll position should invalidate selector_map cache because it only includes visible elements
|
||||
# cdp_session = await self.browser_session.get_or_create_cdp_session(focus=True)
|
||||
# scroll_info = await cdp_session.cdp_client.send.Runtime.evaluate(
|
||||
# params={'expression': 'JSON.stringify({y: document.body.scrollTop, x: document.body.scrollLeft, width: document.documentElement.clientWidth, height: document.documentElement.clientHeight})'},
|
||||
# session_id=cdp_session.session_id,
|
||||
# )
|
||||
# self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got scroll info: {scroll_info["result"]}')
|
||||
|
||||
try:
|
||||
# Fast path for empty pages
|
||||
if not_a_meaningful_website:
|
||||
self.logger.debug(f'⚡ Skipping BuildDOMTree for empty target: {page_url}')
|
||||
self.logger.debug(f'📸 Not taking screenshot for empty page: {page_url} (non-http/https URL)')
|
||||
|
||||
# Create minimal DOM state
|
||||
content = SerializedDOMState(_root=None, selector_map={})
|
||||
|
||||
# Skip screenshot for empty pages
|
||||
screenshot_b64 = None
|
||||
|
||||
# Try to get page info from CDP, fall back to defaults if unavailable
|
||||
try:
|
||||
page_info = await self._get_page_info()
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to get page info from CDP for empty page: {e}, using fallback')
|
||||
# Use default viewport dimensions
|
||||
viewport = self.browser_session.browser_profile.viewport or {'width': 1280, 'height': 720}
|
||||
page_info = PageInfo(
|
||||
viewport_width=viewport['width'],
|
||||
viewport_height=viewport['height'],
|
||||
page_width=viewport['width'],
|
||||
page_height=viewport['height'],
|
||||
scroll_x=0,
|
||||
scroll_y=0,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
pixels_left=0,
|
||||
pixels_right=0,
|
||||
)
|
||||
|
||||
return BrowserStateSummary(
|
||||
dom_state=content,
|
||||
url=page_url,
|
||||
title='Empty Tab',
|
||||
tabs=tabs_info,
|
||||
screenshot=screenshot_b64,
|
||||
page_info=page_info,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
browser_errors=[],
|
||||
is_pdf_viewer=False,
|
||||
recent_events=self._get_recent_events_str() if event.include_recent_events else None,
|
||||
pending_network_requests=[], # Empty page has no pending requests
|
||||
pagination_buttons=[], # Empty page has no pagination
|
||||
closed_popup_messages=self.browser_session._closed_popup_messages.copy(),
|
||||
)
|
||||
|
||||
# Execute DOM building and screenshot capture in parallel
|
||||
dom_task = None
|
||||
screenshot_task = None
|
||||
|
||||
# Start DOM building task if requested
|
||||
if event.include_dom:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🌳 Starting DOM tree build task...')
|
||||
|
||||
previous_state = (
|
||||
self.browser_session._cached_browser_state_summary.dom_state
|
||||
if self.browser_session._cached_browser_state_summary
|
||||
else None
|
||||
)
|
||||
|
||||
dom_task = create_task_with_error_handling(
|
||||
self._build_dom_tree_without_highlights(previous_state),
|
||||
name='build_dom_tree',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
# Start clean screenshot task if requested (without JS highlights)
|
||||
if event.include_screenshot:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Starting clean screenshot task...')
|
||||
screenshot_task = create_task_with_error_handling(
|
||||
self._capture_clean_screenshot(),
|
||||
name='capture_screenshot',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
# Wait for both tasks to complete
|
||||
content = None
|
||||
screenshot_b64 = None
|
||||
|
||||
if dom_task:
|
||||
try:
|
||||
content = await dom_task
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ DOM tree build completed')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: DOM build failed: {e}, using minimal state')
|
||||
content = SerializedDOMState(_root=None, selector_map={})
|
||||
else:
|
||||
content = SerializedDOMState(_root=None, selector_map={})
|
||||
|
||||
if screenshot_task:
|
||||
try:
|
||||
screenshot_b64 = await screenshot_task
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Clean screenshot captured')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Clean screenshot failed: {e}')
|
||||
screenshot_b64 = None
|
||||
|
||||
# Add browser-side highlights for user visibility
|
||||
if content and content.selector_map and self.browser_session.browser_profile.dom_highlight_elements:
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: 🎨 Adding browser-side highlights...')
|
||||
await self.browser_session.add_highlights(content.selector_map)
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ Added browser highlights for {len(content.selector_map)} elements'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Browser highlighting failed: {e}')
|
||||
|
||||
# Ensure we have valid content
|
||||
if not content:
|
||||
content = SerializedDOMState(_root=None, selector_map={})
|
||||
|
||||
# Tabs info already fetched at the beginning
|
||||
|
||||
# Get target title safely
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting page title...')
|
||||
title = await asyncio.wait_for(self.browser_session.get_current_page_title(), timeout=1.0)
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got title: {title}')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Failed to get title: {e}')
|
||||
title = 'Page'
|
||||
|
||||
# Get comprehensive page info from CDP with timeout
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: Getting page info from CDP...')
|
||||
page_info = await asyncio.wait_for(self._get_page_info(), timeout=1.0)
|
||||
self.logger.debug(f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Got page info from CDP: {page_info}')
|
||||
except Exception as e:
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: Failed to get page info from CDP: {e}, using fallback'
|
||||
)
|
||||
# Fallback to default viewport dimensions
|
||||
viewport = self.browser_session.browser_profile.viewport or {'width': 1280, 'height': 720}
|
||||
page_info = PageInfo(
|
||||
viewport_width=viewport['width'],
|
||||
viewport_height=viewport['height'],
|
||||
page_width=viewport['width'],
|
||||
page_height=viewport['height'],
|
||||
scroll_x=0,
|
||||
scroll_y=0,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
pixels_left=0,
|
||||
pixels_right=0,
|
||||
)
|
||||
|
||||
# Check for PDF viewer
|
||||
is_pdf_viewer = page_url.endswith('.pdf') or '/pdf/' in page_url
|
||||
|
||||
# Detect pagination buttons from the DOM
|
||||
pagination_buttons_data = []
|
||||
if content and content.selector_map:
|
||||
pagination_buttons_data = self._detect_pagination_buttons(content.selector_map)
|
||||
|
||||
# Build and cache the browser state summary
|
||||
if screenshot_b64:
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Creating BrowserStateSummary with screenshot, length: {len(screenshot_b64)}'
|
||||
)
|
||||
else:
|
||||
self.logger.debug(
|
||||
'🔍 DOMWatchdog.on_BrowserStateRequestEvent: 📸 Creating BrowserStateSummary WITHOUT screenshot'
|
||||
)
|
||||
|
||||
browser_state = BrowserStateSummary(
|
||||
dom_state=content,
|
||||
url=page_url,
|
||||
title=title,
|
||||
tabs=tabs_info,
|
||||
screenshot=screenshot_b64,
|
||||
page_info=page_info,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
browser_errors=[],
|
||||
is_pdf_viewer=is_pdf_viewer,
|
||||
recent_events=self._get_recent_events_str() if event.include_recent_events else None,
|
||||
pending_network_requests=pending_requests,
|
||||
pagination_buttons=pagination_buttons_data,
|
||||
closed_popup_messages=self.browser_session._closed_popup_messages.copy(),
|
||||
)
|
||||
|
||||
# Cache the state
|
||||
self.browser_session._cached_browser_state_summary = browser_state
|
||||
|
||||
# Cache viewport size for coordinate conversion (if llm_screenshot_size is enabled)
|
||||
if page_info:
|
||||
self.browser_session._original_viewport_size = (page_info.viewport_width, page_info.viewport_height)
|
||||
|
||||
self.logger.debug('🔍 DOMWatchdog.on_BrowserStateRequestEvent: ✅ COMPLETED - Returning browser state')
|
||||
return browser_state
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'Failed to get browser state: {e}')
|
||||
|
||||
# Return minimal recovery state
|
||||
return BrowserStateSummary(
|
||||
dom_state=SerializedDOMState(_root=None, selector_map={}),
|
||||
url=page_url if 'page_url' in locals() else '',
|
||||
title='Error',
|
||||
tabs=[],
|
||||
screenshot=None,
|
||||
page_info=PageInfo(
|
||||
viewport_width=1280,
|
||||
viewport_height=720,
|
||||
page_width=1280,
|
||||
page_height=720,
|
||||
scroll_x=0,
|
||||
scroll_y=0,
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
pixels_left=0,
|
||||
pixels_right=0,
|
||||
),
|
||||
pixels_above=0,
|
||||
pixels_below=0,
|
||||
browser_errors=[str(e)],
|
||||
is_pdf_viewer=False,
|
||||
recent_events=None,
|
||||
pending_network_requests=[], # Error state has no pending requests
|
||||
pagination_buttons=[], # Error state has no pagination
|
||||
closed_popup_messages=self.browser_session._closed_popup_messages.copy()
|
||||
if hasattr(self, 'browser_session') and self.browser_session is not None
|
||||
else [],
|
||||
)
|
||||
|
||||
@time_execution_async('build_dom_tree_without_highlights')
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='build_dom_tree_without_highlights')
|
||||
async def _build_dom_tree_without_highlights(self, previous_state: SerializedDOMState | None = None) -> SerializedDOMState:
|
||||
"""Build DOM tree without injecting JavaScript highlights (for parallel execution)."""
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: STARTING DOM tree build')
|
||||
|
||||
# Create or reuse DOM service
|
||||
if self._dom_service is None:
|
||||
self._dom_service = DomService(
|
||||
browser_session=self.browser_session,
|
||||
logger=self.logger,
|
||||
cross_origin_iframes=self.browser_session.browser_profile.cross_origin_iframes,
|
||||
paint_order_filtering=self.browser_session.browser_profile.paint_order_filtering,
|
||||
max_iframes=self.browser_session.browser_profile.max_iframes,
|
||||
max_iframe_depth=self.browser_session.browser_profile.max_iframe_depth,
|
||||
)
|
||||
|
||||
# Get serialized DOM tree using the service
|
||||
self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Calling DomService.get_serialized_dom_tree...')
|
||||
start = time.time()
|
||||
self.current_dom_state, self.enhanced_dom_tree, timing_info = await self._dom_service.get_serialized_dom_tree(
|
||||
previous_cached_state=previous_state,
|
||||
)
|
||||
end = time.time()
|
||||
total_time_ms = (end - start) * 1000
|
||||
self.logger.debug(
|
||||
'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ DomService.get_serialized_dom_tree completed'
|
||||
)
|
||||
|
||||
# Build hierarchical timing breakdown as single multi-line string
|
||||
timing_lines = [f'⏱️ Total DOM tree time: {total_time_ms:.2f}ms', '📊 Timing breakdown:']
|
||||
|
||||
# get_all_trees breakdown
|
||||
get_all_trees_ms = timing_info.get('get_all_trees_total_ms', 0)
|
||||
if get_all_trees_ms > 0:
|
||||
timing_lines.append(f' ├─ get_all_trees: {get_all_trees_ms:.2f}ms')
|
||||
iframe_scroll_ms = timing_info.get('iframe_scroll_detection_ms', 0)
|
||||
cdp_parallel_ms = timing_info.get('cdp_parallel_calls_ms', 0)
|
||||
snapshot_proc_ms = timing_info.get('snapshot_processing_ms', 0)
|
||||
if iframe_scroll_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ iframe_scroll_detection: {iframe_scroll_ms:.2f}ms')
|
||||
if cdp_parallel_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ cdp_parallel_calls: {cdp_parallel_ms:.2f}ms')
|
||||
if snapshot_proc_ms > 0.01:
|
||||
timing_lines.append(f' │ └─ snapshot_processing: {snapshot_proc_ms:.2f}ms')
|
||||
|
||||
# build_ax_lookup
|
||||
build_ax_ms = timing_info.get('build_ax_lookup_ms', 0)
|
||||
if build_ax_ms > 0.01:
|
||||
timing_lines.append(f' ├─ build_ax_lookup: {build_ax_ms:.2f}ms')
|
||||
|
||||
# build_snapshot_lookup
|
||||
build_snapshot_ms = timing_info.get('build_snapshot_lookup_ms', 0)
|
||||
if build_snapshot_ms > 0.01:
|
||||
timing_lines.append(f' ├─ build_snapshot_lookup: {build_snapshot_ms:.2f}ms')
|
||||
|
||||
# construct_enhanced_tree
|
||||
construct_tree_ms = timing_info.get('construct_enhanced_tree_ms', 0)
|
||||
if construct_tree_ms > 0.01:
|
||||
timing_lines.append(f' ├─ construct_enhanced_tree: {construct_tree_ms:.2f}ms')
|
||||
|
||||
# serialize_accessible_elements breakdown
|
||||
serialize_total_ms = timing_info.get('serialize_accessible_elements_total_ms', 0)
|
||||
if serialize_total_ms > 0.01:
|
||||
timing_lines.append(f' ├─ serialize_accessible_elements: {serialize_total_ms:.2f}ms')
|
||||
create_simp_ms = timing_info.get('create_simplified_tree_ms', 0)
|
||||
paint_order_ms = timing_info.get('calculate_paint_order_ms', 0)
|
||||
optimize_ms = timing_info.get('optimize_tree_ms', 0)
|
||||
bbox_ms = timing_info.get('bbox_filtering_ms', 0)
|
||||
assign_idx_ms = timing_info.get('assign_interactive_indices_ms', 0)
|
||||
clickable_ms = timing_info.get('clickable_detection_time_ms', 0)
|
||||
|
||||
if create_simp_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ create_simplified_tree: {create_simp_ms:.2f}ms')
|
||||
if clickable_ms > 0.01:
|
||||
timing_lines.append(f' │ │ └─ clickable_detection: {clickable_ms:.2f}ms')
|
||||
if paint_order_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ calculate_paint_order: {paint_order_ms:.2f}ms')
|
||||
if optimize_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ optimize_tree: {optimize_ms:.2f}ms')
|
||||
if bbox_ms > 0.01:
|
||||
timing_lines.append(f' │ ├─ bbox_filtering: {bbox_ms:.2f}ms')
|
||||
if assign_idx_ms > 0.01:
|
||||
timing_lines.append(f' │ └─ assign_interactive_indices: {assign_idx_ms:.2f}ms')
|
||||
|
||||
# Overheads
|
||||
get_dom_overhead_ms = timing_info.get('get_dom_tree_overhead_ms', 0)
|
||||
serialize_overhead_ms = timing_info.get('serialization_overhead_ms', 0)
|
||||
get_serialized_overhead_ms = timing_info.get('get_serialized_dom_tree_overhead_ms', 0)
|
||||
|
||||
if get_dom_overhead_ms > 0.1:
|
||||
timing_lines.append(f' ├─ get_dom_tree_overhead: {get_dom_overhead_ms:.2f}ms')
|
||||
if serialize_overhead_ms > 0.1:
|
||||
timing_lines.append(f' ├─ serialization_overhead: {serialize_overhead_ms:.2f}ms')
|
||||
if get_serialized_overhead_ms > 0.1:
|
||||
timing_lines.append(f' └─ get_serialized_dom_tree_overhead: {get_serialized_overhead_ms:.2f}ms')
|
||||
|
||||
# Calculate total tracked time for validation
|
||||
main_operations_ms = (
|
||||
get_all_trees_ms
|
||||
+ build_ax_ms
|
||||
+ build_snapshot_ms
|
||||
+ construct_tree_ms
|
||||
+ serialize_total_ms
|
||||
+ get_dom_overhead_ms
|
||||
+ serialize_overhead_ms
|
||||
+ get_serialized_overhead_ms
|
||||
)
|
||||
untracked_time_ms = total_time_ms - main_operations_ms
|
||||
|
||||
if untracked_time_ms > 1.0: # Only log if significant
|
||||
timing_lines.append(f' ⚠️ untracked_time: {untracked_time_ms:.2f}ms')
|
||||
|
||||
# Single log call with all timing info
|
||||
self.logger.debug('\n'.join(timing_lines))
|
||||
|
||||
# Update selector map for other watchdogs
|
||||
self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: Updating selector maps...')
|
||||
self.selector_map = self.current_dom_state.selector_map
|
||||
# Update BrowserSession's cached selector map
|
||||
if self.browser_session:
|
||||
self.browser_session.update_cached_selector_map(self.selector_map)
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ Selector maps updated, {len(self.selector_map)} elements'
|
||||
)
|
||||
|
||||
# Skip JavaScript highlighting injection - Python highlighting will be applied later
|
||||
self.logger.debug('🔍 DOMWatchdog._build_dom_tree_without_highlights: ✅ COMPLETED DOM tree build (no JS highlights)')
|
||||
return self.current_dom_state
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'Failed to build DOM tree without highlights: {e}')
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='DOMBuildFailed',
|
||||
message=str(e),
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
@time_execution_async('capture_clean_screenshot')
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='capture_clean_screenshot')
|
||||
async def _capture_clean_screenshot(self) -> str:
|
||||
"""Capture a clean screenshot without JavaScript highlights."""
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: Capturing clean screenshot...')
|
||||
|
||||
await self.browser_session.get_or_create_cdp_session(target_id=self.browser_session.agent_focus_target_id, focus=True)
|
||||
|
||||
# Check if handler is registered
|
||||
handlers = self.event_bus.handlers.get('ScreenshotEvent', [])
|
||||
handler_names = [getattr(h, '__name__', str(h)) for h in handlers]
|
||||
self.logger.debug(f'📸 ScreenshotEvent handlers registered: {len(handlers)} - {handler_names}')
|
||||
|
||||
screenshot_event = self.event_bus.dispatch(ScreenshotEvent(full_page=False))
|
||||
self.logger.debug('📸 Dispatched ScreenshotEvent, waiting for event to complete...')
|
||||
|
||||
# Wait for the event itself to complete (this waits for all handlers)
|
||||
await screenshot_event
|
||||
|
||||
# Get the single handler result
|
||||
screenshot_b64 = await screenshot_event.event_result(raise_if_any=True, raise_if_none=True)
|
||||
if screenshot_b64 is None:
|
||||
raise RuntimeError('Screenshot handler returned None')
|
||||
self.logger.debug('🔍 DOMWatchdog._capture_clean_screenshot: ✅ Clean screenshot captured successfully')
|
||||
return str(screenshot_b64)
|
||||
|
||||
except TimeoutError:
|
||||
self.logger.warning('📸 Clean screenshot timed out after 6 seconds - no handler registered or slow page?')
|
||||
raise
|
||||
except Exception as e:
|
||||
self.logger.warning(f'📸 Clean screenshot failed: {type(e).__name__}: {e}')
|
||||
raise
|
||||
|
||||
def _detect_pagination_buttons(self, selector_map: dict[int, EnhancedDOMTreeNode]) -> list['PaginationButton']:
|
||||
"""Detect pagination buttons from the DOM selector map.
|
||||
|
||||
Args:
|
||||
selector_map: Dictionary mapping element indices to DOM tree nodes
|
||||
|
||||
Returns:
|
||||
List of PaginationButton instances found in the DOM
|
||||
"""
|
||||
from browser_use.browser.views import PaginationButton
|
||||
|
||||
pagination_buttons_data = []
|
||||
try:
|
||||
self.logger.debug('🔍 DOMWatchdog._detect_pagination_buttons: Detecting pagination buttons...')
|
||||
pagination_buttons_raw = DomService.detect_pagination_buttons(selector_map)
|
||||
# Convert to PaginationButton instances
|
||||
pagination_buttons_data = [
|
||||
PaginationButton(
|
||||
button_type=btn['button_type'], # type: ignore
|
||||
backend_node_id=btn['backend_node_id'], # type: ignore
|
||||
text=btn['text'], # type: ignore
|
||||
selector=btn['selector'], # type: ignore
|
||||
is_disabled=btn['is_disabled'], # type: ignore
|
||||
)
|
||||
for btn in pagination_buttons_raw
|
||||
]
|
||||
if pagination_buttons_data:
|
||||
self.logger.debug(
|
||||
f'🔍 DOMWatchdog._detect_pagination_buttons: Found {len(pagination_buttons_data)} pagination buttons'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f'🔍 DOMWatchdog._detect_pagination_buttons: Pagination detection failed: {e}')
|
||||
|
||||
return pagination_buttons_data
|
||||
|
||||
async def _get_page_info(self) -> 'PageInfo':
|
||||
"""Get comprehensive page information using a single CDP call.
|
||||
|
||||
TODO: should we make this an event as well?
|
||||
|
||||
Returns:
|
||||
PageInfo with all viewport, page dimensions, and scroll information
|
||||
"""
|
||||
|
||||
from browser_use.browser.views import PageInfo
|
||||
|
||||
# get_or_create_cdp_session() handles focus validation automatically
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(
|
||||
target_id=self.browser_session.agent_focus_target_id, focus=True
|
||||
)
|
||||
|
||||
# Get layout metrics which includes all the information we need
|
||||
metrics = await asyncio.wait_for(
|
||||
cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id), timeout=10.0
|
||||
)
|
||||
|
||||
# Extract different viewport types
|
||||
layout_viewport = metrics.get('layoutViewport', {})
|
||||
visual_viewport = metrics.get('visualViewport', {})
|
||||
css_visual_viewport = metrics.get('cssVisualViewport', {})
|
||||
css_layout_viewport = metrics.get('cssLayoutViewport', {})
|
||||
content_size = metrics.get('contentSize', {})
|
||||
|
||||
# Calculate device pixel ratio to convert between device pixels and CSS pixels
|
||||
# This matches the approach in dom/service.py _get_viewport_ratio method
|
||||
css_width = css_visual_viewport.get('clientWidth', css_layout_viewport.get('clientWidth', 1280.0))
|
||||
device_width = visual_viewport.get('clientWidth', css_width)
|
||||
device_pixel_ratio = device_width / css_width if css_width > 0 else 1.0
|
||||
|
||||
# For viewport dimensions, use CSS pixels (what JavaScript sees)
|
||||
# Prioritize CSS layout viewport, then fall back to layout viewport
|
||||
viewport_width = int(css_layout_viewport.get('clientWidth') or layout_viewport.get('clientWidth', 1280))
|
||||
viewport_height = int(css_layout_viewport.get('clientHeight') or layout_viewport.get('clientHeight', 720))
|
||||
|
||||
# For total page dimensions, content size is typically in device pixels, so convert to CSS pixels
|
||||
# by dividing by device pixel ratio
|
||||
raw_page_width = content_size.get('width', viewport_width * device_pixel_ratio)
|
||||
raw_page_height = content_size.get('height', viewport_height * device_pixel_ratio)
|
||||
page_width = int(raw_page_width / device_pixel_ratio)
|
||||
page_height = int(raw_page_height / device_pixel_ratio)
|
||||
|
||||
# For scroll position, use CSS visual viewport if available, otherwise CSS layout viewport
|
||||
# These should already be in CSS pixels
|
||||
scroll_x = int(css_visual_viewport.get('pageX') or css_layout_viewport.get('pageX', 0))
|
||||
scroll_y = int(css_visual_viewport.get('pageY') or css_layout_viewport.get('pageY', 0))
|
||||
|
||||
# Calculate scroll information - pixels that are above/below/left/right of current viewport
|
||||
pixels_above = scroll_y
|
||||
pixels_below = max(0, page_height - viewport_height - scroll_y)
|
||||
pixels_left = scroll_x
|
||||
pixels_right = max(0, page_width - viewport_width - scroll_x)
|
||||
|
||||
page_info = PageInfo(
|
||||
viewport_width=viewport_width,
|
||||
viewport_height=viewport_height,
|
||||
page_width=page_width,
|
||||
page_height=page_height,
|
||||
scroll_x=scroll_x,
|
||||
scroll_y=scroll_y,
|
||||
pixels_above=pixels_above,
|
||||
pixels_below=pixels_below,
|
||||
pixels_left=pixels_left,
|
||||
pixels_right=pixels_right,
|
||||
)
|
||||
|
||||
return page_info
|
||||
|
||||
# ========== Public Helper Methods ==========
|
||||
|
||||
async def get_element_by_index(self, index: int) -> EnhancedDOMTreeNode | None:
|
||||
"""Get DOM element by index from cached selector map.
|
||||
|
||||
Builds DOM if not cached.
|
||||
|
||||
Returns:
|
||||
EnhancedDOMTreeNode or None if index not found
|
||||
"""
|
||||
if not self.selector_map:
|
||||
# Build DOM if not cached
|
||||
await self._build_dom_tree_without_highlights()
|
||||
|
||||
return self.selector_map.get(index) if self.selector_map else None
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""Clear cached DOM state to force rebuild on next access."""
|
||||
self.selector_map = None
|
||||
self.current_dom_state = None
|
||||
self.enhanced_dom_tree = None
|
||||
# Keep the DOM service instance to reuse its CDP client connection
|
||||
|
||||
def is_file_input(self, element: EnhancedDOMTreeNode) -> bool:
|
||||
"""Check if element is a file input."""
|
||||
return element.node_name.upper() == 'INPUT' and element.attributes.get('type', '').lower() == 'file'
|
||||
|
||||
@staticmethod
|
||||
def is_element_visible_according_to_all_parents(node: EnhancedDOMTreeNode, html_frames: list[EnhancedDOMTreeNode]) -> bool:
|
||||
"""Check if the element is visible according to all its parent HTML frames.
|
||||
|
||||
Delegates to the DomService static method.
|
||||
"""
|
||||
return DomService.is_element_visible_according_to_all_parents(node, html_frames)
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
"""Clean up DOM service on exit."""
|
||||
if self._dom_service:
|
||||
await self._dom_service.__aexit__(exc_type, exc_value, traceback)
|
||||
self._dom_service = None
|
||||
|
||||
def __del__(self):
|
||||
"""Clean up DOM service on deletion."""
|
||||
super().__del__()
|
||||
# DOM service will clean up its own CDP client
|
||||
self._dom_service = None
|
||||
1382
.agent/vendor/browser_use/browser_use/browser/watchdogs/downloads_watchdog.py
vendored
Normal file
1382
.agent/vendor/browser_use/browser_use/browser/watchdogs/downloads_watchdog.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
779
.agent/vendor/browser_use/browser_use/browser/watchdogs/har_recording_watchdog.py
vendored
Normal file
779
.agent/vendor/browser_use/browser_use/browser/watchdogs/har_recording_watchdog.py
vendored
Normal file
@@ -0,0 +1,779 @@
|
||||
"""HAR Recording Watchdog for Browser-Use sessions.
|
||||
|
||||
Captures HTTPS network activity via CDP Network domain and writes a HAR 1.2
|
||||
file on browser shutdown. Respects `record_har_content` (omit/embed/attach)
|
||||
and `record_har_mode` (full/minimal).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from importlib import metadata as importlib_metadata
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.network.events import (
|
||||
DataReceivedEvent,
|
||||
LoadingFailedEvent,
|
||||
LoadingFinishedEvent,
|
||||
RequestWillBeSentEvent,
|
||||
ResponseReceivedEvent,
|
||||
)
|
||||
from cdp_use.cdp.page.events import FrameNavigatedEvent, LifecycleEventEvent
|
||||
|
||||
from browser_use.browser.events import BrowserConnectedEvent, BrowserStopEvent
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
|
||||
@dataclass
|
||||
class _HarContent:
|
||||
mime_type: str | None = None
|
||||
text_b64: str | None = None # for embed
|
||||
file_rel: str | None = None # for attach
|
||||
size: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _HarEntryBuilder:
|
||||
request_id: str = ''
|
||||
frame_id: str | None = None
|
||||
document_url: str | None = None
|
||||
url: str | None = None
|
||||
method: str | None = None
|
||||
request_headers: dict = field(default_factory=dict)
|
||||
request_body: bytes | None = None
|
||||
post_data: str | None = None # CDP postData field
|
||||
status: int | None = None
|
||||
status_text: str | None = None
|
||||
response_headers: dict = field(default_factory=dict)
|
||||
mime_type: str | None = None
|
||||
encoded_data: bytearray = field(default_factory=bytearray)
|
||||
failed: bool = False
|
||||
# timing info (CDP timestamps are monotonic seconds); wallTime is epoch seconds
|
||||
ts_request: float | None = None
|
||||
wall_time_request: float | None = None
|
||||
ts_response: float | None = None
|
||||
ts_finished: float | None = None
|
||||
encoded_data_length: int | None = None
|
||||
response_body: bytes | None = None
|
||||
content_length: int | None = None # From Content-Length header
|
||||
protocol: str | None = None
|
||||
server_ip_address: str | None = None
|
||||
server_port: int | None = None
|
||||
security_details: dict | None = None
|
||||
transfer_size: int | None = None
|
||||
|
||||
|
||||
def _is_https(url: str | None) -> bool:
|
||||
return bool(url and url.lower().startswith('https://'))
|
||||
|
||||
|
||||
def _origin(url: str) -> str:
|
||||
# Very small origin extractor, assumes https URLs
|
||||
# https://host[:port]/...
|
||||
if not url:
|
||||
return ''
|
||||
try:
|
||||
without_scheme = url.split('://', 1)[1]
|
||||
host_port = without_scheme.split('/', 1)[0]
|
||||
return f'https://{host_port}'
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
def _mime_to_extension(mime_type: str | None) -> str:
|
||||
"""Map MIME type to file extension, matching Playwright's behavior."""
|
||||
if not mime_type:
|
||||
return 'bin'
|
||||
|
||||
mime_lower = mime_type.lower().split(';')[0].strip()
|
||||
|
||||
# Common MIME type to extension mapping
|
||||
mime_map = {
|
||||
'text/html': 'html',
|
||||
'text/css': 'css',
|
||||
'text/javascript': 'js',
|
||||
'application/javascript': 'js',
|
||||
'application/x-javascript': 'js',
|
||||
'application/json': 'json',
|
||||
'application/xml': 'xml',
|
||||
'text/xml': 'xml',
|
||||
'text/plain': 'txt',
|
||||
'image/png': 'png',
|
||||
'image/jpeg': 'jpg',
|
||||
'image/jpg': 'jpg',
|
||||
'image/gif': 'gif',
|
||||
'image/webp': 'webp',
|
||||
'image/svg+xml': 'svg',
|
||||
'image/x-icon': 'ico',
|
||||
'font/woff': 'woff',
|
||||
'font/woff2': 'woff2',
|
||||
'application/font-woff': 'woff',
|
||||
'application/font-woff2': 'woff2',
|
||||
'application/x-font-woff': 'woff',
|
||||
'application/x-font-woff2': 'woff2',
|
||||
'font/ttf': 'ttf',
|
||||
'application/x-font-ttf': 'ttf',
|
||||
'font/otf': 'otf',
|
||||
'application/x-font-opentype': 'otf',
|
||||
'application/pdf': 'pdf',
|
||||
'application/zip': 'zip',
|
||||
'application/x-zip-compressed': 'zip',
|
||||
'video/mp4': 'mp4',
|
||||
'video/webm': 'webm',
|
||||
'audio/mpeg': 'mp3',
|
||||
'audio/mp3': 'mp3',
|
||||
'audio/wav': 'wav',
|
||||
'audio/ogg': 'ogg',
|
||||
}
|
||||
|
||||
return mime_map.get(mime_lower, 'bin')
|
||||
|
||||
|
||||
def _generate_har_filename(content: bytes, mime_type: str | None) -> str:
|
||||
"""Generate a hash-based filename for HAR attach mode, matching Playwright's format."""
|
||||
content_hash = hashlib.sha1(content).hexdigest()
|
||||
extension = _mime_to_extension(mime_type)
|
||||
return f'{content_hash}.{extension}'
|
||||
|
||||
|
||||
class HarRecordingWatchdog(BaseWatchdog):
|
||||
"""Collects HTTPS requests/responses and writes a HAR 1.2 file on stop."""
|
||||
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [BrowserConnectedEvent, BrowserStopEvent]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = []
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._enabled: bool = False
|
||||
self._entries: dict[str, _HarEntryBuilder] = {}
|
||||
self._top_level_pages: dict[
|
||||
str, dict
|
||||
] = {} # frameId -> {url, title, startedDateTime, monotonic_start, onContentLoad, onLoad}
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
profile = self.browser_session.browser_profile
|
||||
if not profile.record_har_path:
|
||||
return
|
||||
|
||||
# Normalize config
|
||||
self._content_mode = (profile.record_har_content or 'embed').lower()
|
||||
self._mode = (profile.record_har_mode or 'full').lower()
|
||||
self._har_path = Path(str(profile.record_har_path)).expanduser().resolve()
|
||||
self._har_dir = self._har_path.parent
|
||||
self._har_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
# Enable Network and Page domains for events
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
||||
await cdp_session.cdp_client.send.Network.enable(session_id=cdp_session.session_id)
|
||||
await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
|
||||
|
||||
# Query browser version for HAR log.browser
|
||||
try:
|
||||
version_info = await self.browser_session.cdp_client.send.Browser.getVersion()
|
||||
self._browser_name = version_info.get('product') or 'Chromium'
|
||||
self._browser_version = version_info.get('jsVersion') or ''
|
||||
except Exception:
|
||||
self._browser_name = 'Chromium'
|
||||
self._browser_version = ''
|
||||
|
||||
cdp = self.browser_session.cdp_client.register
|
||||
cdp.Network.requestWillBeSent(self._on_request_will_be_sent)
|
||||
cdp.Network.responseReceived(self._on_response_received)
|
||||
cdp.Network.dataReceived(self._on_data_received)
|
||||
cdp.Network.loadingFinished(self._on_loading_finished)
|
||||
cdp.Network.loadingFailed(self._on_loading_failed)
|
||||
cdp.Page.lifecycleEvent(self._on_lifecycle_event)
|
||||
cdp.Page.frameNavigated(self._on_frame_navigated)
|
||||
|
||||
self._enabled = True
|
||||
self.logger.info(f'📊 Starting HAR recording to {self._har_path}')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Failed to enable HAR recording: {e}')
|
||||
self._enabled = False
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
if not self._enabled:
|
||||
return
|
||||
try:
|
||||
await self._write_har()
|
||||
self.logger.info(f'📊 HAR file saved: {self._har_path}')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Failed to write HAR: {e}')
|
||||
|
||||
# =============== CDP Event Handlers (sync) ==================
|
||||
def _on_request_will_be_sent(self, params: RequestWillBeSentEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
req = params.get('request', {}) if hasattr(params, 'get') else getattr(params, 'request', {})
|
||||
url = req.get('url') if isinstance(req, dict) else getattr(req, 'url', None)
|
||||
if not _is_https(url):
|
||||
return # HTTPS-only requirement (only HTTPS requests are recorded for now)
|
||||
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if not request_id:
|
||||
return
|
||||
|
||||
entry = self._entries.setdefault(request_id, _HarEntryBuilder(request_id=request_id))
|
||||
entry.url = url
|
||||
entry.method = req.get('method') if isinstance(req, dict) else getattr(req, 'method', None)
|
||||
entry.post_data = req.get('postData') if isinstance(req, dict) else getattr(req, 'postData', None)
|
||||
|
||||
# Convert headers to plain dict, handling various formats
|
||||
headers_raw = req.get('headers') if isinstance(req, dict) else getattr(req, 'headers', None)
|
||||
if headers_raw is None:
|
||||
entry.request_headers = {}
|
||||
elif isinstance(headers_raw, dict):
|
||||
entry.request_headers = {k.lower(): str(v) for k, v in headers_raw.items()}
|
||||
elif isinstance(headers_raw, list):
|
||||
entry.request_headers = {
|
||||
h.get('name', '').lower(): str(h.get('value') or '') for h in headers_raw if isinstance(h, dict)
|
||||
}
|
||||
else:
|
||||
# Handle Headers type or other formats - convert to dict
|
||||
try:
|
||||
headers_dict = dict(headers_raw) if hasattr(headers_raw, '__iter__') else {}
|
||||
entry.request_headers = {k.lower(): str(v) for k, v in headers_dict.items()}
|
||||
except Exception:
|
||||
entry.request_headers = {}
|
||||
|
||||
entry.frame_id = params.get('frameId') if hasattr(params, 'get') else getattr(params, 'frameId', None)
|
||||
entry.document_url = (
|
||||
params.get('documentURL')
|
||||
if hasattr(params, 'get')
|
||||
else getattr(params, 'documentURL', None) or entry.document_url
|
||||
)
|
||||
|
||||
# Timing anchors
|
||||
entry.ts_request = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
|
||||
entry.wall_time_request = params.get('wallTime') if hasattr(params, 'get') else getattr(params, 'wallTime', None)
|
||||
|
||||
# Track top-level navigations for page context
|
||||
req_type = params.get('type') if hasattr(params, 'get') else getattr(params, 'type', None)
|
||||
is_same_doc = (
|
||||
params.get('isSameDocument', False) if hasattr(params, 'get') else getattr(params, 'isSameDocument', False)
|
||||
)
|
||||
if req_type == 'Document' and not is_same_doc:
|
||||
# best-effort: consider as navigation
|
||||
if entry.frame_id and url:
|
||||
if entry.frame_id not in self._top_level_pages:
|
||||
self._top_level_pages[entry.frame_id] = {
|
||||
'url': str(url),
|
||||
'title': str(url), # Default to URL, will be updated from DOM
|
||||
'startedDateTime': entry.wall_time_request,
|
||||
'monotonic_start': entry.ts_request, # Track monotonic start time for timing calculations
|
||||
'onContentLoad': -1,
|
||||
'onLoad': -1,
|
||||
}
|
||||
else:
|
||||
# Update startedDateTime and monotonic_start if this is earlier
|
||||
page_info = self._top_level_pages[entry.frame_id]
|
||||
if entry.wall_time_request and (
|
||||
page_info['startedDateTime'] is None or entry.wall_time_request < page_info['startedDateTime']
|
||||
):
|
||||
page_info['startedDateTime'] = entry.wall_time_request
|
||||
page_info['monotonic_start'] = entry.ts_request
|
||||
except Exception as e:
|
||||
self.logger.debug(f'requestWillBeSent handling error: {e}')
|
||||
|
||||
def _on_response_received(self, params: ResponseReceivedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if not request_id or request_id not in self._entries:
|
||||
return
|
||||
response = params.get('response', {}) if hasattr(params, 'get') else getattr(params, 'response', {})
|
||||
entry = self._entries[request_id]
|
||||
entry.status = response.get('status') if isinstance(response, dict) else getattr(response, 'status', None)
|
||||
entry.status_text = (
|
||||
response.get('statusText') if isinstance(response, dict) else getattr(response, 'statusText', None)
|
||||
)
|
||||
|
||||
# Extract Content-Length for compression calculation (before converting headers)
|
||||
headers_raw = response.get('headers') if isinstance(response, dict) else getattr(response, 'headers', None)
|
||||
if headers_raw:
|
||||
if isinstance(headers_raw, dict):
|
||||
cl_str = headers_raw.get('content-length') or headers_raw.get('Content-Length')
|
||||
elif isinstance(headers_raw, list):
|
||||
cl_header = next(
|
||||
(h for h in headers_raw if isinstance(h, dict) and h.get('name', '').lower() == 'content-length'), None
|
||||
)
|
||||
cl_str = cl_header.get('value') if cl_header else None
|
||||
else:
|
||||
cl_str = None
|
||||
if cl_str:
|
||||
try:
|
||||
entry.content_length = int(cl_str)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Convert headers to plain dict, handling various formats
|
||||
if headers_raw is None:
|
||||
entry.response_headers = {}
|
||||
elif isinstance(headers_raw, dict):
|
||||
entry.response_headers = {k.lower(): str(v) for k, v in headers_raw.items()}
|
||||
elif isinstance(headers_raw, list):
|
||||
entry.response_headers = {
|
||||
h.get('name', '').lower(): str(h.get('value') or '') for h in headers_raw if isinstance(h, dict)
|
||||
}
|
||||
else:
|
||||
# Handle Headers type or other formats - convert to dict
|
||||
try:
|
||||
headers_dict = dict(headers_raw) if hasattr(headers_raw, '__iter__') else {}
|
||||
entry.response_headers = {k.lower(): str(v) for k, v in headers_dict.items()}
|
||||
except Exception:
|
||||
entry.response_headers = {}
|
||||
|
||||
entry.mime_type = response.get('mimeType') if isinstance(response, dict) else getattr(response, 'mimeType', None)
|
||||
entry.ts_response = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
|
||||
|
||||
protocol_raw = response.get('protocol') if isinstance(response, dict) else getattr(response, 'protocol', None)
|
||||
if protocol_raw:
|
||||
protocol_lower = str(protocol_raw).lower()
|
||||
if protocol_lower == 'h2' or protocol_lower.startswith('http/2'):
|
||||
entry.protocol = 'HTTP/2.0'
|
||||
elif protocol_lower.startswith('http/1.1'):
|
||||
entry.protocol = 'HTTP/1.1'
|
||||
elif protocol_lower.startswith('http/1.0'):
|
||||
entry.protocol = 'HTTP/1.0'
|
||||
else:
|
||||
entry.protocol = str(protocol_raw).upper()
|
||||
|
||||
entry.server_ip_address = (
|
||||
response.get('remoteIPAddress') if isinstance(response, dict) else getattr(response, 'remoteIPAddress', None)
|
||||
)
|
||||
server_port_raw = response.get('remotePort') if isinstance(response, dict) else getattr(response, 'remotePort', None)
|
||||
if server_port_raw is not None:
|
||||
try:
|
||||
entry.server_port = int(server_port_raw)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# Extract security details (TLS info)
|
||||
security_details_raw = (
|
||||
response.get('securityDetails') if isinstance(response, dict) else getattr(response, 'securityDetails', None)
|
||||
)
|
||||
if security_details_raw:
|
||||
try:
|
||||
entry.security_details = dict(security_details_raw)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.logger.debug(f'responseReceived handling error: {e}')
|
||||
|
||||
def _on_data_received(self, params: DataReceivedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if not request_id or request_id not in self._entries:
|
||||
return
|
||||
data = params.get('data') if hasattr(params, 'get') else getattr(params, 'data', None)
|
||||
if isinstance(data, str):
|
||||
try:
|
||||
self._entries[request_id].encoded_data.extend(data.encode('latin1'))
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.logger.debug(f'dataReceived handling error: {e}')
|
||||
|
||||
def _on_loading_finished(self, params: LoadingFinishedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if not request_id or request_id not in self._entries:
|
||||
return
|
||||
entry = self._entries[request_id]
|
||||
entry.ts_finished = params.get('timestamp')
|
||||
# Fetch response body via CDP as dataReceived may be incomplete
|
||||
import asyncio as _asyncio
|
||||
|
||||
async def _fetch_body(self_ref, req_id, sess_id):
|
||||
try:
|
||||
resp = await self_ref.browser_session.cdp_client.send.Network.getResponseBody(
|
||||
params={'requestId': req_id}, session_id=sess_id
|
||||
)
|
||||
data = resp.get('body', b'')
|
||||
if resp.get('base64Encoded'):
|
||||
import base64 as _b64
|
||||
|
||||
data = _b64.b64decode(data)
|
||||
else:
|
||||
# Ensure data is bytes even if CDP returns a string
|
||||
if isinstance(data, str):
|
||||
data = data.encode('utf-8', errors='replace')
|
||||
# Ensure we always have bytes
|
||||
if not isinstance(data, bytes):
|
||||
data = bytes(data) if data else b''
|
||||
entry.response_body = data
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Always schedule the response body fetch task
|
||||
_asyncio.create_task(_fetch_body(self, request_id, session_id))
|
||||
|
||||
encoded_length = (
|
||||
params.get('encodedDataLength') if hasattr(params, 'get') else getattr(params, 'encodedDataLength', None)
|
||||
)
|
||||
if encoded_length is not None:
|
||||
try:
|
||||
entry.encoded_data_length = int(encoded_length)
|
||||
entry.transfer_size = entry.encoded_data_length
|
||||
except Exception:
|
||||
entry.encoded_data_length = None
|
||||
except Exception as e:
|
||||
self.logger.debug(f'loadingFinished handling error: {e}')
|
||||
|
||||
def _on_loading_failed(self, params: LoadingFailedEvent, session_id: str | None) -> None:
|
||||
try:
|
||||
request_id = params.get('requestId') if hasattr(params, 'get') else getattr(params, 'requestId', None)
|
||||
if request_id and request_id in self._entries:
|
||||
self._entries[request_id].failed = True
|
||||
except Exception as e:
|
||||
self.logger.debug(f'loadingFailed handling error: {e}')
|
||||
|
||||
# ===================== HAR Writing ==========================
|
||||
def _on_lifecycle_event(self, params: LifecycleEventEvent, session_id: str | None) -> None:
|
||||
"""Handle Page.lifecycleEvent for tracking page load timings."""
|
||||
try:
|
||||
frame_id = params.get('frameId') if hasattr(params, 'get') else getattr(params, 'frameId', None)
|
||||
name = params.get('name') if hasattr(params, 'get') else getattr(params, 'name', None)
|
||||
timestamp = params.get('timestamp') if hasattr(params, 'get') else getattr(params, 'timestamp', None)
|
||||
|
||||
if not frame_id or not name or frame_id not in self._top_level_pages:
|
||||
return
|
||||
|
||||
page_info = self._top_level_pages[frame_id]
|
||||
# Use monotonic_start instead of startedDateTime (wall-clock) for timing calculations
|
||||
monotonic_start = page_info.get('monotonic_start')
|
||||
|
||||
if name == 'DOMContentLoaded' and monotonic_start is not None:
|
||||
# Calculate milliseconds since page start using monotonic timestamps
|
||||
try:
|
||||
elapsed_ms = int(round((timestamp - monotonic_start) * 1000))
|
||||
page_info['onContentLoad'] = max(0, elapsed_ms)
|
||||
except Exception:
|
||||
pass
|
||||
elif name == 'load' and monotonic_start is not None:
|
||||
try:
|
||||
elapsed_ms = int(round((timestamp - monotonic_start) * 1000))
|
||||
page_info['onLoad'] = max(0, elapsed_ms)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.logger.debug(f'lifecycleEvent handling error: {e}')
|
||||
|
||||
def _on_frame_navigated(self, params: FrameNavigatedEvent, session_id: str | None) -> None:
|
||||
"""Handle Page.frameNavigated to update page title from DOM."""
|
||||
try:
|
||||
frame = params.get('frame') if hasattr(params, 'get') else getattr(params, 'frame', None)
|
||||
if not frame:
|
||||
return
|
||||
|
||||
frame_id = frame.get('id') if isinstance(frame, dict) else getattr(frame, 'id', None)
|
||||
title = (
|
||||
frame.get('name') or frame.get('url')
|
||||
if isinstance(frame, dict)
|
||||
else getattr(frame, 'name', None) or getattr(frame, 'url', None)
|
||||
)
|
||||
|
||||
if frame_id and frame_id in self._top_level_pages:
|
||||
# Try to get actual page title via Runtime.evaluate if possible
|
||||
# For now, use frame name or URL as fallback
|
||||
if title:
|
||||
self._top_level_pages[frame_id]['title'] = str(title)
|
||||
except Exception as e:
|
||||
self.logger.debug(f'frameNavigated handling error: {e}')
|
||||
|
||||
# ===================== HAR Writing ==========================
|
||||
async def _write_har(self) -> None:
|
||||
# Filter by mode and HTTPS already respected at collection time
|
||||
entries = [e for e in self._entries.values() if self._include_entry(e)]
|
||||
|
||||
har_entries = []
|
||||
sidecar_dir: Path | None = None
|
||||
if self._content_mode == 'attach':
|
||||
sidecar_dir = self._har_dir / f'{self._har_path.stem}_har_parts'
|
||||
sidecar_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for e in entries:
|
||||
content_obj: dict = {'mimeType': e.mime_type or ''}
|
||||
|
||||
# Get body data, preferring response_body over encoded_data
|
||||
if e.response_body is not None:
|
||||
body_data = e.response_body
|
||||
else:
|
||||
body_data = e.encoded_data
|
||||
|
||||
# Defensive conversion: ensure body_data is always bytes
|
||||
if isinstance(body_data, str):
|
||||
body_bytes = body_data.encode('utf-8', errors='replace')
|
||||
elif isinstance(body_data, bytearray):
|
||||
body_bytes = bytes(body_data)
|
||||
elif isinstance(body_data, bytes):
|
||||
body_bytes = body_data
|
||||
else:
|
||||
# Fallback: try to convert to bytes
|
||||
try:
|
||||
body_bytes = bytes(body_data) if body_data else b''
|
||||
except (TypeError, ValueError):
|
||||
body_bytes = b''
|
||||
|
||||
content_size = len(body_bytes)
|
||||
|
||||
# Calculate compression (bytes saved by compression)
|
||||
compression = 0
|
||||
if e.content_length is not None and e.encoded_data_length is not None:
|
||||
compression = max(0, e.content_length - e.encoded_data_length)
|
||||
|
||||
if self._content_mode == 'embed' and content_size > 0:
|
||||
# Prefer plain text; fallback to base64 only if decoding fails
|
||||
try:
|
||||
text_decoded = body_bytes.decode('utf-8')
|
||||
content_obj['text'] = text_decoded
|
||||
content_obj['size'] = content_size
|
||||
content_obj['compression'] = compression
|
||||
except UnicodeDecodeError:
|
||||
content_obj['text'] = base64.b64encode(body_bytes).decode('ascii')
|
||||
content_obj['encoding'] = 'base64'
|
||||
content_obj['size'] = content_size
|
||||
content_obj['compression'] = compression
|
||||
elif self._content_mode == 'attach' and content_size > 0 and sidecar_dir is not None:
|
||||
filename = _generate_har_filename(body_bytes, e.mime_type)
|
||||
(sidecar_dir / filename).write_bytes(body_bytes)
|
||||
content_obj['_file'] = filename
|
||||
content_obj['size'] = content_size
|
||||
content_obj['compression'] = compression
|
||||
else:
|
||||
# omit or empty
|
||||
content_obj['size'] = content_size
|
||||
if content_size > 0:
|
||||
content_obj['compression'] = compression
|
||||
|
||||
started_date_time, total_time_ms, timings = self._compute_timings(e)
|
||||
req_headers_list = [{'name': k, 'value': str(v)} for k, v in (e.request_headers or {}).items()]
|
||||
resp_headers_list = [{'name': k, 'value': str(v)} for k, v in (e.response_headers or {}).items()]
|
||||
request_headers_size = self._calc_headers_size(e.method or 'GET', e.url or '', req_headers_list)
|
||||
response_headers_size = self._calc_headers_size(None, None, resp_headers_list)
|
||||
request_body_size = self._calc_request_body_size(e)
|
||||
request_post_data = None
|
||||
if e.post_data and self._content_mode != 'omit':
|
||||
if self._content_mode == 'embed':
|
||||
request_post_data = {'mimeType': e.request_headers.get('content-type', ''), 'text': e.post_data}
|
||||
elif self._content_mode == 'attach' and sidecar_dir is not None:
|
||||
post_data_bytes = e.post_data.encode('utf-8')
|
||||
req_mime_type = e.request_headers.get('content-type', 'text/plain')
|
||||
req_filename = _generate_har_filename(post_data_bytes, req_mime_type)
|
||||
(sidecar_dir / req_filename).write_bytes(post_data_bytes)
|
||||
request_post_data = {
|
||||
'mimeType': req_mime_type,
|
||||
'_file': req_filename,
|
||||
}
|
||||
|
||||
http_version = e.protocol if e.protocol else 'HTTP/1.1'
|
||||
|
||||
response_body_size = e.transfer_size
|
||||
if response_body_size is None:
|
||||
response_body_size = e.encoded_data_length
|
||||
if response_body_size is None:
|
||||
response_body_size = content_size if content_size > 0 else -1
|
||||
|
||||
entry_dict = {
|
||||
'startedDateTime': started_date_time,
|
||||
'time': total_time_ms,
|
||||
'request': {
|
||||
'method': e.method or 'GET',
|
||||
'url': e.url or '',
|
||||
'httpVersion': http_version,
|
||||
'headers': req_headers_list,
|
||||
'queryString': [],
|
||||
'cookies': [],
|
||||
'headersSize': request_headers_size,
|
||||
'bodySize': request_body_size,
|
||||
'postData': request_post_data,
|
||||
},
|
||||
'response': {
|
||||
'status': e.status or 0,
|
||||
'statusText': e.status_text or '',
|
||||
'httpVersion': http_version,
|
||||
'headers': resp_headers_list,
|
||||
'cookies': [],
|
||||
'content': content_obj,
|
||||
'redirectURL': '',
|
||||
'headersSize': response_headers_size,
|
||||
'bodySize': response_body_size,
|
||||
},
|
||||
'cache': {},
|
||||
'timings': timings,
|
||||
'pageref': self._page_ref_for_entry(e),
|
||||
}
|
||||
|
||||
# Add security/TLS details if available
|
||||
if e.server_ip_address:
|
||||
entry_dict['serverIPAddress'] = e.server_ip_address
|
||||
if e.server_port is not None:
|
||||
entry_dict['_serverPort'] = e.server_port
|
||||
if e.security_details:
|
||||
# Filter to match Playwright's minimal security details set
|
||||
security_filtered = {}
|
||||
if 'protocol' in e.security_details:
|
||||
security_filtered['protocol'] = e.security_details['protocol']
|
||||
if 'subjectName' in e.security_details:
|
||||
security_filtered['subjectName'] = e.security_details['subjectName']
|
||||
if 'issuer' in e.security_details:
|
||||
security_filtered['issuer'] = e.security_details['issuer']
|
||||
if 'validFrom' in e.security_details:
|
||||
security_filtered['validFrom'] = e.security_details['validFrom']
|
||||
if 'validTo' in e.security_details:
|
||||
security_filtered['validTo'] = e.security_details['validTo']
|
||||
if security_filtered:
|
||||
entry_dict['_securityDetails'] = security_filtered
|
||||
if e.transfer_size is not None:
|
||||
entry_dict['response']['_transferSize'] = e.transfer_size
|
||||
|
||||
har_entries.append(entry_dict)
|
||||
|
||||
# Try to include our library version in creator
|
||||
try:
|
||||
bu_version = importlib_metadata.version('browser-use')
|
||||
except Exception:
|
||||
# Fallback when running from source without installed package metadata
|
||||
bu_version = 'dev'
|
||||
|
||||
har_obj = {
|
||||
'log': {
|
||||
'version': '1.2',
|
||||
'creator': {'name': 'browser-use', 'version': bu_version},
|
||||
'browser': {'name': self._browser_name, 'version': self._browser_version},
|
||||
'pages': [
|
||||
{
|
||||
'id': f'page@{pid}', # Use Playwright format: "page@{frame_id}"
|
||||
'title': page_info.get('title', page_info.get('url', '')),
|
||||
'startedDateTime': self._format_page_started_datetime(page_info.get('startedDateTime')),
|
||||
'pageTimings': (
|
||||
(lambda _ocl, _ol: ({k: v for k, v in (('onContentLoad', _ocl), ('onLoad', _ol)) if v is not None}))(
|
||||
(page_info.get('onContentLoad') if page_info.get('onContentLoad', -1) >= 0 else None),
|
||||
(page_info.get('onLoad') if page_info.get('onLoad', -1) >= 0 else None),
|
||||
)
|
||||
),
|
||||
}
|
||||
for pid, page_info in self._top_level_pages.items()
|
||||
],
|
||||
'entries': har_entries,
|
||||
}
|
||||
}
|
||||
|
||||
tmp_path = self._har_path.with_suffix(self._har_path.suffix + '.tmp')
|
||||
# Write as bytes explicitly to avoid any text/binary mode confusion in different environments
|
||||
tmp_path.write_bytes(json.dumps(har_obj, indent=2, ensure_ascii=False).encode('utf-8'))
|
||||
tmp_path.replace(self._har_path)
|
||||
|
||||
def _format_page_started_datetime(self, timestamp: float | None) -> str:
|
||||
"""Format page startedDateTime from timestamp."""
|
||||
if timestamp is None:
|
||||
return ''
|
||||
try:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
return datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat().replace('+00:00', 'Z')
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
def _page_ref_for_entry(self, e: _HarEntryBuilder) -> str | None:
|
||||
# Use Playwright format: "page@{frame_id}" if frame_id is known
|
||||
if e.frame_id and e.frame_id in self._top_level_pages:
|
||||
return f'page@{e.frame_id}'
|
||||
return None
|
||||
|
||||
def _include_entry(self, e: _HarEntryBuilder) -> bool:
|
||||
if not _is_https(e.url):
|
||||
return False
|
||||
# Filter out favicon requests (matching Playwright behavior)
|
||||
if e.url and '/favicon.ico' in e.url.lower():
|
||||
return False
|
||||
if getattr(self, '_mode', 'full') == 'full':
|
||||
return True
|
||||
# minimal: include main document and same-origin subresources
|
||||
if e.frame_id and e.frame_id in self._top_level_pages:
|
||||
page_info = self._top_level_pages[e.frame_id]
|
||||
page_url = page_info.get('url') if isinstance(page_info, dict) else page_info
|
||||
return _origin(e.url or '') == _origin(page_url or '')
|
||||
return False
|
||||
|
||||
# ===================== Helpers ==============================
|
||||
def _compute_timings(self, e: _HarEntryBuilder) -> tuple[str, int, dict]:
|
||||
# startedDateTime from wall_time_request in ISO8601 Z
|
||||
started = ''
|
||||
try:
|
||||
if e.wall_time_request is not None:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
started = datetime.fromtimestamp(e.wall_time_request, tz=timezone.utc).isoformat().replace('+00:00', 'Z')
|
||||
except Exception:
|
||||
started = ''
|
||||
|
||||
# Calculate timings - CDP doesn't always provide DNS/connect/SSL breakdown
|
||||
# Default to 0 for unavailable timings, calculate what we can from timestamps
|
||||
dns_ms = 0
|
||||
connect_ms = 0
|
||||
ssl_ms = 0
|
||||
send_ms = 0
|
||||
wait_ms = 0
|
||||
receive_ms = 0
|
||||
|
||||
if e.ts_request is not None and e.ts_response is not None:
|
||||
wait_ms = max(0, int(round((e.ts_response - e.ts_request) * 1000)))
|
||||
|
||||
if e.ts_response is not None and e.ts_finished is not None:
|
||||
receive_ms = max(0, int(round((e.ts_finished - e.ts_response) * 1000)))
|
||||
|
||||
# Note: DNS, connect, and SSL timings would require additional CDP events or ResourceTiming API
|
||||
# For now, we structure the timings dict to match Playwright format
|
||||
# but leave DNS/connect/SSL as 0 since CDP doesn't provide this breakdown directly
|
||||
|
||||
total = dns_ms + connect_ms + ssl_ms + send_ms + wait_ms + receive_ms
|
||||
return (
|
||||
started,
|
||||
total,
|
||||
{
|
||||
'dns': dns_ms,
|
||||
'connect': connect_ms,
|
||||
'ssl': ssl_ms,
|
||||
'send': send_ms,
|
||||
'wait': wait_ms,
|
||||
'receive': receive_ms,
|
||||
},
|
||||
)
|
||||
|
||||
def _calc_headers_size(self, method: str | None, url: str | None, headers_list: list[dict]) -> int:
|
||||
try:
|
||||
# Approximate per RFC: sum of header lines + CRLF; include request/status line only for request
|
||||
size = 0
|
||||
if method and url:
|
||||
# Use HTTP/1.1 request line approximation
|
||||
size += len(f'{method} {url} HTTP/1.1\r\n'.encode('latin1'))
|
||||
for h in headers_list:
|
||||
size += len(f'{h.get("name", "")}: {h.get("value", "")}\r\n'.encode('latin1'))
|
||||
size += len(b'\r\n')
|
||||
return size
|
||||
except Exception:
|
||||
return -1
|
||||
|
||||
def _calc_request_body_size(self, e: _HarEntryBuilder) -> int:
|
||||
# Try Content-Length header first; else post_data; else request_body; else 0 for GET/HEAD, -1 if unknown
|
||||
try:
|
||||
cl = None
|
||||
if e.request_headers:
|
||||
cl = e.request_headers.get('content-length') or e.request_headers.get('Content-Length')
|
||||
if cl is not None:
|
||||
return int(cl)
|
||||
if e.post_data:
|
||||
return len(e.post_data.encode('utf-8'))
|
||||
if e.request_body is not None:
|
||||
return len(e.request_body)
|
||||
# GET/HEAD requests typically have no body
|
||||
if e.method and e.method.upper() in ('GET', 'HEAD'):
|
||||
return 0
|
||||
except Exception:
|
||||
pass
|
||||
return -1
|
||||
506
.agent/vendor/browser_use/browser_use/browser/watchdogs/local_browser_watchdog.py
vendored
Normal file
506
.agent/vendor/browser_use/browser_use/browser/watchdogs/local_browser_watchdog.py
vendored
Normal file
@@ -0,0 +1,506 @@
|
||||
"""Local browser watchdog for managing browser subprocess lifecycle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import psutil
|
||||
from bubus import BaseEvent
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserKillEvent,
|
||||
BrowserLaunchEvent,
|
||||
BrowserLaunchResult,
|
||||
BrowserStopEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.observability import observe_debug
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.browser.profile import BrowserChannel
|
||||
|
||||
|
||||
class LocalBrowserWatchdog(BaseWatchdog):
|
||||
"""Manages local browser subprocess lifecycle."""
|
||||
|
||||
# Events this watchdog listens to
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [
|
||||
BrowserLaunchEvent,
|
||||
BrowserKillEvent,
|
||||
BrowserStopEvent,
|
||||
]
|
||||
|
||||
# Events this watchdog emits
|
||||
EMITS: ClassVar[list[type[BaseEvent[Any]]]] = []
|
||||
|
||||
# Private state for subprocess management
|
||||
_subprocess: psutil.Process | None = PrivateAttr(default=None)
|
||||
_owns_browser_resources: bool = PrivateAttr(default=True)
|
||||
_temp_dirs_to_cleanup: list[Path] = PrivateAttr(default_factory=list)
|
||||
_original_user_data_dir: str | None = PrivateAttr(default=None)
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='browser_launch_event')
|
||||
async def on_BrowserLaunchEvent(self, event: BrowserLaunchEvent) -> BrowserLaunchResult:
|
||||
"""Launch a local browser process."""
|
||||
|
||||
try:
|
||||
self.logger.debug('[LocalBrowserWatchdog] Received BrowserLaunchEvent, launching local browser...')
|
||||
|
||||
# self.logger.debug('[LocalBrowserWatchdog] Calling _launch_browser...')
|
||||
process, cdp_url = await self._launch_browser()
|
||||
self._subprocess = process
|
||||
# self.logger.debug(f'[LocalBrowserWatchdog] _launch_browser returned: process={process}, cdp_url={cdp_url}')
|
||||
|
||||
return BrowserLaunchResult(cdp_url=cdp_url)
|
||||
except Exception as e:
|
||||
self.logger.error(f'[LocalBrowserWatchdog] Exception in on_BrowserLaunchEvent: {e}', exc_info=True)
|
||||
raise
|
||||
|
||||
async def on_BrowserKillEvent(self, event: BrowserKillEvent) -> None:
|
||||
"""Kill the local browser subprocess."""
|
||||
self.logger.debug('[LocalBrowserWatchdog] Killing local browser process')
|
||||
|
||||
if self._subprocess:
|
||||
await self._cleanup_process(self._subprocess)
|
||||
self._subprocess = None
|
||||
|
||||
# Clean up temp directories if any were created
|
||||
for temp_dir in self._temp_dirs_to_cleanup:
|
||||
self._cleanup_temp_dir(temp_dir)
|
||||
self._temp_dirs_to_cleanup.clear()
|
||||
|
||||
# Restore original user_data_dir if it was modified
|
||||
if self._original_user_data_dir is not None:
|
||||
self.browser_session.browser_profile.user_data_dir = self._original_user_data_dir
|
||||
self._original_user_data_dir = None
|
||||
|
||||
self.logger.debug('[LocalBrowserWatchdog] Browser cleanup completed')
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
"""Listen for BrowserStopEvent and dispatch BrowserKillEvent without awaiting it."""
|
||||
if self.browser_session.is_local and self._subprocess:
|
||||
self.logger.debug('[LocalBrowserWatchdog] BrowserStopEvent received, dispatching BrowserKillEvent')
|
||||
# Dispatch BrowserKillEvent without awaiting so it gets processed after all BrowserStopEvent handlers
|
||||
self.event_bus.dispatch(BrowserKillEvent())
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='launch_browser_process')
|
||||
async def _launch_browser(self, max_retries: int = 3) -> tuple[psutil.Process, str]:
|
||||
"""Launch browser process and return (process, cdp_url).
|
||||
|
||||
Handles launch errors by falling back to temporary directories if needed.
|
||||
|
||||
Returns:
|
||||
Tuple of (psutil.Process, cdp_url)
|
||||
"""
|
||||
# Keep track of original user_data_dir to restore if needed
|
||||
profile = self.browser_session.browser_profile
|
||||
self._original_user_data_dir = str(profile.user_data_dir) if profile.user_data_dir else None
|
||||
self._temp_dirs_to_cleanup = []
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
# Get launch args from profile
|
||||
launch_args = profile.get_args()
|
||||
|
||||
# Add debugging port
|
||||
debug_port = self._find_free_port()
|
||||
launch_args.extend(
|
||||
[
|
||||
f'--remote-debugging-port={debug_port}',
|
||||
]
|
||||
)
|
||||
assert '--user-data-dir' in str(launch_args), (
|
||||
'User data dir must be set somewhere in launch args to a non-default path, otherwise Chrome will not let us attach via CDP'
|
||||
)
|
||||
|
||||
# Get browser executable
|
||||
# Priority: custom executable > fallback paths > playwright subprocess
|
||||
if profile.executable_path:
|
||||
browser_path = profile.executable_path
|
||||
self.logger.debug(f'[LocalBrowserWatchdog] 📦 Using custom local browser executable_path= {browser_path}')
|
||||
else:
|
||||
# self.logger.debug('[LocalBrowserWatchdog] 🔍 Looking for local browser binary path...')
|
||||
# Try fallback paths first (system browsers preferred)
|
||||
browser_path = self._find_installed_browser_path(channel=profile.channel)
|
||||
if not browser_path:
|
||||
self.logger.error(
|
||||
'[LocalBrowserWatchdog] ⚠️ No local browser binary found, installing browser using playwright subprocess...'
|
||||
)
|
||||
browser_path = await self._install_browser_with_playwright()
|
||||
|
||||
self.logger.debug(f'[LocalBrowserWatchdog] 📦 Found local browser installed at executable_path= {browser_path}')
|
||||
if not browser_path:
|
||||
raise RuntimeError('No local Chrome/Chromium install found, and failed to install with playwright')
|
||||
|
||||
# Launch browser subprocess directly
|
||||
self.logger.debug(f'[LocalBrowserWatchdog] 🚀 Launching browser subprocess with {len(launch_args)} args...')
|
||||
self.logger.debug(
|
||||
f'[LocalBrowserWatchdog] 📂 user_data_dir={profile.user_data_dir}, profile_directory={profile.profile_directory}'
|
||||
)
|
||||
subprocess = await asyncio.create_subprocess_exec(
|
||||
browser_path,
|
||||
*launch_args,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
self.logger.debug(
|
||||
f'[LocalBrowserWatchdog] 🎭 Browser running with browser_pid= {subprocess.pid} 🔗 listening on CDP port :{debug_port}'
|
||||
)
|
||||
|
||||
# Convert to psutil.Process
|
||||
process = psutil.Process(subprocess.pid)
|
||||
|
||||
# Wait for CDP to be ready and get the URL
|
||||
cdp_url = await self._wait_for_cdp_url(debug_port)
|
||||
|
||||
# Success! Clean up only the temp dirs we created but didn't use
|
||||
currently_used_dir = str(profile.user_data_dir)
|
||||
unused_temp_dirs = [tmp_dir for tmp_dir in self._temp_dirs_to_cleanup if str(tmp_dir) != currently_used_dir]
|
||||
|
||||
for tmp_dir in unused_temp_dirs:
|
||||
try:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Keep only the in-use directory for cleanup during browser kill
|
||||
if currently_used_dir and 'browseruse-tmp-' in currently_used_dir:
|
||||
self._temp_dirs_to_cleanup = [Path(currently_used_dir)]
|
||||
else:
|
||||
self._temp_dirs_to_cleanup = []
|
||||
|
||||
return process, cdp_url
|
||||
|
||||
except Exception as e:
|
||||
error_str = str(e).lower()
|
||||
|
||||
# Check if this is a user_data_dir related error
|
||||
if any(err in error_str for err in ['singletonlock', 'user data directory', 'cannot create', 'already in use']):
|
||||
self.logger.warning(f'Browser launch failed (attempt {attempt + 1}/{max_retries}): {e}')
|
||||
|
||||
if attempt < max_retries - 1:
|
||||
# Create a temporary directory for next attempt
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix='browseruse-tmp-'))
|
||||
self._temp_dirs_to_cleanup.append(tmp_dir)
|
||||
|
||||
# Update profile to use temp directory
|
||||
profile.user_data_dir = str(tmp_dir)
|
||||
self.logger.debug(f'Retrying with temporary user_data_dir: {tmp_dir}')
|
||||
|
||||
# Small delay before retry
|
||||
await asyncio.sleep(0.5)
|
||||
continue
|
||||
|
||||
# Not a recoverable error or last attempt failed
|
||||
# Restore original user_data_dir before raising
|
||||
if self._original_user_data_dir is not None:
|
||||
profile.user_data_dir = self._original_user_data_dir
|
||||
|
||||
# Clean up any temp dirs we created
|
||||
for tmp_dir in self._temp_dirs_to_cleanup:
|
||||
try:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
raise
|
||||
|
||||
# Should not reach here, but just in case
|
||||
if self._original_user_data_dir is not None:
|
||||
profile.user_data_dir = self._original_user_data_dir
|
||||
raise RuntimeError(f'Failed to launch browser after {max_retries} attempts')
|
||||
|
||||
@staticmethod
|
||||
def _find_installed_browser_path(channel: BrowserChannel | None = None) -> str | None:
|
||||
"""Try to find browser executable from common fallback locations.
|
||||
|
||||
If a channel is specified, paths for that browser are searched first.
|
||||
Falls back to all known browser paths if the channel-specific search fails.
|
||||
|
||||
Prioritizes:
|
||||
1. Channel-specific paths (if channel is set)
|
||||
2. System Chrome stable
|
||||
3. Playwright chromium
|
||||
4. Other system native browsers (Chromium -> Chrome Canary/Dev -> Brave -> Edge)
|
||||
5. Playwright headless-shell fallback
|
||||
|
||||
Returns:
|
||||
Path to browser executable or None if not found
|
||||
"""
|
||||
import glob
|
||||
import platform
|
||||
from pathlib import Path
|
||||
|
||||
from browser_use.browser.profile import BROWSERUSE_DEFAULT_CHANNEL, BrowserChannel
|
||||
|
||||
system = platform.system()
|
||||
|
||||
# Get playwright browsers path from environment variable if set
|
||||
playwright_path = os.environ.get('PLAYWRIGHT_BROWSERS_PATH')
|
||||
|
||||
# Build tagged pattern lists per OS: (browser_group, path)
|
||||
# browser_group is used to match against the requested channel
|
||||
if system == 'Darwin': # macOS
|
||||
if not playwright_path:
|
||||
playwright_path = '~/Library/Caches/ms-playwright'
|
||||
all_patterns = [
|
||||
('chrome', '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'),
|
||||
('chromium', f'{playwright_path}/chromium-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium'),
|
||||
('chromium', '/Applications/Chromium.app/Contents/MacOS/Chromium'),
|
||||
('chrome-canary', '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary'),
|
||||
('brave', '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'),
|
||||
('msedge', '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge'),
|
||||
('chromium', f'{playwright_path}/chromium_headless_shell-*/chrome-mac/Chromium.app/Contents/MacOS/Chromium'),
|
||||
]
|
||||
elif system == 'Linux':
|
||||
if not playwright_path:
|
||||
playwright_path = '~/.cache/ms-playwright'
|
||||
all_patterns = [
|
||||
('chrome', '/usr/bin/google-chrome-stable'),
|
||||
('chrome', '/usr/bin/google-chrome'),
|
||||
('chrome', '/usr/local/bin/google-chrome'),
|
||||
('chromium', f'{playwright_path}/chromium-*/chrome-linux*/chrome'),
|
||||
('chromium', '/usr/bin/chromium'),
|
||||
('chromium', '/usr/bin/chromium-browser'),
|
||||
('chromium', '/usr/local/bin/chromium'),
|
||||
('chromium', '/snap/bin/chromium'),
|
||||
('chrome-beta', '/usr/bin/google-chrome-beta'),
|
||||
('chrome-dev', '/usr/bin/google-chrome-dev'),
|
||||
('brave', '/usr/bin/brave-browser'),
|
||||
('msedge', '/usr/bin/microsoft-edge-stable'),
|
||||
('msedge', '/usr/bin/microsoft-edge'),
|
||||
('chromium', f'{playwright_path}/chromium_headless_shell-*/chrome-linux*/chrome'),
|
||||
]
|
||||
elif system == 'Windows':
|
||||
if not playwright_path:
|
||||
playwright_path = r'%LOCALAPPDATA%\ms-playwright'
|
||||
all_patterns = [
|
||||
('chrome', r'C:\Program Files\Google\Chrome\Application\chrome.exe'),
|
||||
('chrome', r'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe'),
|
||||
('chrome', r'%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe'),
|
||||
('chrome', r'%PROGRAMFILES%\Google\Chrome\Application\chrome.exe'),
|
||||
('chrome', r'%PROGRAMFILES(X86)%\Google\Chrome\Application\chrome.exe'),
|
||||
('chromium', f'{playwright_path}\\chromium-*\\chrome-win\\chrome.exe'),
|
||||
('chromium', r'C:\Program Files\Chromium\Application\chrome.exe'),
|
||||
('chromium', r'C:\Program Files (x86)\Chromium\Application\chrome.exe'),
|
||||
('chromium', r'%LOCALAPPDATA%\Chromium\Application\chrome.exe'),
|
||||
('brave', r'C:\Program Files\BraveSoftware\Brave-Browser\Application\brave.exe'),
|
||||
('brave', r'C:\Program Files (x86)\BraveSoftware\Brave-Browser\Application\brave.exe'),
|
||||
('msedge', r'C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe'),
|
||||
('msedge', r'C:\Program Files\Microsoft\Edge\Application\msedge.exe'),
|
||||
('msedge', r'%LOCALAPPDATA%\Microsoft\Edge\Application\msedge.exe'),
|
||||
('chromium', f'{playwright_path}\\chromium_headless_shell-*\\chrome-win\\chrome.exe'),
|
||||
]
|
||||
else:
|
||||
all_patterns = []
|
||||
|
||||
# Map channel enum values to browser group tags
|
||||
_channel_to_group: dict[BrowserChannel, str] = {
|
||||
BrowserChannel.CHROME: 'chrome',
|
||||
BrowserChannel.CHROME_BETA: 'chrome-beta',
|
||||
BrowserChannel.CHROME_DEV: 'chrome-dev',
|
||||
BrowserChannel.CHROME_CANARY: 'chrome-canary',
|
||||
BrowserChannel.CHROMIUM: 'chromium',
|
||||
BrowserChannel.MSEDGE: 'msedge',
|
||||
BrowserChannel.MSEDGE_BETA: 'msedge',
|
||||
BrowserChannel.MSEDGE_DEV: 'msedge',
|
||||
BrowserChannel.MSEDGE_CANARY: 'msedge',
|
||||
}
|
||||
|
||||
# If a non-default channel is specified, put matching patterns first, then the rest as fallback
|
||||
if channel and channel != BROWSERUSE_DEFAULT_CHANNEL and channel in _channel_to_group:
|
||||
target_group = _channel_to_group[channel]
|
||||
prioritized = [p for g, p in all_patterns if g == target_group]
|
||||
rest = [p for g, p in all_patterns if g != target_group]
|
||||
patterns = prioritized + rest
|
||||
else:
|
||||
patterns = [p for _, p in all_patterns]
|
||||
|
||||
for pattern in patterns:
|
||||
# Expand user home directory
|
||||
expanded_pattern = Path(pattern).expanduser()
|
||||
|
||||
# Handle Windows environment variables
|
||||
if system == 'Windows':
|
||||
pattern_str = str(expanded_pattern)
|
||||
for env_var in ['%LOCALAPPDATA%', '%PROGRAMFILES%', '%PROGRAMFILES(X86)%']:
|
||||
if env_var in pattern_str:
|
||||
env_key = env_var.strip('%').replace('(X86)', ' (x86)')
|
||||
env_value = os.environ.get(env_key, '')
|
||||
if env_value:
|
||||
pattern_str = pattern_str.replace(env_var, env_value)
|
||||
expanded_pattern = Path(pattern_str)
|
||||
|
||||
# Convert to string for glob
|
||||
pattern_str = str(expanded_pattern)
|
||||
|
||||
# Check if pattern contains wildcards
|
||||
if '*' in pattern_str:
|
||||
# Use glob to expand the pattern
|
||||
matches = glob.glob(pattern_str)
|
||||
if matches:
|
||||
# Sort matches and take the last one (alphanumerically highest version)
|
||||
matches.sort()
|
||||
browser_path = matches[-1]
|
||||
if Path(browser_path).exists() and Path(browser_path).is_file():
|
||||
return browser_path
|
||||
else:
|
||||
# Direct path check
|
||||
if expanded_pattern.exists() and expanded_pattern.is_file():
|
||||
return str(expanded_pattern)
|
||||
|
||||
return None
|
||||
|
||||
async def _install_browser_with_playwright(self) -> str:
|
||||
"""Get browser executable path from playwright in a subprocess to avoid thread issues."""
|
||||
import platform
|
||||
|
||||
# Build command - only use --with-deps on Linux (it fails on Windows/macOS)
|
||||
cmd = ['uvx', 'playwright', 'install', 'chrome']
|
||||
if platform.system() == 'Linux':
|
||||
cmd.append('--with-deps')
|
||||
|
||||
# Run in subprocess with timeout
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=60.0)
|
||||
self.logger.debug(f'[LocalBrowserWatchdog] 📦 Playwright install output: {stdout}')
|
||||
browser_path = self._find_installed_browser_path()
|
||||
if browser_path:
|
||||
return browser_path
|
||||
self.logger.error(f'[LocalBrowserWatchdog] ❌ Playwright local browser installation error: \n{stdout}\n{stderr}')
|
||||
raise RuntimeError('No local browser path found after: uvx playwright install chrome')
|
||||
except TimeoutError:
|
||||
# Kill the subprocess if it times out
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise RuntimeError('Timeout getting browser path from playwright')
|
||||
except Exception as e:
|
||||
# Make sure subprocess is terminated
|
||||
if process.returncode is None:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise RuntimeError(f'Error getting browser path: {e}')
|
||||
|
||||
@staticmethod
|
||||
def _find_free_port() -> int:
|
||||
"""Find a free port for the debugging interface."""
|
||||
import socket
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(('127.0.0.1', 0))
|
||||
s.listen(1)
|
||||
port = s.getsockname()[1]
|
||||
return port
|
||||
|
||||
@staticmethod
|
||||
async def _wait_for_cdp_url(port: int, timeout: float = 30) -> str:
|
||||
"""Wait for the browser to start and return the CDP URL."""
|
||||
import aiohttp
|
||||
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
|
||||
while asyncio.get_event_loop().time() - start_time < timeout:
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(f'http://127.0.0.1:{port}/json/version') as resp:
|
||||
if resp.status == 200:
|
||||
# Chrome is ready
|
||||
return f'http://127.0.0.1:{port}/'
|
||||
else:
|
||||
# Chrome is starting up and returning 502/500 errors
|
||||
await asyncio.sleep(0.1)
|
||||
except Exception:
|
||||
# Connection error - Chrome might not be ready yet
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
raise TimeoutError(f'Browser did not start within {timeout} seconds')
|
||||
|
||||
@staticmethod
|
||||
async def _cleanup_process(process: psutil.Process) -> None:
|
||||
"""Clean up browser process.
|
||||
|
||||
Args:
|
||||
process: psutil.Process to terminate
|
||||
"""
|
||||
if not process:
|
||||
return
|
||||
|
||||
try:
|
||||
# Try graceful shutdown first
|
||||
process.terminate()
|
||||
|
||||
# Use async wait instead of blocking wait
|
||||
for _ in range(50): # Wait up to 5 seconds (50 * 0.1)
|
||||
if not process.is_running():
|
||||
return
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# If still running after 5 seconds, force kill
|
||||
if process.is_running():
|
||||
process.kill()
|
||||
# Give it a moment to die
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
except psutil.NoSuchProcess:
|
||||
# Process already gone
|
||||
pass
|
||||
except Exception:
|
||||
# Ignore any other errors during cleanup
|
||||
pass
|
||||
|
||||
def _cleanup_temp_dir(self, temp_dir: Path | str) -> None:
|
||||
"""Clean up temporary directory.
|
||||
|
||||
Args:
|
||||
temp_dir: Path to temporary directory to remove
|
||||
"""
|
||||
if not temp_dir:
|
||||
return
|
||||
|
||||
try:
|
||||
temp_path = Path(temp_dir)
|
||||
# Only remove if it's actually a temp directory we created
|
||||
if 'browseruse-tmp-' in str(temp_path):
|
||||
shutil.rmtree(temp_path, ignore_errors=True)
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to cleanup temp dir {temp_dir}: {e}')
|
||||
|
||||
@property
|
||||
def browser_pid(self) -> int | None:
|
||||
"""Get the browser process ID."""
|
||||
if self._subprocess:
|
||||
return self._subprocess.pid
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def get_browser_pid_via_cdp(browser) -> int | None:
|
||||
"""Get the browser process ID via CDP SystemInfo.getProcessInfo.
|
||||
|
||||
Args:
|
||||
browser: Playwright Browser instance
|
||||
|
||||
Returns:
|
||||
Process ID or None if failed
|
||||
"""
|
||||
try:
|
||||
cdp_session = await browser.new_browser_cdp_session()
|
||||
result = await cdp_session.send('SystemInfo.getProcessInfo')
|
||||
process_info = result.get('processInfo', {})
|
||||
pid = process_info.get('id')
|
||||
await cdp_session.detach()
|
||||
return pid
|
||||
except Exception:
|
||||
# If we can't get PID via CDP, it's not critical
|
||||
return None
|
||||
43
.agent/vendor/browser_use/browser_use/browser/watchdogs/permissions_watchdog.py
vendored
Normal file
43
.agent/vendor/browser_use/browser_use/browser/watchdogs/permissions_watchdog.py
vendored
Normal file
@@ -0,0 +1,43 @@
|
||||
"""Permissions watchdog for granting browser permissions on connection."""
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
|
||||
from browser_use.browser.events import BrowserConnectedEvent
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class PermissionsWatchdog(BaseWatchdog):
|
||||
"""Grants browser permissions when browser connects."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserConnectedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = []
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""Grant permissions when browser connects."""
|
||||
permissions = self.browser_session.browser_profile.permissions
|
||||
|
||||
if not permissions:
|
||||
self.logger.debug('No permissions to grant')
|
||||
return
|
||||
|
||||
self.logger.debug(f'🔓 Granting browser permissions: {permissions}')
|
||||
|
||||
try:
|
||||
# Grant permissions using CDP Browser.grantPermissions
|
||||
# origin=None means grant to all origins
|
||||
# Browser domain commands don't use session_id
|
||||
await self.browser_session.cdp_client.send.Browser.grantPermissions(
|
||||
params={'permissions': permissions} # type: ignore
|
||||
)
|
||||
self.logger.debug(f'✅ Successfully granted permissions: {permissions}')
|
||||
except Exception as e:
|
||||
self.logger.error(f'❌ Failed to grant permissions: {str(e)}')
|
||||
# Don't raise - permissions are not critical to browser operation
|
||||
145
.agent/vendor/browser_use/browser_use/browser/watchdogs/popups_watchdog.py
vendored
Normal file
145
.agent/vendor/browser_use/browser_use/browser/watchdogs/popups_watchdog.py
vendored
Normal file
@@ -0,0 +1,145 @@
|
||||
"""Watchdog for handling JavaScript dialogs (alert, confirm, prompt) automatically."""
|
||||
|
||||
import asyncio
|
||||
from typing import ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from browser_use.browser.events import TabCreatedEvent
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
|
||||
class PopupsWatchdog(BaseWatchdog):
|
||||
"""Handles JavaScript dialogs (alert, confirm, prompt) by automatically accepting them immediately."""
|
||||
|
||||
# Events this watchdog listens to and emits
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [TabCreatedEvent]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = []
|
||||
|
||||
# Track which targets have dialog handlers registered
|
||||
_dialog_listeners_registered: set[str] = PrivateAttr(default_factory=set)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.logger.debug(f'🚀 PopupsWatchdog initialized with browser_session={self.browser_session}, ID={id(self)}')
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
"""Set up JavaScript dialog handling when a new tab is created."""
|
||||
target_id = event.target_id
|
||||
self.logger.debug(f'🎯 PopupsWatchdog received TabCreatedEvent for target {target_id}')
|
||||
|
||||
# Skip if we've already registered for this target
|
||||
if target_id in self._dialog_listeners_registered:
|
||||
self.logger.debug(f'Already registered dialog handlers for target {target_id}')
|
||||
return
|
||||
|
||||
self.logger.debug(f'📌 Starting dialog handler setup for target {target_id}')
|
||||
try:
|
||||
# Get all CDP sessions for this target and any child frames
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(
|
||||
target_id, focus=False
|
||||
) # don't auto-focus new tabs! sometimes we need to open tabs in background
|
||||
|
||||
# CRITICAL: Enable Page domain to receive dialog events
|
||||
try:
|
||||
await cdp_session.cdp_client.send.Page.enable(session_id=cdp_session.session_id)
|
||||
self.logger.debug(f'✅ Enabled Page domain for session {cdp_session.session_id[-8:]}')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to enable Page domain: {e}')
|
||||
|
||||
# Also register for the root CDP client to catch dialogs from any frame
|
||||
if self.browser_session._cdp_client_root:
|
||||
self.logger.debug('📌 Also registering handler on root CDP client')
|
||||
try:
|
||||
# Enable Page domain on root client too
|
||||
await self.browser_session._cdp_client_root.send.Page.enable()
|
||||
self.logger.debug('✅ Enabled Page domain on root CDP client')
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to enable Page domain on root: {e}')
|
||||
|
||||
# Set up async handler for JavaScript dialogs - accept immediately without event dispatch
|
||||
async def handle_dialog(event_data, session_id: str | None = None):
|
||||
"""Handle JavaScript dialog events - accept immediately."""
|
||||
try:
|
||||
dialog_type = event_data.get('type', 'alert')
|
||||
message = event_data.get('message', '')
|
||||
|
||||
# Store the popup message in browser session for inclusion in browser state
|
||||
if message:
|
||||
formatted_message = f'[{dialog_type}] {message}'
|
||||
self.browser_session._closed_popup_messages.append(formatted_message)
|
||||
self.logger.debug(f'📝 Stored popup message: {formatted_message[:100]}')
|
||||
|
||||
# Choose action based on dialog type:
|
||||
# - alert: accept=true (click OK to dismiss)
|
||||
# - confirm: accept=true (click OK to proceed - safer for automation)
|
||||
# - prompt: accept=false (click Cancel since we can't provide input)
|
||||
# - beforeunload: accept=true (allow navigation)
|
||||
should_accept = dialog_type in ('alert', 'confirm', 'beforeunload')
|
||||
|
||||
action_str = 'accepting (OK)' if should_accept else 'dismissing (Cancel)'
|
||||
self.logger.info(f"🔔 JavaScript {dialog_type} dialog: '{message[:100]}' - {action_str}...")
|
||||
|
||||
dismissed = False
|
||||
|
||||
# Approach 1: Use the session that detected the dialog (most reliable)
|
||||
if self.browser_session._cdp_client_root and session_id:
|
||||
try:
|
||||
self.logger.debug(f'🔄 Approach 1: Using detecting session {session_id[-8:]}')
|
||||
await asyncio.wait_for(
|
||||
self.browser_session._cdp_client_root.send.Page.handleJavaScriptDialog(
|
||||
params={'accept': should_accept},
|
||||
session_id=session_id,
|
||||
),
|
||||
timeout=0.5,
|
||||
)
|
||||
dismissed = True
|
||||
self.logger.info('✅ Dialog handled successfully via detecting session')
|
||||
except (TimeoutError, Exception) as e:
|
||||
self.logger.debug(f'Approach 1 failed: {type(e).__name__}')
|
||||
|
||||
# Approach 2: Try with current agent focus session
|
||||
if not dismissed and self.browser_session._cdp_client_root and self.browser_session.agent_focus_target_id:
|
||||
try:
|
||||
# Use public API with focus=False to avoid changing focus during popup dismissal
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(
|
||||
self.browser_session.agent_focus_target_id, focus=False
|
||||
)
|
||||
self.logger.debug(f'🔄 Approach 2: Using agent focus session {cdp_session.session_id[-8:]}')
|
||||
await asyncio.wait_for(
|
||||
self.browser_session._cdp_client_root.send.Page.handleJavaScriptDialog(
|
||||
params={'accept': should_accept},
|
||||
session_id=cdp_session.session_id,
|
||||
),
|
||||
timeout=0.5,
|
||||
)
|
||||
dismissed = True
|
||||
self.logger.info('✅ Dialog handled successfully via agent focus session')
|
||||
except (TimeoutError, Exception) as e:
|
||||
self.logger.debug(f'Approach 2 failed: {type(e).__name__}')
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'❌ Critical error in dialog handler: {type(e).__name__}: {e}')
|
||||
|
||||
# Register handler on the specific session
|
||||
cdp_session.cdp_client.register.Page.javascriptDialogOpening(handle_dialog) # type: ignore[arg-type]
|
||||
self.logger.debug(
|
||||
f'Successfully registered Page.javascriptDialogOpening handler for session {cdp_session.session_id}'
|
||||
)
|
||||
|
||||
# Also register on root CDP client to catch dialogs from any frame
|
||||
if hasattr(self.browser_session._cdp_client_root, 'register'):
|
||||
try:
|
||||
self.browser_session._cdp_client_root.register.Page.javascriptDialogOpening(handle_dialog) # type: ignore[arg-type]
|
||||
self.logger.debug('Successfully registered dialog handler on root CDP client for all frames')
|
||||
except Exception as root_error:
|
||||
self.logger.warning(f'Failed to register on root CDP client: {root_error}')
|
||||
|
||||
# Mark this target as having dialog handling set up
|
||||
self._dialog_listeners_registered.add(target_id)
|
||||
|
||||
self.logger.debug(f'Set up JavaScript dialog handling for tab {target_id}')
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Failed to set up popup handling for tab {target_id}: {e}')
|
||||
176
.agent/vendor/browser_use/browser_use/browser/watchdogs/recording_watchdog.py
vendored
Normal file
176
.agent/vendor/browser_use/browser_use/browser/watchdogs/recording_watchdog.py
vendored
Normal file
@@ -0,0 +1,176 @@
|
||||
"""Recording Watchdog for Browser Use Sessions."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.page.events import ScreencastFrameEvent
|
||||
from pydantic import PrivateAttr
|
||||
from uuid_extensions import uuid7str
|
||||
|
||||
from browser_use.browser.events import AgentFocusChangedEvent, BrowserConnectedEvent, BrowserStopEvent
|
||||
from browser_use.browser.profile import ViewportSize
|
||||
from browser_use.browser.video_recorder import VideoRecorderService
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.utils import create_task_with_error_handling
|
||||
|
||||
|
||||
class RecordingWatchdog(BaseWatchdog):
|
||||
"""
|
||||
Manages video recording of a browser session using CDP screencasting.
|
||||
"""
|
||||
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [BrowserConnectedEvent, BrowserStopEvent, AgentFocusChangedEvent]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = []
|
||||
|
||||
_recorder: VideoRecorderService | None = PrivateAttr(default=None)
|
||||
_current_session_id: str | None = PrivateAttr(default=None)
|
||||
_screencast_params: dict[str, Any] | None = PrivateAttr(default=None)
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""
|
||||
Starts video recording if it is configured in the browser profile.
|
||||
"""
|
||||
profile = self.browser_session.browser_profile
|
||||
if not profile.record_video_dir:
|
||||
return
|
||||
|
||||
# Dynamically determine video size
|
||||
size = profile.record_video_size
|
||||
if not size:
|
||||
self.logger.debug('record_video_size not specified, detecting viewport size...')
|
||||
size = await self._get_current_viewport_size()
|
||||
|
||||
if not size:
|
||||
self.logger.warning('Cannot start video recording: viewport size could not be determined.')
|
||||
return
|
||||
|
||||
video_format = getattr(profile, 'record_video_format', 'mp4').strip('.')
|
||||
output_path = Path(profile.record_video_dir) / f'{uuid7str()}.{video_format}'
|
||||
|
||||
self.logger.debug(f'Initializing video recorder for format: {video_format}')
|
||||
self._recorder = VideoRecorderService(output_path=output_path, size=size, framerate=profile.record_video_framerate)
|
||||
self._recorder.start()
|
||||
|
||||
if not self._recorder._is_active:
|
||||
self._recorder = None
|
||||
return
|
||||
|
||||
self.browser_session.cdp_client.register.Page.screencastFrame(self.on_screencastFrame)
|
||||
|
||||
self._screencast_params = {
|
||||
'format': 'png',
|
||||
'quality': 90,
|
||||
'maxWidth': size['width'],
|
||||
'maxHeight': size['height'],
|
||||
'everyNthFrame': 1,
|
||||
}
|
||||
|
||||
await self._start_screencast()
|
||||
|
||||
async def on_AgentFocusChangedEvent(self, event: AgentFocusChangedEvent) -> None:
|
||||
"""
|
||||
Switches video recording to the new tab.
|
||||
"""
|
||||
if self._recorder:
|
||||
self.logger.debug(f'Agent focus changed to {event.target_id}, switching screencast...')
|
||||
await self._start_screencast()
|
||||
|
||||
async def _start_screencast(self) -> None:
|
||||
"""Starts screencast on the currently focused tab."""
|
||||
if not self._recorder or not self._screencast_params:
|
||||
return
|
||||
|
||||
try:
|
||||
# Get the current session (for the focused target)
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
||||
|
||||
# If we are already recording this session, do nothing
|
||||
if self._current_session_id == cdp_session.session_id:
|
||||
return
|
||||
|
||||
# Stop recording on the previous session
|
||||
if self._current_session_id:
|
||||
try:
|
||||
# Use the root client to stop screencast on the specific session
|
||||
await self.browser_session.cdp_client.send.Page.stopScreencast(session_id=self._current_session_id)
|
||||
except Exception as e:
|
||||
# It's possible the session is already closed
|
||||
self.logger.debug(f'Failed to stop screencast on old session {self._current_session_id}: {e}')
|
||||
|
||||
self._current_session_id = cdp_session.session_id
|
||||
|
||||
# Start recording on the new session
|
||||
await cdp_session.cdp_client.send.Page.startScreencast(
|
||||
params=self._screencast_params, # type: ignore
|
||||
session_id=cdp_session.session_id,
|
||||
)
|
||||
self.logger.info(f'📹 Started/Switched video recording to target {cdp_session.target_id}')
|
||||
except Exception as e:
|
||||
self.logger.error(f'Failed to switch screencast via CDP: {e}')
|
||||
# If we fail to start on the new tab, we reset current session id
|
||||
self._current_session_id = None
|
||||
|
||||
async def _get_current_viewport_size(self) -> ViewportSize | None:
|
||||
"""Gets the current viewport size directly from the browser via CDP."""
|
||||
try:
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session()
|
||||
metrics = await cdp_session.cdp_client.send.Page.getLayoutMetrics(session_id=cdp_session.session_id)
|
||||
|
||||
# Use cssVisualViewport for the most accurate representation of the visible area
|
||||
viewport = metrics.get('cssVisualViewport', {})
|
||||
width = viewport.get('clientWidth')
|
||||
height = viewport.get('clientHeight')
|
||||
|
||||
if width and height:
|
||||
self.logger.debug(f'Detected viewport size: {width}x{height}')
|
||||
return ViewportSize(width=int(width), height=int(height))
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Failed to get viewport size from browser: {e}')
|
||||
|
||||
return None
|
||||
|
||||
def on_screencastFrame(self, event: ScreencastFrameEvent, session_id: str | None) -> None:
|
||||
"""
|
||||
Synchronous handler for incoming screencast frames.
|
||||
"""
|
||||
# Only process frames from the current session we intend to record
|
||||
# This handles race conditions where old session might still send frames before stop completes
|
||||
if self._current_session_id and session_id != self._current_session_id:
|
||||
return
|
||||
|
||||
if not self._recorder:
|
||||
return
|
||||
self._recorder.add_frame(event['data'])
|
||||
create_task_with_error_handling(
|
||||
self._ack_screencast_frame(event, session_id),
|
||||
name='ack_screencast_frame',
|
||||
logger_instance=self.logger,
|
||||
suppress_exceptions=True,
|
||||
)
|
||||
|
||||
async def _ack_screencast_frame(self, event: ScreencastFrameEvent, session_id: str | None) -> None:
|
||||
"""
|
||||
Asynchronously acknowledges a screencast frame.
|
||||
"""
|
||||
try:
|
||||
await self.browser_session.cdp_client.send.Page.screencastFrameAck(
|
||||
params={'sessionId': event['sessionId']}, session_id=session_id
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.debug(f'Failed to acknowledge screencast frame: {e}')
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
"""
|
||||
Stops the video recording and finalizes the video file.
|
||||
"""
|
||||
if self._recorder:
|
||||
recorder = self._recorder
|
||||
self._recorder = None
|
||||
self._current_session_id = None
|
||||
self._screencast_params = None
|
||||
|
||||
self.logger.debug('Stopping video recording and saving file...')
|
||||
loop = asyncio.get_event_loop()
|
||||
await loop.run_in_executor(None, recorder.stop_and_save)
|
||||
88
.agent/vendor/browser_use/browser_use/browser/watchdogs/screenshot_watchdog.py
vendored
Normal file
88
.agent/vendor/browser_use/browser_use/browser/watchdogs/screenshot_watchdog.py
vendored
Normal file
@@ -0,0 +1,88 @@
|
||||
"""Screenshot watchdog for handling screenshot requests using CDP."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.page import CaptureScreenshotParameters
|
||||
|
||||
from browser_use.browser.events import ScreenshotEvent
|
||||
from browser_use.browser.views import BrowserError
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.observability import observe_debug
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
class ScreenshotWatchdog(BaseWatchdog):
|
||||
"""Handles screenshot requests using CDP."""
|
||||
|
||||
# Events this watchdog listens to
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent[Any]]]] = [ScreenshotEvent]
|
||||
|
||||
# Events this watchdog emits
|
||||
EMITS: ClassVar[list[type[BaseEvent[Any]]]] = []
|
||||
|
||||
@observe_debug(ignore_input=True, ignore_output=True, name='screenshot_event_handler')
|
||||
async def on_ScreenshotEvent(self, event: ScreenshotEvent) -> str:
|
||||
"""Handle screenshot request using CDP.
|
||||
|
||||
Args:
|
||||
event: ScreenshotEvent with optional full_page and clip parameters
|
||||
|
||||
Returns:
|
||||
Dict with 'screenshot' key containing base64-encoded screenshot or None
|
||||
"""
|
||||
self.logger.debug('[ScreenshotWatchdog] Handler START - on_ScreenshotEvent called')
|
||||
try:
|
||||
# Validate focused target is a top-level page (not iframe/worker)
|
||||
# CDP Page.captureScreenshot only works on page/tab targets
|
||||
focused_target = self.browser_session.get_focused_target()
|
||||
|
||||
if focused_target and focused_target.target_type in ('page', 'tab'):
|
||||
target_id = focused_target.target_id
|
||||
else:
|
||||
# Focused target is iframe/worker/missing - fall back to any page target
|
||||
target_type_str = focused_target.target_type if focused_target else 'None'
|
||||
self.logger.warning(f'[ScreenshotWatchdog] Focused target is {target_type_str}, falling back to page target')
|
||||
page_targets = self.browser_session.get_page_targets()
|
||||
if not page_targets:
|
||||
raise BrowserError('[ScreenshotWatchdog] No page targets available for screenshot')
|
||||
target_id = page_targets[-1].target_id
|
||||
|
||||
cdp_session = await self.browser_session.get_or_create_cdp_session(target_id, focus=True)
|
||||
|
||||
# Remove highlights BEFORE taking the screenshot so they don't appear in the image.
|
||||
# Done here (not in finally) so CancelledError is never swallowed — any await in a
|
||||
# finally block can suppress external task cancellation.
|
||||
# remove_highlights() has its own asyncio.timeout(3.0) internally so it won't block.
|
||||
try:
|
||||
await self.browser_session.remove_highlights()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Prepare screenshot parameters
|
||||
params_dict: dict[str, Any] = {'format': 'png', 'captureBeyondViewport': event.full_page}
|
||||
if event.clip:
|
||||
params_dict['clip'] = {
|
||||
'x': event.clip['x'],
|
||||
'y': event.clip['y'],
|
||||
'width': event.clip['width'],
|
||||
'height': event.clip['height'],
|
||||
'scale': 1,
|
||||
}
|
||||
params = CaptureScreenshotParameters(**params_dict)
|
||||
|
||||
# Take screenshot using CDP
|
||||
self.logger.debug(f'[ScreenshotWatchdog] Taking screenshot with params: {params}')
|
||||
result = await cdp_session.cdp_client.send.Page.captureScreenshot(params=params, session_id=cdp_session.session_id)
|
||||
|
||||
# Return base64-encoded screenshot data
|
||||
if result and 'data' in result:
|
||||
self.logger.debug('[ScreenshotWatchdog] Screenshot captured successfully')
|
||||
return result['data']
|
||||
|
||||
raise BrowserError('[ScreenshotWatchdog] Screenshot result missing data')
|
||||
except Exception as e:
|
||||
self.logger.error(f'[ScreenshotWatchdog] Screenshot failed: {e}')
|
||||
raise
|
||||
278
.agent/vendor/browser_use/browser_use/browser/watchdogs/security_watchdog.py
vendored
Normal file
278
.agent/vendor/browser_use/browser_use/browser/watchdogs/security_watchdog.py
vendored
Normal file
@@ -0,0 +1,278 @@
|
||||
"""Security watchdog for enforcing URL access policies."""
|
||||
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserErrorEvent,
|
||||
NavigateToUrlEvent,
|
||||
NavigationCompleteEvent,
|
||||
TabCreatedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
# Track if we've shown the glob warning
|
||||
_GLOB_WARNING_SHOWN = False
|
||||
|
||||
|
||||
class SecurityWatchdog(BaseWatchdog):
|
||||
"""Monitors and enforces security policies for URL access."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
NavigateToUrlEvent,
|
||||
NavigationCompleteEvent,
|
||||
TabCreatedEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserErrorEvent,
|
||||
]
|
||||
|
||||
async def on_NavigateToUrlEvent(self, event: NavigateToUrlEvent) -> None:
|
||||
"""Check if navigation URL is allowed before navigation starts."""
|
||||
# Security check BEFORE navigation
|
||||
if not self._is_url_allowed(event.url):
|
||||
self.logger.warning(f'⛔️ Blocking navigation to disallowed URL: {event.url}')
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='NavigationBlocked',
|
||||
message=f'Navigation blocked to disallowed URL: {event.url}',
|
||||
details={'url': event.url, 'reason': 'not_in_allowed_domains'},
|
||||
)
|
||||
)
|
||||
# Stop event propagation by raising exception
|
||||
raise ValueError(f'Navigation to {event.url} blocked by security policy')
|
||||
|
||||
async def on_NavigationCompleteEvent(self, event: NavigationCompleteEvent) -> None:
|
||||
"""Check if navigated URL is allowed (catches redirects to blocked domains)."""
|
||||
# Check if the navigated URL is allowed (in case of redirects)
|
||||
if not self._is_url_allowed(event.url):
|
||||
self.logger.warning(f'⛔️ Navigation to non-allowed URL detected: {event.url}')
|
||||
|
||||
# Dispatch browser error
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='NavigationBlocked',
|
||||
message=f'Navigation blocked to non-allowed URL: {event.url} - redirecting to about:blank',
|
||||
details={'url': event.url, 'target_id': event.target_id},
|
||||
)
|
||||
)
|
||||
# Navigate to about:blank to keep session alive
|
||||
# Agent will see the error and can continue with other tasks
|
||||
try:
|
||||
session = await self.browser_session.get_or_create_cdp_session(target_id=event.target_id)
|
||||
await session.cdp_client.send.Page.navigate(params={'url': 'about:blank'}, session_id=session.session_id)
|
||||
self.logger.info(f'⛔️ Navigated to about:blank after blocked URL: {event.url}')
|
||||
except Exception as e:
|
||||
self.logger.error(f'⛔️ Failed to navigate to about:blank: {type(e).__name__} {e}')
|
||||
|
||||
async def on_TabCreatedEvent(self, event: TabCreatedEvent) -> None:
|
||||
"""Check if new tab URL is allowed."""
|
||||
if not self._is_url_allowed(event.url):
|
||||
self.logger.warning(f'⛔️ New tab created with disallowed URL: {event.url}')
|
||||
|
||||
# Dispatch error and try to close the tab
|
||||
self.event_bus.dispatch(
|
||||
BrowserErrorEvent(
|
||||
error_type='TabCreationBlocked',
|
||||
message=f'Tab created with non-allowed URL: {event.url}',
|
||||
details={'url': event.url, 'target_id': event.target_id},
|
||||
)
|
||||
)
|
||||
|
||||
# Try to close the offending tab
|
||||
try:
|
||||
await self.browser_session._cdp_close_page(event.target_id)
|
||||
self.logger.info(f'⛔️ Closed new tab with non-allowed URL: {event.url}')
|
||||
except Exception as e:
|
||||
self.logger.error(f'⛔️ Failed to close new tab with non-allowed URL: {type(e).__name__} {e}')
|
||||
|
||||
def _is_root_domain(self, domain: str) -> bool:
|
||||
"""Check if a domain is a root domain (no subdomain present).
|
||||
|
||||
Simple heuristic: only add www for domains with exactly 1 dot (domain.tld).
|
||||
For complex cases like country TLDs or subdomains, users should configure explicitly.
|
||||
|
||||
Args:
|
||||
domain: The domain to check
|
||||
|
||||
Returns:
|
||||
True if it's a simple root domain, False otherwise
|
||||
"""
|
||||
# Skip if it contains wildcards or protocol
|
||||
if '*' in domain or '://' in domain:
|
||||
return False
|
||||
|
||||
return domain.count('.') == 1
|
||||
|
||||
def _log_glob_warning(self) -> None:
|
||||
"""Log a warning about glob patterns in allowed_domains."""
|
||||
global _GLOB_WARNING_SHOWN
|
||||
if not _GLOB_WARNING_SHOWN:
|
||||
_GLOB_WARNING_SHOWN = True
|
||||
self.logger.warning(
|
||||
'⚠️ Using glob patterns in allowed_domains. '
|
||||
'Note: Patterns like "*.example.com" will match both subdomains AND the main domain.'
|
||||
)
|
||||
|
||||
def _get_domain_variants(self, host: str) -> tuple[str, str]:
|
||||
"""Get both variants of a domain (with and without www prefix).
|
||||
|
||||
Args:
|
||||
host: The hostname to process
|
||||
|
||||
Returns:
|
||||
Tuple of (original_host, variant_host)
|
||||
- If host starts with www., variant is without www.
|
||||
- Otherwise, variant is with www. prefix
|
||||
"""
|
||||
if host.startswith('www.'):
|
||||
return (host, host[4:]) # ('www.example.com', 'example.com')
|
||||
else:
|
||||
return (host, f'www.{host}') # ('example.com', 'www.example.com')
|
||||
|
||||
def _is_ip_address(self, host: str) -> bool:
|
||||
"""Check if a hostname is an IP address (IPv4 or IPv6).
|
||||
|
||||
Args:
|
||||
host: The hostname to check
|
||||
|
||||
Returns:
|
||||
True if the host is an IP address, False otherwise
|
||||
"""
|
||||
import ipaddress
|
||||
|
||||
try:
|
||||
# Try to parse as IP address (handles both IPv4 and IPv6)
|
||||
ipaddress.ip_address(host)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _is_url_allowed(self, url: str) -> bool:
|
||||
"""Check if a URL is allowed based on the allowed_domains configuration.
|
||||
|
||||
Args:
|
||||
url: The URL to check
|
||||
|
||||
Returns:
|
||||
True if the URL is allowed, False otherwise
|
||||
"""
|
||||
|
||||
# Always allow internal browser targets (before any other checks)
|
||||
if url in ['about:blank', 'chrome://new-tab-page/', 'chrome://new-tab-page', 'chrome://newtab/']:
|
||||
return True
|
||||
|
||||
# Parse the URL to extract components
|
||||
from urllib.parse import urlparse
|
||||
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except Exception:
|
||||
# Invalid URL
|
||||
return False
|
||||
|
||||
# Allow data: and blob: URLs (they don't have hostnames)
|
||||
if parsed.scheme in ['data', 'blob']:
|
||||
return True
|
||||
|
||||
# Get the actual host (domain)
|
||||
host = parsed.hostname
|
||||
if not host:
|
||||
return False
|
||||
|
||||
# Check if IP addresses should be blocked (before domain checks)
|
||||
if self.browser_session.browser_profile.block_ip_addresses:
|
||||
if self._is_ip_address(host):
|
||||
return False
|
||||
|
||||
# If no allowed_domains specified, allow all URLs
|
||||
if (
|
||||
not self.browser_session.browser_profile.allowed_domains
|
||||
and not self.browser_session.browser_profile.prohibited_domains
|
||||
):
|
||||
return True
|
||||
|
||||
# Check allowed domains (fast path for sets, slow path for lists with patterns)
|
||||
if self.browser_session.browser_profile.allowed_domains:
|
||||
allowed_domains = self.browser_session.browser_profile.allowed_domains
|
||||
|
||||
if isinstance(allowed_domains, set):
|
||||
# Fast path: O(1) exact hostname match - check both www and non-www variants
|
||||
host_variant, host_alt = self._get_domain_variants(host)
|
||||
return host_variant in allowed_domains or host_alt in allowed_domains
|
||||
else:
|
||||
# Slow path: O(n) pattern matching for lists
|
||||
for pattern in allowed_domains:
|
||||
if self._is_url_match(url, host, parsed.scheme, pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
# Check prohibited domains (fast path for sets, slow path for lists with patterns)
|
||||
if self.browser_session.browser_profile.prohibited_domains:
|
||||
prohibited_domains = self.browser_session.browser_profile.prohibited_domains
|
||||
|
||||
if isinstance(prohibited_domains, set):
|
||||
# Fast path: O(1) exact hostname match - check both www and non-www variants
|
||||
host_variant, host_alt = self._get_domain_variants(host)
|
||||
return host_variant not in prohibited_domains and host_alt not in prohibited_domains
|
||||
else:
|
||||
# Slow path: O(n) pattern matching for lists
|
||||
for pattern in prohibited_domains:
|
||||
if self._is_url_match(url, host, parsed.scheme, pattern):
|
||||
return False
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
def _is_url_match(self, url: str, host: str, scheme: str, pattern: str) -> bool:
|
||||
"""Check if a URL matches a pattern."""
|
||||
|
||||
# Full URL for matching (scheme + host)
|
||||
full_url_pattern = f'{scheme}://{host}'
|
||||
|
||||
# Handle glob patterns
|
||||
if '*' in pattern:
|
||||
self._log_glob_warning()
|
||||
import fnmatch
|
||||
|
||||
# Check if pattern matches the host
|
||||
if pattern.startswith('*.'):
|
||||
# Pattern like *.example.com should match subdomains and main domain
|
||||
domain_part = pattern[2:] # Remove *.
|
||||
if host == domain_part or host.endswith('.' + domain_part):
|
||||
# Only match http/https URLs for domain-only patterns
|
||||
if scheme in ['http', 'https']:
|
||||
return True
|
||||
elif pattern.endswith('/*'):
|
||||
# Pattern like brave://* or http*://example.com/*
|
||||
if fnmatch.fnmatch(url, pattern):
|
||||
return True
|
||||
else:
|
||||
# Use fnmatch for other glob patterns
|
||||
if fnmatch.fnmatch(
|
||||
full_url_pattern if '://' in pattern else host,
|
||||
pattern,
|
||||
):
|
||||
return True
|
||||
else:
|
||||
# Exact match
|
||||
if '://' in pattern:
|
||||
# Full URL pattern
|
||||
if url.startswith(pattern):
|
||||
return True
|
||||
else:
|
||||
# Domain-only pattern (case-insensitive comparison)
|
||||
if host.lower() == pattern.lower():
|
||||
return True
|
||||
# If pattern is a root domain, also check www subdomain
|
||||
if self._is_root_domain(pattern) and host.lower() == f'www.{pattern.lower()}':
|
||||
return True
|
||||
|
||||
return False
|
||||
373
.agent/vendor/browser_use/browser_use/browser/watchdogs/storage_state_watchdog.py
vendored
Normal file
373
.agent/vendor/browser_use/browser_use/browser/watchdogs/storage_state_watchdog.py
vendored
Normal file
@@ -0,0 +1,373 @@
|
||||
"""Storage state watchdog for managing browser cookies and storage persistence."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from bubus import BaseEvent
|
||||
from cdp_use.cdp.network import Cookie
|
||||
from pydantic import Field, PrivateAttr
|
||||
|
||||
from browser_use.browser.events import (
|
||||
BrowserConnectedEvent,
|
||||
BrowserStopEvent,
|
||||
LoadStorageStateEvent,
|
||||
SaveStorageStateEvent,
|
||||
StorageStateLoadedEvent,
|
||||
StorageStateSavedEvent,
|
||||
)
|
||||
from browser_use.browser.watchdog_base import BaseWatchdog
|
||||
from browser_use.utils import create_task_with_error_handling
|
||||
|
||||
|
||||
class StorageStateWatchdog(BaseWatchdog):
|
||||
"""Monitors and persists browser storage state including cookies and localStorage."""
|
||||
|
||||
# Event contracts
|
||||
LISTENS_TO: ClassVar[list[type[BaseEvent]]] = [
|
||||
BrowserConnectedEvent,
|
||||
BrowserStopEvent,
|
||||
SaveStorageStateEvent,
|
||||
LoadStorageStateEvent,
|
||||
]
|
||||
EMITS: ClassVar[list[type[BaseEvent]]] = [
|
||||
StorageStateSavedEvent,
|
||||
StorageStateLoadedEvent,
|
||||
]
|
||||
|
||||
# Configuration
|
||||
auto_save_interval: float = Field(default=30.0) # Auto-save every 30 seconds
|
||||
save_on_change: bool = Field(default=True) # Save immediately when cookies change
|
||||
|
||||
# Private state
|
||||
_monitoring_task: asyncio.Task | None = PrivateAttr(default=None)
|
||||
_last_cookie_state: list[dict] = PrivateAttr(default_factory=list)
|
||||
_save_lock: asyncio.Lock = PrivateAttr(default_factory=asyncio.Lock)
|
||||
|
||||
async def on_BrowserConnectedEvent(self, event: BrowserConnectedEvent) -> None:
|
||||
"""Start monitoring when browser starts."""
|
||||
self.logger.debug('[StorageStateWatchdog] 🍪 Initializing auth/cookies sync <-> with storage_state.json file')
|
||||
|
||||
# Start monitoring
|
||||
await self._start_monitoring()
|
||||
|
||||
# Automatically load storage state after browser start
|
||||
await self.event_bus.dispatch(LoadStorageStateEvent())
|
||||
|
||||
async def on_BrowserStopEvent(self, event: BrowserStopEvent) -> None:
|
||||
"""Stop monitoring when browser stops."""
|
||||
self.logger.debug('[StorageStateWatchdog] Stopping storage_state monitoring')
|
||||
await self._stop_monitoring()
|
||||
|
||||
async def on_SaveStorageStateEvent(self, event: SaveStorageStateEvent) -> None:
|
||||
"""Handle storage state save request."""
|
||||
# Use provided path or fall back to profile default
|
||||
path = event.path
|
||||
if path is None:
|
||||
# Use profile default path if available
|
||||
if self.browser_session.browser_profile.storage_state:
|
||||
path = str(self.browser_session.browser_profile.storage_state)
|
||||
else:
|
||||
path = None # Skip saving if no path available
|
||||
await self._save_storage_state(path)
|
||||
|
||||
async def on_LoadStorageStateEvent(self, event: LoadStorageStateEvent) -> None:
|
||||
"""Handle storage state load request."""
|
||||
# Use provided path or fall back to profile default
|
||||
path = event.path
|
||||
if path is None:
|
||||
# Use profile default path if available
|
||||
if self.browser_session.browser_profile.storage_state:
|
||||
path = str(self.browser_session.browser_profile.storage_state)
|
||||
else:
|
||||
path = None # Skip loading if no path available
|
||||
await self._load_storage_state(path)
|
||||
|
||||
async def _start_monitoring(self) -> None:
|
||||
"""Start the monitoring task."""
|
||||
if self._monitoring_task and not self._monitoring_task.done():
|
||||
return
|
||||
|
||||
assert self.browser_session.cdp_client is not None
|
||||
|
||||
self._monitoring_task = create_task_with_error_handling(
|
||||
self._monitor_storage_changes(), name='monitor_storage_changes', logger_instance=self.logger, suppress_exceptions=True
|
||||
)
|
||||
# self.logger'[StorageStateWatchdog] Started storage monitoring task')
|
||||
|
||||
async def _stop_monitoring(self) -> None:
|
||||
"""Stop the monitoring task."""
|
||||
if self._monitoring_task and not self._monitoring_task.done():
|
||||
self._monitoring_task.cancel()
|
||||
try:
|
||||
await self._monitoring_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
# self.logger.debug('[StorageStateWatchdog] Stopped storage monitoring task')
|
||||
|
||||
async def _check_for_cookie_changes_cdp(self, event: dict) -> None:
|
||||
"""Check if a CDP network event indicates cookie changes.
|
||||
|
||||
This would be called by Network.responseReceivedExtraInfo events
|
||||
if we set up CDP event listeners.
|
||||
"""
|
||||
try:
|
||||
# Check for Set-Cookie headers in the response
|
||||
headers = event.get('headers', {})
|
||||
if 'set-cookie' in headers or 'Set-Cookie' in headers:
|
||||
self.logger.debug('[StorageStateWatchdog] Cookie change detected via CDP')
|
||||
|
||||
# If save on change is enabled, trigger save immediately
|
||||
if self.save_on_change:
|
||||
await self._save_storage_state()
|
||||
except Exception as e:
|
||||
self.logger.warning(f'[StorageStateWatchdog] Error checking for cookie changes: {e}')
|
||||
|
||||
async def _monitor_storage_changes(self) -> None:
|
||||
"""Periodically check for storage changes and auto-save."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(self.auto_save_interval)
|
||||
|
||||
# Check if cookies have changed
|
||||
if await self._have_cookies_changed():
|
||||
self.logger.debug('[StorageStateWatchdog] Detected changes to sync with storage_state.json')
|
||||
await self._save_storage_state()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Error in monitoring loop: {e}')
|
||||
|
||||
async def _have_cookies_changed(self) -> bool:
|
||||
"""Check if cookies have changed since last save."""
|
||||
if not self.browser_session.cdp_client:
|
||||
return False
|
||||
|
||||
try:
|
||||
# Get current cookies using CDP
|
||||
current_cookies = await self.browser_session._cdp_get_cookies()
|
||||
|
||||
# Convert to comparable format, using .get() for optional fields
|
||||
current_cookie_set = {
|
||||
(c.get('name', ''), c.get('domain', ''), c.get('path', '')): c.get('value', '') for c in current_cookies
|
||||
}
|
||||
|
||||
last_cookie_set = {
|
||||
(c.get('name', ''), c.get('domain', ''), c.get('path', '')): c.get('value', '') for c in self._last_cookie_state
|
||||
}
|
||||
|
||||
return current_cookie_set != last_cookie_set
|
||||
except Exception as e:
|
||||
self.logger.debug(f'[StorageStateWatchdog] Error comparing cookies: {e}')
|
||||
return False
|
||||
|
||||
async def _save_storage_state(self, path: str | None = None) -> None:
|
||||
"""Save browser storage state to file."""
|
||||
async with self._save_lock:
|
||||
# Check if CDP client is available
|
||||
assert await self.browser_session.get_or_create_cdp_session(target_id=None)
|
||||
|
||||
save_path = path or self.browser_session.browser_profile.storage_state
|
||||
if not save_path:
|
||||
return
|
||||
|
||||
# Skip saving if the storage state is already a dict (indicates it was loaded from memory)
|
||||
# We only save to file if it started as a file path
|
||||
if isinstance(save_path, dict):
|
||||
self.logger.debug('[StorageStateWatchdog] Storage state is already a dict, skipping file save')
|
||||
return
|
||||
|
||||
try:
|
||||
# Get current storage state using CDP
|
||||
storage_state = await self.browser_session._cdp_get_storage_state()
|
||||
|
||||
# Update our last known state
|
||||
self._last_cookie_state = storage_state.get('cookies', []).copy()
|
||||
|
||||
# Convert path to Path object
|
||||
json_path = Path(save_path).expanduser().resolve()
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Merge with existing state if file exists
|
||||
merged_state = storage_state
|
||||
if json_path.exists():
|
||||
try:
|
||||
existing_state = json.loads(json_path.read_text())
|
||||
merged_state = self._merge_storage_states(existing_state, dict(storage_state))
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to merge with existing state: {e}')
|
||||
|
||||
# Write atomically
|
||||
temp_path = json_path.with_suffix('.json.tmp')
|
||||
temp_path.write_text(json.dumps(merged_state, indent=4, ensure_ascii=False), encoding='utf-8')
|
||||
|
||||
# Backup existing file
|
||||
if json_path.exists():
|
||||
backup_path = json_path.with_suffix('.json.bak')
|
||||
json_path.replace(backup_path)
|
||||
|
||||
# Move temp to final
|
||||
temp_path.replace(json_path)
|
||||
|
||||
# Emit success event
|
||||
self.event_bus.dispatch(
|
||||
StorageStateSavedEvent(
|
||||
path=str(json_path),
|
||||
cookies_count=len(merged_state.get('cookies', [])),
|
||||
origins_count=len(merged_state.get('origins', [])),
|
||||
)
|
||||
)
|
||||
|
||||
self.logger.debug(
|
||||
f'[StorageStateWatchdog] Saved storage state to {json_path} '
|
||||
f'({len(merged_state.get("cookies", []))} cookies, '
|
||||
f'{len(merged_state.get("origins", []))} origins)'
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to save storage state: {e}')
|
||||
|
||||
async def _load_storage_state(self, path: str | None = None) -> None:
|
||||
"""Load browser storage state from file."""
|
||||
if not self.browser_session.cdp_client:
|
||||
self.logger.warning('[StorageStateWatchdog] No CDP client available for loading')
|
||||
return
|
||||
|
||||
load_path = path or self.browser_session.browser_profile.storage_state
|
||||
if not load_path or not os.path.exists(str(load_path)):
|
||||
return
|
||||
|
||||
try:
|
||||
# Read the storage state file asynchronously
|
||||
import anyio
|
||||
|
||||
content = await anyio.Path(str(load_path)).read_text()
|
||||
storage = json.loads(content)
|
||||
|
||||
# Apply cookies if present
|
||||
if 'cookies' in storage and storage['cookies']:
|
||||
# Playwright exports session cookies with expires=0/-1. CDP treats expires=0 as expired.
|
||||
# Normalize session cookies by omitting expires
|
||||
normalized_cookies: list[Cookie] = []
|
||||
for cookie in storage['cookies']:
|
||||
if not isinstance(cookie, dict):
|
||||
normalized_cookies.append(cookie) # type: ignore[arg-type]
|
||||
continue
|
||||
c = dict(cookie)
|
||||
expires = c.get('expires')
|
||||
if expires in (0, 0.0, -1, -1.0):
|
||||
c.pop('expires', None)
|
||||
normalized_cookies.append(Cookie(**c))
|
||||
|
||||
await self.browser_session._cdp_set_cookies(normalized_cookies)
|
||||
self._last_cookie_state = storage['cookies'].copy()
|
||||
self.logger.debug(f'[StorageStateWatchdog] Added {len(storage["cookies"])} cookies from storage state')
|
||||
|
||||
# Apply origins (localStorage/sessionStorage) if present
|
||||
if 'origins' in storage and storage['origins']:
|
||||
for origin in storage['origins']:
|
||||
origin_value = origin.get('origin')
|
||||
if not origin_value:
|
||||
continue
|
||||
|
||||
# Scope storage restoration to its origin to avoid cross-site pollution.
|
||||
if origin.get('localStorage'):
|
||||
lines = []
|
||||
for item in origin['localStorage']:
|
||||
lines.append(f'window.localStorage.setItem({json.dumps(item["name"])}, {json.dumps(item["value"])});')
|
||||
script = (
|
||||
'(function(){\n'
|
||||
f' if (window.location && window.location.origin !== {json.dumps(origin_value)}) return;\n'
|
||||
' try {\n'
|
||||
f' {" ".join(lines)}\n'
|
||||
' } catch (e) {}\n'
|
||||
'})();'
|
||||
)
|
||||
await self.browser_session._cdp_add_init_script(script)
|
||||
|
||||
if origin.get('sessionStorage'):
|
||||
lines = []
|
||||
for item in origin['sessionStorage']:
|
||||
lines.append(
|
||||
f'window.sessionStorage.setItem({json.dumps(item["name"])}, {json.dumps(item["value"])});'
|
||||
)
|
||||
script = (
|
||||
'(function(){\n'
|
||||
f' if (window.location && window.location.origin !== {json.dumps(origin_value)}) return;\n'
|
||||
' try {\n'
|
||||
f' {" ".join(lines)}\n'
|
||||
' } catch (e) {}\n'
|
||||
'})();'
|
||||
)
|
||||
await self.browser_session._cdp_add_init_script(script)
|
||||
self.logger.debug(
|
||||
f'[StorageStateWatchdog] Applied localStorage/sessionStorage from {len(storage["origins"])} origins'
|
||||
)
|
||||
|
||||
self.event_bus.dispatch(
|
||||
StorageStateLoadedEvent(
|
||||
path=str(load_path),
|
||||
cookies_count=len(storage.get('cookies', [])),
|
||||
origins_count=len(storage.get('origins', [])),
|
||||
)
|
||||
)
|
||||
|
||||
self.logger.debug(f'[StorageStateWatchdog] Loaded storage state from: {load_path}')
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to load storage state: {e}')
|
||||
|
||||
@staticmethod
|
||||
def _merge_storage_states(existing: dict[str, Any], new: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Merge two storage states, with new values taking precedence."""
|
||||
merged = existing.copy()
|
||||
|
||||
# Merge cookies
|
||||
existing_cookies = {(c['name'], c['domain'], c['path']): c for c in existing.get('cookies', [])}
|
||||
|
||||
for cookie in new.get('cookies', []):
|
||||
key = (cookie['name'], cookie['domain'], cookie['path'])
|
||||
existing_cookies[key] = cookie
|
||||
|
||||
merged['cookies'] = list(existing_cookies.values())
|
||||
|
||||
# Merge origins
|
||||
existing_origins = {origin['origin']: origin for origin in existing.get('origins', [])}
|
||||
|
||||
for origin in new.get('origins', []):
|
||||
existing_origins[origin['origin']] = origin
|
||||
|
||||
merged['origins'] = list(existing_origins.values())
|
||||
|
||||
return merged
|
||||
|
||||
async def get_current_cookies(self) -> list[dict[str, Any]]:
|
||||
"""Get current cookies using CDP."""
|
||||
if not self.browser_session.cdp_client:
|
||||
return []
|
||||
|
||||
try:
|
||||
cookies = await self.browser_session._cdp_get_cookies()
|
||||
# Cookie is a TypedDict, cast to dict for compatibility
|
||||
return [dict(cookie) for cookie in cookies]
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to get cookies: {e}')
|
||||
return []
|
||||
|
||||
async def add_cookies(self, cookies: list[dict[str, Any]]) -> None:
|
||||
"""Add cookies using CDP."""
|
||||
if not self.browser_session.cdp_client:
|
||||
self.logger.warning('[StorageStateWatchdog] No CDP client available for adding cookies')
|
||||
return
|
||||
|
||||
try:
|
||||
# Convert dicts to Cookie objects
|
||||
cookie_objects = [Cookie(**cookie_dict) if isinstance(cookie_dict, dict) else cookie_dict for cookie_dict in cookies]
|
||||
# Set cookies using CDP
|
||||
await self.browser_session._cdp_set_cookies(cookie_objects)
|
||||
self.logger.debug(f'[StorageStateWatchdog] Added {len(cookies)} cookies')
|
||||
except Exception as e:
|
||||
self.logger.error(f'[StorageStateWatchdog] Failed to add cookies: {e}')
|
||||
2362
.agent/vendor/browser_use/browser_use/cli.py
vendored
Normal file
2362
.agent/vendor/browser_use/browser_use/cli.py
vendored
Normal file
File diff suppressed because it is too large
Load Diff
525
.agent/vendor/browser_use/browser_use/config.py
vendored
Normal file
525
.agent/vendor/browser_use/browser_use/config.py
vendored
Normal file
@@ -0,0 +1,525 @@
|
||||
"""Configuration system for browser-use with automatic migration support."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import psutil
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@cache
|
||||
def is_running_in_docker() -> bool:
|
||||
"""Detect if we are running in a docker container, for the purpose of optimizing chrome launch flags (dev shm usage, gpu settings, etc.)"""
|
||||
try:
|
||||
if Path('/.dockerenv').exists() or 'docker' in Path('/proc/1/cgroup').read_text().lower():
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
# if init proc (PID 1) looks like uvicorn/python/uv/etc. then we're in Docker
|
||||
# if init proc (PID 1) looks like bash/systemd/init/etc. then we're probably NOT in Docker
|
||||
init_cmd = ' '.join(psutil.Process(1).cmdline())
|
||||
if ('py' in init_cmd) or ('uv' in init_cmd) or ('app' in init_cmd):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
# if less than 10 total running procs, then we're almost certainly in a container
|
||||
if len(psutil.pids()) < 10:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
class OldConfig:
|
||||
"""Original lazy-loading configuration class for environment variables."""
|
||||
|
||||
# Cache for directory creation tracking
|
||||
_dirs_created = False
|
||||
|
||||
@property
|
||||
def BROWSER_USE_LOGGING_LEVEL(self) -> str:
|
||||
return os.getenv('BROWSER_USE_LOGGING_LEVEL', 'info').lower()
|
||||
|
||||
@property
|
||||
def ANONYMIZED_TELEMETRY(self) -> bool:
|
||||
return os.getenv('ANONYMIZED_TELEMETRY', 'true').lower()[:1] in 'ty1'
|
||||
|
||||
@property
|
||||
def BROWSER_USE_CLOUD_SYNC(self) -> bool:
|
||||
return os.getenv('BROWSER_USE_CLOUD_SYNC', str(self.ANONYMIZED_TELEMETRY)).lower()[:1] in 'ty1'
|
||||
|
||||
@property
|
||||
def BROWSER_USE_CLOUD_API_URL(self) -> str:
|
||||
url = os.getenv('BROWSER_USE_CLOUD_API_URL', 'https://api.browser-use.com')
|
||||
assert '://' in url, 'BROWSER_USE_CLOUD_API_URL must be a valid URL'
|
||||
return url
|
||||
|
||||
@property
|
||||
def BROWSER_USE_CLOUD_UI_URL(self) -> str:
|
||||
url = os.getenv('BROWSER_USE_CLOUD_UI_URL', '')
|
||||
# Allow empty string as default, only validate if set
|
||||
if url and '://' not in url:
|
||||
raise AssertionError('BROWSER_USE_CLOUD_UI_URL must be a valid URL if set')
|
||||
return url
|
||||
|
||||
@property
|
||||
def BROWSER_USE_MODEL_PRICING_URL(self) -> str:
|
||||
url = os.getenv('BROWSER_USE_MODEL_PRICING_URL', '')
|
||||
if url and '://' not in url:
|
||||
raise AssertionError('BROWSER_USE_MODEL_PRICING_URL must be a valid URL if set')
|
||||
return url
|
||||
|
||||
# Path configuration
|
||||
@property
|
||||
def XDG_CACHE_HOME(self) -> Path:
|
||||
return Path(os.getenv('XDG_CACHE_HOME', '~/.cache')).expanduser().resolve()
|
||||
|
||||
@property
|
||||
def XDG_CONFIG_HOME(self) -> Path:
|
||||
return Path(os.getenv('XDG_CONFIG_HOME', '~/.config')).expanduser().resolve()
|
||||
|
||||
@property
|
||||
def BROWSER_USE_CONFIG_DIR(self) -> Path:
|
||||
path = Path(os.getenv('BROWSER_USE_CONFIG_DIR', str(self.XDG_CONFIG_HOME / 'browseruse'))).expanduser().resolve()
|
||||
self._ensure_dirs()
|
||||
return path
|
||||
|
||||
@property
|
||||
def BROWSER_USE_CONFIG_FILE(self) -> Path:
|
||||
return self.BROWSER_USE_CONFIG_DIR / 'config.json'
|
||||
|
||||
@property
|
||||
def BROWSER_USE_PROFILES_DIR(self) -> Path:
|
||||
path = self.BROWSER_USE_CONFIG_DIR / 'profiles'
|
||||
self._ensure_dirs()
|
||||
return path
|
||||
|
||||
@property
|
||||
def BROWSER_USE_DEFAULT_USER_DATA_DIR(self) -> Path:
|
||||
return self.BROWSER_USE_PROFILES_DIR / 'default'
|
||||
|
||||
@property
|
||||
def BROWSER_USE_EXTENSIONS_DIR(self) -> Path:
|
||||
path = self.BROWSER_USE_CONFIG_DIR / 'extensions'
|
||||
self._ensure_dirs()
|
||||
return path
|
||||
|
||||
def _ensure_dirs(self) -> None:
|
||||
"""Create directories if they don't exist (only once)"""
|
||||
if not self._dirs_created:
|
||||
config_dir = (
|
||||
Path(os.getenv('BROWSER_USE_CONFIG_DIR', str(self.XDG_CONFIG_HOME / 'browseruse'))).expanduser().resolve()
|
||||
)
|
||||
config_dir.mkdir(parents=True, exist_ok=True)
|
||||
(config_dir / 'profiles').mkdir(parents=True, exist_ok=True)
|
||||
(config_dir / 'extensions').mkdir(parents=True, exist_ok=True)
|
||||
self._dirs_created = True
|
||||
|
||||
# LLM API key configuration
|
||||
@property
|
||||
def OPENAI_API_KEY(self) -> str:
|
||||
return os.getenv('OPENAI_API_KEY', '')
|
||||
|
||||
@property
|
||||
def ANTHROPIC_API_KEY(self) -> str:
|
||||
return os.getenv('ANTHROPIC_API_KEY', '')
|
||||
|
||||
@property
|
||||
def GOOGLE_API_KEY(self) -> str:
|
||||
return os.getenv('GOOGLE_API_KEY', '')
|
||||
|
||||
@property
|
||||
def DEEPSEEK_API_KEY(self) -> str:
|
||||
return os.getenv('DEEPSEEK_API_KEY', '')
|
||||
|
||||
@property
|
||||
def GROK_API_KEY(self) -> str:
|
||||
return os.getenv('GROK_API_KEY', '')
|
||||
|
||||
@property
|
||||
def NOVITA_API_KEY(self) -> str:
|
||||
return os.getenv('NOVITA_API_KEY', '')
|
||||
|
||||
@property
|
||||
def AZURE_OPENAI_ENDPOINT(self) -> str:
|
||||
return os.getenv('AZURE_OPENAI_ENDPOINT', '')
|
||||
|
||||
@property
|
||||
def AZURE_OPENAI_KEY(self) -> str:
|
||||
return os.getenv('AZURE_OPENAI_KEY', '')
|
||||
|
||||
@property
|
||||
def SKIP_LLM_API_KEY_VERIFICATION(self) -> bool:
|
||||
return os.getenv('SKIP_LLM_API_KEY_VERIFICATION', 'false').lower()[:1] in 'ty1'
|
||||
|
||||
@property
|
||||
def DEFAULT_LLM(self) -> str:
|
||||
return os.getenv('DEFAULT_LLM', '')
|
||||
|
||||
# Runtime hints
|
||||
@property
|
||||
def IN_DOCKER(self) -> bool:
|
||||
return os.getenv('IN_DOCKER', 'false').lower()[:1] in 'ty1' or is_running_in_docker()
|
||||
|
||||
@property
|
||||
def IS_IN_EVALS(self) -> bool:
|
||||
return os.getenv('IS_IN_EVALS', 'false').lower()[:1] in 'ty1'
|
||||
|
||||
@property
|
||||
def BROWSER_USE_VERSION_CHECK(self) -> bool:
|
||||
return os.getenv('BROWSER_USE_VERSION_CHECK', 'true').lower()[:1] in 'ty1'
|
||||
|
||||
@property
|
||||
def WIN_FONT_DIR(self) -> str:
|
||||
return os.getenv('WIN_FONT_DIR', 'C:\\Windows\\Fonts')
|
||||
|
||||
|
||||
class FlatEnvConfig(BaseSettings):
|
||||
"""All environment variables in a flat namespace."""
|
||||
|
||||
model_config = SettingsConfigDict(env_file='.env', env_file_encoding='utf-8', case_sensitive=True, extra='allow')
|
||||
|
||||
# Logging and telemetry
|
||||
BROWSER_USE_LOGGING_LEVEL: str = Field(default='info')
|
||||
CDP_LOGGING_LEVEL: str = Field(default='WARNING')
|
||||
BROWSER_USE_DEBUG_LOG_FILE: str | None = Field(default=None)
|
||||
BROWSER_USE_INFO_LOG_FILE: str | None = Field(default=None)
|
||||
ANONYMIZED_TELEMETRY: bool = Field(default=True)
|
||||
BROWSER_USE_CLOUD_SYNC: bool | None = Field(default=None)
|
||||
BROWSER_USE_CLOUD_API_URL: str = Field(default='https://api.browser-use.com')
|
||||
BROWSER_USE_CLOUD_UI_URL: str = Field(default='')
|
||||
BROWSER_USE_MODEL_PRICING_URL: str = Field(default='')
|
||||
|
||||
# Path configuration
|
||||
XDG_CACHE_HOME: str = Field(default='~/.cache')
|
||||
XDG_CONFIG_HOME: str = Field(default='~/.config')
|
||||
BROWSER_USE_CONFIG_DIR: str | None = Field(default=None)
|
||||
|
||||
# LLM API keys
|
||||
OPENAI_API_KEY: str = Field(default='')
|
||||
ANTHROPIC_API_KEY: str = Field(default='')
|
||||
GOOGLE_API_KEY: str = Field(default='')
|
||||
DEEPSEEK_API_KEY: str = Field(default='')
|
||||
GROK_API_KEY: str = Field(default='')
|
||||
NOVITA_API_KEY: str = Field(default='')
|
||||
AZURE_OPENAI_ENDPOINT: str = Field(default='')
|
||||
AZURE_OPENAI_KEY: str = Field(default='')
|
||||
SKIP_LLM_API_KEY_VERIFICATION: bool = Field(default=False)
|
||||
DEFAULT_LLM: str = Field(default='')
|
||||
|
||||
# Runtime hints
|
||||
IN_DOCKER: bool | None = Field(default=None)
|
||||
IS_IN_EVALS: bool = Field(default=False)
|
||||
WIN_FONT_DIR: str = Field(default='C:\\Windows\\Fonts')
|
||||
BROWSER_USE_VERSION_CHECK: bool = Field(default=True)
|
||||
|
||||
# MCP-specific env vars
|
||||
BROWSER_USE_CONFIG_PATH: str | None = Field(default=None)
|
||||
BROWSER_USE_HEADLESS: bool | None = Field(default=None)
|
||||
BROWSER_USE_ALLOWED_DOMAINS: str | None = Field(default=None)
|
||||
BROWSER_USE_LLM_MODEL: str | None = Field(default=None)
|
||||
|
||||
# Proxy env vars
|
||||
BROWSER_USE_PROXY_URL: str | None = Field(default=None)
|
||||
BROWSER_USE_NO_PROXY: str | None = Field(default=None)
|
||||
BROWSER_USE_PROXY_USERNAME: str | None = Field(default=None)
|
||||
BROWSER_USE_PROXY_PASSWORD: str | None = Field(default=None)
|
||||
|
||||
# Extension env vars
|
||||
BROWSER_USE_DISABLE_EXTENSIONS: bool | None = Field(default=None)
|
||||
|
||||
|
||||
class DBStyleEntry(BaseModel):
|
||||
"""Database-style entry with UUID and metadata."""
|
||||
|
||||
id: str = Field(default_factory=lambda: str(uuid4()))
|
||||
default: bool = Field(default=False)
|
||||
created_at: str = Field(default_factory=lambda: datetime.utcnow().isoformat())
|
||||
|
||||
|
||||
class BrowserProfileEntry(DBStyleEntry):
|
||||
"""Browser profile configuration entry - accepts any BrowserProfile fields."""
|
||||
|
||||
model_config = ConfigDict(extra='allow')
|
||||
|
||||
# Common browser profile fields for reference
|
||||
headless: bool | None = None
|
||||
user_data_dir: str | None = None
|
||||
allowed_domains: list[str] | None = None
|
||||
downloads_path: str | None = None
|
||||
|
||||
|
||||
class LLMEntry(DBStyleEntry):
|
||||
"""LLM configuration entry."""
|
||||
|
||||
api_key: str | None = None
|
||||
model: str | None = None
|
||||
temperature: float | None = None
|
||||
max_tokens: int | None = None
|
||||
|
||||
|
||||
class AgentEntry(DBStyleEntry):
|
||||
"""Agent configuration entry."""
|
||||
|
||||
max_steps: int | None = None
|
||||
use_vision: bool | None = None
|
||||
system_prompt: str | None = None
|
||||
|
||||
|
||||
class DBStyleConfigJSON(BaseModel):
|
||||
"""New database-style configuration format."""
|
||||
|
||||
browser_profile: dict[str, BrowserProfileEntry] = Field(default_factory=dict)
|
||||
llm: dict[str, LLMEntry] = Field(default_factory=dict)
|
||||
agent: dict[str, AgentEntry] = Field(default_factory=dict)
|
||||
|
||||
|
||||
def create_default_config() -> DBStyleConfigJSON:
|
||||
"""Create a fresh default configuration."""
|
||||
logger.debug('Creating fresh default config.json')
|
||||
|
||||
new_config = DBStyleConfigJSON()
|
||||
|
||||
# Generate default IDs
|
||||
profile_id = str(uuid4())
|
||||
llm_id = str(uuid4())
|
||||
agent_id = str(uuid4())
|
||||
|
||||
# Create default browser profile entry
|
||||
new_config.browser_profile[profile_id] = BrowserProfileEntry(id=profile_id, default=True, headless=False, user_data_dir=None)
|
||||
|
||||
# Create default LLM entry
|
||||
new_config.llm[llm_id] = LLMEntry(id=llm_id, default=True, model='gpt-4.1-mini', api_key='your-openai-api-key-here')
|
||||
|
||||
# Create default agent entry
|
||||
new_config.agent[agent_id] = AgentEntry(id=agent_id, default=True)
|
||||
|
||||
return new_config
|
||||
|
||||
|
||||
def load_and_migrate_config(config_path: Path) -> DBStyleConfigJSON:
|
||||
"""Load config.json or create fresh one if old format detected."""
|
||||
if not config_path.exists():
|
||||
# Create fresh config with defaults
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_config = create_default_config()
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(new_config.model_dump(), f, indent=2)
|
||||
return new_config
|
||||
|
||||
try:
|
||||
with open(config_path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Check if it's already in DB-style format
|
||||
if all(key in data for key in ['browser_profile', 'llm', 'agent']) and all(
|
||||
isinstance(data.get(key, {}), dict) for key in ['browser_profile', 'llm', 'agent']
|
||||
):
|
||||
# Check if the values are DB-style entries (have UUIDs as keys)
|
||||
if data.get('browser_profile') and all(isinstance(v, dict) and 'id' in v for v in data['browser_profile'].values()):
|
||||
# Already in new format
|
||||
return DBStyleConfigJSON(**data)
|
||||
|
||||
# Old format detected - delete it and create fresh config
|
||||
logger.debug(f'Old config format detected at {config_path}, creating fresh config')
|
||||
new_config = create_default_config()
|
||||
|
||||
# Overwrite with new config
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(new_config.model_dump(), f, indent=2)
|
||||
|
||||
logger.debug(f'Created fresh config.json at {config_path}')
|
||||
return new_config
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to load config from {config_path}: {e}, creating fresh config')
|
||||
# On any error, create fresh config
|
||||
new_config = create_default_config()
|
||||
try:
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(new_config.model_dump(), f, indent=2)
|
||||
except Exception as write_error:
|
||||
logger.error(f'Failed to write fresh config: {write_error}')
|
||||
return new_config
|
||||
|
||||
|
||||
class Config:
|
||||
"""Backward-compatible configuration class that merges all config sources.
|
||||
|
||||
Re-reads environment variables on every access to maintain compatibility.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Cache for directory creation tracking only
|
||||
self._dirs_created = False
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
"""Dynamically proxy all attributes to fresh instances.
|
||||
|
||||
This ensures env vars are re-read on every access.
|
||||
"""
|
||||
# Special handling for internal attributes
|
||||
if name.startswith('_'):
|
||||
raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
|
||||
|
||||
# Create fresh instances on every access
|
||||
old_config = OldConfig()
|
||||
|
||||
# Always use old config for all attributes (it handles env vars with proper transformations)
|
||||
if hasattr(old_config, name):
|
||||
return getattr(old_config, name)
|
||||
|
||||
# For new MCP-specific attributes not in old config
|
||||
env_config = FlatEnvConfig()
|
||||
if hasattr(env_config, name):
|
||||
return getattr(env_config, name)
|
||||
|
||||
# Handle special methods
|
||||
if name == 'get_default_profile':
|
||||
return lambda: self._get_default_profile()
|
||||
elif name == 'get_default_llm':
|
||||
return lambda: self._get_default_llm()
|
||||
elif name == 'get_default_agent':
|
||||
return lambda: self._get_default_agent()
|
||||
elif name == 'load_config':
|
||||
return lambda: self._load_config()
|
||||
elif name == '_ensure_dirs':
|
||||
return lambda: old_config._ensure_dirs()
|
||||
|
||||
raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
|
||||
|
||||
def _get_config_path(self) -> Path:
|
||||
"""Get config path from fresh env config."""
|
||||
env_config = FlatEnvConfig()
|
||||
if env_config.BROWSER_USE_CONFIG_PATH:
|
||||
return Path(env_config.BROWSER_USE_CONFIG_PATH).expanduser()
|
||||
elif env_config.BROWSER_USE_CONFIG_DIR:
|
||||
return Path(env_config.BROWSER_USE_CONFIG_DIR).expanduser() / 'config.json'
|
||||
else:
|
||||
xdg_config = Path(env_config.XDG_CONFIG_HOME).expanduser()
|
||||
return xdg_config / 'browseruse' / 'config.json'
|
||||
|
||||
def _get_db_config(self) -> DBStyleConfigJSON:
|
||||
"""Load and migrate config.json."""
|
||||
config_path = self._get_config_path()
|
||||
return load_and_migrate_config(config_path)
|
||||
|
||||
def _get_default_profile(self) -> dict[str, Any]:
|
||||
"""Get the default browser profile configuration."""
|
||||
db_config = self._get_db_config()
|
||||
for profile in db_config.browser_profile.values():
|
||||
if profile.default:
|
||||
return profile.model_dump(exclude_none=True)
|
||||
|
||||
# Return first profile if no default
|
||||
if db_config.browser_profile:
|
||||
return next(iter(db_config.browser_profile.values())).model_dump(exclude_none=True)
|
||||
|
||||
return {}
|
||||
|
||||
def _get_default_llm(self) -> dict[str, Any]:
|
||||
"""Get the default LLM configuration."""
|
||||
db_config = self._get_db_config()
|
||||
for llm in db_config.llm.values():
|
||||
if llm.default:
|
||||
return llm.model_dump(exclude_none=True)
|
||||
|
||||
# Return first LLM if no default
|
||||
if db_config.llm:
|
||||
return next(iter(db_config.llm.values())).model_dump(exclude_none=True)
|
||||
|
||||
return {}
|
||||
|
||||
def _get_default_agent(self) -> dict[str, Any]:
|
||||
"""Get the default agent configuration."""
|
||||
db_config = self._get_db_config()
|
||||
for agent in db_config.agent.values():
|
||||
if agent.default:
|
||||
return agent.model_dump(exclude_none=True)
|
||||
|
||||
# Return first agent if no default
|
||||
if db_config.agent:
|
||||
return next(iter(db_config.agent.values())).model_dump(exclude_none=True)
|
||||
|
||||
return {}
|
||||
|
||||
def _load_config(self) -> dict[str, Any]:
|
||||
"""Load configuration with env var overrides for MCP components."""
|
||||
config = {
|
||||
'browser_profile': self._get_default_profile(),
|
||||
'llm': self._get_default_llm(),
|
||||
'agent': self._get_default_agent(),
|
||||
}
|
||||
|
||||
# Fresh env config for overrides
|
||||
env_config = FlatEnvConfig()
|
||||
|
||||
# Apply MCP-specific env var overrides
|
||||
if env_config.BROWSER_USE_HEADLESS is not None:
|
||||
config['browser_profile']['headless'] = env_config.BROWSER_USE_HEADLESS
|
||||
|
||||
if env_config.BROWSER_USE_ALLOWED_DOMAINS:
|
||||
domains = [d.strip() for d in env_config.BROWSER_USE_ALLOWED_DOMAINS.split(',') if d.strip()]
|
||||
config['browser_profile']['allowed_domains'] = domains
|
||||
|
||||
# Proxy settings (Chromium) -> consolidated `proxy` dict
|
||||
proxy_dict: dict[str, Any] = {}
|
||||
if env_config.BROWSER_USE_PROXY_URL:
|
||||
proxy_dict['server'] = env_config.BROWSER_USE_PROXY_URL
|
||||
if env_config.BROWSER_USE_NO_PROXY:
|
||||
# store bypass as comma-separated string to match Chrome flag
|
||||
proxy_dict['bypass'] = ','.join([d.strip() for d in env_config.BROWSER_USE_NO_PROXY.split(',') if d.strip()])
|
||||
if env_config.BROWSER_USE_PROXY_USERNAME:
|
||||
proxy_dict['username'] = env_config.BROWSER_USE_PROXY_USERNAME
|
||||
if env_config.BROWSER_USE_PROXY_PASSWORD:
|
||||
proxy_dict['password'] = env_config.BROWSER_USE_PROXY_PASSWORD
|
||||
if proxy_dict:
|
||||
# ensure section exists
|
||||
config.setdefault('browser_profile', {})
|
||||
config['browser_profile']['proxy'] = proxy_dict
|
||||
|
||||
if env_config.OPENAI_API_KEY:
|
||||
config['llm']['api_key'] = env_config.OPENAI_API_KEY
|
||||
|
||||
if env_config.BROWSER_USE_LLM_MODEL:
|
||||
config['llm']['model'] = env_config.BROWSER_USE_LLM_MODEL
|
||||
|
||||
# Extension settings
|
||||
if env_config.BROWSER_USE_DISABLE_EXTENSIONS is not None:
|
||||
config['browser_profile']['enable_default_extensions'] = not env_config.BROWSER_USE_DISABLE_EXTENSIONS
|
||||
|
||||
return config
|
||||
|
||||
|
||||
# Create singleton instance
|
||||
CONFIG = Config()
|
||||
|
||||
|
||||
# Helper functions for MCP components
|
||||
def load_browser_use_config() -> dict[str, Any]:
|
||||
"""Load browser-use configuration for MCP components."""
|
||||
return CONFIG.load_config()
|
||||
|
||||
|
||||
def get_default_profile(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Get default browser profile from config dict."""
|
||||
return config.get('browser_profile', {})
|
||||
|
||||
|
||||
def get_default_llm(config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Get default LLM config from config dict."""
|
||||
return config.get('llm', {})
|
||||
3
.agent/vendor/browser_use/browser_use/controller/__init__.py
vendored
Normal file
3
.agent/vendor/browser_use/browser_use/controller/__init__.py
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
from browser_use.tools.service import Controller
|
||||
|
||||
__all__ = ['Controller']
|
||||
175
.agent/vendor/browser_use/browser_use/dom/enhanced_snapshot.py
vendored
Normal file
175
.agent/vendor/browser_use/browser_use/dom/enhanced_snapshot.py
vendored
Normal file
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
Enhanced snapshot processing for browser-use DOM tree extraction.
|
||||
|
||||
This module provides stateless functions for parsing Chrome DevTools Protocol (CDP) DOMSnapshot data
|
||||
to extract visibility, clickability, cursor styles, and other layout information.
|
||||
"""
|
||||
|
||||
from cdp_use.cdp.domsnapshot.commands import CaptureSnapshotReturns
|
||||
from cdp_use.cdp.domsnapshot.types import (
|
||||
LayoutTreeSnapshot,
|
||||
NodeTreeSnapshot,
|
||||
RareBooleanData,
|
||||
)
|
||||
|
||||
from browser_use.dom.views import DOMRect, EnhancedSnapshotNode
|
||||
|
||||
# Only the ESSENTIAL computed styles for interactivity and visibility detection
|
||||
REQUIRED_COMPUTED_STYLES = [
|
||||
# Only styles actually accessed in the codebase (prevents Chrome crashes on heavy sites)
|
||||
'display', # Used in service.py visibility detection
|
||||
'visibility', # Used in service.py visibility detection
|
||||
'opacity', # Used in service.py visibility detection
|
||||
'overflow', # Used in views.py scrollability detection
|
||||
'overflow-x', # Used in views.py scrollability detection
|
||||
'overflow-y', # Used in views.py scrollability detection
|
||||
'cursor', # Used in enhanced_snapshot.py cursor extraction
|
||||
'pointer-events', # Used for clickability logic
|
||||
'position', # Used for visibility logic
|
||||
'background-color', # Used for visibility logic
|
||||
]
|
||||
|
||||
|
||||
def _parse_rare_boolean_data(rare_data: RareBooleanData, index: int) -> bool | None:
|
||||
"""Parse rare boolean data from snapshot - returns True if index is in the rare data."""
|
||||
return index in rare_data['index']
|
||||
|
||||
|
||||
def _parse_computed_styles(strings: list[str], style_indices: list[int]) -> dict[str, str]:
|
||||
"""Parse computed styles from layout tree using string indices."""
|
||||
styles = {}
|
||||
for i, style_index in enumerate(style_indices):
|
||||
if i < len(REQUIRED_COMPUTED_STYLES) and 0 <= style_index < len(strings):
|
||||
styles[REQUIRED_COMPUTED_STYLES[i]] = strings[style_index]
|
||||
return styles
|
||||
|
||||
|
||||
def build_snapshot_lookup(
|
||||
snapshot: CaptureSnapshotReturns,
|
||||
device_pixel_ratio: float = 1.0,
|
||||
) -> dict[int, EnhancedSnapshotNode]:
|
||||
"""Build a lookup table of backend node ID to enhanced snapshot data with everything calculated upfront."""
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger('browser_use.dom.enhanced_snapshot')
|
||||
snapshot_lookup: dict[int, EnhancedSnapshotNode] = {}
|
||||
|
||||
if not snapshot['documents']:
|
||||
return snapshot_lookup
|
||||
|
||||
strings = snapshot['strings']
|
||||
logger.debug(f'🔍 SNAPSHOT: Processing {len(snapshot["documents"])} documents with {len(strings)} strings')
|
||||
|
||||
for doc_idx, document in enumerate(snapshot['documents']):
|
||||
nodes: NodeTreeSnapshot = document['nodes']
|
||||
layout: LayoutTreeSnapshot = document['layout']
|
||||
|
||||
# Build backend node id to snapshot index lookup
|
||||
backend_node_to_snapshot_index = {}
|
||||
if 'backendNodeId' in nodes:
|
||||
for i, backend_node_id in enumerate(nodes['backendNodeId']):
|
||||
backend_node_to_snapshot_index[backend_node_id] = i
|
||||
|
||||
# Log document info
|
||||
doc_url = strings[document.get('documentURL', 0)] if document.get('documentURL', 0) < len(strings) else 'N/A'
|
||||
logger.debug(
|
||||
f'🔍 SNAPSHOT doc[{doc_idx}]: url={doc_url[:80]}... has {len(backend_node_to_snapshot_index)} nodes, '
|
||||
f'layout has {len(layout.get("nodeIndex", []))} entries'
|
||||
)
|
||||
|
||||
# PERFORMANCE: Pre-build layout index map to eliminate O(n²) double lookups
|
||||
# Preserve original behavior: use FIRST occurrence for duplicates
|
||||
layout_index_map = {}
|
||||
if layout and 'nodeIndex' in layout:
|
||||
for layout_idx, node_index in enumerate(layout['nodeIndex']):
|
||||
if node_index not in layout_index_map: # Only store first occurrence
|
||||
layout_index_map[node_index] = layout_idx
|
||||
|
||||
# Build snapshot lookup for each backend node id
|
||||
for backend_node_id, snapshot_index in backend_node_to_snapshot_index.items():
|
||||
is_clickable = None
|
||||
if 'isClickable' in nodes:
|
||||
is_clickable = _parse_rare_boolean_data(nodes['isClickable'], snapshot_index)
|
||||
|
||||
# Find corresponding layout node
|
||||
cursor_style = None
|
||||
is_visible = None
|
||||
bounding_box = None
|
||||
computed_styles = {}
|
||||
|
||||
# Look for layout tree node that corresponds to this snapshot node
|
||||
paint_order = None
|
||||
client_rects = None
|
||||
scroll_rects = None
|
||||
stacking_contexts = None
|
||||
if snapshot_index in layout_index_map:
|
||||
layout_idx = layout_index_map[snapshot_index]
|
||||
if layout_idx < len(layout.get('bounds', [])):
|
||||
# Parse bounding box
|
||||
bounds = layout['bounds'][layout_idx]
|
||||
if len(bounds) >= 4:
|
||||
# IMPORTANT: CDP coordinates are in device pixels, convert to CSS pixels
|
||||
# by dividing by the device pixel ratio
|
||||
raw_x, raw_y, raw_width, raw_height = bounds[0], bounds[1], bounds[2], bounds[3]
|
||||
|
||||
# Apply device pixel ratio scaling to convert device pixels to CSS pixels
|
||||
bounding_box = DOMRect(
|
||||
x=raw_x / device_pixel_ratio,
|
||||
y=raw_y / device_pixel_ratio,
|
||||
width=raw_width / device_pixel_ratio,
|
||||
height=raw_height / device_pixel_ratio,
|
||||
)
|
||||
|
||||
# Parse computed styles for this layout node
|
||||
if layout_idx < len(layout.get('styles', [])):
|
||||
style_indices = layout['styles'][layout_idx]
|
||||
computed_styles = _parse_computed_styles(strings, style_indices)
|
||||
cursor_style = computed_styles.get('cursor')
|
||||
|
||||
# Extract paint order if available
|
||||
if layout_idx < len(layout.get('paintOrders', [])):
|
||||
paint_order = layout.get('paintOrders', [])[layout_idx]
|
||||
|
||||
# Extract client rects if available
|
||||
client_rects_data = layout.get('clientRects', [])
|
||||
if layout_idx < len(client_rects_data):
|
||||
client_rect_data = client_rects_data[layout_idx]
|
||||
if client_rect_data and len(client_rect_data) >= 4:
|
||||
client_rects = DOMRect(
|
||||
x=client_rect_data[0],
|
||||
y=client_rect_data[1],
|
||||
width=client_rect_data[2],
|
||||
height=client_rect_data[3],
|
||||
)
|
||||
|
||||
# Extract scroll rects if available
|
||||
scroll_rects_data = layout.get('scrollRects', [])
|
||||
if layout_idx < len(scroll_rects_data):
|
||||
scroll_rect_data = scroll_rects_data[layout_idx]
|
||||
if scroll_rect_data and len(scroll_rect_data) >= 4:
|
||||
scroll_rects = DOMRect(
|
||||
x=scroll_rect_data[0],
|
||||
y=scroll_rect_data[1],
|
||||
width=scroll_rect_data[2],
|
||||
height=scroll_rect_data[3],
|
||||
)
|
||||
|
||||
# Extract stacking contexts if available
|
||||
if layout_idx < len(layout.get('stackingContexts', [])):
|
||||
stacking_contexts = layout.get('stackingContexts', {}).get('index', [])[layout_idx]
|
||||
|
||||
snapshot_lookup[backend_node_id] = EnhancedSnapshotNode(
|
||||
is_clickable=is_clickable,
|
||||
cursor_style=cursor_style,
|
||||
bounds=bounding_box,
|
||||
clientRects=client_rects,
|
||||
scrollRects=scroll_rects,
|
||||
computed_styles=computed_styles if computed_styles else None,
|
||||
paint_order=paint_order,
|
||||
stacking_contexts=stacking_contexts,
|
||||
)
|
||||
|
||||
# Count how many have bounds (are actually visible/laid out)
|
||||
with_bounds = sum(1 for n in snapshot_lookup.values() if n.bounds)
|
||||
logger.debug(f'🔍 SNAPSHOT: Built lookup with {len(snapshot_lookup)} total entries, {with_bounds} have bounds')
|
||||
return snapshot_lookup
|
||||
534
.agent/vendor/browser_use/browser_use/dom/markdown_extractor.py
vendored
Normal file
534
.agent/vendor/browser_use/browser_use/dom/markdown_extractor.py
vendored
Normal file
@@ -0,0 +1,534 @@
|
||||
"""
|
||||
Shared markdown extraction utilities for browser content processing.
|
||||
|
||||
This module provides a unified interface for extracting clean markdown from browser content,
|
||||
used by both the tools service and page actor.
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from browser_use.dom.serializer.html_serializer import HTMLSerializer
|
||||
from browser_use.dom.service import DomService
|
||||
from browser_use.dom.views import MarkdownChunk
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from browser_use.browser.session import BrowserSession
|
||||
from browser_use.browser.watchdogs.dom_watchdog import DOMWatchdog
|
||||
|
||||
|
||||
async def extract_clean_markdown(
|
||||
browser_session: 'BrowserSession | None' = None,
|
||||
dom_service: DomService | None = None,
|
||||
target_id: str | None = None,
|
||||
extract_links: bool = False,
|
||||
extract_images: bool = False,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
"""Extract clean markdown from browser content using enhanced DOM tree.
|
||||
|
||||
This unified function can extract markdown using either a browser session (for tools service)
|
||||
or a DOM service with target ID (for page actor).
|
||||
|
||||
Args:
|
||||
browser_session: Browser session to extract content from (tools service path)
|
||||
dom_service: DOM service instance (page actor path)
|
||||
target_id: Target ID for the page (required when using dom_service)
|
||||
extract_links: Whether to preserve links in markdown
|
||||
extract_images: Whether to preserve inline image src URLs in markdown
|
||||
|
||||
Returns:
|
||||
tuple: (clean_markdown_content, content_statistics)
|
||||
|
||||
Raises:
|
||||
ValueError: If neither browser_session nor (dom_service + target_id) are provided
|
||||
"""
|
||||
# Validate input parameters
|
||||
if browser_session is not None:
|
||||
if dom_service is not None or target_id is not None:
|
||||
raise ValueError('Cannot specify both browser_session and dom_service/target_id')
|
||||
# Browser session path (tools service)
|
||||
enhanced_dom_tree = await _get_enhanced_dom_tree_from_browser_session(browser_session)
|
||||
current_url = await browser_session.get_current_page_url()
|
||||
method = 'enhanced_dom_tree'
|
||||
elif dom_service is not None and target_id is not None:
|
||||
# DOM service path (page actor)
|
||||
# Lazy fetch all_frames inside get_dom_tree if needed (for cross-origin iframes)
|
||||
enhanced_dom_tree, _ = await dom_service.get_dom_tree(target_id=target_id, all_frames=None)
|
||||
current_url = None # Not available via DOM service
|
||||
method = 'dom_service'
|
||||
else:
|
||||
raise ValueError('Must provide either browser_session or both dom_service and target_id')
|
||||
|
||||
# Use the HTML serializer with the enhanced DOM tree
|
||||
html_serializer = HTMLSerializer(extract_links=extract_links)
|
||||
page_html = html_serializer.serialize(enhanced_dom_tree)
|
||||
|
||||
original_html_length = len(page_html)
|
||||
|
||||
# Use markdownify for clean markdown conversion
|
||||
from markdownify import markdownify as md
|
||||
|
||||
# 'td', 'th', and headings are the only elements where markdownify sets the _inline context,
|
||||
# which causes img elements to be stripped to just alt text when keep_inline_images_in=[]
|
||||
_keep_inline_images_in = ['td', 'th', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'] if extract_images else []
|
||||
content = md(
|
||||
page_html,
|
||||
heading_style='ATX', # Use # style headings
|
||||
strip=['script', 'style'], # Remove these tags
|
||||
bullets='-', # Use - for unordered lists
|
||||
code_language='', # Don't add language to code blocks
|
||||
escape_asterisks=False, # Don't escape asterisks (cleaner output)
|
||||
escape_underscores=False, # Don't escape underscores (cleaner output)
|
||||
escape_misc=False, # Don't escape other characters (cleaner output)
|
||||
autolinks=False, # Don't convert URLs to <> format
|
||||
default_title=False, # Don't add default title attributes
|
||||
keep_inline_images_in=_keep_inline_images_in, # Include image src URLs when extract_images=True
|
||||
)
|
||||
|
||||
initial_markdown_length = len(content)
|
||||
|
||||
# Minimal cleanup - markdownify already does most of the work
|
||||
content = re.sub(r'%[0-9A-Fa-f]{2}', '', content) # Remove any remaining URL encoding
|
||||
|
||||
# Apply light preprocessing to clean up excessive whitespace
|
||||
content, chars_filtered = _preprocess_markdown_content(content)
|
||||
|
||||
final_filtered_length = len(content)
|
||||
|
||||
# Content statistics
|
||||
stats = {
|
||||
'method': method,
|
||||
'original_html_chars': original_html_length,
|
||||
'initial_markdown_chars': initial_markdown_length,
|
||||
'filtered_chars_removed': chars_filtered,
|
||||
'final_filtered_chars': final_filtered_length,
|
||||
}
|
||||
|
||||
# Add URL to stats if available
|
||||
if current_url:
|
||||
stats['url'] = current_url
|
||||
|
||||
return content, stats
|
||||
|
||||
|
||||
async def _get_enhanced_dom_tree_from_browser_session(browser_session: 'BrowserSession'):
|
||||
"""Get enhanced DOM tree from browser session via DOMWatchdog."""
|
||||
# Get the enhanced DOM tree from DOMWatchdog
|
||||
# This captures the current state of the page including dynamic content, shadow roots, etc.
|
||||
dom_watchdog: DOMWatchdog | None = browser_session._dom_watchdog
|
||||
assert dom_watchdog is not None, 'DOMWatchdog not available'
|
||||
|
||||
# Use cached enhanced DOM tree if available, otherwise build it
|
||||
if dom_watchdog.enhanced_dom_tree is not None:
|
||||
return dom_watchdog.enhanced_dom_tree
|
||||
|
||||
# Build the enhanced DOM tree if not cached
|
||||
await dom_watchdog._build_dom_tree_without_highlights()
|
||||
enhanced_dom_tree = dom_watchdog.enhanced_dom_tree
|
||||
assert enhanced_dom_tree is not None, 'Enhanced DOM tree not available'
|
||||
|
||||
return enhanced_dom_tree
|
||||
|
||||
|
||||
# Legacy aliases removed - all code now uses the unified extract_clean_markdown function
|
||||
|
||||
|
||||
def _preprocess_markdown_content(content: str, max_newlines: int = 3) -> tuple[str, int]:
|
||||
"""
|
||||
Light preprocessing of markdown output - minimal cleanup with JSON blob removal.
|
||||
|
||||
Args:
|
||||
content: Markdown content to lightly filter
|
||||
max_newlines: Maximum consecutive newlines to allow
|
||||
|
||||
Returns:
|
||||
tuple: (filtered_content, chars_filtered)
|
||||
"""
|
||||
original_length = len(content)
|
||||
|
||||
# Remove JSON blobs (common in SPAs like LinkedIn, Facebook, etc.)
|
||||
# These are often embedded as `{"key":"value",...}` and can be massive
|
||||
# Match JSON objects/arrays that are at least 100 chars long
|
||||
# This catches SPA state/config data without removing small inline JSON
|
||||
content = re.sub(r'`\{["\w].*?\}`', '', content, flags=re.DOTALL) # Remove JSON in code blocks
|
||||
content = re.sub(r'\{"\$type":[^}]{100,}\}', '', content) # Remove JSON with $type fields (common pattern)
|
||||
content = re.sub(r'\{"[^"]{5,}":\{[^}]{100,}\}', '', content) # Remove nested JSON objects
|
||||
|
||||
# Compress consecutive newlines (4+ newlines become max_newlines)
|
||||
content = re.sub(r'\n{4,}', '\n' * max_newlines, content)
|
||||
|
||||
# Remove lines that are only whitespace
|
||||
lines = content.split('\n')
|
||||
filtered_lines = []
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
# Keep all non-empty lines
|
||||
if stripped:
|
||||
# Skip lines that look like JSON (start with { or [ and are very long)
|
||||
if (stripped.startswith('{') or stripped.startswith('[')) and len(stripped) > 100:
|
||||
continue
|
||||
filtered_lines.append(line)
|
||||
|
||||
content = '\n'.join(filtered_lines)
|
||||
content = content.strip()
|
||||
|
||||
chars_filtered = original_length - len(content)
|
||||
return content, chars_filtered
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Structure-aware markdown chunking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _BlockType(Enum):
|
||||
HEADER = auto()
|
||||
CODE_FENCE = auto()
|
||||
TABLE = auto()
|
||||
LIST_ITEM = auto()
|
||||
PARAGRAPH = auto()
|
||||
BLANK = auto()
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _AtomicBlock:
|
||||
block_type: _BlockType
|
||||
lines: list[str]
|
||||
char_start: int # offset in original content
|
||||
char_end: int # offset in original content (exclusive)
|
||||
|
||||
|
||||
_TABLE_ROW_RE = re.compile(r'^\s*\|.*\|\s*$')
|
||||
_LIST_ITEM_RE = re.compile(r'^(\s*)([-*+]|\d+[.)]) ')
|
||||
_LIST_CONTINUATION_RE = re.compile(r'^(\s{2,}|\t)')
|
||||
|
||||
|
||||
def _parse_atomic_blocks(content: str) -> list[_AtomicBlock]:
|
||||
"""Phase 1: Walk lines, group into unsplittable blocks."""
|
||||
lines = content.split('\n')
|
||||
blocks: list[_AtomicBlock] = []
|
||||
i = 0
|
||||
offset = 0 # char offset tracking
|
||||
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
line_len = len(line) + 1 # +1 for the newline we split on
|
||||
|
||||
# BLANK
|
||||
if not line.strip():
|
||||
blocks.append(
|
||||
_AtomicBlock(
|
||||
block_type=_BlockType.BLANK,
|
||||
lines=[line],
|
||||
char_start=offset,
|
||||
char_end=offset + line_len,
|
||||
)
|
||||
)
|
||||
offset += line_len
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# CODE FENCE
|
||||
if line.strip().startswith('```'):
|
||||
fence_lines = [line]
|
||||
fence_end = offset + line_len
|
||||
i += 1
|
||||
# Consume until closing fence or EOF
|
||||
while i < len(lines):
|
||||
fence_line = lines[i]
|
||||
fence_line_len = len(fence_line) + 1
|
||||
fence_lines.append(fence_line)
|
||||
fence_end += fence_line_len
|
||||
i += 1
|
||||
if fence_line.strip().startswith('```') and len(fence_lines) > 1:
|
||||
break
|
||||
blocks.append(
|
||||
_AtomicBlock(
|
||||
block_type=_BlockType.CODE_FENCE,
|
||||
lines=fence_lines,
|
||||
char_start=offset,
|
||||
char_end=fence_end,
|
||||
)
|
||||
)
|
||||
offset = fence_end
|
||||
continue
|
||||
|
||||
# HEADER
|
||||
if line.lstrip().startswith('#'):
|
||||
blocks.append(
|
||||
_AtomicBlock(
|
||||
block_type=_BlockType.HEADER,
|
||||
lines=[line],
|
||||
char_start=offset,
|
||||
char_end=offset + line_len,
|
||||
)
|
||||
)
|
||||
offset += line_len
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# TABLE (consecutive |...| lines)
|
||||
# Header + separator row stay together; each data row is its own block
|
||||
if _TABLE_ROW_RE.match(line):
|
||||
# Collect header line
|
||||
header_lines = [line]
|
||||
header_end = offset + line_len
|
||||
i += 1
|
||||
# Check if next line is separator (contains ---)
|
||||
if i < len(lines) and _TABLE_ROW_RE.match(lines[i]) and '---' in lines[i]:
|
||||
sep = lines[i]
|
||||
sep_len = len(sep) + 1
|
||||
header_lines.append(sep)
|
||||
header_end += sep_len
|
||||
i += 1
|
||||
# Emit header+separator as one atomic block
|
||||
blocks.append(
|
||||
_AtomicBlock(
|
||||
block_type=_BlockType.TABLE,
|
||||
lines=header_lines,
|
||||
char_start=offset,
|
||||
char_end=header_end,
|
||||
)
|
||||
)
|
||||
offset = header_end
|
||||
# Each subsequent table row is its own TABLE block (splittable between rows)
|
||||
while i < len(lines) and _TABLE_ROW_RE.match(lines[i]):
|
||||
row = lines[i]
|
||||
row_len = len(row) + 1
|
||||
blocks.append(
|
||||
_AtomicBlock(
|
||||
block_type=_BlockType.TABLE,
|
||||
lines=[row],
|
||||
char_start=offset,
|
||||
char_end=offset + row_len,
|
||||
)
|
||||
)
|
||||
offset += row_len
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# LIST ITEM (with indented continuations)
|
||||
if _LIST_ITEM_RE.match(line):
|
||||
list_lines = [line]
|
||||
list_end = offset + line_len
|
||||
i += 1
|
||||
# Consume continuation lines (indented or blank between items)
|
||||
while i < len(lines):
|
||||
next_line = lines[i]
|
||||
next_len = len(next_line) + 1
|
||||
# Another list item at same or deeper indent → still part of this block
|
||||
if _LIST_ITEM_RE.match(next_line):
|
||||
list_lines.append(next_line)
|
||||
list_end += next_len
|
||||
i += 1
|
||||
continue
|
||||
# Indented continuation
|
||||
if next_line.strip() and _LIST_CONTINUATION_RE.match(next_line):
|
||||
list_lines.append(next_line)
|
||||
list_end += next_len
|
||||
i += 1
|
||||
continue
|
||||
break
|
||||
blocks.append(
|
||||
_AtomicBlock(
|
||||
block_type=_BlockType.LIST_ITEM,
|
||||
lines=list_lines,
|
||||
char_start=offset,
|
||||
char_end=list_end,
|
||||
)
|
||||
)
|
||||
offset = list_end
|
||||
continue
|
||||
|
||||
# PARAGRAPH (everything else, up to next blank line)
|
||||
para_lines = [line]
|
||||
para_end = offset + line_len
|
||||
i += 1
|
||||
while i < len(lines) and lines[i].strip():
|
||||
# Stop if next line starts a different block type
|
||||
nl = lines[i]
|
||||
if nl.lstrip().startswith('#') or nl.strip().startswith('```') or _TABLE_ROW_RE.match(nl) or _LIST_ITEM_RE.match(nl):
|
||||
break
|
||||
nl_len = len(nl) + 1
|
||||
para_lines.append(nl)
|
||||
para_end += nl_len
|
||||
i += 1
|
||||
blocks.append(
|
||||
_AtomicBlock(
|
||||
block_type=_BlockType.PARAGRAPH,
|
||||
lines=para_lines,
|
||||
char_start=offset,
|
||||
char_end=para_end,
|
||||
)
|
||||
)
|
||||
offset = para_end
|
||||
|
||||
# Fix last block char_end: content may not end with \n
|
||||
if blocks and content and not content.endswith('\n'):
|
||||
blocks[-1] = _AtomicBlock(
|
||||
block_type=blocks[-1].block_type,
|
||||
lines=blocks[-1].lines,
|
||||
char_start=blocks[-1].char_start,
|
||||
char_end=len(content),
|
||||
)
|
||||
|
||||
return blocks
|
||||
|
||||
|
||||
def _block_text(block: _AtomicBlock) -> str:
|
||||
return '\n'.join(block.lines)
|
||||
|
||||
|
||||
def _get_table_header(block: _AtomicBlock) -> str | None:
|
||||
"""Extract table header + separator rows from a TABLE block."""
|
||||
assert block.block_type == _BlockType.TABLE
|
||||
if len(block.lines) < 2:
|
||||
return None
|
||||
# Header is first line, separator is second line (must contain ---)
|
||||
sep_line = block.lines[1]
|
||||
if '---' in sep_line or '- -' in sep_line:
|
||||
return block.lines[0] + '\n' + block.lines[1]
|
||||
return None
|
||||
|
||||
|
||||
def chunk_markdown_by_structure(
|
||||
content: str,
|
||||
max_chunk_chars: int = 100_000,
|
||||
overlap_lines: int = 5,
|
||||
start_from_char: int = 0,
|
||||
) -> list[MarkdownChunk]:
|
||||
"""Split markdown into structure-aware chunks.
|
||||
|
||||
Algorithm:
|
||||
Phase 1 — Parse atomic blocks (headers, code fences, tables, list items, paragraphs).
|
||||
Phase 2 — Greedy chunk assembly: accumulate blocks until exceeding max_chunk_chars.
|
||||
A single block exceeding the limit is allowed (soft limit).
|
||||
Phase 3 — Build overlap prefixes for context carry between chunks.
|
||||
|
||||
Args:
|
||||
content: Full markdown string.
|
||||
max_chunk_chars: Target maximum chars per chunk (soft limit for single blocks).
|
||||
overlap_lines: Number of trailing lines from previous chunk to prepend.
|
||||
start_from_char: Return chunks starting from the chunk that contains this offset.
|
||||
|
||||
Returns:
|
||||
List of MarkdownChunk. Empty if start_from_char is past end of content.
|
||||
"""
|
||||
if not content:
|
||||
return [
|
||||
MarkdownChunk(
|
||||
content='',
|
||||
chunk_index=0,
|
||||
total_chunks=1,
|
||||
char_offset_start=0,
|
||||
char_offset_end=0,
|
||||
overlap_prefix='',
|
||||
has_more=False,
|
||||
)
|
||||
]
|
||||
|
||||
if start_from_char >= len(content):
|
||||
return []
|
||||
|
||||
# Phase 1: parse atomic blocks
|
||||
blocks = _parse_atomic_blocks(content)
|
||||
if not blocks:
|
||||
return []
|
||||
|
||||
# Phase 2: greedy chunk assembly with header-preferred splitting
|
||||
raw_chunks: list[list[_AtomicBlock]] = []
|
||||
current_chunk: list[_AtomicBlock] = []
|
||||
current_size = 0
|
||||
|
||||
for block in blocks:
|
||||
block_size = block.char_end - block.char_start
|
||||
# If adding this block would exceed limit AND we already have content, emit chunk
|
||||
if current_size + block_size > max_chunk_chars and current_chunk:
|
||||
# Prefer splitting at a header boundary within the current chunk.
|
||||
# Scan backwards for the last HEADER block; if found and it wouldn't
|
||||
# create a tiny chunk (< 50% of limit), split right before it so the
|
||||
# header starts the next chunk for better semantic coherence.
|
||||
best_split = len(current_chunk)
|
||||
for j in range(len(current_chunk) - 1, 0, -1):
|
||||
if current_chunk[j].block_type == _BlockType.HEADER:
|
||||
prefix_size = sum(b.char_end - b.char_start for b in current_chunk[:j])
|
||||
if prefix_size >= max_chunk_chars * 0.5:
|
||||
best_split = j
|
||||
break
|
||||
raw_chunks.append(current_chunk[:best_split])
|
||||
# Carry remaining blocks (from the header onward) into the next chunk
|
||||
current_chunk = current_chunk[best_split:]
|
||||
current_size = sum(b.char_end - b.char_start for b in current_chunk)
|
||||
current_chunk.append(block)
|
||||
current_size += block_size
|
||||
|
||||
if current_chunk:
|
||||
raw_chunks.append(current_chunk)
|
||||
|
||||
total_chunks = len(raw_chunks)
|
||||
|
||||
# Phase 3: build MarkdownChunk objects with overlap prefixes
|
||||
chunks: list[MarkdownChunk] = []
|
||||
# Track table header from previous chunk for table continuations
|
||||
prev_chunk_last_table_header: str | None = None
|
||||
|
||||
for idx, chunk_blocks in enumerate(raw_chunks):
|
||||
chunk_text = '\n'.join(_block_text(b) for b in chunk_blocks)
|
||||
char_start = chunk_blocks[0].char_start
|
||||
char_end = chunk_blocks[-1].char_end
|
||||
|
||||
# Build overlap prefix
|
||||
overlap = ''
|
||||
if idx > 0:
|
||||
prev_blocks = raw_chunks[idx - 1]
|
||||
prev_text = '\n'.join(_block_text(b) for b in prev_blocks)
|
||||
prev_lines = prev_text.split('\n')
|
||||
|
||||
# Check if current chunk starts with a table continuation
|
||||
first_block = chunk_blocks[0]
|
||||
if first_block.block_type == _BlockType.TABLE and prev_chunk_last_table_header:
|
||||
# Always prepend table header for continuation
|
||||
trailing = prev_lines[-(overlap_lines):] if overlap_lines > 0 else []
|
||||
header_lines = prev_chunk_last_table_header.split('\n')
|
||||
# Deduplicate: don't repeat header lines if they're already in trailing
|
||||
combined = list(header_lines)
|
||||
for tl in trailing:
|
||||
if tl not in combined:
|
||||
combined.append(tl)
|
||||
overlap = '\n'.join(combined)
|
||||
elif overlap_lines > 0:
|
||||
overlap = '\n'.join(prev_lines[-(overlap_lines):])
|
||||
|
||||
# Track table header from this chunk for next iteration.
|
||||
# Only overwrite if this chunk contains a new header+separator block;
|
||||
# otherwise preserve the previous header so tables spanning 3+ chunks
|
||||
# still get the header carried forward.
|
||||
for b in chunk_blocks:
|
||||
if b.block_type == _BlockType.TABLE:
|
||||
hdr = _get_table_header(b)
|
||||
if hdr is not None:
|
||||
prev_chunk_last_table_header = hdr
|
||||
|
||||
has_more = idx < total_chunks - 1
|
||||
chunks.append(
|
||||
MarkdownChunk(
|
||||
content=chunk_text,
|
||||
chunk_index=idx,
|
||||
total_chunks=total_chunks,
|
||||
char_offset_start=char_start,
|
||||
char_offset_end=char_end,
|
||||
overlap_prefix=overlap,
|
||||
has_more=has_more,
|
||||
)
|
||||
)
|
||||
|
||||
# Apply start_from_char filter: return chunks from the one containing that offset
|
||||
if start_from_char > 0:
|
||||
for i, chunk in enumerate(chunks):
|
||||
if chunk.char_offset_end > start_from_char:
|
||||
return chunks[i:]
|
||||
return [] # offset past all chunks
|
||||
|
||||
return chunks
|
||||
312
.agent/vendor/browser_use/browser_use/dom/playground/extraction.py
vendored
Normal file
312
.agent/vendor/browser_use/browser_use/dom/playground/extraction.py
vendored
Normal file
@@ -0,0 +1,312 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
import anyio
|
||||
import pyperclip
|
||||
import tiktoken
|
||||
|
||||
from browser_use.agent.prompts import AgentMessagePrompt
|
||||
from browser_use.browser import BrowserProfile, BrowserSession
|
||||
from browser_use.browser.events import ClickElementEvent, TypeTextEvent
|
||||
from browser_use.browser.profile import ViewportSize
|
||||
from browser_use.dom.service import DomService
|
||||
from browser_use.dom.views import DEFAULT_INCLUDE_ATTRIBUTES
|
||||
from browser_use.filesystem.file_system import FileSystem
|
||||
|
||||
TIMEOUT = 60
|
||||
|
||||
|
||||
async def test_focus_vs_all_elements():
|
||||
browser_session = BrowserSession(
|
||||
browser_profile=BrowserProfile(
|
||||
# executable_path='/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
||||
window_size=ViewportSize(width=1100, height=1000),
|
||||
disable_security=False,
|
||||
wait_for_network_idle_page_load_time=1,
|
||||
headless=False,
|
||||
args=['--incognito'],
|
||||
paint_order_filtering=True,
|
||||
),
|
||||
)
|
||||
|
||||
# 10 Sample websites with various interactive elements
|
||||
sample_websites = [
|
||||
'https://browser-use.github.io/stress-tests/challenges/iframe-inception-level2.html',
|
||||
'https://www.google.com/travel/flights',
|
||||
'https://v0-simple-ui-test-site.vercel.app',
|
||||
'https://browser-use.github.io/stress-tests/challenges/iframe-inception-level1.html',
|
||||
'https://browser-use.github.io/stress-tests/challenges/angular-form.html',
|
||||
'https://www.google.com/travel/flights',
|
||||
'https://www.amazon.com/s?k=laptop',
|
||||
'https://github.com/trending',
|
||||
'https://www.reddit.com',
|
||||
'https://www.ycombinator.com/companies',
|
||||
'https://www.kayak.com/flights',
|
||||
'https://www.booking.com',
|
||||
'https://www.airbnb.com',
|
||||
'https://www.linkedin.com/jobs',
|
||||
'https://stackoverflow.com/questions',
|
||||
]
|
||||
|
||||
# 5 Difficult websites with complex elements (iframes, canvas, dropdowns, etc.)
|
||||
difficult_websites = [
|
||||
'https://www.w3schools.com/html/tryit.asp?filename=tryhtml_iframe', # Nested iframes
|
||||
'https://semantic-ui.com/modules/dropdown.html', # Complex dropdowns
|
||||
'https://www.dezlearn.com/nested-iframes-example/', # Cross-origin nested iframes
|
||||
'https://codepen.io/towc/pen/mJzOWJ', # Canvas elements with interactions
|
||||
'https://jqueryui.com/accordion/', # Complex accordion/dropdown widgets
|
||||
'https://v0-simple-landing-page-seven-xi.vercel.app/', # Simple landing page with iframe
|
||||
'https://www.unesco.org/en',
|
||||
]
|
||||
|
||||
# Descriptions for difficult websites
|
||||
difficult_descriptions = {
|
||||
'https://www.w3schools.com/html/tryit.asp?filename=tryhtml_iframe': '🔸 NESTED IFRAMES: Multiple iframe layers',
|
||||
'https://semantic-ui.com/modules/dropdown.html': '🔸 COMPLEX DROPDOWNS: Custom dropdown components',
|
||||
'https://www.dezlearn.com/nested-iframes-example/': '🔸 CROSS-ORIGIN IFRAMES: Different domain iframes',
|
||||
'https://codepen.io/towc/pen/mJzOWJ': '🔸 CANVAS ELEMENTS: Interactive canvas graphics',
|
||||
'https://jqueryui.com/accordion/': '🔸 ACCORDION WIDGETS: Collapsible content sections',
|
||||
}
|
||||
|
||||
websites = sample_websites + difficult_websites
|
||||
current_website_index = 0
|
||||
|
||||
def get_website_list_for_prompt() -> str:
|
||||
"""Get a compact website list for the input prompt."""
|
||||
lines = []
|
||||
lines.append('📋 Websites:')
|
||||
|
||||
# Sample websites (1-10)
|
||||
for i, site in enumerate(sample_websites, 1):
|
||||
current_marker = ' ←' if (i - 1) == current_website_index else ''
|
||||
domain = site.replace('https://', '').split('/')[0]
|
||||
lines.append(f' {i:2d}.{domain[:15]:<15}{current_marker}')
|
||||
|
||||
# Difficult websites (11-15)
|
||||
for i, site in enumerate(difficult_websites, len(sample_websites) + 1):
|
||||
current_marker = ' ←' if (i - 1) == current_website_index else ''
|
||||
domain = site.replace('https://', '').split('/')[0]
|
||||
desc = difficult_descriptions.get(site, '')
|
||||
challenge = desc.split(': ')[1][:15] if ': ' in desc else ''
|
||||
lines.append(f' {i:2d}.{domain[:15]:<15} ({challenge}){current_marker}')
|
||||
|
||||
return '\n'.join(lines)
|
||||
|
||||
await browser_session.start()
|
||||
|
||||
# Show startup info
|
||||
print('\n🌐 BROWSER-USE DOM EXTRACTION TESTER')
|
||||
print(f'📊 {len(websites)} websites total: {len(sample_websites)} standard + {len(difficult_websites)} complex')
|
||||
print('🔧 Controls: Type 1-15 to jump | Enter to re-run | "n" next | "q" quit')
|
||||
print('💾 Outputs: tmp/user_message.txt & tmp/element_tree.json\n')
|
||||
|
||||
dom_service = DomService(browser_session)
|
||||
|
||||
while True:
|
||||
# Cycle through websites
|
||||
if current_website_index >= len(websites):
|
||||
current_website_index = 0
|
||||
print('Cycled back to first website!')
|
||||
|
||||
website = websites[current_website_index]
|
||||
# sleep 2
|
||||
await browser_session._cdp_navigate(website)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
last_clicked_index = None # Track the index for text input
|
||||
while True:
|
||||
try:
|
||||
# all_elements_state = await dom_service.get_serialized_dom_tree()
|
||||
|
||||
website_type = 'DIFFICULT' if website in difficult_websites else 'SAMPLE'
|
||||
print(f'\n{"=" * 60}')
|
||||
print(f'[{current_website_index + 1}/{len(websites)}] [{website_type}] Testing: {website}')
|
||||
if website in difficult_descriptions:
|
||||
print(f'{difficult_descriptions[website]}')
|
||||
print(f'{"=" * 60}')
|
||||
|
||||
# Get/refresh the state (includes removing old highlights)
|
||||
print('\nGetting page state...')
|
||||
|
||||
start_time = time.time()
|
||||
all_elements_state = await browser_session.get_browser_state_summary(True)
|
||||
end_time = time.time()
|
||||
get_state_time = end_time - start_time
|
||||
print(f'get_state_summary took {get_state_time:.2f} seconds')
|
||||
|
||||
# Get detailed timing info from DOM service
|
||||
print('\nGetting detailed DOM timing...')
|
||||
serialized_state, _, timing_info = await dom_service.get_serialized_dom_tree()
|
||||
|
||||
# Combine all timing info
|
||||
all_timing = {'get_state_summary_total': get_state_time, **timing_info}
|
||||
|
||||
selector_map = all_elements_state.dom_state.selector_map
|
||||
total_elements = len(selector_map.keys())
|
||||
print(f'Total number of elements: {total_elements}')
|
||||
|
||||
# print(all_elements_state.element_tree.clickable_elements_to_string())
|
||||
prompt = AgentMessagePrompt(
|
||||
browser_state_summary=all_elements_state,
|
||||
file_system=FileSystem(base_dir='./tmp'),
|
||||
include_attributes=DEFAULT_INCLUDE_ATTRIBUTES,
|
||||
step_info=None,
|
||||
)
|
||||
# Write the user message to a file for analysis
|
||||
user_message = prompt.get_user_message(use_vision=False).text
|
||||
|
||||
# clickable_elements_str = all_elements_state.element_tree.clickable_elements_to_string()
|
||||
|
||||
text_to_save = user_message
|
||||
|
||||
os.makedirs('./tmp', exist_ok=True)
|
||||
async with await anyio.open_file('./tmp/user_message.txt', 'w', encoding='utf-8') as f:
|
||||
await f.write(text_to_save)
|
||||
|
||||
# save pure clickable elements to a file
|
||||
if all_elements_state.dom_state._root:
|
||||
async with await anyio.open_file('./tmp/simplified_element_tree.json', 'w', encoding='utf-8') as f:
|
||||
await f.write(json.dumps(all_elements_state.dom_state._root.__json__(), indent=2))
|
||||
|
||||
async with await anyio.open_file('./tmp/original_element_tree.json', 'w', encoding='utf-8') as f:
|
||||
await f.write(json.dumps(all_elements_state.dom_state._root.original_node.__json__(), indent=2))
|
||||
|
||||
# copy the user message to the clipboard
|
||||
# pyperclip.copy(text_to_save)
|
||||
|
||||
encoding = tiktoken.encoding_for_model('gpt-4.1-mini')
|
||||
token_count = len(encoding.encode(text_to_save))
|
||||
print(f'Token count: {token_count}')
|
||||
|
||||
print('User message written to ./tmp/user_message.txt')
|
||||
print('Element tree written to ./tmp/simplified_element_tree.json')
|
||||
print('Original element tree written to ./tmp/original_element_tree.json')
|
||||
|
||||
# Save timing information
|
||||
timing_text = '🔍 DOM EXTRACTION PERFORMANCE ANALYSIS\n'
|
||||
timing_text += f'{"=" * 50}\n\n'
|
||||
timing_text += f'📄 Website: {website}\n'
|
||||
timing_text += f'📊 Total Elements: {total_elements}\n'
|
||||
timing_text += f'🎯 Token Count: {token_count}\n\n'
|
||||
|
||||
timing_text += '⏱️ TIMING BREAKDOWN:\n'
|
||||
timing_text += f'{"─" * 30}\n'
|
||||
for key, value in all_timing.items():
|
||||
timing_text += f'{key:<35}: {value * 1000:>8.2f} ms\n'
|
||||
|
||||
# Calculate percentages
|
||||
total_time = all_timing.get('get_state_summary_total', 0)
|
||||
if total_time > 0 and total_elements > 0:
|
||||
timing_text += '\n📈 PERCENTAGE BREAKDOWN:\n'
|
||||
timing_text += f'{"─" * 30}\n'
|
||||
for key, value in all_timing.items():
|
||||
if key != 'get_state_summary_total':
|
||||
percentage = (value / total_time) * 100
|
||||
timing_text += f'{key:<35}: {percentage:>7.1f}%\n'
|
||||
|
||||
timing_text += '\n🎯 CLICKABLE DETECTION ANALYSIS:\n'
|
||||
timing_text += f'{"─" * 35}\n'
|
||||
clickable_time = all_timing.get('clickable_detection_time', 0)
|
||||
if clickable_time > 0 and total_elements > 0:
|
||||
avg_per_element = (clickable_time / total_elements) * 1000000 # microseconds
|
||||
timing_text += f'Total clickable detection time: {clickable_time * 1000:.2f} ms\n'
|
||||
timing_text += f'Average per element: {avg_per_element:.2f} μs\n'
|
||||
timing_text += f'Clickable detection calls: ~{total_elements} (approx)\n'
|
||||
|
||||
async with await anyio.open_file('./tmp/timing_analysis.txt', 'w', encoding='utf-8') as f:
|
||||
await f.write(timing_text)
|
||||
|
||||
print('Timing analysis written to ./tmp/timing_analysis.txt')
|
||||
|
||||
# also save all_elements_state.element_tree.clickable_elements_to_string() to a file
|
||||
# with open('./tmp/clickable_elements.json', 'w', encoding='utf-8') as f:
|
||||
# f.write(json.dumps(all_elements_state.element_tree.__json__(), indent=2))
|
||||
# print('Clickable elements written to ./tmp/clickable_elements.json')
|
||||
|
||||
website_list = get_website_list_for_prompt()
|
||||
answer = input(
|
||||
"🎮 Enter: element index | 'index' click (clickable) | 'index,text' input | 'c,index' copy | Enter re-run | 'n' next | 'q' quit: "
|
||||
)
|
||||
|
||||
if answer.lower() == 'q':
|
||||
return # Exit completely
|
||||
elif answer.lower() == 'n':
|
||||
print('Moving to next website...')
|
||||
current_website_index += 1
|
||||
break # Break inner loop to go to next website
|
||||
elif answer.strip() == '':
|
||||
print('Re-running extraction on current page state...')
|
||||
continue # Continue inner loop to re-extract DOM without reloading page
|
||||
elif answer.strip().isdigit():
|
||||
# Click element format: index
|
||||
try:
|
||||
clicked_index = int(answer)
|
||||
if clicked_index in selector_map:
|
||||
element_node = selector_map[clicked_index]
|
||||
print(f'Clicking element {clicked_index}: {element_node.tag_name}')
|
||||
event = browser_session.event_bus.dispatch(ClickElementEvent(node=element_node))
|
||||
await event
|
||||
print('Click successful.')
|
||||
except ValueError:
|
||||
print(f"Invalid input: '{answer}'. Enter an index, 'index,text', 'c,index', or 'q'.")
|
||||
continue
|
||||
|
||||
try:
|
||||
if answer.lower().startswith('c,'):
|
||||
# Copy element JSON format: c,index
|
||||
parts = answer.split(',', 1)
|
||||
if len(parts) == 2:
|
||||
try:
|
||||
target_index = int(parts[1].strip())
|
||||
if target_index in selector_map:
|
||||
element_node = selector_map[target_index]
|
||||
element_json = json.dumps(element_node.__json__(), indent=2, default=str)
|
||||
pyperclip.copy(element_json)
|
||||
print(f'Copied element {target_index} JSON to clipboard: {element_node.tag_name}')
|
||||
else:
|
||||
print(f'Invalid index: {target_index}')
|
||||
except ValueError:
|
||||
print(f'Invalid index format: {parts[1]}')
|
||||
else:
|
||||
print("Invalid input format. Use 'c,index'.")
|
||||
elif ',' in answer:
|
||||
# Input text format: index,text
|
||||
parts = answer.split(',', 1)
|
||||
if len(parts) == 2:
|
||||
try:
|
||||
target_index = int(parts[0].strip())
|
||||
text_to_input = parts[1]
|
||||
if target_index in selector_map:
|
||||
element_node = selector_map[target_index]
|
||||
print(
|
||||
f"Inputting text '{text_to_input}' into element {target_index}: {element_node.tag_name}"
|
||||
)
|
||||
|
||||
event = await browser_session.event_bus.dispatch(
|
||||
TypeTextEvent(node=element_node, text=text_to_input)
|
||||
)
|
||||
|
||||
print('Input successful.')
|
||||
else:
|
||||
print(f'Invalid index: {target_index}')
|
||||
except ValueError:
|
||||
print(f'Invalid index format: {parts[0]}')
|
||||
else:
|
||||
print("Invalid input format. Use 'index,text'.")
|
||||
|
||||
except Exception as action_e:
|
||||
print(f'Action failed: {action_e}')
|
||||
|
||||
# No explicit highlight removal here, get_state handles it at the start of the loop
|
||||
|
||||
except Exception as e:
|
||||
print(f'Error in loop: {e}')
|
||||
# Optionally add a small delay before retrying
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(test_focus_vs_all_elements())
|
||||
# asyncio.run(test_process_html_file()) # Commented out the other test
|
||||
32
.agent/vendor/browser_use/browser_use/dom/playground/multi_act.py
vendored
Normal file
32
.agent/vendor/browser_use/browser_use/dom/playground/multi_act.py
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
from browser_use import Agent
|
||||
from browser_use.browser import BrowserProfile, BrowserSession
|
||||
from browser_use.browser.profile import ViewportSize
|
||||
from browser_use.llm import ChatAzureOpenAI
|
||||
|
||||
# Initialize the Azure OpenAI client
|
||||
llm = ChatAzureOpenAI(
|
||||
model='gpt-4.1-mini',
|
||||
)
|
||||
|
||||
|
||||
TASK = """
|
||||
Go to https://browser-use.github.io/stress-tests/challenges/react-native-web-form.html and complete the React Native Web form by filling in all required fields and submitting.
|
||||
"""
|
||||
|
||||
|
||||
async def main():
|
||||
browser = BrowserSession(
|
||||
browser_profile=BrowserProfile(
|
||||
window_size=ViewportSize(width=1100, height=1000),
|
||||
)
|
||||
)
|
||||
|
||||
agent = Agent(task=TASK, llm=llm)
|
||||
|
||||
await agent.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
246
.agent/vendor/browser_use/browser_use/dom/serializer/clickable_elements.py
vendored
Normal file
246
.agent/vendor/browser_use/browser_use/dom/serializer/clickable_elements.py
vendored
Normal file
@@ -0,0 +1,246 @@
|
||||
from browser_use.dom.views import EnhancedDOMTreeNode, NodeType
|
||||
|
||||
|
||||
class ClickableElementDetector:
|
||||
@staticmethod
|
||||
def is_interactive(node: EnhancedDOMTreeNode) -> bool:
|
||||
"""Check if this node is clickable/interactive using enhanced scoring."""
|
||||
|
||||
def has_form_control_descendant(element: EnhancedDOMTreeNode, max_depth: int = 2) -> bool:
|
||||
"""Detect nested form controls within limited depth (handles label/span wrappers)."""
|
||||
if max_depth <= 0:
|
||||
return False
|
||||
|
||||
for child in element.children_and_shadow_roots:
|
||||
if child.node_type != NodeType.ELEMENT_NODE:
|
||||
continue
|
||||
|
||||
tag_name = child.tag_name
|
||||
if tag_name in {'input', 'select', 'textarea'}:
|
||||
return True
|
||||
|
||||
if has_form_control_descendant(child, max_depth=max_depth - 1):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# Skip non-element nodes
|
||||
if node.node_type != NodeType.ELEMENT_NODE:
|
||||
return False
|
||||
|
||||
# # if ax ignored skip
|
||||
# if node.ax_node and node.ax_node.ignored:
|
||||
# return False
|
||||
|
||||
# remove html and body nodes
|
||||
if node.tag_name in {'html', 'body'}:
|
||||
return False
|
||||
|
||||
# Check for JavaScript click event listeners detected via CDP (without DOM mutation)
|
||||
# this handles vue.js @click, react onClick, angular (click), etc.
|
||||
if node.has_js_click_listener:
|
||||
return True
|
||||
|
||||
# IFRAME elements should be interactive if they're large enough to potentially need scrolling
|
||||
# Small iframes (< 100px width or height) are unlikely to have scrollable content
|
||||
if node.tag_name and node.tag_name.upper() == 'IFRAME' or node.tag_name.upper() == 'FRAME':
|
||||
if node.snapshot_node and node.snapshot_node.bounds:
|
||||
width = node.snapshot_node.bounds.width
|
||||
height = node.snapshot_node.bounds.height
|
||||
# Only include iframes larger than 100x100px
|
||||
if width > 100 and height > 100:
|
||||
return True
|
||||
|
||||
# RELAXED SIZE CHECK: Allow all elements including size 0 (they might be interactive overlays, etc.)
|
||||
# Note: Size 0 elements can still be interactive (e.g., invisible clickable overlays)
|
||||
# Visibility is determined separately by CSS styles, not just bounding box size
|
||||
|
||||
# Specialized handling for labels used as component wrappers (e.g., Ant Design radio/checkbox)
|
||||
if node.tag_name == 'label':
|
||||
# Skip labels that proxy via "for" to avoid double-activating external inputs
|
||||
if node.attributes and node.attributes.get('for'):
|
||||
return False
|
||||
|
||||
# Detect labels that wrap form controls up to two levels deep (label > span > input)
|
||||
if has_form_control_descendant(node, max_depth=2):
|
||||
return True
|
||||
# Fall through to pointer/role/attribute heuristics for other label cases
|
||||
|
||||
# Span wrappers for UI components (detect clear interactive signals only)
|
||||
if node.tag_name == 'span':
|
||||
if has_form_control_descendant(node, max_depth=2):
|
||||
return True
|
||||
# Allow other heuristics (aria roles, event handlers, pointer) to decide
|
||||
|
||||
# SEARCH ELEMENT DETECTION: Check for search-related classes and attributes
|
||||
if node.attributes:
|
||||
search_indicators = {
|
||||
'search',
|
||||
'magnify',
|
||||
'glass',
|
||||
'lookup',
|
||||
'find',
|
||||
'query',
|
||||
'search-icon',
|
||||
'search-btn',
|
||||
'search-button',
|
||||
'searchbox',
|
||||
}
|
||||
|
||||
# Check class names for search indicators
|
||||
class_list = node.attributes.get('class', '').lower().split()
|
||||
if any(indicator in ' '.join(class_list) for indicator in search_indicators):
|
||||
return True
|
||||
|
||||
# Check id for search indicators
|
||||
element_id = node.attributes.get('id', '').lower()
|
||||
if any(indicator in element_id for indicator in search_indicators):
|
||||
return True
|
||||
|
||||
# Check data attributes for search functionality
|
||||
for attr_name, attr_value in node.attributes.items():
|
||||
if attr_name.startswith('data-') and any(indicator in attr_value.lower() for indicator in search_indicators):
|
||||
return True
|
||||
|
||||
# Enhanced accessibility property checks - direct clear indicators only
|
||||
if node.ax_node and node.ax_node.properties:
|
||||
for prop in node.ax_node.properties:
|
||||
try:
|
||||
# aria disabled
|
||||
if prop.name == 'disabled' and prop.value:
|
||||
return False
|
||||
|
||||
# aria hidden
|
||||
if prop.name == 'hidden' and prop.value:
|
||||
return False
|
||||
|
||||
# Direct interactiveness indicators
|
||||
if prop.name in ['focusable', 'editable', 'settable'] and prop.value:
|
||||
return True
|
||||
|
||||
# Interactive state properties (presence indicates interactive widget)
|
||||
if prop.name in ['checked', 'expanded', 'pressed', 'selected']:
|
||||
# These properties only exist on interactive elements
|
||||
return True
|
||||
|
||||
# Form-related interactiveness
|
||||
if prop.name in ['required', 'autocomplete'] and prop.value:
|
||||
return True
|
||||
|
||||
# Elements with keyboard shortcuts are interactive
|
||||
if prop.name == 'keyshortcuts' and prop.value:
|
||||
return True
|
||||
except (AttributeError, ValueError):
|
||||
# Skip properties we can't process
|
||||
continue
|
||||
|
||||
# ENHANCED TAG CHECK: Include truly interactive elements
|
||||
# Note: 'label' removed - labels are handled by other attribute checks below - other wise labels with "for" attribute can destroy the real clickable element on apartments.com
|
||||
interactive_tags = {
|
||||
'button',
|
||||
'input',
|
||||
'select',
|
||||
'textarea',
|
||||
'a',
|
||||
'details',
|
||||
'summary',
|
||||
'option',
|
||||
'optgroup',
|
||||
}
|
||||
# Check with case-insensitive comparison
|
||||
if node.tag_name and node.tag_name.lower() in interactive_tags:
|
||||
return True
|
||||
|
||||
# SVG elements need special handling - only interactive if they have explicit handlers
|
||||
# svg_tags = {'svg', 'path', 'circle', 'rect', 'polygon', 'ellipse', 'line', 'polyline', 'g'}
|
||||
# if node.tag_name in svg_tags:
|
||||
# # Only consider SVG elements interactive if they have:
|
||||
# # 1. Explicit event handlers
|
||||
# # 2. Interactive role attributes
|
||||
# # 3. Cursor pointer style
|
||||
# if node.attributes:
|
||||
# # Check for event handlers
|
||||
# if any(attr.startswith('on') for attr in node.attributes):
|
||||
# return True
|
||||
# # Check for interactive roles
|
||||
# if node.attributes.get('role') in {'button', 'link', 'menuitem'}:
|
||||
# return True
|
||||
# # Check for cursor pointer (indicating clickability)
|
||||
# if node.attributes.get('style') and 'cursor: pointer' in node.attributes.get('style', ''):
|
||||
# return True
|
||||
# # Otherwise, SVG elements are decorative
|
||||
# return False
|
||||
|
||||
# Tertiary check: elements with interactive attributes
|
||||
if node.attributes:
|
||||
# Check for event handlers or interactive attributes
|
||||
interactive_attributes = {'onclick', 'onmousedown', 'onmouseup', 'onkeydown', 'onkeyup', 'tabindex'}
|
||||
if any(attr in node.attributes for attr in interactive_attributes):
|
||||
return True
|
||||
|
||||
# Check for interactive ARIA roles
|
||||
if 'role' in node.attributes:
|
||||
interactive_roles = {
|
||||
'button',
|
||||
'link',
|
||||
'menuitem',
|
||||
'option',
|
||||
'radio',
|
||||
'checkbox',
|
||||
'tab',
|
||||
'textbox',
|
||||
'combobox',
|
||||
'slider',
|
||||
'spinbutton',
|
||||
'search',
|
||||
'searchbox',
|
||||
'row',
|
||||
'cell',
|
||||
'gridcell',
|
||||
}
|
||||
if node.attributes['role'] in interactive_roles:
|
||||
return True
|
||||
|
||||
# Quaternary check: accessibility tree roles
|
||||
if node.ax_node and node.ax_node.role:
|
||||
interactive_ax_roles = {
|
||||
'button',
|
||||
'link',
|
||||
'menuitem',
|
||||
'option',
|
||||
'radio',
|
||||
'checkbox',
|
||||
'tab',
|
||||
'textbox',
|
||||
'combobox',
|
||||
'slider',
|
||||
'spinbutton',
|
||||
'listbox',
|
||||
'search',
|
||||
'searchbox',
|
||||
'row',
|
||||
'cell',
|
||||
'gridcell',
|
||||
}
|
||||
if node.ax_node.role in interactive_ax_roles:
|
||||
return True
|
||||
|
||||
# ICON AND SMALL ELEMENT CHECK: Elements that might be icons
|
||||
if (
|
||||
node.snapshot_node
|
||||
and node.snapshot_node.bounds
|
||||
and 10 <= node.snapshot_node.bounds.width <= 50 # Icon-sized elements
|
||||
and 10 <= node.snapshot_node.bounds.height <= 50
|
||||
):
|
||||
# Check if this small element has interactive properties
|
||||
if node.attributes:
|
||||
# Small elements with these attributes are likely interactive icons
|
||||
icon_attributes = {'class', 'role', 'onclick', 'data-action', 'aria-label'}
|
||||
if any(attr in node.attributes for attr in icon_attributes):
|
||||
return True
|
||||
|
||||
# Final fallback: cursor style indicates interactivity (for cases Chrome missed)
|
||||
if node.snapshot_node and node.snapshot_node.cursor_style and node.snapshot_node.cursor_style == 'pointer':
|
||||
return True
|
||||
|
||||
return False
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user