import {
puppeteer,
connect,
ExtensionTransport,
} from 'puppeteer-core/lib/cjs/puppeteer/puppeteer-core-browser.js';
import { getexecute } from '@/api/agentapi.js'
export const navigator=()=> {
let browser = null;
async function browsercomm() {
const tabitem = await getActiveTabId();
// console.log(tabitem)
let tabId=tabitem.id
// console.log(tabId)
const connectOptions = {
transport: await ExtensionTransport.connectTab(tabId), // 替换为实际标签ID
defaultViewport: null,
protocol: 'cdp'
};
// console.log(connectOptions)
// let browser;
try {
// 连接到浏览器
browser = await connect(connectOptions);
const [page] = await browser.pages();
await chrome.scripting.executeScript({
target: {tabId: tabId},
files: ['buildDomTree1.js'], // 注入外部文件
world: "MAIN"
});
// await page.waitForFunction(() => typeof window.yourFunc === 'function');
const domState = await page.evaluate(() => {
const buildDomTree = window.buildDomTree({
doHighlightElements: true,
focusHighlightIndex: -1,
viewportExpansion: 0,
debugMode: false,
})
function findDomByXpath(xpath) {
try {
return document.evaluate('/' + xpath.replace(/^\/+/, ''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
} catch (e) {
return null;
}
}
function cleanHtmlTags(htmlStr, attrs = ['name', 'href']) {
const doc = new DOMParser().parseFromString(htmlStr.replace(/[\r\n]+/g, ''), 'text/html');
// 遍历所有元素
const walker = doc.createTreeWalker(doc.body, NodeFilter.SHOW_ELEMENT);
while (walker.nextNode()) {
const node = walker.currentNode;
// 移除指定属性
attrs.forEach(attr => node.removeAttribute(attr));
// 只保留当前节点和其文本内容
node.innerHTML = node.textContent;
}
return doc.body.innerHTML;
}
const domList = Object.values(buildDomTree.map)
.filter(n => typeof n === "object" && n.highlightIndex !== undefined)
.map(n => findDomByXpath(n.xpath))
.filter(x => !!x)
.map((item, index) => `[${index}] ${item.outerHTML}`)
.map((item)=>{
const attrs = ['name', 'href','src','class','id','target',]
return cleanHtmlTags(item, attrs);
})
.join("\n")
// console.log(domList)
return {buildDomTree:buildDomTree,domList:domList}
});
return {domState:domState, page:page,tabitem:tabitem};
} finally {
// if (browser) await browser.disconnect();
}
}
async function browserautomate(domState,page,type,targetIndex,text) {
// console.log(domState,page,type,targetIndex,browser)
// 查找目标元素
const targetElement = findElementByIndex(domState.map, targetIndex);
if (!targetElement) throw new Error('Element not found');
// 生成增强选择器
const selector = generateSelector(targetElement);
// console.log(domState, targetElement, selector)
// const [page] = await browser.pages()
try {
await page.waitForSelector(selector, {visible: true, timeout: 10000});
switch (type) {
case 'click':
await page.click(selector, {delay: 100});
break;
case 'key_enter':
await page.click(selector, {delay: 100});
break;
case 'navigate':
await page.goto('https://example.com', {waitUntil: 'networkidle2'});
break;
case 'getcontent':
console.log(await page.content());
break;
case 'input_text':
await page.type(selector, text, {delay: 600});
break;
// 添加更多命令支持，如：
case 'form':
const formData = [{
text: 'your-username',
index: 'your-password',
email: 'your-email@example.com',
question2: 'option2', // 假设有一个单选题，选项为 option2
question3: '我非常满意这次体验', // 假设有一个文本框，答案为 “我非常满意这次体验”
}];
await page.type(selector, 'your-username', {delay: 600});
await page.type(selector, 'your-password', {delay: 600});
await page.click(selector); // 或者其他登录按钮
break;
case '':
page.type(selector, 'World', {delay: 600})
break
default:
console.log('未知指令。');
}
// 验证点击结果
// await page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 15000 });
// console.log(`Successfully clicked element ${targetIndex} on attempt ${attempt}`);
await page.evaluate(() => {
// document.getElementById('playwright-highlight-container')?.remove();
});
return true;
} catch (clickError) {
// console.warn(`Attempt ${attempt} failed: ${clickError.message}`);
// if (attempt === retries) throw clickError;
// await page.waitForTimeout(delay);
// 刷新 DOM 状态
await page.evaluate(() => {
document.getElementById('playwright-highlight-container')?.remove();
});
if (browser) await browser.disconnect();
}finally {
// console.log(123456789)
await page.evaluate(() => {
document.getElementById('playwright-highlight-container')?.remove();
});
if (browser) await browser.disconnect();
}
}
function allTabs(){
return new Promise( (resolve, reject) => {
chrome.tabs.query({}, (tabs) => {
// console.log('tabs', tabs)
resolve(tabs);
});
})
}
function getActiveTabId() {
return new Promise((resolve) => {
setTimeout(() => {
chrome.tabs.query({active: true}, (tabs) => {
// console.log('tabs', tabs)
resolve(tabs[0]);
});
},500)
});
}
// 辅助函数：通过索引查找元素
function findElementByIndex(domMap, targetIndex) {
return Object.values(domMap).find(
node => node.highlightIndex === targetIndex
);
}
// 辅助函数：生成 CSS 选择器
function generateSelector(element) {
const attrs = Object.entries(element.attributes)
.map(([k, v]) => `[${k}="${v.replace(/"/g, '\\"')}"]`)
.join('');
return `${element.tagName}${attrs}`;
}
function gfe() {
chrome.tabs.query({active: true, currentWindow: true}, async (tabs) => {
const tabId = tabs[0].id;
await smartClick(tabId, 1, {
type: 'click',
text: 3000
});
})
}
// 背景脚本
function switchTabOrOpenNew(url) {
return new Promise(resolve => {
// 查询是否存在指定的标签页
// chrome.tabs.query({windowId: chrome.windows.WINDOW_ID_CURRENT}, async (tabs) => {
chrome.tabs.query({}, async (tabs) => {
let tabId = tabs.find(tabs => tabs.url.replace(/\/$/, '') === url);
// console.log(tabs, tabId,url)
if (tabId) {
// 如果存在，则激活该标签页
await chrome.tabs.update(tabId.id, {active: true});
resolve()
} else {
// 如果不存在，则新打开一个标签页
await chrome.tabs.create({url: url});
resolve()
}
});
})
}
function getexecuteapi(params) {
return getexecute(params).then((res)=>{
return res;
})
}
async function agent(payload) {
for(let item of payload.steps){
console.log(item)
if(item.stepIndex === 0){
let params = {
conversationId: payload.conversationId,
messageId: payload.messageId,
messageContent: payload.messageContent,
planId: payload.id,
currentPlanStepId: item.id,
envData: {
osName: "Google Chrome",
osVersion: "137.0.7151.69（正式版本）",
osArch: "arm64",
url: "",
title:"",
tabs: [],
interactiveElements: ""
},
resultSummary: null,
needSummary: true,
success: false
}
let add = await getexecuteapi(params)
console.log(add)
let toolExecution = JSON.parse(add.data.plan.steps[0].toolExecution)[0].toolParameters
chrome.runtime.sendMessage({
type: "FROM_STEP",
payload: JSON.parse(toolExecution)
});
if (JSON.parse(toolExecution).action == 'navigate') {
await switchTabOrOpenNew(JSON.parse(toolExecution).url)
}
}else {
let browdata= await browsercomm()
console.log(browdata,'browdata')
const tabs = await allTabs()
let params = {
conversationId: payload.conversationId,
messageId: payload.messageId,
messageContent: payload.messageContent,
planId: payload.id,
currentPlanStepId: item.id,
envData: {
osName: "Google Chrome",
osVersion: "137.0.7151.69（正式版本）",
osArch: "arm64",
url: browdata.tabitem.url,
title: browdata.tabitem.title,
tabs: tabs,
interactiveElements: browdata.domState.domList
// interactiveElements: "[0] 新闻\n [1] hao123\n [2] 地图\n [3] 贴吧\n [4] 视频\n [5] 图片\n [6] 网盘\n [7] 文库\n [8] \n [9] 更多\n [10] \n [11] 登录\n [12] \n [13] \n [14] AI搜索已支持DeepSeek R1最新版立即体验\n [16] 换一换\n [17] 0让“干坡坡”变“金窝窝”\n [18] 5福建一楼房发生爆炸 有人员"
},
resultSummary: null,
needSummary: true,
success: false
}
let add = await getexecuteapi(params)
console.log(add,'add')
let toolExecution = add.data.plan.steps[item.stepIndex].toolExecution
if (toolExecution) {
let toolExecution=JSON.parse(add.data.plan.steps[item.stepIndex].toolExecution)[0].toolParameters
chrome.runtime.sendMessage({
type: "FROM_STEP",
payload: JSON.parse(toolExecution)
});
await browserautomate(browdata.domState.buildDomTree, browdata.page, JSON.parse(toolExecution).action, JSON.parse(toolExecution).index, JSON.parse(toolExecution).text)
}else {
return
}
}
}
}
chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
console.log(request, sender, sendResponse);
if (request.type == 'FROM_PLAN') {
agent(request.payload)
}
})
}