|
|
@@ -4,12 +4,12 @@ import {
|
|
|
ExtensionTransport,
|
|
|
} from 'puppeteer-core/lib/cjs/puppeteer/puppeteer-core-browser.js';
|
|
|
import { getexecute } from '@/api/agentapi.js'
|
|
|
-export const navigator=()=> {
|
|
|
+export const navigator = () => {
|
|
|
let browser = null;
|
|
|
async function browsercomm() {
|
|
|
const tabitem = await getActiveTabId();
|
|
|
console.log(tabitem)
|
|
|
- let tabId=tabitem.id
|
|
|
+ let tabId = tabitem.id
|
|
|
console.log(tabId)
|
|
|
const connectOptions = {
|
|
|
transport: await ExtensionTransport.connectTab(tabId), // 替换为实际标签ID
|
|
|
@@ -24,7 +24,7 @@ export const navigator=()=> {
|
|
|
browser = await connect(connectOptions);
|
|
|
const [page] = await browser.pages();
|
|
|
await chrome.scripting.executeScript({
|
|
|
- target: {tabId: tabId},
|
|
|
+ target: { tabId: tabId },
|
|
|
files: ['buildDomTree1.js'], // 注入外部文件
|
|
|
world: "MAIN"
|
|
|
});
|
|
|
@@ -64,22 +64,22 @@ export const navigator=()=> {
|
|
|
.map(n => findDomByXpath(n.xpath))
|
|
|
.filter(x => !!x)
|
|
|
.map((item, index) => `[${index}] ${item.outerHTML}`)
|
|
|
- .map((item)=>{
|
|
|
- const attrs = ['name', 'href','src','class','id','target',]
|
|
|
+ .map((item) => {
|
|
|
+ const attrs = ['name', 'href', 'src', 'class', 'id', 'target',]
|
|
|
return cleanHtmlTags(item, attrs);
|
|
|
})
|
|
|
.join("\n")
|
|
|
console.log(domList)
|
|
|
- return {buildDomTree:buildDomTree,domList:domList}
|
|
|
+ return { buildDomTree: buildDomTree, domList: domList }
|
|
|
});
|
|
|
- return {domState:domState, page:page,tabitem:tabitem};
|
|
|
+ return { domState: domState, page: page, tabitem: tabitem };
|
|
|
} finally {
|
|
|
// if (browser) await browser.disconnect();
|
|
|
|
|
|
}
|
|
|
}
|
|
|
- async function browserautomate(domState,page,type,targetIndex,text) {
|
|
|
-console.log(domState,page,type,targetIndex,browser)
|
|
|
+ async function browserautomate(domState, page, type, targetIndex, text) {
|
|
|
+ console.log(domState, page, type, targetIndex, browser)
|
|
|
// 查找目标元素
|
|
|
const targetElement = findElementByIndex(domState.map, targetIndex);
|
|
|
if (!targetElement) throw new Error('Element not found');
|
|
|
@@ -89,22 +89,22 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
console.log(domState, targetElement, selector)
|
|
|
// const [page] = await browser.pages()
|
|
|
try {
|
|
|
- await page.waitForSelector(selector, {visible: true, timeout: 10000});
|
|
|
+ await page.waitForSelector(selector, { visible: true, timeout: 10000 });
|
|
|
switch (type) {
|
|
|
case 'click':
|
|
|
- await page.click(selector, {delay: 100});
|
|
|
+ await page.click(selector, { delay: 100 });
|
|
|
break;
|
|
|
case 'key_enter':
|
|
|
- await page.click(selector, {delay: 100});
|
|
|
+ await page.click(selector, { delay: 100 });
|
|
|
break;
|
|
|
case 'navigate':
|
|
|
- await page.goto('https://example.com', {waitUntil: 'networkidle2'});
|
|
|
+ await page.goto('https://example.com', { waitUntil: 'networkidle2' });
|
|
|
break;
|
|
|
case 'getcontent':
|
|
|
console.log(await page.content());
|
|
|
break;
|
|
|
case 'input_text':
|
|
|
- await page.type(selector, text, {delay: 600});
|
|
|
+ await page.type(selector, text, { delay: 600 });
|
|
|
break;
|
|
|
// 添加更多命令支持，如：
|
|
|
case 'form':
|
|
|
@@ -115,12 +115,12 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
question2: 'option2', // 假设有一个单选题，选项为 option2
|
|
|
question3: '我非常满意这次体验', // 假设有一个文本框，答案为 “我非常满意这次体验”
|
|
|
}];
|
|
|
- await page.type(selector, 'your-username', {delay: 600});
|
|
|
- await page.type(selector, 'your-password', {delay: 600});
|
|
|
+ await page.type(selector, 'your-username', { delay: 600 });
|
|
|
+ await page.type(selector, 'your-password', { delay: 600 });
|
|
|
await page.click(selector); // 或者其他登录按钮
|
|
|
break;
|
|
|
case '':
|
|
|
- page.type(selector, 'World', {delay: 600})
|
|
|
+ page.type(selector, 'World', { delay: 600 })
|
|
|
break
|
|
|
default:
|
|
|
console.log('未知指令。');
|
|
|
@@ -145,7 +145,7 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
document.getElementById('playwright-highlight-container')?.remove();
|
|
|
});
|
|
|
if (browser) await browser.disconnect();
|
|
|
- }finally {
|
|
|
+ } finally {
|
|
|
console.log(123456789)
|
|
|
await page.evaluate(() => {
|
|
|
document.getElementById('playwright-highlight-container')?.remove();
|
|
|
@@ -153,8 +153,8 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
if (browser) await browser.disconnect();
|
|
|
}
|
|
|
}
|
|
|
- function allTabs(){
|
|
|
- return new Promise( (resolve, reject) => {
|
|
|
+ function allTabs() {
|
|
|
+ return new Promise((resolve, reject) => {
|
|
|
chrome.tabs.query({}, (tabs) => {
|
|
|
console.log('tabs', tabs)
|
|
|
resolve(tabs);
|
|
|
@@ -165,21 +165,21 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
function getActiveTabId() {
|
|
|
return new Promise((resolve) => {
|
|
|
setTimeout(() => {
|
|
|
- chrome.tabs.query({active: true}, (tabs) => {
|
|
|
+ chrome.tabs.query({ active: true }, (tabs) => {
|
|
|
console.log('tabs', tabs)
|
|
|
resolve(tabs[0]);
|
|
|
});
|
|
|
- },500)
|
|
|
+ }, 500)
|
|
|
});
|
|
|
}
|
|
|
-// 辅助函数：通过索引查找元素
|
|
|
+ // 辅助函数：通过索引查找元素
|
|
|
function findElementByIndex(domMap, targetIndex) {
|
|
|
return Object.values(domMap).find(
|
|
|
node => node.highlightIndex === targetIndex
|
|
|
);
|
|
|
}
|
|
|
|
|
|
-// 辅助函数：生成 CSS 选择器
|
|
|
+ // 辅助函数：生成 CSS 选择器
|
|
|
function generateSelector(element) {
|
|
|
const attrs = Object.entries(element.attributes)
|
|
|
.map(([k, v]) => `[${k}="${v.replace(/"/g, '\\"')}"]`)
|
|
|
@@ -188,7 +188,7 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
}
|
|
|
|
|
|
function gfe() {
|
|
|
- chrome.tabs.query({active: true, currentWindow: true}, async (tabs) => {
|
|
|
+ chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => {
|
|
|
const tabId = tabs[0].id;
|
|
|
await smartClick(tabId, 1, {
|
|
|
type: 'click',
|
|
|
@@ -197,21 +197,21 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
})
|
|
|
}
|
|
|
|
|
|
-// 背景脚本
|
|
|
+ // 背景脚本
|
|
|
function switchTabOrOpenNew(url) {
|
|
|
return new Promise(resolve => {
|
|
|
// 查询是否存在指定的标签页
|
|
|
// chrome.tabs.query({windowId: chrome.windows.WINDOW_ID_CURRENT}, async (tabs) => {
|
|
|
- chrome.tabs.query({}, async (tabs) => {
|
|
|
+ chrome.tabs.query({}, async (tabs) => {
|
|
|
let tabId = tabs.find(tabs => tabs.url.replace(/\/$/, '') === url);
|
|
|
- console.log(tabs, tabId,url)
|
|
|
+ console.log(tabs, tabId, url)
|
|
|
if (tabId) {
|
|
|
// 如果存在，则激活该标签页
|
|
|
- await chrome.tabs.update(tabId.id, {active: true});
|
|
|
+ await chrome.tabs.update(tabId.id, { active: true });
|
|
|
resolve()
|
|
|
} else {
|
|
|
// 如果不存在，则新打开一个标签页
|
|
|
- await chrome.tabs.create({url: url});
|
|
|
+ await chrome.tabs.create({ url: url });
|
|
|
resolve()
|
|
|
}
|
|
|
});
|
|
|
@@ -219,16 +219,16 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
}
|
|
|
|
|
|
function getexecuteapi(params) {
|
|
|
- return getexecute(params).then((res)=>{
|
|
|
- return res;
|
|
|
+ return getexecute(params).then((res) => {
|
|
|
+ return res;
|
|
|
})
|
|
|
}
|
|
|
|
|
|
async function agent(payload) {
|
|
|
|
|
|
- for(let item of payload.steps){
|
|
|
+ for (let item of payload.steps) {
|
|
|
console.log(item)
|
|
|
- if(item.stepIndex === 0){
|
|
|
+ if (item.stepIndex === 0) {
|
|
|
let params = {
|
|
|
conversationId: payload.conversationId,
|
|
|
messageId: payload.messageId,
|
|
|
@@ -240,7 +240,7 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
osVersion: "137.0.7151.69（正式版本）",
|
|
|
osArch: "arm64",
|
|
|
url: "",
|
|
|
- title:"",
|
|
|
+ title: "",
|
|
|
tabs: [],
|
|
|
interactiveElements: ""
|
|
|
},
|
|
|
@@ -251,56 +251,62 @@ console.log(domState,page,type,targetIndex,browser)
|
|
|
let add = await getexecuteapi(params)
|
|
|
console.log(add)
|
|
|
|
|
|
- let toolExecution=JSON.parse(add.data.plan.steps[0].toolExecution)[0].toolParameters
|
|
|
+ let toolExecution = JSON.parse(add.data.plan.steps[0].toolExecution)[0].toolParameters
|
|
|
console.log(JSON.parse(toolExecution).action)
|
|
|
if (JSON.parse(toolExecution).action == 'navigate') {
|
|
|
await switchTabOrOpenNew(JSON.parse(toolExecution).url)
|
|
|
}
|
|
|
- }else {
|
|
|
- let browdata= await browsercomm()
|
|
|
- console.log(browdata)
|
|
|
- const tabs = await allTabs()
|
|
|
- let params = {
|
|
|
- conversationId: payload.conversationId,
|
|
|
- messageId: payload.messageId,
|
|
|
- messageContent: payload.messageContent,
|
|
|
- planId: payload.id,
|
|
|
- currentPlanStepId: item.id,
|
|
|
- envData: {
|
|
|
- osName: "Google Chrome",
|
|
|
- osVersion: "137.0.7151.69（正式版本）",
|
|
|
- osArch: "arm64",
|
|
|
- url: browdata.tabitem.url,
|
|
|
- title: browdata.tabitem.title,
|
|
|
- tabs: tabs,
|
|
|
- interactiveElements: browdata.domState.domList
|
|
|
- // interactiveElements: "[0] <a>新闻</a>\n [1] <a>hao123</a>\n [2] <a>地图</a>\n [3] <a>贴吧</a>\n [4] <a>视频</a>\n [5] <a>图片</a>\n [6] <a>网盘</a>\n [7] <a>文库</a>\n [8] <a id=\"csaitab\"></a>\n [9] <a name=\"tj_briicon\">更多</a>\n [10] <a></a>\n [11] <a name=\"tj_login\" id=\"s-top-loginbtn\">登录</a>\n [12] <input placeholder=\"梁靖崑称优势是有王楚钦\" name=\"wd\" id=\"kw\" value=\"\"></input>\n [13] <input type=\"submit\" id=\"su\" value=\"百度一下\"></input>\n [14] <a>AI搜索已支持DeepSeek R1最新版立即体验</a>\n [16] <a id=\"hotsearch-refresh-btn\">换一换</a>\n [17] <a>0让“干坡坡”变“金窝窝”</a>\n [18] <a>5福建一楼房发生爆炸 有人员"
|
|
|
- },
|
|
|
- resultSummary: null,
|
|
|
- needSummary: true,
|
|
|
- success: false
|
|
|
- }
|
|
|
- let add = await getexecuteapi(params)
|
|
|
+ } else {
|
|
|
+ let browdata = await browsercomm()
|
|
|
+ console.log(browdata)
|
|
|
+ const tabs = await allTabs()
|
|
|
+ let params = {
|
|
|
+ conversationId: payload.conversationId,
|
|
|
+ messageId: payload.messageId,
|
|
|
+ messageContent: payload.messageContent,
|
|
|
+ planId: payload.id,
|
|
|
+ currentPlanStepId: item.id,
|
|
|
+ envData: {
|
|
|
+ osName: "Google Chrome",
|
|
|
+ osVersion: "137.0.7151.69（正式版本）",
|
|
|
+ osArch: "arm64",
|
|
|
+ url: browdata.tabitem.url,
|
|
|
+ title: browdata.tabitem.title,
|
|
|
+ tabs: tabs,
|
|
|
+ interactiveElements: browdata.domState.domList
|
|
|
+ // interactiveElements: "[0] <a>新闻</a>\n [1] <a>hao123</a>\n [2] <a>地图</a>\n [3] <a>贴吧</a>\n [4] <a>视频</a>\n [5] <a>图片</a>\n [6] <a>网盘</a>\n [7] <a>文库</a>\n [8] <a id=\"csaitab\"></a>\n [9] <a name=\"tj_briicon\">更多</a>\n [10] <a></a>\n [11] <a name=\"tj_login\" id=\"s-top-loginbtn\">登录</a>\n [12] <input placeholder=\"梁靖崑称优势是有王楚钦\" name=\"wd\" id=\"kw\" value=\"\"></input>\n [13] <input type=\"submit\" id=\"su\" value=\"百度一下\"></input>\n [14] <a>AI搜索已支持DeepSeek R1最新版立即体验</a>\n [16] <a id=\"hotsearch-refresh-btn\">换一换</a>\n [17] <a>0让“干坡坡”变“金窝窝”</a>\n [18] <a>5福建一楼房发生爆炸 有人员"
|
|
|
+ },
|
|
|
+ resultSummary: null,
|
|
|
+ needSummary: true,
|
|
|
+ success: false
|
|
|
+ }
|
|
|
+ let add = await getexecuteapi(params)
|
|
|
console.log(add)
|
|
|
- let toolExecution=add.data.plan.steps[item.stepIndex].toolExecution
|
|
|
- if(toolExecution) {
|
|
|
- let toolExecution=JSON.parse(add.data.plan.steps[item.stepIndex].toolExecution)[0].toolParameters
|
|
|
- await browserautomate(browdata.domState.buildDomTree, browdata.page, JSON.parse(toolExecution).action, JSON.parse(toolExecution).index,JSON.parse(toolExecution).text)
|
|
|
+ let toolExecution = add.data.plan.steps[item.stepIndex].toolExecution
|
|
|
+ if (toolExecution) {
|
|
|
+ let toolExecution = JSON.parse(add.data.plan.steps[item.stepIndex].toolExecution)[0].toolParameters
|
|
|
+ await browserautomate(browdata.domState.buildDomTree, browdata.page, JSON.parse(toolExecution).action, JSON.parse(toolExecution).index, JSON.parse(toolExecution).text)
|
|
|
|
|
|
- }else {
|
|
|
+ } else {
|
|
|
return
|
|
|
}
|
|
|
}
|
|
|
chrome.runtime.sendMessage({
|
|
|
type: "FROM_STEP",
|
|
|
- payload: item.stepIndex});
|
|
|
+ payload: item.stepIndex
|
|
|
+ });
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
- chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
|
|
|
+ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
|
|
|
console.log(request, sender, sendResponse);
|
|
|
if (request.type == 'FROM_PLAN') {
|
|
|
- agent(request.payload)
|
|
|
+ // agent(request.payload)
|
|
|
+ chrome.runtime.sendMessage({
|
|
|
+ type: "FROM_STEP",
|
|
|
+ payload: 111
|
|
|
+ });
|
|
|
+ console.log(7575);
|
|
|
+
|
|
|
}
|
|
|
})
|
|
|
}
|
