自动化

src/utils/navigator.js

@@ -8,17 +8,12 @@ export const navigator=()=> {
     let browser = null;
     async function browsercomm() {
         const tabitem = await getActiveTabId();
-        console.log(tabitem)
         let tabId=tabitem.id
-        console.log(tabId)
         const connectOptions = {
             transport: await ExtensionTransport.connectTab(tabId), // 替换为实际标签ID
             defaultViewport: null,
             protocol: 'cdp'
         };
-        console.log(connectOptions)
-        // let browser;
-
         try {
             // 连接到浏览器
             browser = await connect(connectOptions);
@@ -69,8 +64,7 @@ export const navigator=()=> {
                         return cleanHtmlTags(item, attrs);
                     })
                     .join("\n")
-                console.log(domList)
-                return {buildDomTree:buildDomTree,domList:domList}
+                return {buildDomTree:buildDomTree,domList:domList,body:document.body.innerText}
             });
             return {domState:domState, page:page,tabitem:tabitem};
         } finally {
@@ -79,14 +73,14 @@ export const navigator=()=> {
         }
     }
     async function browserautomate(domState,page,type,targetIndex,text) {
-console.log(domState,page,type,targetIndex,browser)
+// console.log(domState,page,type,targetIndex,browser)
         // 查找目标元素
         const targetElement = findElementByIndex(domState.map, targetIndex);
         if (!targetElement) throw new Error('Element not found');
 
         // 生成增强选择器
         const selector = generateSelector(targetElement);
-        console.log(domState, targetElement, selector)
+        // console.log(domState, targetElement, selector)
         // const [page] = await browser.pages()
         try {
             await page.waitForSelector(selector, {visible: true, timeout: 10000});
@@ -103,6 +97,9 @@ console.log(domState,page,type,targetIndex,browser)
                 case 'getcontent':
                     console.log(await page.content());
                     break;
+                case 'get_text':
+                    console.log(await page.content());
+                    break;
                 case 'input_text':
                     await page.type(selector, text, {delay: 600});
                     break;
@@ -119,9 +116,6 @@ console.log(domState,page,type,targetIndex,browser)
                     await page.type(selector, 'your-password', {delay: 600});
                     await page.click(selector); // 或者其他登录按钮
                     break;
-                case '':
-                    page.type(selector, 'World', {delay: 600})
-                    break
                 default:
                     console.log('未知指令。');
             }
@@ -140,11 +134,10 @@ console.log(domState,page,type,targetIndex,browser)
             // if (attempt === retries) throw clickError;
             // await page.waitForTimeout(delay);
 
-            // 刷新 DOM 状态
-            await page.evaluate(() => {
-                document.getElementById('playwright-highlight-container')?.remove();
-            });
-            if (browser) await browser.disconnect();
+            // await page.evaluate(() => {
+            //     document.getElementById('playwright-highlight-container')?.remove();
+            // });
+            // if (browser) await browser.disconnect();
         }finally {
             console.log(123456789)
             await page.evaluate(() => {
@@ -156,7 +149,6 @@ console.log(domState,page,type,targetIndex,browser)
     function allTabs(){
         return new Promise( (resolve, reject) => {
             chrome.tabs.query({}, (tabs) => {
-                console.log('tabs', tabs)
                 resolve(tabs);
             });
         })
@@ -166,7 +158,6 @@ console.log(domState,page,type,targetIndex,browser)
         return new Promise((resolve) => {
             setTimeout(() => {
                 chrome.tabs.query({active: true}, (tabs) => {
-                    console.log('tabs', tabs)
                     resolve(tabs[0]);
                 });
             },500)
@@ -204,7 +195,6 @@ console.log(domState,page,type,targetIndex,browser)
             // chrome.tabs.query({windowId: chrome.windows.WINDOW_ID_CURRENT}, async (tabs) => {
                 chrome.tabs.query({}, async (tabs) => {
                 let tabId = tabs.find(tabs => tabs.url.replace(/\/$/, '') === url);
-                console.log(tabs, tabId,url)
                 if (tabId) {
                     // 如果存在，则激活该标签页
                     await chrome.tabs.update(tabId.id, {active: true});
@@ -225,9 +215,8 @@ console.log(domState,page,type,targetIndex,browser)
     }
 
     async function agent(payload) {
-
+        let isDOMType="element"
         for(let item of payload.steps){
-            console.log(item)
             if(item.stepIndex === 0){
                 let params = {
                     conversationId: payload.conversationId,
@@ -249,46 +238,43 @@ console.log(domState,page,type,targetIndex,browser)
                     success: false
                 }
                 let add = await getexecuteapi(params)
-                console.log(add)
-
                 let toolExecution=JSON.parse(add.data.plan.steps[0].toolExecution)[0].toolParameters
-                console.log(JSON.parse(toolExecution).action)
                 if (JSON.parse(toolExecution).action == 'navigate') {
                     await switchTabOrOpenNew(JSON.parse(toolExecution).url)
                 }
             }else {
-                    let browdata=  await  browsercomm()
-        console.log(browdata)
-               const tabs = await allTabs()
-        let params = {
-            conversationId: payload.conversationId,
-            messageId: payload.messageId,
-            messageContent: payload.messageContent,
-            planId: payload.id,
-            currentPlanStepId: item.id,
-            envData: {
-                osName: "Google Chrome",
-                osVersion: "137.0.7151.69（正式版本）",
-                osArch: "arm64",
-                url: browdata.tabitem.url,
-                title: browdata.tabitem.title,
-                tabs: tabs,
-                interactiveElements: browdata.domState.domList
-                // interactiveElements: "[0] <a>新闻</a>\n        [1] <a>hao123</a>\n        [2] <a>地图</a>\n        [3] <a>贴吧</a>\n        [4] <a>视频</a>\n        [5] <a>图片</a>\n        [6] <a>网盘</a>\n        [7] <a>文库</a>\n        [8] <a id=\"csaitab\"></a>\n        [9] <a name=\"tj_briicon\">更多</a>\n        [10] <a></a>\n        [11] <a name=\"tj_login\" id=\"s-top-loginbtn\">登录</a>\n        [12] <input placeholder=\"梁靖崑称优势是有王楚钦\" name=\"wd\" id=\"kw\" value=\"\"></input>\n        [13] <input type=\"submit\" id=\"su\" value=\"百度一下\"></input>\n        [14] <a>AI搜索已支持DeepSeek R1最新版立即体验</a>\n        [16] <a id=\"hotsearch-refresh-btn\">换一换</a>\n        [17] <a>0让“干坡坡”变“金窝窝”</a>\n        [18] <a>5福建一楼房发生爆炸 有人员"
-            },
-            resultSummary: null,
-            needSummary: true,
-            success: false
-        }
-        let add = await getexecuteapi(params)
-                console.log(add)
-                let toolExecution=add.data.plan.steps[item.stepIndex].toolExecution
-                if(toolExecution) {
-                    let toolExecution=JSON.parse(add.data.plan.steps[item.stepIndex].toolExecution)[0].toolParameters
-                    await browserautomate(browdata.domState.buildDomTree, browdata.page, JSON.parse(toolExecution).action, JSON.parse(toolExecution).index,JSON.parse(toolExecution).text)
-
-                }else {
-                    return
+                let browdata=  await  browsercomm()
+                console.log(browdata)
+                const tabs = await allTabs()
+                let params = {
+                    conversationId: payload.conversationId,
+                    messageId: payload.messageId,
+                    messageContent: payload.messageContent,
+                    planId: payload.id,
+                    currentPlanStepId: item.id,
+                    envData: {
+                        osName: "Google Chrome",
+                        osVersion: "137.0.7151.69（正式版本）",
+                        osArch: "arm64",
+                        url: browdata.tabitem.url,
+                        title: browdata.tabitem.title,
+                        tabs: tabs,
+                        interactiveElements:isDOMType=='text'?browdata.domState.body: browdata.domState.domList
+                        // interactiveElements: "[0] <a>新闻</a>\n        [1] <a>hao123</a>\n        [2] <a>地图</a>\n        [3] <a>贴吧</a>\n        [4] <a>视频</a>\n        [5] <a>图片</a>\n        [6] <a>网盘</a>\n        [7] <a>文库</a>\n        [8] <a id=\"csaitab\"></a>\n        [9] <a name=\"tj_briicon\">更多</a>\n        [10] <a></a>\n        [11] <a name=\"tj_login\" id=\"s-top-loginbtn\">登录</a>\n        [12] <input placeholder=\"梁靖崑称优势是有王楚钦\" name=\"wd\" id=\"kw\" value=\"\"></input>\n        [13] <input type=\"submit\" id=\"su\" value=\"百度一下\"></input>\n        [14] <a>AI搜索已支持DeepSeek R1最新版立即体验</a>\n        [16] <a id=\"hotsearch-refresh-btn\">换一换</a>\n        [17] <a>0让“干坡坡”变“金窝窝”</a>\n        [18] <a>5福建一楼房发生爆炸 有人员"
+                    },
+                    resultSummary: null,
+                    needSummary: true,
+                    success: false
+                }
+                let getexecutedaat = await getexecuteapi(params)
+                let toolExecution=JSON.parse(getexecutedaat.data.plan.steps[item.stepIndex].toolExecution)
+                for(let item of toolExecution){
+                    if(JSON.parse(item.toolParameters).action=='get_text'){
+                        isDOMType='text'
+                    }else {
+                        isDOMType='element'
+                    }
+                    await browserautomate(browdata.domState.buildDomTree, browdata.page, JSON.parse(item.toolParameters).action, JSON.parse(item.toolParameters).index||0,JSON.parse(item.toolParameters).text||'')
                 }
             }
             chrome.runtime.sendMessage({

wxt.config.ts

@@ -33,6 +33,10 @@ export default defineConfig({
       'webRequest',
       'sidePanel',
       "debugger",
+        'scripting'
+    ],
+    host_permissions: [
+      "<all_urls>"
     ],
     content_security_policy: {
       extension_pages: "script-src 'self'; object-src 'self';"