src/entrypoints/sidepanel/component/StepsDisplay.vue

@@ -1,5 +1,5 @@
 <template>
-  <div class="">
+  <div class="min-w-60">
     <div  class="steps-content">
       <div class="mb-2">
         <div class="text-x" v-for="step,i in props.steps" :key="step.id">
@@ -18,7 +18,6 @@
     </div>
   </div>
 </template>
-  
 <script lang="js" setup>
 import { onMounted } from 'vue'
 import { actionMap } from '@/utils/actionMap'

src/utils/navigator.js

@@ -8,17 +8,12 @@ export const navigator = () => {
     let browser = null;
     async function browsercomm() {
         const tabitem = await getActiveTabId();
-        // console.log(tabitem)
         let tabId = tabitem.id
-        // console.log(tabId)
         const connectOptions = {
             transport: await ExtensionTransport.connectTab(tabId), // 替换为实际标签ID
             defaultViewport: null,
             protocol: 'cdp'
         };
-        // console.log(connectOptions)
-        // let browser;
-
         try {
             // 连接到浏览器
             browser = await connect(connectOptions);
@@ -69,8 +64,7 @@ export const navigator = () => {
                         return cleanHtmlTags(item, attrs);
                     })
                     .join("\n")
-                // console.log(domList)
-                return { buildDomTree: buildDomTree, domList: domList }
+                return { buildDomTree: buildDomTree, domList: domList, body: document.body.innerText }
             });
             return { domState: domState, page: page, tabitem: tabitem };
         } finally {
@@ -87,6 +81,7 @@ export const navigator = () => {
         // 生成增强选择器
         const selector = generateSelector(targetElement);
         // console.log(domState, targetElement, selector)
+        // console.log(domState, targetElement, selector)
         // const [page] = await browser.pages()
         try {
             await page.waitForSelector(selector, { visible: true, timeout: 10000 });
@@ -103,6 +98,9 @@ export const navigator = () => {
                 case 'getcontent':
                     console.log(await page.content());
                     break;
+                case 'get_text':
+                    console.log(await page.content());
+                    break;
                 case 'input_text':
                     await page.type(selector, text, { delay: 600 });
                     break;
@@ -119,9 +117,6 @@ export const navigator = () => {
                     await page.type(selector, 'your-password', { delay: 600 });
                     await page.click(selector); // 或者其他登录按钮
                     break;
-                case '':
-                    page.type(selector, 'World', { delay: 600 })
-                    break
                 default:
                     console.log('未知指令。');
             }
@@ -140,13 +135,12 @@ export const navigator = () => {
             // if (attempt === retries) throw clickError;
             // await page.waitForTimeout(delay);
 
-            // 刷新 DOM 状态
-            await page.evaluate(() => {
-                document.getElementById('playwright-highlight-container')?.remove();
-            });
-            if (browser) await browser.disconnect();
+            // await page.evaluate(() => {
+            //     document.getElementById('playwright-highlight-container')?.remove();
+            // });
+            // if (browser) await browser.disconnect();
         } finally {
-            // console.log(123456789)
+            console.log(123456789)
             await page.evaluate(() => {
                 document.getElementById('playwright-highlight-container')?.remove();
             });
@@ -156,7 +150,6 @@ export const navigator = () => {
     function allTabs() {
         return new Promise((resolve, reject) => {
             chrome.tabs.query({}, (tabs) => {
-                // console.log('tabs', tabs)
                 resolve(tabs);
             });
         })
@@ -166,7 +159,6 @@ export const navigator = () => {
         return new Promise((resolve) => {
             setTimeout(() => {
                 chrome.tabs.query({ active: true }, (tabs) => {
-                    // console.log('tabs', tabs)
                     resolve(tabs[0]);
                 });
             }, 500)
@@ -204,7 +196,6 @@ export const navigator = () => {
             // chrome.tabs.query({windowId: chrome.windows.WINDOW_ID_CURRENT}, async (tabs) => {
             chrome.tabs.query({}, async (tabs) => {
                 let tabId = tabs.find(tabs => tabs.url.replace(/\/$/, '') === url);
-                // console.log(tabs, tabId,url)
                 if (tabId) {
                     // 如果存在，则激活该标签页
                     await chrome.tabs.update(tabId.id, { active: true });
@@ -225,14 +216,14 @@ export const navigator = () => {
     }
 
     async function agent(payload) {
-
+        let isDOMType = "element"
         for (let item of payload.steps) {
             console.log(item)
             chrome.runtime.sendMessage({
                 type: "PLAN_STEP",
                 payload: { dimension: item.directions, stepIndex: item.stepIndex },
             }, function (response) {
-                
+
             });
             if (item.stepIndex === 0) {
                 let params = {
@@ -267,7 +258,7 @@ export const navigator = () => {
                 }
             } else {
                 let browdata = await browsercomm()
-                // console.log(browdata, 'browdata')
+                console.log(browdata)
                 const tabs = await allTabs()
                 let params = {
                     conversationId: payload.conversationId,
@@ -282,43 +273,45 @@ export const navigator = () => {
                         url: browdata.tabitem.url,
                         title: browdata.tabitem.title,
                         tabs: tabs,
-                        interactiveElements: browdata.domState.domList
+                        interactiveElements: isDOMType == 'text' ? browdata.domState.body : browdata.domState.domList
                         // interactiveElements: "[0] <a>新闻</a>\n        [1] <a>hao123</a>\n        [2] <a>地图</a>\n        [3] <a>贴吧</a>\n        [4] <a>视频</a>\n        [5] <a>图片</a>\n        [6] <a>网盘</a>\n        [7] <a>文库</a>\n        [8] <a id=\"csaitab\"></a>\n        [9] <a name=\"tj_briicon\">更多</a>\n        [10] <a></a>\n        [11] <a name=\"tj_login\" id=\"s-top-loginbtn\">登录</a>\n        [12] <input placeholder=\"梁靖崑称优势是有王楚钦\" name=\"wd\" id=\"kw\" value=\"\"></input>\n        [13] <input type=\"submit\" id=\"su\" value=\"百度一下\"></input>\n        [14] <a>AI搜索已支持DeepSeek R1最新版立即体验</a>\n        [16] <a id=\"hotsearch-refresh-btn\">换一换</a>\n        [17] <a>0让“干坡坡”变“金窝窝”</a>\n        [18] <a>5福建一楼房发生爆炸 有人员"
                     },
                     resultSummary: null,
                     needSummary: true,
                     success: false
                 }
-                let add = await getexecuteapi(params)
-                // console.log(add, 'add')
-                let toolExecution = add.data.plan.steps[item.stepIndex].toolExecution
-                if (toolExecution) {
-
-                    let toolExecution = JSON.parse(add.data.plan.steps[item.stepIndex].toolExecution)[0].toolParameters
+                let getexecutedaat = await getexecuteapi(params)
+                let toolExecution = JSON.parse(getexecutedaat.data.plan.steps[item.stepIndex].toolExecution)
+                // chrome.runtime.sendMessage({
+                //     type: "FROM_STEP",
+                //     payload: toolExecution
+                // });
+                for (let item of toolExecution) {
+                    if (JSON.parse(item.toolParameters).action == 'get_text') {
+                        isDOMType = 'text'
+                    } else {
+                        isDOMType = 'element'
+                    }
                     chrome.runtime.sendMessage({
                         type: "FROM_STEP",
-                        payload: JSON.parse(toolExecution)
+                        payload: JSON.parse(item.toolParameters)
                     });
-                    await browserautomate(browdata.domState.buildDomTree, browdata.page, JSON.parse(toolExecution).action, JSON.parse(toolExecution).index, JSON.parse(toolExecution).text)
-
-                } else {
-                    return
+                    await browserautomate(browdata.domState.buildDomTree, browdata.page, JSON.parse(item.toolParameters).action, JSON.parse(item.toolParameters).index || 0, JSON.parse(item.toolParameters).text || '')
                 }
             }
 
         }
-        
+
     }
 
-    chrome.runtime.onMessage.addListener( (request, sender, sendResponse) => {
+    chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
         // console.log(request, sender, sendResponse);
         if (request.type == 'FROM_PLAN') {
             agent(request.payload).then(res => {
                 console.log('完成11111');
-                
-                sendResponse({status:'1'})
+                sendResponse({ status: '1' })
             }).catch(async res => {
-                console.log('错误信息'+ res);
+                console.log('错误信息' + res);
                 sendResponse({ status: '0' })
                 if (browser) await browser.disconnect();
             })

wxt.config.ts

@@ -36,6 +36,10 @@ export default defineConfig({
       'webRequest',
       'sidePanel',
       "debugger",
+        'scripting'
+    ],
+    host_permissions: [
+      "<all_urls>"
     ],
     content_security_policy: {
       extension_pages: "script-src 'self'; object-src 'self';"