2 Commits 7599d4fd47 ... 0c1cb06d0c

Autore SHA1 Messaggio Data
  wzg 0c1cb06d0c refactor(.env): 更新 Vite 客户端 ID 1 mese fa
  zjh 4bf0710357 后端联调 1 mese fa

+ 3 - 3
.env

@@ -10,8 +10,8 @@ VITE_APP_BASE_API='http://192.168.1.5:8000'
 # VITE_API_WS_URL = 'ws://180.76.147.97:13088'
 VITE_API_WS_URL = 'ws://192.168.1.5:8000'
 #VITE_API_WS_URL = 'ws://192.168.1.166:5555'
+
 # 终端ID
-# VITE_CLIENT_ID = '765be25e4e78b101b896cb3ecac39b1b'
-VITE_CLIENT_ID = '7d489df3ed56b51fcbf4a4f7139885c2'
-#VITE_CLIENT_ID = '7a1958fafed3268a7515b083bd6f144f'
+VITE_CLIENT_ID = '7a1958fafed3268a7515b083bd6f144f'
+
 

+ 21 - 21
package-lock.json

@@ -25,7 +25,7 @@
         "moment": "^2.30.1",
         "openai": "^4.85.4",
         "pinia": "^3.0.1",
-        "puppeteer-core": "^24.10.2",
+        "puppeteer-core": "^24.8.2",
         "sass": "^1.85.1",
         "uuid": "^11.1.0",
         "vant": "^4.9.17",
@@ -1185,16 +1185,16 @@
       }
     },
     "node_modules/@puppeteer/browsers": {
-      "version": "2.10.5",
-      "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.5.tgz",
-      "integrity": "sha512-eifa0o+i8dERnngJwKrfp3dEq7ia5XFyoqB17S4gK8GhsQE4/P8nxOfQSE0zQHxzzLo/cmF+7+ywEQ7wK7Fb+w==",
+      "version": "2.10.4",
+      "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.4.tgz",
+      "integrity": "sha512-9DxbZx+XGMNdjBynIs4BRSz+M3iRDeB7qRcAr6UORFLphCIM2x3DXgOucvADiifcqCE4XePFUKcnaAMyGbrDlQ==",
       "license": "Apache-2.0",
       "dependencies": {
-        "debug": "^4.4.1",
+        "debug": "^4.4.0",
         "extract-zip": "^2.0.1",
         "progress": "^2.0.3",
         "proxy-agent": "^6.5.0",
-        "semver": "^7.7.2",
+        "semver": "^7.7.1",
         "tar-fs": "^3.0.8",
         "yargs": "^17.7.2"
       },
@@ -4208,9 +4208,9 @@
       }
     },
     "node_modules/devtools-protocol": {
-      "version": "0.0.1452169",
-      "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1452169.tgz",
-      "integrity": "sha512-FOFDVMGrAUNp0dDKsAU1TorWJUx2JOU1k9xdgBKKJF3IBh/Uhl2yswG5r3TEAOrCiGY2QRp1e6LVDQrCsTKO4g==",
+      "version": "0.0.1439962",
+      "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1439962.tgz",
+      "integrity": "sha512-jJF48UdryzKiWhJ1bLKr7BFWUQCEIT5uCNbDLqkQJBtkFxYzILJH44WN0PDKMIlGDN7Utb8vyUY85C3w4R/t2g==",
       "license": "BSD-3-Clause"
     },
     "node_modules/didyoumean": {
@@ -9684,15 +9684,15 @@
       }
     },
     "node_modules/puppeteer-core": {
-      "version": "24.10.2",
-      "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.10.2.tgz",
-      "integrity": "sha512-CnzhOgrZj8DvkDqI+Yx+9or33i3Y9uUYbKyYpP4C13jWwXx/keQ38RMTMmxuLCWQlxjZrOH0Foq7P2fGP7adDQ==",
+      "version": "24.8.2",
+      "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.8.2.tgz",
+      "integrity": "sha512-wNw5cRZOHiFibWc0vdYCYO92QuKTbJ8frXiUfOq/UGJWMqhPoBThTKkV+dJ99YyWfzJ2CfQQ4T1nhhR0h8FlVw==",
       "license": "Apache-2.0",
       "dependencies": {
-        "@puppeteer/browsers": "2.10.5",
+        "@puppeteer/browsers": "2.10.4",
         "chromium-bidi": "5.1.0",
-        "debug": "^4.4.1",
-        "devtools-protocol": "0.0.1452169",
+        "debug": "^4.4.0",
+        "devtools-protocol": "0.0.1439962",
         "typed-query-selector": "^2.12.0",
         "ws": "^8.18.2"
       },
@@ -9701,9 +9701,9 @@
       }
     },
     "node_modules/puppeteer-core/node_modules/ws": {
-      "version": "8.18.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz",
-      "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==",
+      "version": "8.18.3",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz",
+      "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
       "license": "MIT",
       "engines": {
         "node": ">=10.0.0"
@@ -11913,9 +11913,9 @@
       }
     },
     "node_modules/tar-fs": {
-      "version": "3.0.10",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.10.tgz",
-      "integrity": "sha512-C1SwlQGNLe/jPNqapK8epDsXME7CAJR5RL3GcE6KWx1d9OUByzoHVcbu1VPI8tevg9H8Alae0AApHHFGzrD5zA==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.0.tgz",
+      "integrity": "sha512-5Mty5y/sOF1YWj1J6GiBodjlDc05CUR8PKXrsnFAiSG0xA+GHeWLovaZPYUDXkH/1iKRf2+M5+OrRgzC7O9b7w==",
       "license": "MIT",
       "dependencies": {
         "pump": "^3.0.0",

+ 1 - 1
package.json

@@ -32,7 +32,7 @@
     "moment": "^2.30.1",
     "openai": "^4.85.4",
     "pinia": "^3.0.1",
-    "puppeteer-core": "^24.10.2",
+    "puppeteer-core": "^24.8.2",
     "sass": "^1.85.1",
     "uuid": "^11.1.0",
     "vant": "^4.9.17",

+ 3 - 3
src/api/index.js

@@ -22,7 +22,7 @@ export function getUserInfo() {
 }
 export function getChatList(data) {
     return request({
-        url: '/messages/history/selConversationList',
+        url: '/api/aigc/conversation',
         method: 'get',
         data: data
     })
@@ -123,8 +123,8 @@ export function getDomainList(data) {
 
 export function getModalList(data) {
     return request({
-        url: `/ai/model/pageList`,
-        method: 'get',
+        url: `/aigc/model/selModelAndSecrt`,
+        method: 'post',
         data
     })
 }

+ 33 - 32
src/entrypoints/background/index.ts

@@ -1,15 +1,16 @@
 import { onMessage, sendMessage } from 'webext-bridge/background'
-import { navigator } from '@/utils/navigator.js';
-import {plan} from './plan'
+import { navigator } from '@/utils/navigator.js'
+import { plan, messageList } from './plan'
+
 export default defineBackground(() => {
   navigator()
   let currentTabId
   let arr = []
   let token = ''
   let userId = ''
-  console.log();
+  console.log()
   chrome.runtime.getPlatformInfo().then(res => console.log(res))
-  const url = import.meta.env.VITE_APP_BASE_API +  '/behavior/user/adds'
+  const url = import.meta.env.VITE_APP_BASE_API + '/behavior/user/adds'
   // Executed when background is loaded
   chrome.sidePanel
     .setPanelBehavior({ openPanelOnActionClick: true })
@@ -20,7 +21,7 @@ export default defineBackground(() => {
       if (token) {
         arr.push({
           ...message.data,
-          userId:token
+          userId: token
         })
         if (arr.length > 10) {
           fetch(url, {
@@ -57,21 +58,21 @@ export default defineBackground(() => {
               data: 1
             }, (response) => {
               if (chrome.runtime.lastError) {
-                console.error(`Tab ${tab.id} 错误: ${chrome.runtime.lastError.message}`);
+                console.error(`Tab ${tab.id} 错误: ${chrome.runtime.lastError.message}`)
               } else {
-                console.log(`Tab ${tab.id} 响应:`, response);
+                console.log(`Tab ${tab.id} 响应:`, response)
               }
-            });
+            })
           }
-        });
-      });
+        })
+      })
       chrome.runtime.sendMessage({
         type: 'SET_SELECT_TO_CENTENT',
-        data:1
+        data: 1
       })
     }
     if (message.type === 'GET_SETTING_INFO') {
-      console.log(message);
+      console.log(message)
 
       chrome.storage.local.get('token', (result) => {
         sendResponse(result.token || '')
@@ -124,7 +125,7 @@ export default defineBackground(() => {
         const tabId = tabs[1].id // 获取当前活动的 tabId
         chrome.tabs.sendMessage(
           tabId,
-          { type: 'INPUT_FORM'},
+          { type: 'INPUT_FORM' },
           (response) => {
             if (chrome.runtime.lastError) {
               console.error('消息发送失败:', chrome.runtime.lastError.message)
@@ -157,7 +158,7 @@ export default defineBackground(() => {
     }
     // 截图
     if (message.type === 'SCREENSHOT') {
-      chrome.tabs.query({ active: true, currentWindow: true}, (tabs) => {
+      chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
         if (tabs.length === 0) return // 确保有活动标签页
         const tabId = tabs[0].id // 获取当前活动的 tabId
         chrome.tabs.sendMessage(tabId, message, (response) => {
@@ -189,33 +190,33 @@ export default defineBackground(() => {
   //   }, 0)
   // });
   chrome.storage.onChanged.addListener((changes, areaName) => {
-    console.log('[Storage Changed] Area:', areaName);
+    console.log('[Storage Changed] Area:', areaName)
     // 遍历所有被修改的键
     for (const [key, { oldValue, newValue }] of Object.entries(changes)) {
-      console.log(key);
+      console.log(key)
 
       if (key === 'token') {
         token = newValue
         if (!newValue) {
           chrome.runtime.sendMessage({
-            type: 'USER_LOGOUT',
-          }, function (response) {
-          });
+            type: 'USER_LOGOUT'
+          }, function(response) {
+          })
           continue
         }
       }
       if (key === 'userInfo') {
         if (!newValue) {
           chrome.runtime.sendMessage({
-            type: 'USER_LOGOUT',
-          }, function (response) {
-          });
+            type: 'USER_LOGOUT'
+          }, function(response) {
+          })
           continue
         }
         userId = JSON.parse(newValue).id
       }
     }
-  });
+  })
   chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => {
     if (changeInfo.status === 'complete' && tab.url) {
       chrome.runtime.sendMessage({
@@ -240,15 +241,15 @@ export default defineBackground(() => {
     })
   })
 
-  onMessage('FROM_PLAN', (message) => {
-    plan(message.data)
-    console.log('side-panel ------> background', message);
-  });
-
-  setTimeout(async ()=>{
-    const res = await browsercomm()
-    console.log('background',res)
-  },10000)
+  onMessage('FROM_PLAN',  async (message) => {
+    messageList.push({
+      id:message.data.payload.id,
+      type:'ALL',
+      params:message.data.params,
+      result:message.data.payload
+    })
+    await plan()
+  })
 })
 
 // Allows users to open the side panel by clicking on the action toolbar icon

+ 99 - 104
src/entrypoints/background/plan.ts

@@ -1,128 +1,123 @@
-import { browsercomm, getExecuteApi, switchTabOrOpenNew } from '@/utils/navigator.js'
+import { browsercomm, getExecuteApi, switchTabOrOpenNew, tabsList, getActiveTabId } from '@/utils/utils.js'
 import { browser } from 'wxt/browser'
+import {tootIp} from "./tool/index"
 
-const messageList = []
+interface MessageList {
+  id: string,
+  status: 'success' | 'error',
+  type: 'ALL'|'PLAN'|'EXECUTE',
+  params: any,
+  result: any
+}
 
-export const plan = async (data: any) => {
-  console.log('plan', data)
-  const stepsList = data.payload.steps.map((item: any) => {
-    return {
-      conversationId: data.payload.conversationId,
-      messageId: data.payload.messageId,
-      messageContent: data.payload.messageContent,
-      ...item
-    }
-  })
+export const messageList: MessageList[] = []
 
-  for (let i = 0; i < stepsList.length; i++) {
+export const plan = async () => {
+  console.log('plan', messageList)
+  const { steps } = messageList[0].result
 
+  for (let i = 0; i < steps.length; i++) {
     // 计划生成
-    const res = await generator(stepsList[i], i)
-    console.log('第一步:计划生成', res)
+    const res = await generator(messageList[0], steps[i], i)
+    if (!res){
+      return new Error('计划生成失败')
+    }
 
     // 执行计划
-    const res2 = await execute(res.data.plan)
-    console.log('第二步:执行计划', res2)
+    const res2 = await execute(messageList.at(-1))
+    if (!res2){
+      return new Error('执行计划失败')
+    }
 
+    // 验证分步计划
 
-    // let continueLoop = true
-    // let maxLoop = 5
-    // let nowLoop = 0
-    // try {
-    //   while (continueLoop) {
-    //     nowLoop++
-    //     // 计划生成
-    //     const res = await generator(stepsList[i], i)
-    //     console.log('第一步:计划生成', res)
-    //
-    //     // 执行计划
-    //     const res2 = await execute(res.data.plan)
-    //     console.log('第二步:执行计划', res2)
-    //
-    //     if (res2.nowStep === true){
-    //       continueLoop = false
-    //     }
-    //     // // 验证程序
-    //     // if (res2.completed === true) {
-    //     //   // 进入验证程序
-    //     //   continueLoop = false
-    //     // }
-    //     if (nowLoop >= maxLoop)
-    //       continueLoop = false
-    //   }
-    // } catch (e) {
-    //   console.log('stop', e)
-    //   return {
-    //     completed: false,
-    //     status: 'error',
-    //     message: '执行计划失败'
-    //   }
-    // }
   }
 
+  // 验证最终计划
+  // if (messageList.at(-1).result.completed) {
+  //   // 执行最终验证程序
+  //   break
+  // }
+
 }
 
 // 执行计划
-export const execute = async (data) => {
-  console.log('execute', data)
-  const tal = data.steps
-  let toolList = []
-  for (let i = 0; i < tal.length; i++) {
-    if (!tal[i].toolParameters)
-      return {
-        completed: data.completed,
+export const execute = async (data:MessageList) => {
+  console.log('execute', data,tootIp)
+  const t = data.result.steps.find(item => item.id === data.params.currentPlanStepId)
+  if (!t.toolExecution) {
+    messageList.push({
+      id: data.id,
+      status: 'error',
+      type: 'PLAN',
+      params: t.toolExecution,
+      result: t
+    })
+    console.error('计划不存在')
+    return false
+  }
+  const toolExecution = JSON.parse(t.toolExecution)
+  console.log(toolExecution)
+  for (let i = 0; i < toolExecution.length; i++) {
+    const toolParameters = JSON.parse(toolExecution[i].toolParameters)
+    const r = await tootIp[toolParameters.action].func(toolParameters)
+    if (!r) {
+      messageList.push({
+        id: data.id,
         status: 'error',
-        message: '没有可执行的计划'
-      }
-    const toolParameters = JSON.parse(tal[i].toolParameters)
-    // 执行动作
-    if (toolParameters.action === 'navigate') {
-      const r = await switchTabOrOpenNew(toolParameters.url)
-      if (!r) {
-        return {
-          completed: data.completed,
-          status: 'error',
-          message: '页面跳转失败'
-        }
-      } else {
-        toolList.push(toolParameters)
-      }
+        type: 'EXECUTE',
+        params: toolExecution[i].toolParameters,
+        result: r
+      })
+      console.error('执行计划失败')
+      return false
     }
-    // 判断页面是否发生变化
-    if (1) {
-      // 页面发生变化,需要重新计划 completed 改为 true
-      return {
-        nowStep: true,
-        completed: true,
-        status: 'success',
-        toolList,
-        message: '页面已发生变化'
-      }
-    } else {
-      return {
-        nowStep: true,
-        completed: data.completed,
-        status: 'success',
-        toolList,
-        message: '当前所有步骤执行完毕'
-      }
-    }
-
+    messageList.push({
+      id: data.id,
+      status: 'success',
+      type: 'EXECUTE',
+      params: toolExecution[i].toolParameters,
+      result: r
+    })
   }
+  return true
 }
 
 // 计划生成
-export const generator = async (data, index) => {
-  messageList.push(data)
-  let dom, tabs = []
-  if (index !== 0) {
-    tabs = await browser.tabs.query({ active: true, currentWindow: true })
-
-    // const dom = await browsercomm()
-    // console.log('dom元素',dom)
-    dom = '[0] <a>新闻</a >\n        [1] <a>hao123</a >\n        [2] <a>地图</a >\n        [3] <a>贴吧</a >\n        [4] <a>视频</a >\n        [5] <a>图片</a >\n        [6] <a>网盘</a >\n        [7] <a>文库</a >\n        [8] <a id="csaitab"></a >\n        [9] <a name="tj_briicon">更多</a >\n        [10] <a></a >\n        [11] <a name="tj_login" id="s-top-loginbtn">登录</a >\n        [12] <input placeholder="梁靖崑称优势是有王楚钦" name="wd" id="kw" value=""></input>\n        [13] <input type="submit" id="su" value="百度一下"></input>\n        [14] <a>AI搜索已支持DeepSeek R1最新版立即体验</a >\n        [16] <a id="hotsearch-refresh-btn">换一换</a >\n        [17] <a>0让“干坡坡”变“金窝窝”</a >\n        [18] <a>5福建一楼房发生爆炸 有人员'
+export const generator = async (masterPlan: MessageList, data: MessageList, index: number) => {
+  let tab,tabs=[], dom
+  if (index !== 0){
+    tab = await getActiveTabId()
+    dom = await browsercomm()
+    tabs = await tabsList()
+  }
+  let params = {
+    conversationId: masterPlan.conversationId,
+    messageId: masterPlan.messageId,
+    messageContent: masterPlan.messageContent,
+    planId: data?.planId,
+    currentPlanStepId: data.id,
+    envData: {
+      osName: 'Google Chrome',
+      osVersion: '137.0.7151.69(正式版本)',
+      osArch: 'arm64',
+      url: index === 0 ? '' : tab.url,
+      title: index === 0 ? '' : tab.title,
+      tabs: index === 0 ? [] : tabs,
+      interactiveElements: index === 0 ? null : dom
+    },
+    resultSummary: null,
+    needSummary: true,
+    success: false
   }
 
-  return await getExecuteApi(data, dom, tabs)
-
+  const res = await getExecuteApi(params)
+  messageList.push({
+    id: data.id,
+    status: 'success',
+    type: 'PLAN',
+    params,
+    result: res.data.plan
+  })
+  return true
 }

+ 74 - 0
src/entrypoints/background/tool/index.ts

@@ -0,0 +1,74 @@
+// - 'navigate':访问特定URL,默认使用https://baidu.com
+// - 'click':按索引点击元素
+// - 'input_text':在元素中输入文本,对于百度(Baidu),输入框的索引是
+// - 'key_enter':按回车键
+// - 'screenshot':捕获屏幕截图
+// - 'get_html':获取页面HTML内容
+// - 'get_text':获取页面文本内容
+// - 'execute_js':执行JavaScript代码
+// - 'scroll':滚动页面
+// - 'switch_tab':切换到特定标签页
+// - 'new_tab':打开新标签页
+// - 'close_tab':关闭当前标签页
+// - 'refresh':刷新当前页面
+
+import { switchTabOrOpenNew } from "@/utils/utils.js"
+
+
+export const tootIp = {
+  navigate: {
+    name: "访问特定URL",
+    description: "访问特定URL,默认使用https://baidu.com",
+    func: async (params) => {
+      return await switchTabOrOpenNew(params.url)
+    },
+  },
+  click: {
+    name: "按索引点击元素",
+    description: "按索引点击元素",
+  },
+  input_text: {
+    name: "在元素中输入文本",
+    description: "在元素中输入文本,对于百度(Baidu),输入框的索引是",
+  },
+  key_enter: {
+    name: "按回车键",
+    description: "按回车键",
+  },
+  screenshot: {
+    name: "捕获屏幕截图",
+    description: "捕获屏幕截图",
+  },
+  get_html: {
+    name: "获取页面HTML内容",
+    description: "获取页面HTML内容",
+  },
+  get_text: {
+    name: "获取页面文本内容",
+    description: "获取页面文本内容",
+  },
+  execute_js: {
+    name: "执行JavaScript代码",
+    description: "执行JavaScript代码",
+  },
+  scroll: {
+    name: "滚动页面",
+    description: "滚动页面",
+  },
+  switch_tab: {
+    name: "切换到特定标签页",
+    description: "切换到特定标签页",
+  },
+  new_tab: {
+    name: "打开新标签页",
+    description: "打开新标签页",
+  },
+  close_tab: {
+    name: "关闭当前标签页",
+    description: "关闭当前标签页",
+  },
+  refresh: {
+    name: "刷新当前页面",
+    description: "刷新当前页面",
+  }
+}

+ 0 - 0
src/entrypoints/background/tool/tool.ts


+ 39 - 23
src/entrypoints/content.js

@@ -14,6 +14,7 @@ import {
 import { ZoomOut } from '@element-plus/icons-vue'
 import { defineContentScript } from 'wxt/sandbox'
 import { domToCanvas } from '@/entrypoints/sidepanel/utils/index.js'
+import { onMessage } from 'webext-bridge/content-script'
 
 export default defineContentScript({
   matches: ['<all_urls>'],
@@ -22,10 +23,10 @@ export default defineContentScript({
     const src = chrome.runtime.getURL('images/begin.png')
     window.pageAnalyzer = new PageAnalyzer()
     let iframe = null
-    let iframeDoc = null;
+    let iframeDoc = null
     let select = false
     window.onload = () => {
-      iframe = document.querySelector('iframe');
+      iframe = document.querySelector('iframe')
       iframeDoc = iframe?.contentWindow?.document
       // chrome.runtime.sendMessage({
       //   type: 'GET_SETTING_INFO',
@@ -36,16 +37,16 @@ export default defineContentScript({
       //   console.log(response,88896);
 
       // });
-      chrome.storage.local.get(['domainList','collect'], (result) => {
+      chrome.storage.local.get(['domainList', 'collect'], (result) => {
         const urls = result.domainList && Object.values(result.domainList)
         select = result.collect && !!urls?.find(_ => location.href.includes(_))
-        console.log(select);
+        console.log(select)
 
         if (select) {
           document.addEventListener('click', handleClick)
           document.addEventListener('mouseover', handleMouseover)
         } else {
-          console.log(2222);
+          console.log(2222)
 
           document.removeEventListener('click', handleClick)
           document.removeEventListener('mouseover', handleMouseover)
@@ -56,6 +57,7 @@ export default defineContentScript({
     let formChildren = []
     let excelDataA = {}
     let timer
+
     function handleMouseover(e) {
       clearTimeout(timer)
       timer = setTimeout(() => {
@@ -73,15 +75,16 @@ export default defineContentScript({
             tag: e.target.tagName.toLowerCase(),
             clickTime: +new Date()
           }
-        }, function (response) {
-        });
-      }, 1000);
+        }, function(response) {
+        })
+      }, 1000)
     }
 
     let tag
     let innerText
+
     function handleClick(e) {
-      console.log(e);
+      console.log(e)
 
       if (tag === e.target.outerHTML && innerText === e.target.innerText) return
       if (e.target.outerHTML.length > 1000) return
@@ -98,9 +101,10 @@ export default defineContentScript({
           tag: e.target.tagName.toLowerCase(),
           clickTime: +new Date()
         }
-      }, function (response) {
-      });
+      }, function(response) {
+      })
     }
+
     document.addEventListener('click', handleClick)
     document.addEventListener('mouseover', handleMouseover)
     chrome.runtime.onMessage.addListener(
@@ -138,7 +142,7 @@ export default defineContentScript({
         }
 
         if (message.type === 'SET_SELECT_TO_CENTENT') {
-          console.log(2);
+          console.log(2)
 
           chrome.storage.local.get(['domainList', 'collect'], (result) => {
             const urls = Object.values(result.domainList)
@@ -177,6 +181,7 @@ export default defineContentScript({
           })
           for (const item of forms) {
             item.style.border = '2px solid red'
+
             function handleClick(e) {
               e.stopPropagation()
               // console.log(this.outerHTML)
@@ -198,6 +203,7 @@ export default defineContentScript({
                 data: cloneForm.outerHTML
               })
             }
+
             item.addEventListener('click', handleClick, true)
           }
           // if (!form) {
@@ -223,7 +229,7 @@ export default defineContentScript({
           // }
         }
         if (message.type === 'INPUT_FORM') {
-          iframe = document.querySelector('iframe');
+          iframe = document.querySelector('iframe')
           // iframeDoc = iframe.contentWindow.document
           // await simulateUserInput(iframeDoc.getElementById('vehicleLicenseCodeId'), '冀D-06121')
           // await simulateUserInput(iframeDoc.getElementById('engineNo'), 'EK04656-06127')
@@ -233,14 +239,14 @@ export default defineContentScript({
           handleFillInput(formData, 0)
 
         }
-        if (message.type === 'SCREENSHOT') {
-          domToCanvas().then(res => {
-            sendResponse({ status: 'ok', data: res })
-          }).catch(err => {
-            sendResponse({ status: 'error', message: err })
-          })
-          // return true
-        }
+        // if (message.type === 'SCREENSHOT') {
+        //   domToCanvas().then(res => {
+        //     sendResponse({ status: 'ok', data: res })
+        //   }).catch(err => {
+        //     sendResponse({ status: 'error', message: err })
+        //   })
+        //   // return true
+        // }
         return true
       })
     const handleFillInput = async (data, index) => {
@@ -284,7 +290,7 @@ export default defineContentScript({
                 const span = findLabelForSpan(label)
                 span.forEach((span) => {
                   span.innerText === excelDataA[item.excelColumn] &&
-                    span.click()
+                  span.click()
                 })
               }
             }
@@ -380,9 +386,19 @@ export default defineContentScript({
         await new Promise((res) => {
           setTimeout(() => {
             res()
-          }, 100);
+          }, 100)
         })
       }
     }
+
+    // 截图功能 消息接受/发送
+    onMessage('SCREENSHOT',  (message) => {
+      console.log(message)
+      domToCanvas().then(res => {
+        return sendResponse({ status: 'ok', data: res })
+      }).catch(err => {
+        return sendResponse({ status: 'error', message: err })
+      })
+    })
   }
 })

+ 3 - 3
src/entrypoints/sidepanel/AdvancedMode.vue

@@ -339,18 +339,18 @@ async function createWS(msg) {
         scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
       }
     })
-    let param = {
+    let params = {
       conversationId: Math.floor(Math.random() * 1000) + 1,
       messageId: Math.floor(Math.random() * 1000) + 1,
       messageContent: msg.rawContent
     }
-    getPlan2(param).then(res => {
+    getPlan2(params).then(res => {
       sendLoading.value = false
       obj.type = 'plan'
       console.log(res, 5558)
       obj.rawContent = res.data.plan
       sendMessage('FROM_PLAN', {
-        param,
+        params,
         payload: { ...res.data.plan }
       }, 'background')
     }).catch(res => {

+ 217 - 186
src/entrypoints/sidepanel/Chat.vue

@@ -15,14 +15,15 @@
         </div>
         <div class="messages" ref="messagesContainer">
           <div v-for="(message, index) in messages" :key="index"
-            :class="['message-item', message.role === 'user' ? 'self' : 'other']">
+               :class="['message-item', message.role === 'user' ? 'self' : 'other']">
             <el-avatar :size="32" :src="message.role === 'user' ? userAvatar : avatar" />
             <div class="message-content">
               <div class="content" v-if="message.role === 'system'"
-                :class="{ 'loading-content': message.content === '' }">
+                   :class="{ 'loading-content': message.content === '' }">
                 <formTable v-if="message.type === 'form'" :content="message.rawContent" />
                 <div v-else>
-                  <div v-html="compiledMarkdown(message.rawContent)" class="markdown-body" style="font-size: small"></div>
+                  <div v-html="compiledMarkdown(message.rawContent)" class="markdown-body"
+                       style="font-size: small"></div>
                   <!-- <link rel="stylesheet" href="/node_modules/github-markdown-css/github-markdown-dark.css"> -->
                 </div>
                 <span class="loading-indicator" v-if="sendLoading && index === messages.length - 1">
@@ -32,7 +33,7 @@
                 </span>
               </div>
               <document v-else-if="message.type === 'document' && message.role === 'user'" :content="message.content"
-                :rawContent="message.rawContent" />
+                        :rawContent="message.rawContent" />
               <div v-else class="content">
                 <span v-if="message.type === ''">{{ message.content }}</span>
                 <span class="loading-indicator" v-if="sendLoading && index === messages.length - 1">
@@ -51,15 +52,17 @@
       <ScrollToBottom :target="scrollbar" ref="scrollToBottomRef" />
     </div>
 
-    <Tools v-if="selectModal" :disHistory="sendLoading" :upload="type === FunctionList.File_Operation || !!formInfo" @read-click="readClick" @upload-file="(file) => createFileObj(file)" @handle-capture="handleCapture" @his-records="hisRecords"
-      @add-new-dialogue="addNewDialogue" @handle-current-change="handleCurrentChange"
-      @handel-intelligent-filling-click="handelIntelligentFillingClick" />
+    <Tools :disHistory="sendLoading" :upload="type === FunctionList.File_Operation || !!formInfo"
+           @read-click="readClick" @upload-file="(file) => createFileObj(file)" @handle-capture="handleCapture"
+           @his-records="hisRecords"
+           @add-new-dialogue="addNewDialogue" @handle-current-change="handleCurrentChange"
+           @handel-intelligent-filling-click="handelIntelligentFillingClick" />
 
     <div class="w-full max-w-[720px] p-[0_12px_12px]">
       <!-- 输入区域 -->
       <div class="input-area w-full">
         <el-icon class="closeShow" :style="{ display: isShowPage ? 'block' : 'none' }" size="16px" color="#909399"
-          @click="closePageInfo">
+                 @click="closePageInfo">
           <CircleClose />
         </el-icon>
         <div v-show="isShowPage" style="border-bottom: 1px solid #F0F0F0;">
@@ -67,7 +70,7 @@
             <template v-for="(v, i) in pageInfoList" :key="i">
               <div class="card_image " v-if="v.type === 'image'">
                 <el-image v-loading="v.isUpload" class="w-full h-full p-0.5 object-cover" :src="v.url" :zoom-rate="1.2"
-                  :max-scale="7" :min-scale="0.2" :preview-src-list="[v.url]" :initial-index="4" fit="cover" />
+                          :max-scale="7" :min-scale="0.2" :preview-src-list="[v.url]" :initial-index="4" fit="cover" />
                 <el-icon class="closeIcon" size="16px" color="#909399" @click="deletePageInfo(i)">
                   <CircleClose />
                 </el-icon>
@@ -92,27 +95,32 @@
           </div>
 
           <div class="card-btn">
-            <el-tooltip  content="总结" placement="top">
-              <el-button :disabled="disabledBtn" v-if="type !== FunctionList.Intelligent_Form_filling"  round @click="handleSummary">总结</el-button>
+            <el-tooltip content="总结" placement="top">
+              <el-button :disabled="disabledBtn" v-if="type !== FunctionList.Intelligent_Form_filling" round
+                         @click="handleSummary">总结
+              </el-button>
             </el-tooltip>
-             <el-tooltip   content="抽取表单信息" placement="top">
-              <el-button :disabled="disabledBtn" v-if="type === FunctionList.Intelligent_Form_filling && pageInfoList.length" round @click="handleSummaryFile">抽取</el-button>
+            <el-tooltip content="抽取表单信息" placement="top">
+              <el-button :disabled="disabledBtn"
+                         v-if="type === FunctionList.Intelligent_Form_filling && pageInfoList.length" round
+                         @click="handleSummaryFile">抽取
+              </el-button>
             </el-tooltip>
           </div>
         </div>
 
         <el-input ref="textareaRef" v-model="inputMessage" type="textarea" :rows="3" placeholder="输入消息..."
-          @keyup.enter="() => handleAsk()" />
+                  @keyup.enter="() => handleAsk()" />
         <div class="chat_area_op">
-        
-          <el-button v-if="sendLoading" style="background-color:#ffffff;border:2px solid rgb(134 143 153);"  circle
-            @click="handleStopAsk">
+
+          <el-button v-if="sendLoading" style="background-color:#ffffff;border:2px solid rgb(134 143 153);" circle
+                     @click="handleStopAsk">
             <svg-icon icon-class="stop" color="red" />
           </el-button>
-            <el-button
+          <el-button
             v-else
             :style="`background-color:${inputMessage.trim() ? '#4d6bfe' : '#d6dee8'};border-color:${inputMessage.trim() ? '#4d6bfe' : '#d6dee8'}`"
-             type="primary" circle @click="() => handleAsk()"
+            type="primary" circle @click="() => handleAsk()"
             :disabled="disSend">
             <svg-icon icon-class="send" color="#000000" />
           </el-button>
@@ -128,8 +136,9 @@
 </template>
 
 <script setup>
-import { ref, onMounted, nextTick, inject, useTemplateRef, reactive,onBeforeMount } from 'vue'
+import { ref, onMounted, nextTick, inject, useTemplateRef, reactive, onBeforeMount } from 'vue'
 import { ElScrollbar, ElAvatar, ElInput, ElButton } from 'element-plus'
+import { sendMessage } from 'webext-bridge/side-panel'
 import moment from 'moment'
 import fileLogo from '@/assets/svg/file.svg'
 import {
@@ -149,21 +158,22 @@ import ScrollToBottom from '@/entrypoints/sidepanel/component/ScrollToBottom.vue
 import formTable from '@/entrypoints/sidepanel/component/formTable.vue'
 import userAvatar from '@/assets/images/user.png'
 import avatar from '@/public/icon/icon.png'
-import {  options, FunctionList } from '@/entrypoints/sidepanel/mock'
+import { FunctionList } from '@/entrypoints/sidepanel/mock'
 import { useAutoResizeTextarea } from '@/entrypoints/sidepanel/hook/useAutoResizeTextarea.ts'
-import { getPageInfo, getPageInfoClean, handleInput,downloadFile } from './utils/index.js'
+import { getPageInfo, getPageInfoClean, handleInput, downloadFile } from './utils/index.js'
 import { useMsgStore } from '@/store/modules/msg.ts'
 import { useUserStore } from '@/store/modules/user'
-import { putChat, uploadFile } from "@/api/index.js";
-import {askQues,getFormKey} from '@/api/modal.js'
+import { putChat, uploadFile } from '@/api/index.js'
+import { askQues, getFormKey } from '@/api/modal.js'
 import { debounce } from 'lodash'
-import {renderMarkdown} from './utils/markdown.js'
-import "github-markdown-css"
+import { renderMarkdown } from './utils/markdown.js'
+import 'github-markdown-css'
 // import "/node_modules/github-markdown-css/github-markdown-dark.css"
 const userStore = useUserStore()
 // 在其他状态变量附近添加
 const isLoadingMore = ref(false)
-import { v4 as uuidv4 } from 'uuid';
+import { v4 as uuidv4 } from 'uuid'
+import { browser } from 'wxt/browser'
 // 滚动条引用
 const scrollbar = ref(null)
 const scrollToBottomRef = ref(null)
@@ -179,7 +189,7 @@ const disSend = computed(() => {
 })
 // 获取父组件提供的 Hook 实例
 const msgStore = useMsgStore()
-const { pageInfoList, messages, msgUuid, AIModel,page,hasNext,msgLoading,selectModal } = storeToRefs(useMsgStore())
+const { pageInfoList, messages, msgUuid, AIModel, page, hasNext, msgLoading, selectModal } = storeToRefs(useMsgStore())
 const formInfo = ref('')
 const {
   taklToHtml,
@@ -193,16 +203,17 @@ const pageInfo = ref('')
 const summaryHtml = ref(false)
 
 const compiledMarkdown = computed(() => {
-  return (val)=>{
+  return (val) => {
     return renderMarkdown(val)
   }
 })
+
 function handleStopAsk() {
-    inputMessage.value = ''
-    pageInfoList.value = []
-    isShowPage.value = false
-    taklToHtml.value = false
-    type.value = FunctionList.File_Operation
+  inputMessage.value = ''
+  pageInfoList.value = []
+  isShowPage.value = false
+  taklToHtml.value = false
+  type.value = FunctionList.File_Operation
   controllerList.value[0]?.abort()
   controllerList.value = []
   sendLoading.value = false
@@ -213,16 +224,17 @@ function handleScroll(a) {
   const { wrapRef } = scrollbar.value
   const { scrollHeight, clientHeight } = wrapRef
   scrollToBottomRef.value.showButton = scrollHeight - a.scrollTop - clientHeight >= 350
-  
+
   // 检测是否滚动到顶部
   if (a.scrollTop <= 10 && hasNext.value) {
     handleScrollToTop()
   }
 }
+
 const messagesContainer = ref(null)
 let innerText = ''
 // 处理滚动到顶部的事件
-const handleScrollToTop = debounce(async function () {
+const handleScrollToTop = debounce(async function() {
   console.log('滚动到顶部,可以加载更多历史消息')
   if (!sendLoading.value && !isLoadingMore.value) {
     isLoadingMore.value = true
@@ -234,12 +246,12 @@ const handleScrollToTop = debounce(async function () {
       await msgStore.changePage()
       setTimeout(() => {
         if (firstMessage) {
-          console.log([...messagesContainer.value.querySelectorAll('.timestamp')].find(_ => _.innerText === innerText).offsetTop, innerText, 778);
+          console.log([...messagesContainer.value.querySelectorAll('.timestamp')].find(_ => _.innerText === innerText).offsetTop, innerText, 778)
           const newHeight = [...messagesContainer.value.querySelectorAll('.timestamp')].find(_ => _.innerText === innerText).offsetTop
           const scrollOffset = newHeight - oldHeight
           scrollbar.value?.setScrollTop(scrollOffset)
         }
-      }, 400);
+      }, 400)
       // 恢复滚动位置
 
     } finally {
@@ -247,6 +259,7 @@ const handleScrollToTop = debounce(async function () {
     }
   }
 }, 500, { leading: false, trailing: true })
+
 /**
  * @param {string} msg
  * @param {string} raw
@@ -290,7 +303,10 @@ const handleSummaryFile = async () => {
   if (sendLoading.value) return
   try {
     sendLoading.value = true
-    const userMsg = await addMessage(JSON.stringify([...pageInfoList.value, { type: 'text', value: '抽取表单' }]), '抽取表单', 'document')
+    const userMsg = await addMessage(JSON.stringify([...pageInfoList.value, {
+      type: 'text',
+      value: '抽取表单'
+    }]), '抽取表单', 'document')
     userMsg.fileId = pageInfoList.value[0].fileId
     userMsg.redisKey = pageInfoList.value[0].redisKey
     await putChat(userMsg)
@@ -320,13 +336,14 @@ const handleSummaryFile = async () => {
 
 
   } catch (error) {
-    console.log(error);
+    console.log(error)
 
   } finally {
     //  sendLoading.value = false
     // type.value = FunctionList.File_Operation
   }
 }
+
 async function handelIntelligentFillingClick() {
   handleInput()
   isShowPage.value = true
@@ -339,24 +356,7 @@ async function handelIntelligentFillingClick() {
 }
 
 function handleCurrentChange(e) {
-  options.forEach((item) => {
-    item.options.forEach((item2) => {
-      if (item2.value === e) {
-        msgStore.updateAIModel({ ...item2, api_sk: item['api_sk'], api_url: item['api_url'] })
-      }
-    })
-  })
-  if (AIModel.value.file === true) {
-    isShowPage.value = false
-    taklToHtml.value = false
-    pageInfoList.value = []
-  }
   msgStore.updateAIModel(e)
-  // if (AIModel.value.file === true) {
-  //   isShowPage.value = false
-  //   taklToHtml.value = false
-  //   pageInfoList.value = []
-  // }
 }
 
 async function handleCurrentData(e) {
@@ -372,25 +372,27 @@ async function handleCurrentData(e) {
   chrome.storage.local.set({ msgUuid: msgUuid.value })
   await msgStore.initMsg()
   nextTick(() => {
-  if (scrollbar.value && scrollbar.value.wrapRef) {
-    scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
-  }
-})
+    if (scrollbar.value && scrollbar.value.wrapRef) {
+      scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
+    }
+  })
 }
+
 let SystemMsg = null
 
 watchEffect(() => {
-  console.log(isShowPage.value);
-  
+  console.log(isShowPage.value)
+
   if (!isShowPage.value) {
     summaryHtml.value = false
-   
+
     type.value = FunctionList.File_Operation
     SystemMsg = null
     inputMessage.value = ''
   }
 })
 const htmlIcon = ref('')
+
 async function readClick() {
   if (type.value === FunctionList.Intelligent_Form_filling) {
     pageInfoList.value = []
@@ -410,8 +412,8 @@ async function readClick() {
   // console.log(file);
   // 下载文件到本地
   // downloadFile(file);
-  console.log(tempPageInfo);
-  
+  console.log(tempPageInfo)
+
   await createFileObj(file)
   // await handleUpload(file,tempPageInfo.title + '.txt')
 }
@@ -427,7 +429,7 @@ function deletePageInfo(i) {
       SystemMsg.rawContent = '智能填表流程中断'
       putChat({
         ...SystemMsg,
-        content: '',
+        content: ''
       })
     }
     handleStopAsk()
@@ -450,22 +452,25 @@ function addNewDialogue() {
   hasNext.value = false
   chrome.storage.local.set({ msgUuid: msgUuid.value })
 }
+
 const disabledBtn = computed(() => {
   return !!(pageInfoList.value.find(_ => _.loading))
 })
+
 function closePageInfo() {
   pageInfoList.value = []
-   if (type.value === FunctionList.Intelligent_Form_filling && SystemMsg ) {
-      SystemMsg.content = '智能填表流程中断'
-      SystemMsg.rawContent = '智能填表流程中断'
-      putChat({
-        ...SystemMsg,
-        content: '',
-      })
-    }
-    handleStopAsk()
-  
+  if (type.value === FunctionList.Intelligent_Form_filling && SystemMsg) {
+    SystemMsg.content = '智能填表流程中断'
+    SystemMsg.rawContent = '智能填表流程中断'
+    putChat({
+      ...SystemMsg,
+      content: ''
+    })
+  }
+  handleStopAsk()
+
 }
+
 async function handleAsk(value) {
   if (sendLoading.value) return
   const str = value ?? inputMessage.value.trim()
@@ -475,7 +480,7 @@ async function handleAsk(value) {
     await putChat(msg)
     const [res, obj] = await fetchRes(str)
     if (res.status === 'ok') {
-      console.log(obj);
+      console.log(obj)
       formInfo.value = res.data
       SystemMsg = obj
     } else {
@@ -496,8 +501,9 @@ async function handleAsk(value) {
   await putChat(msg)
   createWS(msg)
 }
+
 /**
- * 
+ *
  * @param msg 用户消息对象,调用askQues时需要
  */
 async function createWS(msg) {
@@ -510,36 +516,36 @@ async function createWS(msg) {
     content: '',
     sortKey: moment().valueOf(),
     role: 'system',
-    conversationId: msgUuid.value,
+    conversationId: msgUuid.value
   })
   try {
-  messages.value.push(obj)
-  nextTick(() => {
-    if (scrollbar.value && scrollbar.value.wrapRef) {
-      scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
-    }
-  })
-  const websocketId= uuidv4()
-   const wsUrl = `${import.meta.env.VITE_API_WS_URL}/webSocket/clue/${websocketId}`;
-    const socket = new WebSocket(wsUrl);
-    console.log(selectModal.value);
-
-     askQues({
-       conversationId: msgUuid.value,
-    websocketId,
-    modelName: selectModal.value.modelName,
-    question: type.value === FunctionList.File_Operation ? msg.rawContent : buildObjPrompt(xlsxData.value,formInfo.value),
-    id: selectModal.value.id,
-    redisKey:msg.redisKey
-     }).catch(res => {
-       obj.rawContent = '接口出错,请重试。'
-       obj.content = '接口出错,请重试。'
+    messages.value.push(obj)
+    nextTick(() => {
+      if (scrollbar.value && scrollbar.value.wrapRef) {
+        scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
+      }
+    })
+    const websocketId = uuidv4()
+    const wsUrl = `${import.meta.env.VITE_API_WS_URL}/webSocket/clue/${websocketId}`
+    const socket = new WebSocket(wsUrl)
+    console.log(selectModal.value)
+
+    askQues({
+      conversationId: msgUuid.value,
+      websocketId,
+      modelName: selectModal.value.modelName,
+      question: type.value === FunctionList.File_Operation ? msg.rawContent : buildObjPrompt(xlsxData.value, formInfo.value),
+      id: selectModal.value.id,
+      redisKey: msg.redisKey
+    }).catch(res => {
+      obj.rawContent = '接口出错,请重试。'
+      obj.content = '接口出错,请重试。'
       socket.close()
     })
     socket.onmessage = (event) => {
       try {
         if (event.data === '') {
-          console.log(23223);
+          console.log(23223)
 
           socket.close()
           if (type.value === FunctionList.Intelligent_Form_filling) {
@@ -548,81 +554,105 @@ async function createWS(msg) {
             if (formResult.includes('json')) form = JSON.parse(formResult.split('json')[1].split('```')[0])
             else form = JSON.parse(formResult)
             sendLoading.value = false
-            console.log(xlsxData.value, form);
+            console.log(xlsxData.value, form)
 
             handleInput(xlsxData.value, form)
           }
           isShowPage.value = false
         } else {
-          obj.rawContent += event.data;
-          obj.content = obj.type === 'form' ? obj.rawContent :  formatMessage(obj.rawContent);
+          obj.rawContent += event.data
+          obj.content = obj.type === 'form' ? obj.rawContent : formatMessage(obj.rawContent)
           // 滚动到底部
           nextTick(() => {
             if (scrollbar.value && scrollbar.value.wrapRef) {
-              scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight);
+              scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
             }
-          });
+          })
         }
       } catch (error) {
-        console.error('解析消息出错:', error);
+        console.error('解析消息出错:', error)
       }
-    };
+    }
     socket.onerror = (error) => {
-      console.error('WebSocket 错误:', error);
-      socket.close();
-    };
+      console.error('WebSocket 错误:', error)
+      socket.close()
+    }
     socket.onclose = () => {
-      console.log('WebSocket 连接已关闭');
-      sendLoading.value = false;
+      console.log('WebSocket 连接已关闭')
+      sendLoading.value = false
       putChat({
-      ...obj,
-      content:''
-     });
+        ...obj,
+        content: ''
+      })
       sendLoading.value = false
       controllerList.value = []
-     type.value = FunctionList.File_Operation
-    };
+      type.value = FunctionList.File_Operation
+    }
     // 添加到控制器列表,以便可以在需要时中断连接
     controllerList.value.push({
       abort: () => {
-        socket.close();
+        socket.close()
       }
-    });
+    })
   } catch (error) {
-    console.error( error);
-    sendLoading.value = false;
+    console.error(error)
+    sendLoading.value = false
     // 显示错误消息
-    obj.content = '连接失败,请重试';
-     putChat({
+    obj.content = '连接失败,请重试'
+    putChat({
       ...obj,
-      content:''
-     });
+      content: ''
+    })
   } finally {
-     
+
   }
 }
+
 function handleCapture() {
-  ElMessage({
-    message: '开发中...',
-    grouping: true,
-    showClose: true
+  browser.tabs.query({ active: true, currentWindow: true }).then(tabs => {
+    if (tabs.length === 0) {
+      ElMessage.warning('请先打开一个页面')
+      return
+    }
+    sendMessage('SCREENSHOT', { t: 'Start taking screenshots' }, `content-script@${tabs[0].id}`)
   })
+
+
+  // chrome.runtime.sendMessage({
+  //   type: 'SCREENSHOT',
+  //   data: 'Start taking screenshots'
+  // }, (response) => {
+  //   if (response.status === 'ok') {
+  //     isShowPage.value = true
+  //     pageInfoList.value.unshift({
+  //       isUpload: false,
+  //       fileId: '',
+  //       type: 'image',
+  //       url: response.data
+  //     })
+  //     // console.log(response);
+  //     // console.log(response, pageInfoList.value)
+  //   }
+  // })
 }
+
 function hisRecords() {
   drawerRef.value.drawer = true
+  console.log(123)
 }
+
 const createFileObj = async (file) => {
   handleUpload(file)
 }
 const handleUpload = async (file) => {
-    const obj = reactive({
-    title: summaryHtml.value ?  file.name.split('.')[0] :  file.name,
+  const obj = reactive({
+    title: summaryHtml.value ? file.name.split('.')[0] : file.name,
     state: '上传中...',
     favIconUrl: fileLogo,
     loading: true,
-    url:'',
+    url: '',
     fileId: '',
-    redisKey:'',
+    redisKey: '',
     content: {
       mainContent: ''
     }
@@ -649,70 +679,72 @@ const handleUpload = async (file) => {
     let formData = new FormData()
     formData.append('avatarFile', file)
     const res = await uploadFile(formData)
-  if (type.value === FunctionList.Intelligent_Form_filling) {
-    obj.state = '解析中...'
-    obj.redisKey = res.data.redisKey
-    obj.id = res.data.file.id
-    obj.url = res.data.file.url
-    // const fileExtension = file.name.split('.').pop().toLowerCase()
-     const result = await getFormKey({
-      body: formInfo.value,
-      input_data:res.data.data
-    })
-    xlsxData.value = JSON.parse(result.data)
-    // if (fileExtension === 'xlsx') {
-    //       const readData = await getXlsxValue(file)
-    //   readData[0].forEach((header, i) => {
-    //     // if (!xlsxData.value[header]) xlsxData.value[header] = []
-    //     xlsxData.value[header] = readData[1][i]
-    //   })
-    // } else {
-    //      const result = await getFormKey({
-    //   body: formInfo.value,
-    //   input_data:res.data.data
-    // })
-    // xlsxData.value = JSON.parse(result.data)
-    //   }
+    if (type.value === FunctionList.Intelligent_Form_filling) {
+      obj.state = '解析中...'
+      obj.redisKey = res.data.redisKey
+      obj.id = res.data.file.id
+      obj.url = res.data.file.url
+      // const fileExtension = file.name.split('.').pop().toLowerCase()
+      const result = await getFormKey({
+        body: formInfo.value,
+        input_data: res.data.data
+      })
+      xlsxData.value = JSON.parse(result.data)
+      // if (fileExtension === 'xlsx') {
+      //       const readData = await getXlsxValue(file)
+      //   readData[0].forEach((header, i) => {
+      //     // if (!xlsxData.value[header]) xlsxData.value[header] = []
+      //     xlsxData.value[header] = readData[1][i]
+      //   })
+      // } else {
+      //      const result = await getFormKey({
+      //   body: formInfo.value,
+      //   input_data:res.data.data
+      // })
+      // xlsxData.value = JSON.parse(result.data)
+      //   }
     }
- 
-   nextTick(() => {
-  if (scrollbar.value && scrollbar.value.wrapRef) {
-    scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
-  }
-})
-  if (type.value === FunctionList.File_Operation) {
-    obj.loading = false
-    obj.url = res.data.url
-    obj.redisKey = res.data.redisKey
-    obj.fileId = res.data.file.id
-    obj.url = res.data.file.url
+
+    nextTick(() => {
+      if (scrollbar.value && scrollbar.value.wrapRef) {
+        scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
+      }
+    })
+    if (type.value === FunctionList.File_Operation) {
+      obj.loading = false
+      obj.url = res.data.url
+      obj.redisKey = res.data.redisKey
+      obj.fileId = res.data.file.id
+      obj.url = res.data.file.url
     }
     obj.loading = false
-  console.log(SystemMsg);
-  
+    console.log(SystemMsg)
+
   } catch (error) {
     if (SystemMsg) {
       SystemMsg.content = '数据上传出错,请重试'
       SystemMsg.rawContent = '数据上传出错,请重试'
     }
-    console.log(error);
+    console.log(error)
     ElMessage.error('上传出错')
     pageInfoList.value = []
     isShowPage.value = false
   } finally {
-      SystemMsg && putChat({
-        ...SystemMsg,
-        content: '',
-      })
+    SystemMsg && putChat({
+      ...SystemMsg,
+      content: ''
+    })
     SystemMsg = null
   }
 }
+
 async function getFileValue(file) {
   let formData = new FormData()
   formData.append('file', file)
   const res = await getFileContent(formData)
   return res.data
 }
+
 let a = null
 // 组件挂载时滚动到底部
 onBeforeMount(async () => {
@@ -720,7 +752,6 @@ onBeforeMount(async () => {
   await msgStore.initModal()
 })
 onMounted(async () => {
-  // msgStore.updateAIModel(options[0].options[0])
   useAutoResizeTextarea(tareRef, inputMessage)
   chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
     if (message.type === 'TO_SIDE_PANEL_PAGE_INFO') {
@@ -735,11 +766,11 @@ onMounted(async () => {
       pageInfo.value = message.data
     }
   })
- nextTick(() => {
-  if (scrollbar.value && scrollbar.value.wrapRef) {
-    scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
-  }
-})
+  nextTick(() => {
+    if (scrollbar.value && scrollbar.value.wrapRef) {
+      scrollbar.value.setScrollTop(scrollbar.value.wrapRef.scrollHeight)
+    }
+  })
 })
 </script>
 
@@ -754,7 +785,7 @@ onMounted(async () => {
   padding: 10px 0;
   color: #909399;
   font-size: 14px;
-  
+
   .loading-spinner {
     width: 20px;
     height: 20px;

+ 23 - 23
src/entrypoints/sidepanel/component/historyComponent.vue

@@ -24,15 +24,14 @@ const props = defineProps({
 })
 watch(drawer, (newVal) => {
   if (newVal) {
-    // console.log(getChatList)
     loading.value = true
-    getChatList({
-      page:1,
-      size: 50,
-      senderId:userStore.userInfo.id
-    }).then(res => {
-      dataList.value = res.data.list
-    }).finally(res => loading.value = false)
+    // getChatList({
+    //   page: 1,
+    //   size: 100,
+    //   sort: ['createTime,desc']
+    // }).then(res => {
+    //   dataList.value = res.data.list
+    // }).finally(res => loading.value = false)
   }
 })
 function handleDeleteStore(e: any, item: any,name:string) {
@@ -65,8 +64,8 @@ function handleDeleteStore(e: any, item: any,name:string) {
       }
       getChatList({
         page: 1,
-        size: 50,
-        senderId:userStore.userInfo.id
+        size: 100,
+        sort: ['createTime,desc']
       }).then(res => {
         dataList.value = res.data.list
       }).finally(res => loading.value = false)
@@ -98,19 +97,20 @@ defineExpose({
         </el-tooltip> -->
       </div>
       <div class="his_content">
-        <template v-for="item in dataList" :key="item.conversationId">
-          <div :class="`his_list ${msgUuid === item.conversationId ? 'his_list_change' : '' }`"
-            @click="emit('currentData', item.conversationId)">
-            <p class="ellipsis" style="color:#000000;font-weight: 900;">{{ item?.content ?? '--' }}</p>
-            <p class="his_list_op">
-              <span style="color: #000">{{ item?.createTime }}</span>
-              <el-tooltip effect="dark" content="删除" placement="top">
-                <el-button :disabled="msgUuid === item.conversationId" :icon="Delete" link
-                  @click="(e: any) => handleDeleteStore(e, item.conversationId, item?.content)" />
-              </el-tooltip>
-            </p>
-          </div>
-        </template>
+        1212
+<!--        <template v-for="item in dataList" :key="item.conversationId">-->
+<!--          <div :class="`his_list ${msgUuid === item.conversationId ? 'his_list_change' : '' }`"-->
+<!--            @click="emit('currentData', item.conversationId)">-->
+<!--            <p class="ellipsis" style="color:#000000;font-weight: 900;">{{ item?.content ?? '&#45;&#45;' }}</p>-->
+<!--            <p class="his_list_op">-->
+<!--              <span style="color: #000">{{ item?.createTime }}</span>-->
+<!--              <el-tooltip effect="dark" content="删除" placement="top">-->
+<!--                <el-button :disabled="msgUuid === item.conversationId" :icon="Delete" link-->
+<!--                  @click="(e: any) => handleDeleteStore(e, item.conversationId, item?.content)" />-->
+<!--              </el-tooltip>-->
+<!--            </p>-->
+<!--          </div>-->
+<!--        </template>-->
       </div>
     </div>
   </el-drawer>

+ 7 - 13
src/entrypoints/sidepanel/component/tools.vue

@@ -6,18 +6,11 @@ import { Reading, Upload, Paperclip, Scissor, AlarmClock, CirclePlus, Edit } fro
 import { useMsgStore } from '@/store/modules/msg'
 import { ElMessage, makeList } from 'element-plus'
 
-const { modelList } = storeToRefs(useMsgStore())
-console.log(modelList,5563);
+const { modelList,selectModal } = storeToRefs(useMsgStore())
 
-const selectInput = ref(null)
+const selectInput = ref(selectModal.value)
 const emit = defineEmits(['readClick', 'uploadFile', 'handleCapture', 'addNewDialogue', 'hisRecords', 'handleCurrentChange', 'handelIntelligentFillingClick'])
 
-watchEffect(() => {
-  // selectInput.value = AIModel.value?.value
-})
-onMounted(() => {
-  selectInput.value = modelList.value[0].id
-})
 const handleUpload = (e) => {
   if (!props.upload) {
     e.stopPropagation()
@@ -35,12 +28,13 @@ const props = defineProps({
 <template>
   <div class="px-3 py-2 flex justify-between items-center w-full max-w-[720px]">
     <div class="flex items-center">
-      <el-select placement="top" v-model="selectInput" placeholder="选择"
+      <el-select placement="top" v-model="selectInput" placeholder="选择" value-key="id"
         @change="(e: any) => emit('handleCurrentChange', e)" style="width: 120px;margin-right: 2px">
-        <!-- <el-option-group v-for="group in options" :key="group.label" :label="group.label"> -->
-          <el-option v-for="item in modelList" :key="item.id" :label="item.modelName" :value="item.id" />
-        <!-- </el-option-group> -->
+        <el-option-group v-for="group in modelList" :key="group.label" :label="group.label">
+          <el-option v-for="item in group.options" :key="item.id" :label="item.name" :value="item" />
+        </el-option-group>
       </el-select>
+
       <el-tooltip effect="dark" content="阅读此页,开启后将会根据左侧网页中的内容做出回答" placement="top">
         <el-button class="tools_btn" link :icon="Reading" @click="emit('readClick')" />
       </el-tooltip>

+ 0 - 0
src/entrypoints/sidepanel/utils/utils.ts


+ 1321 - 0
src/public/buildDomTree.js

@@ -0,0 +1,1321 @@
+
+window.buildDomTree = (
+  args = {
+    doHighlightElements: true,
+    focusHighlightIndex: -1,
+    viewportExpansion: 0,
+    debugMode: false,
+  },
+) => {
+  const { doHighlightElements, focusHighlightIndex, viewportExpansion, debugMode } = args;
+  let highlightIndex = 0; // Reset highlight index
+
+  // Add timing stack to handle recursion
+  const TIMING_STACK = {
+    nodeProcessing: [],
+    treeTraversal: [],
+    highlighting: [],
+    current: null,
+  };
+
+  function pushTiming(type) {
+    TIMING_STACK[type] = TIMING_STACK[type] || [];
+    TIMING_STACK[type].push(performance.now());
+  }
+
+  function popTiming(type) {
+    const start = TIMING_STACK[type].pop();
+    const duration = performance.now() - start;
+    return duration;
+  }
+
+  // Only initialize performance tracking if in debug mode
+  const PERF_METRICS = debugMode
+    ? {
+        buildDomTreeCalls: 0,
+        timings: {
+          buildDomTree: 0,
+          highlightElement: 0,
+          isInteractiveElement: 0,
+          isElementVisible: 0,
+          isTopElement: 0,
+          isInExpandedViewport: 0,
+          isTextNodeVisible: 0,
+          getEffectiveScroll: 0,
+        },
+        cacheMetrics: {
+          boundingRectCacheHits: 0,
+          boundingRectCacheMisses: 0,
+          computedStyleCacheHits: 0,
+          computedStyleCacheMisses: 0,
+          getBoundingClientRectTime: 0,
+          getComputedStyleTime: 0,
+          boundingRectHitRate: 0,
+          computedStyleHitRate: 0,
+          overallHitRate: 0,
+        },
+        nodeMetrics: {
+          totalNodes: 0,
+          processedNodes: 0,
+          skippedNodes: 0,
+        },
+        buildDomTreeBreakdown: {
+          totalTime: 0,
+          totalSelfTime: 0,
+          buildDomTreeCalls: 0,
+          domOperations: {
+            getBoundingClientRect: 0,
+            getComputedStyle: 0,
+          },
+          domOperationCounts: {
+            getBoundingClientRect: 0,
+            getComputedStyle: 0,
+          },
+        },
+      }
+    : null;
+
+  // Simple timing helper that only runs in debug mode
+  function measureTime(fn) {
+    if (!debugMode) return fn;
+    return function (...args) {
+      const start = performance.now();
+      const result = fn.apply(this, args);
+      const duration = performance.now() - start;
+      return result;
+    };
+  }
+
+  // Helper to measure DOM operations
+  function measureDomOperation(operation, name) {
+    if (!debugMode) return operation();
+
+    const start = performance.now();
+    const result = operation();
+    const duration = performance.now() - start;
+
+    if (PERF_METRICS && name in PERF_METRICS.buildDomTreeBreakdown.domOperations) {
+      PERF_METRICS.buildDomTreeBreakdown.domOperations[name] += duration;
+      PERF_METRICS.buildDomTreeBreakdown.domOperationCounts[name]++;
+    }
+
+    return result;
+  }
+
+  // Add caching mechanisms at the top level
+  const DOM_CACHE = {
+    boundingRects: new WeakMap(),
+    computedStyles: new WeakMap(),
+    clearCache: () => {
+      DOM_CACHE.boundingRects = new WeakMap();
+      DOM_CACHE.computedStyles = new WeakMap();
+    },
+  };
+
+  // Cache helper functions
+  function getCachedBoundingRect(element) {
+    if (!element) return null;
+
+    if (DOM_CACHE.boundingRects.has(element)) {
+      if (debugMode && PERF_METRICS) {
+        PERF_METRICS.cacheMetrics.boundingRectCacheHits++;
+      }
+      return DOM_CACHE.boundingRects.get(element);
+    }
+
+    if (debugMode && PERF_METRICS) {
+      PERF_METRICS.cacheMetrics.boundingRectCacheMisses++;
+    }
+
+    let rect;
+    if (debugMode) {
+      const start = performance.now();
+      rect = element.getBoundingClientRect();
+      const duration = performance.now() - start;
+      if (PERF_METRICS) {
+        PERF_METRICS.buildDomTreeBreakdown.domOperations.getBoundingClientRect += duration;
+        PERF_METRICS.buildDomTreeBreakdown.domOperationCounts.getBoundingClientRect++;
+      }
+    } else {
+      rect = element.getBoundingClientRect();
+    }
+
+    if (rect) {
+      DOM_CACHE.boundingRects.set(element, rect);
+    }
+    return rect;
+  }
+
+  function getCachedComputedStyle(element) {
+    if (!element) return null;
+
+    if (DOM_CACHE.computedStyles.has(element)) {
+      if (debugMode && PERF_METRICS) {
+        PERF_METRICS.cacheMetrics.computedStyleCacheHits++;
+      }
+      return DOM_CACHE.computedStyles.get(element);
+    }
+
+    if (debugMode && PERF_METRICS) {
+      PERF_METRICS.cacheMetrics.computedStyleCacheMisses++;
+    }
+
+    let style;
+    if (debugMode) {
+      const start = performance.now();
+      style = window.getComputedStyle(element);
+      const duration = performance.now() - start;
+      if (PERF_METRICS) {
+        PERF_METRICS.buildDomTreeBreakdown.domOperations.getComputedStyle += duration;
+        PERF_METRICS.buildDomTreeBreakdown.domOperationCounts.getComputedStyle++;
+      }
+    } else {
+      style = window.getComputedStyle(element);
+    }
+
+    if (style) {
+      DOM_CACHE.computedStyles.set(element, style);
+    }
+    return style;
+  }
+
+  /**
+   * Hash map of DOM nodes indexed by their highlight index.
+   *
+   * @type {Object<string, any>}
+   */
+  const DOM_HASH_MAP = {};
+
+  const ID = { current: 0 };
+
+  const HIGHLIGHT_CONTAINER_ID = 'playwright-highlight-container';
+
+  /**
+   * Highlights an element in the DOM and returns the index of the next element.
+   */
+  function highlightElement(element, index, parentIframe = null) {
+    if (!element) return index;
+
+    // Store overlays and the single label for updating
+    const overlays = [];
+    let label = null;
+    let labelWidth = 20; // Approximate label width
+    let labelHeight = 16; // Approximate label height
+
+    try {
+      // Create or get highlight container
+      let container = document.getElementById(HIGHLIGHT_CONTAINER_ID);
+      if (!container) {
+        container = document.createElement('div');
+        container.id = HIGHLIGHT_CONTAINER_ID;
+        container.style.position = 'fixed';
+        container.style.pointerEvents = 'none';
+        container.style.top = '0';
+        container.style.left = '0';
+        container.style.width = '100%';
+        container.style.height = '100%';
+        container.style.zIndex = '2147483647';
+        document.body.appendChild(container);
+      }
+
+      // Get element client rects
+      const rects = element.getClientRects(); // Use getClientRects()
+
+      if (!rects || rects.length === 0) return index; // Exit if no rects
+
+      // Generate a color based on the index
+      const colors = [
+        '#FF0000',
+        '#00FF00',
+        '#0000FF',
+        '#FFA500',
+        '#800080',
+        '#008080',
+        '#FF69B4',
+        '#4B0082',
+        '#FF4500',
+        '#2E8B57',
+        '#DC143C',
+        '#4682B4',
+      ];
+      const colorIndex = index % colors.length;
+      const baseColor = colors[colorIndex];
+      const backgroundColor = baseColor + '1A'; // 10% opacity version of the color
+
+      // Get iframe offset if necessary
+      let iframeOffset = { x: 0, y: 0 };
+      if (parentIframe) {
+        const iframeRect = parentIframe.getBoundingClientRect(); // Keep getBoundingClientRect for iframe offset
+        iframeOffset.x = iframeRect.left;
+        iframeOffset.y = iframeRect.top;
+      }
+
+      // Create highlight overlays for each client rect
+      for (const rect of rects) {
+        if (rect.width === 0 || rect.height === 0) continue; // Skip empty rects
+
+        const overlay = document.createElement('div');
+        overlay.style.position = 'fixed';
+        overlay.style.border = `2px solid ${baseColor}`;
+        overlay.style.backgroundColor = backgroundColor;
+        overlay.style.pointerEvents = 'none';
+        overlay.style.boxSizing = 'border-box';
+
+        const top = rect.top + iframeOffset.y;
+        const left = rect.left + iframeOffset.x;
+
+        overlay.style.top = `${top}px`;
+        overlay.style.left = `${left}px`;
+        overlay.style.width = `${rect.width}px`;
+        overlay.style.height = `${rect.height}px`;
+
+        container.appendChild(overlay);
+        overlays.push({ element: overlay, initialRect: rect }); // Store overlay and its rect
+      }
+
+      // Create and position a single label relative to the first rect
+      const firstRect = rects[0];
+      label = document.createElement('div');
+      label.className = 'playwright-highlight-label';
+      label.style.position = 'fixed';
+      label.style.background = baseColor;
+      label.style.color = 'white';
+      label.style.padding = '1px 4px';
+      label.style.borderRadius = '4px';
+      label.style.fontSize = `${Math.min(12, Math.max(8, firstRect.height / 2))}px`;
+      label.textContent = index;
+
+      labelWidth = label.offsetWidth > 0 ? label.offsetWidth : labelWidth; // Update actual width if possible
+      labelHeight = label.offsetHeight > 0 ? label.offsetHeight : labelHeight; // Update actual height if possible
+
+      const firstRectTop = firstRect.top + iframeOffset.y;
+      const firstRectLeft = firstRect.left + iframeOffset.x;
+
+      let labelTop = firstRectTop + 2;
+      let labelLeft = firstRectLeft + firstRect.width - labelWidth - 2;
+
+      // Adjust label position if first rect is too small
+      if (firstRect.width < labelWidth + 4 || firstRect.height < labelHeight + 4) {
+        labelTop = firstRectTop - labelHeight - 2;
+        labelLeft = firstRectLeft + firstRect.width - labelWidth; // Align with right edge
+        if (labelLeft < iframeOffset.x) labelLeft = firstRectLeft; // Prevent going off-left
+      }
+
+      // Ensure label stays within viewport bounds slightly better
+      labelTop = Math.max(0, Math.min(labelTop, window.innerHeight - labelHeight));
+      labelLeft = Math.max(0, Math.min(labelLeft, window.innerWidth - labelWidth));
+
+      label.style.top = `${labelTop}px`;
+      label.style.left = `${labelLeft}px`;
+
+      container.appendChild(label);
+
+      // Update positions on scroll/resize
+      const updatePositions = () => {
+        const newRects = element.getClientRects(); // Get fresh rects
+        let newIframeOffset = { x: 0, y: 0 };
+
+        if (parentIframe) {
+          const iframeRect = parentIframe.getBoundingClientRect(); // Keep getBoundingClientRect for iframe
+          newIframeOffset.x = iframeRect.left;
+          newIframeOffset.y = iframeRect.top;
+        }
+
+        // Update each overlay
+        overlays.forEach((overlayData, i) => {
+          if (i < newRects.length) {
+            // Check if rect still exists
+            const newRect = newRects[i];
+            const newTop = newRect.top + newIframeOffset.y;
+            const newLeft = newRect.left + newIframeOffset.x;
+
+            overlayData.element.style.top = `${newTop}px`;
+            overlayData.element.style.left = `${newLeft}px`;
+            overlayData.element.style.width = `${newRect.width}px`;
+            overlayData.element.style.height = `${newRect.height}px`;
+            overlayData.element.style.display = newRect.width === 0 || newRect.height === 0 ? 'none' : 'block';
+          } else {
+            // If fewer rects now, hide extra overlays
+            overlayData.element.style.display = 'none';
+          }
+        });
+
+        // If there are fewer new rects than overlays, hide the extras
+        if (newRects.length < overlays.length) {
+          for (let i = newRects.length; i < overlays.length; i++) {
+            overlays[i].element.style.display = 'none';
+          }
+        }
+
+        // Update label position based on the first new rect
+        if (label && newRects.length > 0) {
+          const firstNewRect = newRects[0];
+          const firstNewRectTop = firstNewRect.top + newIframeOffset.y;
+          const firstNewRectLeft = firstNewRect.left + newIframeOffset.x;
+
+          let newLabelTop = firstNewRectTop + 2;
+          let newLabelLeft = firstNewRectLeft + firstNewRect.width - labelWidth - 2;
+
+          if (firstNewRect.width < labelWidth + 4 || firstNewRect.height < labelHeight + 4) {
+            newLabelTop = firstNewRectTop - labelHeight - 2;
+            newLabelLeft = firstNewRectLeft + firstNewRect.width - labelWidth;
+            if (newLabelLeft < newIframeOffset.x) newLabelLeft = firstNewRectLeft;
+          }
+
+          // Ensure label stays within viewport bounds
+          newLabelTop = Math.max(0, Math.min(newLabelTop, window.innerHeight - labelHeight));
+          newLabelLeft = Math.max(0, Math.min(newLabelLeft, window.innerWidth - labelWidth));
+
+          label.style.top = `${newLabelTop}px`;
+          label.style.left = `${newLabelLeft}px`;
+          label.style.display = 'block';
+        } else if (label) {
+          // Hide label if element has no rects anymore
+          label.style.display = 'none';
+        }
+      };
+
+      window.addEventListener('scroll', updatePositions, true); // Use capture phase
+      window.addEventListener('resize', updatePositions);
+
+      // TODO: Add cleanup logic to remove listeners and elements when done.
+
+      return index + 1;
+    } finally {
+      // popTiming('highlighting'); // Assuming this was a typo and should be removed or corrected
+    }
+  }
+
+  /**
+   * Returns an XPath tree string for an element.
+   */
+  function getXPathTree(element, stopAtBoundary = true) {
+    const segments = [];
+    let currentElement = element;
+
+    while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE) {
+      // Stop if we hit a shadow root or iframe
+      if (
+        stopAtBoundary &&
+        (currentElement.parentNode instanceof ShadowRoot || currentElement.parentNode instanceof HTMLIFrameElement)
+      ) {
+        break;
+      }
+
+      let index = 0;
+      let sibling = currentElement.previousSibling;
+      while (sibling) {
+        if (sibling.nodeType === Node.ELEMENT_NODE && sibling.nodeName === currentElement.nodeName) {
+          index++;
+        }
+        sibling = sibling.previousSibling;
+      }
+
+      const tagName = currentElement.nodeName.toLowerCase();
+      const xpathIndex = index > 0 ? `[${index + 1}]` : '';
+      segments.unshift(`${tagName}${xpathIndex}`);
+
+      currentElement = currentElement.parentNode;
+    }
+
+    return segments.join('/');
+  }
+
+  /**
+   * Checks if a text node is visible.
+   */
+  function isTextNodeVisible(textNode) {
+    try {
+      const range = document.createRange();
+      range.selectNodeContents(textNode);
+      const rects = range.getClientRects(); // Use getClientRects for Range
+
+      if (!rects || rects.length === 0) {
+        return false;
+      }
+
+      let isAnyRectVisible = false;
+      let isAnyRectInViewport = false;
+
+      for (const rect of rects) {
+        // Check size
+        if (rect.width > 0 && rect.height > 0) {
+          isAnyRectVisible = true;
+
+          // Viewport check for this rect
+          if (
+            !(
+              rect.bottom < -viewportExpansion ||
+              rect.top > window.innerHeight + viewportExpansion ||
+              rect.right < -viewportExpansion ||
+              rect.left > window.innerWidth + viewportExpansion
+            ) ||
+            viewportExpansion === -1
+          ) {
+            isAnyRectInViewport = true;
+            break; // Found a visible rect in viewport, no need to check others
+          }
+        }
+      }
+
+      if (!isAnyRectVisible || !isAnyRectInViewport) {
+        return false;
+      }
+
+      // Check parent visibility
+      const parentElement = textNode.parentElement;
+      if (!parentElement) return false;
+
+      try {
+        return (
+          isInViewport &&
+          parentElement.checkVisibility({
+            checkOpacity: true,
+            checkVisibilityCSS: true,
+          })
+        );
+      } catch (e) {
+        // Fallback if checkVisibility is not supported
+        const style = window.getComputedStyle(parentElement);
+        return isInViewport && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0';
+      }
+    } catch (e) {
+      console.warn('Error checking text node visibility:', e);
+      return false;
+    }
+  }
+
+  // Helper function to check if element is accepted
+  function isElementAccepted(element) {
+    if (!element || !element.tagName) return false;
+
+    // Always accept body and common container elements
+    const alwaysAccept = new Set(['body', 'div', 'main', 'article', 'section', 'nav', 'header', 'footer']);
+    const tagName = element.tagName.toLowerCase();
+
+    if (alwaysAccept.has(tagName)) return true;
+
+    const leafElementDenyList = new Set(['svg', 'script', 'style', 'link', 'meta', 'noscript', 'template']);
+
+    return !leafElementDenyList.has(tagName);
+  }
+
+  /**
+   * Checks if an element is visible.
+   */
+  function isElementVisible(element) {
+    const style = getCachedComputedStyle(element);
+    return (
+      element.offsetWidth > 0 && element.offsetHeight > 0 && style.visibility !== 'hidden' && style.display !== 'none'
+    );
+  }
+
+  /**
+   * Checks if an element is interactive.
+   *
+   * lots of comments, and uncommented code - to show the logic of what we already tried
+   *
+   * One of the things we tried at the beginning was also to use event listeners, and other fancy class, style stuff -> what actually worked best was just combining most things with computed cursor style :)
+   */
+  function isInteractiveElement(element) {
+    if (!element || element.nodeType !== Node.ELEMENT_NODE) {
+      return false;
+    }
+
+    // Define interactive cursors
+    const interactiveCursors = new Set([
+      'pointer', // Link/clickable elements
+      'move', // Movable elements
+      'text', // Text selection
+      'grab', // Grabbable elements
+      'grabbing', // Currently grabbing
+      'cell', // Table cell selection
+      'copy', // Copy operation
+      'alias', // Alias creation
+      'all-scroll', // Scrollable content
+      'col-resize', // Column resize
+      'context-menu', // Context menu available
+      'crosshair', // Precise selection
+      'e-resize', // East resize
+      'ew-resize', // East-west resize
+      'help', // Help available
+      'n-resize', // North resize
+      'ne-resize', // Northeast resize
+      'nesw-resize', // Northeast-southwest resize
+      'ns-resize', // North-south resize
+      'nw-resize', // Northwest resize
+      'nwse-resize', // Northwest-southeast resize
+      'row-resize', // Row resize
+      's-resize', // South resize
+      'se-resize', // Southeast resize
+      'sw-resize', // Southwest resize
+      'vertical-text', // Vertical text selection
+      'w-resize', // West resize
+      'zoom-in', // Zoom in
+      'zoom-out', // Zoom out
+    ]);
+
+    // Define non-interactive cursors
+    const nonInteractiveCursors = new Set([
+      'not-allowed', // Action not allowed
+      'no-drop', // Drop not allowed
+      'wait', // Processing
+      'progress', // In progress
+      'initial', // Initial value
+      'inherit', // Inherited value
+      //? Let's just include all potentially clickable elements that are not specifically blocked
+      // 'none',        // No cursor
+      // 'default',     // Default cursor
+      // 'auto',        // Browser default
+    ]);
+
+    function doesElementHaveInteractivePointer(element) {
+      if (element.tagName.toLowerCase() === 'html') return false;
+      const style = getCachedComputedStyle(element);
+
+      if (interactiveCursors.has(style.cursor)) return true;
+
+      return false;
+    }
+
+    let isInteractiveCursor = doesElementHaveInteractivePointer(element);
+
+    // Genius fix for almost all interactive elements
+    if (isInteractiveCursor) {
+      return true;
+    }
+
+    const interactiveElements = new Set([
+      'a', // Links
+      'button', // Buttons
+      'input', // All input types (text, checkbox, radio, etc.)
+      'select', // Dropdown menus
+      'textarea', // Text areas
+      'details', // Expandable details
+      'summary', // Summary element (clickable part of details)
+      'label', // Form labels (often clickable)
+      'option', // Select options
+      'optgroup', // Option groups
+      'fieldset', // Form fieldsets (can be interactive with legend)
+      'legend', // Fieldset legends
+    ]);
+
+    // Define explicit disable attributes and properties
+    const explicitDisableTags = new Set([
+      'disabled', // Standard disabled attribute
+      // 'aria-disabled',      // ARIA disabled state
+      'readonly', // Read-only state
+      // 'aria-readonly',     // ARIA read-only state
+      // 'aria-hidden',       // Hidden from accessibility
+      // 'hidden',            // Hidden attribute
+      // 'inert',             // Inert attribute
+      // 'aria-inert',        // ARIA inert state
+      // 'tabindex="-1"',     // Removed from tab order
+      // 'aria-hidden="true"' // Hidden from screen readers
+    ]);
+
+    // handle inputs, select, checkbox, radio, textarea, button and make sure they are not cursor style disabled/not-allowed
+    if (interactiveElements.has(element.tagName.toLowerCase())) {
+      const style = getCachedComputedStyle(element);
+
+      // Check for non-interactive cursor
+      if (nonInteractiveCursors.has(style.cursor)) {
+        return false;
+      }
+
+      // Check for explicit disable attributes
+      for (const disableTag of explicitDisableTags) {
+        if (
+          element.hasAttribute(disableTag) ||
+          element.getAttribute(disableTag) === 'true' ||
+          element.getAttribute(disableTag) === ''
+        ) {
+          return false;
+        }
+      }
+
+      // Check for disabled property on form elements
+      if (element.disabled) {
+        return false;
+      }
+
+      // Check for readonly property on form elements
+      if (element.readOnly) {
+        return false;
+      }
+
+      // Check for inert property
+      if (element.inert) {
+        return false;
+      }
+
+      return true;
+    }
+
+    const tagName = element.tagName.toLowerCase();
+    const role = element.getAttribute('role');
+    const ariaRole = element.getAttribute('aria-role');
+
+    // Added enhancement to capture dropdown interactive elements
+    if (
+      element.classList &&
+      (element.classList.contains('button') ||
+        element.classList.contains('dropdown-toggle') ||
+        element.getAttribute('data-index') ||
+        element.getAttribute('data-toggle') === 'dropdown' ||
+        element.getAttribute('aria-haspopup') === 'true')
+    ) {
+      return true;
+    }
+
+    const interactiveRoles = new Set([
+      'button', // Directly clickable element
+      // 'link',            // Clickable link
+      // 'menuitem',        // Clickable menu item
+      'menuitemradio', // Radio-style menu item (selectable)
+      'menuitemcheckbox', // Checkbox-style menu item (toggleable)
+      'radio', // Radio button (selectable)
+      'checkbox', // Checkbox (toggleable)
+      'tab', // Tab (clickable to switch content)
+      'switch', // Toggle switch (clickable to change state)
+      'slider', // Slider control (draggable)
+      'spinbutton', // Number input with up/down controls
+      'combobox', // Dropdown with text input
+      'searchbox', // Search input field
+      'textbox', // Text input field
+      // 'listbox',         // Selectable list
+      'option', // Selectable option in a list
+      'scrollbar', // Scrollable control
+    ]);
+
+    // Basic role/attribute checks
+    const hasInteractiveRole =
+      interactiveElements.has(tagName) || interactiveRoles.has(role) || interactiveRoles.has(ariaRole);
+
+    if (hasInteractiveRole) return true;
+
+    // check whether element has event listeners
+    try {
+      if (typeof getEventListeners === 'function') {
+        const listeners = getEventListeners(element);
+        const mouseEvents = ['click', 'mousedown', 'mouseup', 'dblclick'];
+        for (const eventType of mouseEvents) {
+          if (listeners[eventType] && listeners[eventType].length > 0) {
+            return true; // Found a mouse interaction listener
+          }
+        }
+      } else {
+        // Fallback: Check common event attributes if getEventListeners is not available
+        const commonMouseAttrs = ['onclick', 'onmousedown', 'onmouseup', 'ondblclick'];
+        if (commonMouseAttrs.some(attr => element.hasAttribute(attr))) {
+          return true;
+        }
+      }
+    } catch (e) {
+      // console.warn(`Could not check event listeners for ${element.tagName}:`, e);
+      // If checking listeners fails, rely on other checks
+    }
+
+    return false;
+  }
+
+  /**
+   * Checks if an element is the topmost element at its position.
+   */
+  function isTopElement(element) {
+    const rects = element.getClientRects(); // Use getClientRects
+
+    if (!rects || rects.length === 0) {
+      return false; // No geometry, cannot be top
+    }
+
+    let isAnyRectInViewport = false;
+    for (const rect of rects) {
+      // Use the same logic as isInExpandedViewport check
+      if (
+        (rect.width > 0 &&
+          rect.height > 0 &&
+          !(
+            // Only check non-empty rects
+            (
+              rect.bottom < -viewportExpansion ||
+              rect.top > window.innerHeight + viewportExpansion ||
+              rect.right < -viewportExpansion ||
+              rect.left > window.innerWidth + viewportExpansion
+            )
+          )) ||
+        viewportExpansion === -1
+      ) {
+        isAnyRectInViewport = true;
+        break;
+      }
+    }
+
+    if (!isAnyRectInViewport) {
+      return false; // All rects are outside the viewport area
+    }
+
+    // Find the correct document context and root element
+    let doc = element.ownerDocument;
+
+    // If we're in an iframe, elements are considered top by default
+    if (doc !== window.document) {
+      return true;
+    }
+
+    // For shadow DOM, we need to check within its own root context
+    const shadowRoot = element.getRootNode();
+    if (shadowRoot instanceof ShadowRoot) {
+      const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
+      const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
+
+      try {
+        const topEl = measureDomOperation(() => shadowRoot.elementFromPoint(centerX, centerY), 'elementFromPoint');
+        if (!topEl) return false;
+
+        let current = topEl;
+        while (current && current !== shadowRoot) {
+          if (current === element) return true;
+          current = current.parentElement;
+        }
+        return false;
+      } catch (e) {
+        return true;
+      }
+    }
+
+    // For elements in viewport, check if they're topmost
+    const centerX = rects[Math.floor(rects.length / 2)].left + rects[Math.floor(rects.length / 2)].width / 2;
+    const centerY = rects[Math.floor(rects.length / 2)].top + rects[Math.floor(rects.length / 2)].height / 2;
+
+    try {
+      const topEl = document.elementFromPoint(centerX, centerY);
+      if (!topEl) return false;
+
+      let current = topEl;
+      while (current && current !== document.documentElement) {
+        if (current === element) return true;
+        current = current.parentElement;
+      }
+      return false;
+    } catch (e) {
+      return true;
+    }
+  }
+
+  /**
+   * Checks if an element is within the expanded viewport.
+   */
+  function isInExpandedViewport(element, viewportExpansion) {
+    return true;
+
+    if (viewportExpansion === -1) {
+      return true;
+    }
+
+    const rects = element.getClientRects(); // Use getClientRects
+
+    if (!rects || rects.length === 0) {
+      // Fallback to getBoundingClientRect if getClientRects is empty,
+      // useful for elements like <svg> that might not have client rects but have a bounding box.
+      const boundingRect = getCachedBoundingRect(element);
+      if (!boundingRect || boundingRect.width === 0 || boundingRect.height === 0) {
+        return false;
+      }
+      return !(
+        boundingRect.bottom < -viewportExpansion ||
+        boundingRect.top > window.innerHeight + viewportExpansion ||
+        boundingRect.right < -viewportExpansion ||
+        boundingRect.left > window.innerWidth + viewportExpansion
+      );
+    }
+
+    // Check if *any* client rect is within the viewport
+    for (const rect of rects) {
+      if (rect.width === 0 || rect.height === 0) continue; // Skip empty rects
+
+      if (
+        !(
+          rect.bottom < -viewportExpansion ||
+          rect.top > window.innerHeight + viewportExpansion ||
+          rect.right < -viewportExpansion ||
+          rect.left > window.innerWidth + viewportExpansion
+        )
+      ) {
+        return true; // Found at least one rect in the viewport
+      }
+    }
+
+    return false; // No rects were found in the viewport
+  }
+
+  // Add this new helper function
+  function getEffectiveScroll(element) {
+    let currentEl = element;
+    let scrollX = 0;
+    let scrollY = 0;
+
+    return measureDomOperation(() => {
+      while (currentEl && currentEl !== document.documentElement) {
+        if (currentEl.scrollLeft || currentEl.scrollTop) {
+          scrollX += currentEl.scrollLeft;
+          scrollY += currentEl.scrollTop;
+        }
+        currentEl = currentEl.parentElement;
+      }
+
+      scrollX += window.scrollX;
+      scrollY += window.scrollY;
+
+      return { scrollX, scrollY };
+    }, 'scrollOperations');
+  }
+
+  // Add these helper functions at the top level
+  function isInteractiveCandidate(element) {
+    if (!element || element.nodeType !== Node.ELEMENT_NODE) return false;
+
+    const tagName = element.tagName.toLowerCase();
+
+    // Fast-path for common interactive elements
+    const interactiveElements = new Set(['a', 'button', 'input', 'select', 'textarea', 'details', 'summary']);
+
+    if (interactiveElements.has(tagName)) return true;
+
+    // Quick attribute checks without getting full lists
+    const hasQuickInteractiveAttr =
+      element.hasAttribute('onclick') ||
+      element.hasAttribute('role') ||
+      element.hasAttribute('tabindex') ||
+      element.hasAttribute('aria-') ||
+      element.hasAttribute('data-action') ||
+      element.getAttribute('contenteditable') == 'true';
+
+    return hasQuickInteractiveAttr;
+  }
+
+  // --- Define constants for distinct interaction check ---
+  const DISTINCT_INTERACTIVE_TAGS = new Set([
+    'a',
+    'button',
+    'input',
+    'select',
+    'textarea',
+    'summary',
+    'details',
+    'label',
+    'option',
+  ]);
+  const INTERACTIVE_ROLES = new Set([
+    'button',
+    'link',
+    'menuitem',
+    'menuitemradio',
+    'menuitemcheckbox',
+    'radio',
+    'checkbox',
+    'tab',
+    'switch',
+    'slider',
+    'spinbutton',
+    'combobox',
+    'searchbox',
+    'textbox',
+    'listbox',
+    'option',
+    'scrollbar',
+  ]);
+
+  /**
+   * Checks if an element likely represents a distinct interaction
+   * separate from its parent (if the parent is also interactive).
+   */
+  function isElementDistinctInteraction(element) {
+    if (!element || element.nodeType !== Node.ELEMENT_NODE) {
+      return false;
+    }
+
+    const tagName = element.tagName.toLowerCase();
+    const role = element.getAttribute('role');
+
+    // Check if it's an iframe - always distinct boundary
+    if (tagName === 'iframe') {
+      return true;
+    }
+
+    // Check tag name
+    if (DISTINCT_INTERACTIVE_TAGS.has(tagName)) {
+      return true;
+    }
+    // Check interactive roles
+    if (role && INTERACTIVE_ROLES.has(role)) {
+      return true;
+    }
+    // Check contenteditable
+    if (element.isContentEditable || element.getAttribute('contenteditable') === 'true') {
+      return true;
+    }
+    // Check for common testing/automation attributes
+    if (element.hasAttribute('data-testid') || element.hasAttribute('data-cy') || element.hasAttribute('data-test')) {
+      return true;
+    }
+    // Check for explicit onclick handler (attribute or property)
+    if (element.hasAttribute('onclick') || typeof element.onclick === 'function') {
+      return true;
+    }
+    // Check for other common interaction event listeners
+    try {
+      if (typeof getEventListeners === 'function') {
+        const listeners = getEventListeners(element);
+        const interactionEvents = [
+          'mousedown',
+          'mouseup',
+          'keydown',
+          'keyup',
+          'submit',
+          'change',
+          'input',
+          'focus',
+          'blur',
+        ];
+        for (const eventType of interactionEvents) {
+          if (listeners[eventType] && listeners[eventType].length > 0) {
+            return true; // Found a common interaction listener
+          }
+        }
+      } else {
+        // Fallback: Check common event attributes if getEventListeners is not available
+        const commonEventAttrs = [
+          'onmousedown',
+          'onmouseup',
+          'onkeydown',
+          'onkeyup',
+          'onsubmit',
+          'onchange',
+          'oninput',
+          'onfocus',
+          'onblur',
+        ];
+        if (commonEventAttrs.some(attr => element.hasAttribute(attr))) {
+          return true;
+        }
+      }
+    } catch (e) {
+      // console.warn(`Could not check event listeners for ${element.tagName}:`, e);
+      // If checking listeners fails, rely on other checks
+    }
+
+    // Default to false: if it's interactive but doesn't match above,
+    // assume it triggers the same action as the parent.
+    return false;
+  }
+  // --- End distinct interaction check ---
+
+  /**
+   * Handles the logic for deciding whether to highlight an element and performing the highlight.
+   */
+  function handleHighlighting(nodeData, node, parentIframe, isParentHighlighted) {
+    if (!nodeData.isInteractive) return false; // Not interactive, definitely don't highlight
+
+    let shouldHighlight = false;
+    if (!isParentHighlighted) {
+      // Parent wasn't highlighted, this interactive node can be highlighted.
+      shouldHighlight = true;
+    } else {
+      // Parent *was* highlighted. Only highlight this node if it represents a distinct interaction.
+      if (isElementDistinctInteraction(node)) {
+        shouldHighlight = true;
+      } else {
+        // console.log(`Skipping highlight for ${nodeData.tagName} (parent highlighted)`);
+        shouldHighlight = false;
+      }
+    }
+
+    if (shouldHighlight) {
+      // Check viewport status before assigning index and highlighting
+      nodeData.isInViewport = isInExpandedViewport(node, viewportExpansion);
+      if (nodeData.isInViewport) {
+        nodeData.highlightIndex = highlightIndex++;
+
+        if (doHighlightElements) {
+          if (focusHighlightIndex >= 0) {
+            if (focusHighlightIndex === nodeData.highlightIndex) {
+              highlightElement(node, nodeData.highlightIndex, parentIframe);
+            }
+          } else {
+            highlightElement(node, nodeData.highlightIndex, parentIframe);
+          }
+          return true; // Successfully highlighted
+        }
+      } else {
+        // console.log(`Skipping highlight for ${nodeData.tagName} (outside viewport)`);
+      }
+    }
+
+    return false; // Did not highlight
+  }
+
+  /**
+   * Creates a node data object for a given node and its descendants.
+   */
+  function buildDomTree(node, parentIframe = null, isParentHighlighted = false) {
+    if (debugMode) PERF_METRICS.nodeMetrics.totalNodes++;
+
+    if (!node || node.id === HIGHLIGHT_CONTAINER_ID) {
+      if (debugMode) PERF_METRICS.nodeMetrics.skippedNodes++;
+      return null;
+    }
+
+    // Special handling for root node (body)
+    if (node === document.body) {
+      const nodeData = {
+        tagName: 'body',
+        attributes: {},
+        xpath: '/body',
+        children: [],
+      };
+
+      // Process children of body
+      for (const child of node.childNodes) {
+        const domElement = buildDomTree(child, parentIframe, false); // Body's children have no highlighted parent initially
+        if (domElement) nodeData.children.push(domElement);
+      }
+
+      const id = `${ID.current++}`;
+      DOM_HASH_MAP[id] = nodeData;
+      if (debugMode) PERF_METRICS.nodeMetrics.processedNodes++;
+      return id;
+    }
+
+    // Early bailout for non-element nodes except text
+    if (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE) {
+      if (debugMode) PERF_METRICS.nodeMetrics.skippedNodes++;
+      return null;
+    }
+
+    // Process text nodes
+    if (node.nodeType === Node.TEXT_NODE) {
+      const textContent = node.textContent.trim();
+      if (!textContent) {
+        if (debugMode) PERF_METRICS.nodeMetrics.skippedNodes++;
+        return null;
+      }
+
+      // Only check visibility for text nodes that might be visible
+      const parentElement = node.parentElement;
+      if (!parentElement || parentElement.tagName.toLowerCase() === 'script') {
+        if (debugMode) PERF_METRICS.nodeMetrics.skippedNodes++;
+        return null;
+      }
+
+      const id = `${ID.current++}`;
+      DOM_HASH_MAP[id] = {
+        type: 'TEXT_NODE',
+        text: textContent,
+        isVisible: isTextNodeVisible(node),
+      };
+      if (debugMode) PERF_METRICS.nodeMetrics.processedNodes++;
+      return id;
+    }
+
+    // Quick checks for element nodes
+    if (node.nodeType === Node.ELEMENT_NODE && !isElementAccepted(node)) {
+      if (debugMode) PERF_METRICS.nodeMetrics.skippedNodes++;
+      return null;
+    }
+
+    // Early viewport check - only filter out elements clearly outside viewport
+    if (viewportExpansion !== -1) {
+      const rect = getCachedBoundingRect(node); // Keep for initial quick check
+      const style = getCachedComputedStyle(node);
+
+      // Skip viewport check for fixed/sticky elements as they may appear anywhere
+      const isFixedOrSticky = style && (style.position === 'fixed' || style.position === 'sticky');
+
+      // Check if element has actual dimensions using offsetWidth/Height (quick check)
+      const hasSize = node.offsetWidth > 0 || node.offsetHeight > 0;
+
+      // Use getBoundingClientRect for the quick OUTSIDE check.
+      // isInExpandedViewport will do the more accurate check later if needed.
+      if (
+        !rect ||
+        (!isFixedOrSticky &&
+          !hasSize &&
+          (rect.bottom < -viewportExpansion ||
+            rect.top > window.innerHeight + viewportExpansion ||
+            rect.right < -viewportExpansion ||
+            rect.left > window.innerWidth + viewportExpansion))
+      ) {
+        // console.log("Skipping node outside viewport (quick check):", node.tagName, rect);
+        if (debugMode) PERF_METRICS.nodeMetrics.skippedNodes++;
+        return null;
+      }
+    }
+
+    // Process element node
+    const nodeData = {
+      tagName: node.tagName.toLowerCase(),
+      attributes: {},
+      xpath: getXPathTree(node, true),
+      children: [],
+    };
+
+    // Get attributes for interactive elements or potential text containers
+    if (
+      isInteractiveCandidate(node) ||
+      node.tagName.toLowerCase() === 'iframe' ||
+      node.tagName.toLowerCase() === 'body'
+    ) {
+      const attributeNames = node.getAttributeNames?.() || [];
+      for (const name of attributeNames) {
+        nodeData.attributes[name] = node.getAttribute(name);
+      }
+    }
+
+    let nodeWasHighlighted = false;
+    // Perform visibility, interactivity, and highlighting checks
+    if (node.nodeType === Node.ELEMENT_NODE) {
+      nodeData.isVisible = isElementVisible(node); // isElementVisible uses offsetWidth/Height, which is fine
+      if (nodeData.isVisible) {
+        nodeData.isTopElement = isTopElement(node);
+        if (nodeData.isTopElement) {
+          nodeData.isInteractive = isInteractiveElement(node);
+          // Call the dedicated highlighting function
+          nodeWasHighlighted = handleHighlighting(nodeData, node, parentIframe, isParentHighlighted);
+        }
+      }
+    }
+
+    // Process children, with special handling for iframes and rich text editors
+    if (node.tagName) {
+      const tagName = node.tagName.toLowerCase();
+
+      // Handle iframes
+      if (tagName === 'iframe') {
+        try {
+          const iframeDoc = node.contentDocument || node.contentWindow?.document;
+          if (iframeDoc) {
+            for (const child of iframeDoc.childNodes) {
+              const domElement = buildDomTree(child, node, false);
+              if (domElement) nodeData.children.push(domElement);
+            }
+          }
+        } catch (e) {
+          console.warn('Unable to access iframe:', e);
+        }
+      }
+      // Handle rich text editors and contenteditable elements
+      else if (
+        node.isContentEditable ||
+        node.getAttribute('contenteditable') === 'true' ||
+        node.id === 'tinymce' ||
+        node.classList.contains('mce-content-body') ||
+        (tagName === 'body' && node.getAttribute('data-id')?.startsWith('mce_'))
+      ) {
+        // Process all child nodes to capture formatted text
+        for (const child of node.childNodes) {
+          const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted);
+          if (domElement) nodeData.children.push(domElement);
+        }
+      } else {
+        // Handle shadow DOM
+        if (node.shadowRoot) {
+          nodeData.shadowRoot = true;
+          for (const child of node.shadowRoot.childNodes) {
+            const domElement = buildDomTree(child, parentIframe, nodeWasHighlighted);
+            if (domElement) nodeData.children.push(domElement);
+          }
+        }
+        // Handle regular elements
+        for (const child of node.childNodes) {
+          // Pass the highlighted status of the *current* node to its children
+          const passHighlightStatusToChild = nodeWasHighlighted || isParentHighlighted;
+          const domElement = buildDomTree(child, parentIframe, passHighlightStatusToChild);
+          if (domElement) nodeData.children.push(domElement);
+        }
+      }
+    }
+
+    // Skip empty anchor tags
+    if (nodeData.tagName === 'a' && nodeData.children.length === 0 && !nodeData.attributes.href) {
+      if (debugMode) PERF_METRICS.nodeMetrics.skippedNodes++;
+      return null;
+    }
+
+    const id = `${ID.current++}`;
+    DOM_HASH_MAP[id] = nodeData;
+    if (debugMode) PERF_METRICS.nodeMetrics.processedNodes++;
+    return id;
+  }
+
+  // After all functions are defined, wrap them with performance measurement
+  // Remove buildDomTree from here as we measure it separately
+  highlightElement = measureTime(highlightElement);
+  isInteractiveElement = measureTime(isInteractiveElement);
+  isElementVisible = measureTime(isElementVisible);
+  isTopElement = measureTime(isTopElement);
+  isInExpandedViewport = measureTime(isInExpandedViewport);
+  isTextNodeVisible = measureTime(isTextNodeVisible);
+  getEffectiveScroll = measureTime(getEffectiveScroll);
+
+  const rootId = buildDomTree(document.body);
+
+  // Clear the cache before starting
+  DOM_CACHE.clearCache();
+
+  // Only process metrics in debug mode
+  if (debugMode && PERF_METRICS) {
+    // Convert timings to seconds and add useful derived metrics
+    Object.keys(PERF_METRICS.timings).forEach(key => {
+      PERF_METRICS.timings[key] = PERF_METRICS.timings[key] / 1000;
+    });
+
+    Object.keys(PERF_METRICS.buildDomTreeBreakdown).forEach(key => {
+      if (typeof PERF_METRICS.buildDomTreeBreakdown[key] === 'number') {
+        PERF_METRICS.buildDomTreeBreakdown[key] = PERF_METRICS.buildDomTreeBreakdown[key] / 1000;
+      }
+    });
+
+    // Add some useful derived metrics
+    if (PERF_METRICS.buildDomTreeBreakdown.buildDomTreeCalls > 0) {
+      PERF_METRICS.buildDomTreeBreakdown.averageTimePerNode =
+        PERF_METRICS.buildDomTreeBreakdown.totalTime / PERF_METRICS.buildDomTreeBreakdown.buildDomTreeCalls;
+    }
+
+    PERF_METRICS.buildDomTreeBreakdown.timeInChildCalls =
+      PERF_METRICS.buildDomTreeBreakdown.totalTime - PERF_METRICS.buildDomTreeBreakdown.totalSelfTime;
+
+    // Add average time per operation to the metrics
+    Object.keys(PERF_METRICS.buildDomTreeBreakdown.domOperations).forEach(op => {
+      const time = PERF_METRICS.buildDomTreeBreakdown.domOperations[op];
+      const count = PERF_METRICS.buildDomTreeBreakdown.domOperationCounts[op];
+      if (count > 0) {
+        PERF_METRICS.buildDomTreeBreakdown.domOperations[`${op}Average`] = time / count;
+      }
+    });
+
+    // Calculate cache hit rates
+    const boundingRectTotal =
+      PERF_METRICS.cacheMetrics.boundingRectCacheHits + PERF_METRICS.cacheMetrics.boundingRectCacheMisses;
+    const computedStyleTotal =
+      PERF_METRICS.cacheMetrics.computedStyleCacheHits + PERF_METRICS.cacheMetrics.computedStyleCacheMisses;
+
+    if (boundingRectTotal > 0) {
+      PERF_METRICS.cacheMetrics.boundingRectHitRate =
+        PERF_METRICS.cacheMetrics.boundingRectCacheHits / boundingRectTotal;
+    }
+
+    if (computedStyleTotal > 0) {
+      PERF_METRICS.cacheMetrics.computedStyleHitRate =
+        PERF_METRICS.cacheMetrics.computedStyleCacheHits / computedStyleTotal;
+    }
+
+    if (boundingRectTotal + computedStyleTotal > 0) {
+      PERF_METRICS.cacheMetrics.overallHitRate =
+        (PERF_METRICS.cacheMetrics.boundingRectCacheHits + PERF_METRICS.cacheMetrics.computedStyleCacheHits) /
+        (boundingRectTotal + computedStyleTotal);
+    }
+  }
+
+  return debugMode ? { rootId, map: DOM_HASH_MAP, perfMetrics: PERF_METRICS } : { rootId, map: DOM_HASH_MAP };
+};

+ 23 - 11
src/store/modules/msg.ts

@@ -2,9 +2,11 @@ import { defineStore } from 'pinia'
 import OpenAI from 'openai'
 import { getChatDetail, getModalList } from '@/api/index'
 import {
-  formatMessage,
+  formatMessage
 } from '@/entrypoints/sidepanel/utils/ai-service.js'
-import { v4 as uuidv4 } from 'uuid';
+import { v4 as uuidv4 } from 'uuid'
+import { LLMProviders } from '@/utils/modelSetting'
+
 export const useMsgStore = defineStore('msg', {
   state: () => ({
     msgUuid: <string>'',
@@ -49,31 +51,41 @@ export const useMsgStore = defineStore('msg', {
           if (this.messages.length < res.data.total) {
             this.hasNext = true
             this.page++
-          }
-          else this.hasNext = false
+          } else this.hasNext = false
         } else {
           this.msgUuid = uuidv4()
           chrome.storage.local.set({ msgUuid: this.msgUuid })
         }
       } catch (error) {
-        
+
       } finally {
         this.msgLoading = false
       }
     },
     async initModal() {
-      const res = await getModalList()
-      this.modelList = res.data.list
-      this.selectModal = this.modelList[0]
+      const { data } = await getModalList({ type: 'CHAT' })
+      const temp: any = []
+      LLMProviders.forEach((i) => {
+        const children = data.filter((m: any) => m.provider == i.model)
+        if (children.length === 0) {
+          return
+        }
+        temp.push({
+          isGroup: true,
+          label: i.name,
+          options: children
+        })
+      })
+      this.modelList = temp
+      this.selectModal = this.modelList[0].options[0]
     },
     async changePage() {
       if (this.hasNext) {
         await this.initMsg()
-
       }
     },
-    updateAIModel(id) {
-     this.selectModal = this.modelList.find(_ => _.id === id)
+    updateAIModel(item: any) {
+      this.selectModal = item
     }
   }
 })

+ 218 - 0
src/utils/modelSetting.ts

@@ -0,0 +1,218 @@
+export enum ProviderEnum {
+  OPENAI = 'OPENAI',
+  AZURE_OPENAI = 'AZURE_OPENAI',
+  GEMINI = 'GEMINI',
+  OLLAMA = 'OLLAMA',
+  CLAUDE = 'CLAUDE',
+  Q_FAN = 'Q_FAN',
+  Q_WEN = 'Q_WEN',
+  ZHIPU = 'ZHIPU',
+  GITEEAI = 'GITEEAI',
+  DEEPSEEK = 'DEEPSEEK',
+  DOUYIN = 'DOUYIN',
+  SILICON = 'SILICON',
+  YI = 'YI',
+  SPARK = 'SPARK',
+}
+
+export function getModels(provider: string, providers: Array<any>) {
+  const arr = providers.filter((i) => i.model === provider)
+  if (arr.length === 0) {
+    return []
+  }
+  if (typeof arr[0].models[0] === 'string') {
+    return arr[0].models.map((i) => {
+      return {
+        label: i,
+        value: i
+      }
+    })
+  } else {
+    return arr[0].models
+  }
+}
+
+export function getTitle(provider: string, providers: Array<any>) {
+  return providers.filter((i) => i.model === provider)[0].name
+}
+
+export const LLMProviders: any[] = [
+  {
+    model: ProviderEnum.OPENAI,
+    name: 'OpenAI',
+    models: ['gpt-3.5-turbo', 'gpt-4', 'gpt-4-32k', 'gpt-4-turbo', 'gpt-4o']
+  },
+  {
+    model: ProviderEnum.Q_FAN,
+    name: '百度千帆',
+    models: [
+      'ERNIE-Bot',
+      'ERNIE-Bot 4.0',
+      'ERNIE-Bot-8K',
+      'ERNIE-Bot-turbo',
+      'ERNIE-Speed-128K',
+      'EB-turbo-AppBuilder',
+      'Yi-34B-Chat',
+      'BLOOMZ-7B',
+      'Qianfan-BLOOMZ-7B-compressed',
+      'Mixtral-8x7B-Instruct',
+      'Llama-2-7b-chat',
+      'Llama-2-13b-chat',
+      'Llama-2-70b-chat',
+      'Qianfan-Chinese-Llama-2-7B',
+      'ChatGLM2-6B-32K',
+      'AquilaChat-7B'
+    ]
+  },
+  {
+    model: ProviderEnum.Q_WEN,
+    name: '阿里百炼',
+    models: [
+      'qwen-turbo',
+      'qwen-plus',
+      'qwen-max',
+      'qwen-max-longcontext',
+      'qwen-7b-chat',
+      'qwen-14b-chat',
+      'qwen-72b-chat'
+    ]
+  },
+  {
+    model: ProviderEnum.ZHIPU,
+    name: '智谱清言',
+    models: [
+      'glm-4',
+      'glm-4v',
+      'glm-4-air',
+      'glm-4-airx',
+      'glm-4-flash',
+      'glm-3-turbo',
+      'chatglm_turbo'
+    ]
+  },
+  {
+    model: ProviderEnum.GITEEAI,
+    name: 'Gitee AI',
+    models: [
+      'Qwen2-72B-Instruct',
+      'Qwen2-7B-Instruct',
+      'Qwen2.5-72B-Instruct',
+      'glm-4-9b-chat',
+      'deepseek-coder-33B-instruct',
+      'codegeex4-all-9b',
+      'Yi-34B-Chat',
+      'code-raccoon-v1',
+      'Qwen2.5-Coder-32B-Instruct'
+    ]
+  },
+  {
+    model: ProviderEnum.DEEPSEEK,
+    name: 'DeepSeek',
+    models: ['deepseek-chat', 'deepseek-coder']
+  },
+  {
+    model: ProviderEnum.DOUYIN,
+    name: '抖音豆包',
+    models: []
+  },
+  {
+    model: ProviderEnum.SILICON,
+    name: '硅基流动',
+    models: [
+      'deepseek-ai/DeepSeek-V2-Chat',
+      'deepseek-ai/DeepSeek-Coder-V2-Instruct',
+      'deepseek-ai/DeepSeek-V2.5',
+      'Qwen/Qwen2.5-72B-Instruct-128K',
+      'Qwen/Qwen2.5-72B-Instruct',
+      'Qwen/Qwen2-VL-72B-Instruct',
+      'Qwen/Qwen2.5-32B-Instruct',
+      'Qwen/Qwen2.5-14B-Instruct',
+      'Qwen/Qwen2.5-7B-Instruct',
+      'Qwen/Qwen2.5-Math-72B-Instruct',
+      'Qwen/Qwen2.5-Coder-7B-Instruct',
+      'Qwen/Qwen2-72B-Instruct',
+      'Qwen/Qwen2-7B-Instruct',
+      'Qwen/Qwen2-1.5B-Instruct',
+      'Qwen/Qwen2-57B-A14B-Instruct',
+      'TeleAI/TeleChat2',
+      '01-ai/Yi-1.5-34B-Chat-16K',
+      '01-ai/Yi-1.5-9B-Chat-16K',
+      '01-ai/Yi-1.5-6B-Chat',
+      'THUDM/chatglm3-6b',
+      'THUDM/glm-4-9b-chat',
+      'Vendor-A/Qwen/Qwen2-72B-Instruct',
+      'Vendor-A/Qwen/Qwen2.5-72B-Instruct',
+      'internlm/internlm2_5-7b-chat',
+      'internlm/internlm2_5-20b-chat',
+      'OpenGVLab/InternVL2-Llama3-76B',
+      'OpenGVLab/InternVL2-26B',
+      'nvidia/Llama-3.1-Nemotron-70B-Instruct',
+      'meta-llama/Meta-Llama-3.1-405B-Instruct',
+      'meta-llama/Meta-Llama-3.1-70B-Instruct',
+      'meta-llama/Meta-Llama-3.1-8B-Instruct',
+      'meta-llama/Meta-Llama-3-8B-Instruct',
+      'meta-llama/Meta-Llama-3-70B-Instruct',
+      'google/gemma-2-27b-it',
+      'google/gemma-2-9b-it',
+      'Pro/Qwen/Qwen2.5-7B-Instruct',
+      'Pro/Qwen/Qwen2-7B-Instruct',
+      'Pro/Qwen/Qwen2-1.5B-Instruct',
+      'Pro/Qwen/Qwen2-VL-7B-Instruct',
+      'Pro/01-ai/Yi-1.5-9B-Chat-16K',
+      'Pro/01-ai/Yi-1.5-6B-Chat',
+      'Pro/THUDM/chatglm3-6b',
+      'Pro/THUDM/glm-4-9b-chat',
+      'Pro/internlm/internlm2_5-7b-chat',
+      'Pro/OpenGVLab/InternVL2-8B',
+      'Pro/meta-llama/Meta-Llama-3-8B-Instruct',
+      'Pro/meta-llama/Meta-Llama-3.1-8B-Instruct',
+      'Pro/google/gemma-2-9b-it'
+    ]
+  },
+  {
+    model: ProviderEnum.YI,
+    name: '零一万物',
+    models: [
+      'yi-lightning',
+      'yi-large',
+      'yi-medium',
+      'yi-medium-200k',
+      'yi-spark',
+      'yi-large-rag',
+      'yi-large-turbo'
+    ]
+  },
+  {
+    model: ProviderEnum.SPARK,
+    name: '讯飞星火',
+    // models: ['lite', 'generalv3', 'pro-128k', 'generalv3.5', 'max-32k', '4.0Ultra'],
+    models: [
+      { label: 'Spark Lite', value: 'lite' },
+      { label: 'Spark Pro', value: 'generalv3' },
+      { label: 'Spark Pro-128K', value: 'pro-128k' },
+      { label: 'Spark Max', value: 'generalv3.5' },
+      { label: 'Spark Max-32K', value: 'max-32k' },
+      { label: 'Spark4.0 Ultra', value: '4.0Ultra' }
+    ]
+  },
+  {
+    model: ProviderEnum.OLLAMA,
+    name: 'Ollama',
+    models: []
+  },
+  {
+    model: ProviderEnum.AZURE_OPENAI,
+    name: 'Azure OpenAI',
+    models: ['gpt-3.5-turbo', 'gpt-4', 'gpt-4-32k', 'gpt-4-turbo', 'gpt-4o']
+  },
+  {
+    model: ProviderEnum.GEMINI,
+    name: 'Gemini',
+    models: ['gemini-1.5-pro']
+  },
+  {
+    model: ProviderEnum.CLAUDE,
+    name: 'Claude',
+    models: ['claude-3-opus', 'claude-3-opus-20240229', 'claude-3-sonnet', 'claude-3-haiku']
+  }
+]

+ 236 - 219
src/utils/navigator.js

@@ -1,239 +1,256 @@
-import { connect, ExtensionTransport } from 'puppeteer-core/lib/cjs/puppeteer/puppeteer-core-browser.js'
+import {
+    puppeteer,
+    connect,
+    ExtensionTransport,
+} from 'puppeteer-core/lib/cjs/puppeteer/puppeteer-core-browser.js';
 import { getexecute } from '@/api/agentapi.js'
-import { browser } from 'wxt/browser'
-
-export const navigator = () => {
-  let browser = null
-
-  async function browsercomm() {
-    const tabitem = await getActiveTabId()
-    let tabId = tabitem.id
-    console.log(tabitem)
-    return
-    const connectOptions = {
-      transport: await ExtensionTransport.connectTab(tabId), // 替换为实际标签ID
-      defaultViewport: null,
-      protocol: 'cdp'
-    }
+export const navigator=()=> {
+    let browser = null;
+    async function browsercomm() {
+        const tabitem = await getActiveTabId();
+        console.log(tabitem)
+        let tabId=tabitem.id
+        console.log(tabId)
+        const connectOptions = {
+            transport: await ExtensionTransport.connectTab(tabId), // 替换为实际标签ID
+            defaultViewport: null,
+            protocol: 'cdp'
+        };
+        console.log(connectOptions)
+        let browser;
 
-    let browser
-
-    try {
-      // 连接到浏览器
-      browser = await connect(connectOptions)
-      const [page] = await browser.pages()
-      await chrome.scripting.executeScript({
-        target: { tabId: tabId },
-        files: ['buildDomTree.js'], // 注入外部文件
-        world: 'MAIN'
-      })
-      // await page.waitForFunction(() => typeof window.yourFunc === 'function');
-      const domState = await page.evaluate(() => {
-
-        const buildDomTree = window.buildDomTree({
-          doHighlightElements: true,
-          focusHighlightIndex: -1,
-          viewportExpansion: 0,
-          debugMode: false
-        })
+        try {
+            // 连接到浏览器
+            browser = await connect(connectOptions);
+            const [page] = await browser.pages();
+            await chrome.scripting.executeScript({
+                target: {tabId: tabId},
+                files: ['buildDomTree.js'], // 注入外部文件
+                world: "MAIN"
+            });
+            // await page.waitForFunction(() => typeof window.yourFunc === 'function');
+            const domState = await page.evaluate(() => {
 
-        function findDomByXpath(xpath) {
-          try {
-            return document.evaluate('/' + xpath.replace(/^\/+/, ''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue
-          } catch (e) {
-            return null
-          }
-        }
+                const buildDomTree = window.buildDomTree({
+                    doHighlightElements: true,
+                    focusHighlightIndex: -1,
+                    viewportExpansion: 0,
+                    debugMode: false,
+                })
 
-        const domList = Object.values(buildDomTree.map)
-          .filter(n => typeof n === 'object' && n.highlightIndex !== undefined)
-          .map(n => findDomByXpath(n.xpath))
-          .filter(x => !!x)
-          .map((item, index) => `[${index}] ${item.outerHTML}}`)
-        console.log(domList)
-        return { buildDomTree: buildDomTree, domList: domList }
-      })
-      return { domState: domState, page: page, tabitem: tabitem }
-    } finally {
-      if (browser) await browser.disconnect()
-    }
-  }
-
-  async function browserautomate(domState, page, type, targetIndex) {
-
-    // 查找目标元素
-    const targetElement = findElementByIndex(domState.map, targetIndex)
-    if (!targetElement) throw new Error('Element not found')
-
-    // 生成增强选择器
-    const selector = generateSelector(targetElement)
-    console.log(domState, targetElement, selector)
-
-    try {
-      await page.waitForSelector(selector, { visible: true, timeout: 10000 })
-      switch (type) {
-        case 'click':
-          await page.click(selector, { delay: 100 })
-          break
-        case 'navigate':
-          await page.goto('https://example.com', { waitUntil: 'networkidle2' })
-          break
-        case 'getcontent':
-          console.log(await page.content())
-          break
-        // 添加更多命令支持,如:
-        case 'form':
-          const formData = [{
-            text: 'your-username',
-            index: 'your-password',
-            email: 'your-email@example.com',
-            question2: 'option2', // 假设有一个单选题,选项为 option2
-            question3: '我非常满意这次体验' // 假设有一个文本框,答案为 “我非常满意这次体验”
-          }]
-          await page.type(selector, 'your-username', { delay: 600 })
-          await page.type(selector, 'your-password', { delay: 600 })
-          await page.click(selector) // 或者其他登录按钮
-          break
-        case '':
-          page.type(selector, 'World', { delay: 600 })
-          break
-        default:
-          console.log('未知指令。')
-      }
-
-      // 验证点击结果
-      // await page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 15000 });
-      // console.log(`Successfully clicked element ${targetIndex} on attempt ${attempt}`);
-      await page.evaluate(() => {
-
-        // document.getElementById('playwright-highlight-container')?.remove();
-
-      })
-      return true
-    } catch (clickError) {
-      // console.warn(`Attempt ${attempt} failed: ${clickError.message}`);
-      // if (attempt === retries) throw clickError;
-      // await page.waitForTimeout(delay);
-
-      // 刷新 DOM 状态
-      await page.evaluate(() => {
-        document.getElementById('playwright-highlight-container')?.remove()
-        window.buildDomTree({ doHighlightElements: true })
-      })
+                function findDomByXpath(xpath) {
+                    try {
+                        return document.evaluate('/' + xpath.replace(/^\/+/, ''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                    } catch (e) {
+                        return null;
+                    }
+                }
+
+                const domList = Object.values(buildDomTree.map)
+                    .filter(n => typeof n === "object" && n.highlightIndex !== undefined)
+                    .map(n => findDomByXpath(n.xpath))
+                    .filter(x => !!x)
+                    .map((item, index) => `[${index}] ${item.outerHTML}`)
+                    .join("\n")
+                console.log(domList)
+                return {buildDomTree:buildDomTree,domList:domList}
+            });
+            return {domState:domState, page:page,tabitem:tabitem};
+        } finally {
+            // if (browser) await browser.disconnect();
+        }
     }
-  }
+    async function browserautomate(domState,page,type,targetIndex) {
+console.log(domState,page,type,targetIndex,browser)
+        // 查找目标元素
+        const targetElement = findElementByIndex(domState.map, targetIndex);
+        if (!targetElement) throw new Error('Element not found');
 
-  function getActiveTabId() {
-    return new Promise((resolve) => {
-      chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
-        resolve(tabs[0])
-      })
-    })
-  }
+        // 生成增强选择器
+        const selector = generateSelector(targetElement);
+        console.log(domState, targetElement, selector)
+        // const [page] = await browser.pages()
+        try {
+            await page.waitForSelector(selector, {visible: true, timeout: 10000});
+            switch (type) {
+                case 'click':
+                    await page.click(selector, {delay: 100});
+                    break;
+                case 'navigate':
+                    await page.goto('https://example.com', {waitUntil: 'networkidle2'});
+                    break;
+                case 'getcontent':
+                    console.log(await page.content());
+                    break;
+                // 添加更多命令支持,如:
+                case 'form':
+                    const formData = [{
+                        text: 'your-username',
+                        index: 'your-password',
+                        email: 'your-email@example.com',
+                        question2: 'option2', // 假设有一个单选题,选项为 option2
+                        question3: '我非常满意这次体验', // 假设有一个文本框,答案为 “我非常满意这次体验”
+                    }];
+                    await page.type(selector, 'your-username', {delay: 600});
+                    await page.type(selector, 'your-password', {delay: 600});
+                    await page.click(selector); // 或者其他登录按钮
+                    break;
+                case '':
+                    page.type(selector, 'World', {delay: 600})
+                    break
+                default:
+                    console.log('未知指令。');
+            }
+
+            // 验证点击结果
+            // await page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 15000 });
+            // console.log(`Successfully clicked element ${targetIndex} on attempt ${attempt}`);
+            await page.evaluate(() => {
+
+                // document.getElementById('playwright-highlight-container')?.remove();
 
+            });
+            return true;
+        } catch (clickError) {
+            // console.warn(`Attempt ${attempt} failed: ${clickError.message}`);
+            // if (attempt === retries) throw clickError;
+            // await page.waitForTimeout(delay);
+
+            // 刷新 DOM 状态
+            await page.evaluate(() => {
+                document.getElementById('playwright-highlight-container')?.remove();
+                window.buildDomTree({doHighlightElements: true});
+            });
+        }
+    }
+
+    function getActiveTabId() {
+        return new Promise((resolve) => {
+            setTimeout(() => {
+                chrome.tabs.query({active: true}, (tabs) => {
+                    console.log('tabs', tabs)
+                    resolve(tabs[0]);
+                });
+            },500)
+        });
+    }
 // 辅助函数:通过索引查找元素
-  function findElementByIndex(domMap, targetIndex) {
-    return Object.values(domMap).find(
-      node => node.highlightIndex === targetIndex
-    )
-  }
+    function findElementByIndex(domMap, targetIndex) {
+        return Object.values(domMap).find(
+            node => node.highlightIndex === targetIndex
+        );
+    }
 
 // 辅助函数:生成 CSS 选择器
-  function generateSelector(element) {
-    const attrs = Object.entries(element.attributes)
-      .map(([k, v]) => `[${k}="${v.replace(/"/g, '\\"')}"]`)
-      .join('')
-    return `${element.tagName}${attrs}`
-  }
-
-  function gfe() {
-    chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => {
-      const tabId = tabs[0].id
-      await smartClick(tabId, 1, {
-        type: 'click',
-        text: 3000
-      })
-    })
-  }
+    function generateSelector(element) {
+        const attrs = Object.entries(element.attributes)
+            .map(([k, v]) => `[${k}="${v.replace(/"/g, '\\"')}"]`)
+            .join('');
+        return `${element.tagName}${attrs}`;
+    }
 
-// 背景脚本
-  function switchTabOrOpenNew(url) {
-    return new Promise(resolve => {
-      // 查询是否存在指定的标签页
-      chrome.tabs.query({ windowId: chrome.windows.WINDOW_ID_CURRENT }, async (tabs) => {
-        let tabId = tabs.find(tabs => tabs.url === url)
-        console.log(tabs, tabId)
-        if (tabId) {
-          // 如果存在,则激活该标签页
-          await chrome.tabs.update(tabId.id, { active: true })
-          resolve()
-        } else {
-          // 如果不存在,则新打开一个标签页
-          await chrome.tabs.create({ url: url, active: true })
-          resolve()
-        }
-      })
-    })
-  }
-
-  function getexecuteapi(payload) {
-    console.log(payload)
-    let params = {
-      conversationId: payload.conversationId,
-      messageId: payload.messageId,
-      messageContent: payload.messageContent,
-      planId: payload.id,
-      currentPlanStepId: payload.steps[0].id,
-      envData: {
-        osName: 'Google Chrome',
-        osVersion: '137.0.7151.69(正式版本)',
-        osArch: 'arm64',
-        url: '',
-        title: '',
-        tabs: [],
-        interactiveElements: ''
-      },
-      resultSummary: null,
-      needSummary: true,
-      success: false
+    function gfe() {
+        chrome.tabs.query({active: true, currentWindow: true}, async (tabs) => {
+            const tabId = tabs[0].id;
+            await smartClick(tabId, 1, {
+                type: 'click',
+                text: 3000
+            });
+        })
     }
-    return getexecute(params).then((res) => {
-      return res
-    })
-  }
-
-  async function agent(payload) {
-    for (let item of payload.steps) {
-      console.log(item)
-      if (item.stepIndex === 0) {
-        let add = await getexecuteapi(payload)
-        console.log(add)
-        let toolParameters = add.data.plan.steps[0].toolParameters
-        console.log(JSON.parse(toolParameters).action)
-        if (JSON.parse(toolParameters).action == 'navigate') {
-          await switchTabOrOpenNew(JSON.parse(toolParameters).url)
-        }
-      } else {
-        let browdata = await browsercomm()
-        console.log(browdata)
-      }
+
+// 背景脚本
+    function switchTabOrOpenNew(url) {
+        return new Promise(resolve => {
+            // 查询是否存在指定的标签页
+            chrome.tabs.query({windowId: chrome.windows.WINDOW_ID_CURRENT}, async (tabs) => {
+                let tabId = tabs.find(tabs => tabs.url === url);
+                console.log(tabs, tabId,url)
+                if (tabId) {
+                    // 如果存在,则激活该标签页
+                    await chrome.tabs.update(tabId.id, {active: true});
+                    resolve()
+                } else {
+                    // 如果不存在,则新打开一个标签页
+                    await chrome.tabs.create({url: url});
+                    resolve()
+                }
+            });
+        })
     }
-  }
 
-  chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
-    console.log(request, sender, sendResponse)
-    if (request.type == 'FROM_PLAN') {
-      agent(request.payload)
+    function getexecuteapi(params) {
+       return  getexecute(params).then((res)=>{
+           return res;
+        })
     }
-  })
-}
 
+    async function agent(payload) {
+        for(let item of payload.steps){
+            console.log(item)
+            if(item.stepIndex === 0){
+                let params = {
+                    conversationId: payload.conversationId,
+                    messageId: payload.messageId,
+                    messageContent: payload.messageContent,
+                    planId: payload.id,
+                    currentPlanStepId: item.id,
+                    envData: {
+                        osName: "Google Chrome",
+                        osVersion: "137.0.7151.69(正式版本)",
+                        osArch: "arm64",
+                        url: "",
+                        title:"",
+                        tabs: [],
+                        interactiveElements: ""
+                    },
+                    resultSummary: null,
+                    needSummary: true,
+                    success: false
+                }
+                let add = await getexecuteapi(params)
+                console.log(add)
+                let toolParameters=add.data.plan.steps[0].toolParameters
+                console.log(JSON.parse(toolParameters).action)
+                if (JSON.parse(toolParameters).action == 'navigate') {
+                    await switchTabOrOpenNew(JSON.parse(toolParameters).url)
+                }
+            }else {
 
-function getActiveTabId() {
-  return new Promise((resolve) => {
-    chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
-      resolve(tabs[0])
+                let browdata =  await  browsercomm()
+        console.log(browdata)
+        let params = {
+            conversationId: payload.conversationId,
+            messageId: payload.messageId,
+            messageContent: payload.messageContent,
+            planId: payload.id,
+            currentPlanStepId: item.id,
+            envData: {
+                osName: "Google Chrome",
+                osVersion: "137.0.7151.69(正式版本)",
+                osArch: "arm64",
+                url: browdata.tabitem.url,
+                title: browdata.tabitem.title,
+                tabs: [],
+                interactiveElements: browdata.domState.domList
+            },
+            resultSummary: null,
+            needSummary: true,
+            success: false
+        }
+        let add = await getexecuteapi(params)
+                console.log(add)
+                await  browserautomate(browdata.domState.buildDomTree,browdata.page,'click',1)
+            }
+        }
+    }
+
+    chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
+        console.log(request, sender, sendResponse);
+        if (request.type == 'FROM_PLAN') {
+            agent(request.payload)
+        }
     })
-  })
 }
 
+

+ 21 - 33
src/utils/utils.js

@@ -1,38 +1,25 @@
 import { browser } from 'wxt/browser'
+import { ExtensionTransport,connect } from 'puppeteer-core/lib/cjs/puppeteer/puppeteer-core-browser.js'
+import { getexecute } from '@/api/agentapi.js'
 
 export async function tabsList() {
-  return await browser.tabs.query({})
+  return new Promise((resolve) => {
+    browser.tabs.query({}, (tabs) => {
+      resolve(tabs)
+    })
+  })
 }
 
-export async function activateTab() {
-  return await browser.tabs.query({active: true, currentWindow: true})
+export async function getActiveTabId() {
+  return new Promise((resolve) => {
+    browser.tabs.query({ active: true, currentWindow: true }, (tabs) => {
+      resolve(tabs[0])
+    })
+  })
 }
 
-export function getExecuteApi(payload, dom,tabs) {
-
-  console.log('api', payload)
-  let params = {
-    conversationId: payload.conversationId,
-    messageId: payload.messageId,
-    messageContent: payload.messageContent,
-    planId: payload.planId,
-    currentPlanStepId: payload.id,
-    envData: {
-      osName: 'Google Chrome',
-      osVersion: '137.0.7151.69(正式版本)',
-      osArch: 'arm64',
-      url: '',
-      title: '',
-      tabs: tabs.length ? tabs :  [],
-      interactiveElements: dom ? dom : ''
-    },
-    resultSummary: null,
-    needSummary: true,
-    success: false
-  }
-  return getexecute(params).then((res) => {
-    return res
-  })
+export async function getExecuteApi(params) {
+  return await getexecute(params)
 }
 
 export function switchTabOrOpenNew(url) {
@@ -56,15 +43,15 @@ export function switchTabOrOpenNew(url) {
 
 export async function browsercomm() {
   const tabitem = await getActiveTabId()
-  let tabId = tabitem.id
   console.log(tabitem)
-  return
+  let tabId = tabitem.id
+  console.log(tabId)
   const connectOptions = {
     transport: await ExtensionTransport.connectTab(tabId), // 替换为实际标签ID
     defaultViewport: null,
     protocol: 'cdp'
   }
-
+  console.log(connectOptions)
   let browser
 
   try {
@@ -98,12 +85,13 @@ export async function browsercomm() {
         .filter(n => typeof n === 'object' && n.highlightIndex !== undefined)
         .map(n => findDomByXpath(n.xpath))
         .filter(x => !!x)
-        .map((item, index) => `[${index}] ${item.outerHTML}}`)
+        .map((item, index) => `[${index}] ${item.outerHTML}`)
+        .join('\n')
       console.log(domList)
       return { buildDomTree: buildDomTree, domList: domList }
     })
     return { domState: domState, page: page, tabitem: tabitem }
   } finally {
-    if (browser) await browser.disconnect()
+    // if (browser) await browser.disconnect();
   }
 }

+ 2 - 1
wxt.config.ts

@@ -34,7 +34,8 @@ export default defineConfig({
       'activeTab',
       'webNavigation',
       'webRequest',
-      'sidePanel'
+      'sidePanel',
+      "debugger",
     ],
     content_security_policy: {
       extension_pages: "script-src 'self'; object-src 'self';"