pvs_agent/src/utils/navigator.js

import {
    puppeteer,
    connect,
    ExtensionTransport,
} from 'puppeteer-core/lib/cjs/puppeteer/puppeteer-core-browser.js';
import { getexecute } from '@/api/agentapi.js'
export const navigator=()=> {
    let browser = null;
    async function browsercomm() {
        const tabitem = await getActiveTabId();
        let tabId=tabitem.id
        const connectOptions = {
            transport: await ExtensionTransport.connectTab(tabId), // 替换为实际标签ID
            defaultViewport: null,
            protocol: 'cdp'
        };
        try {
            // 连接到浏览器
            browser = await connect(connectOptions);
            const [page] = await browser.pages();
            await chrome.scripting.executeScript({
                target: {tabId: tabId},
                files: ['buildDomTree1.js'], // 注入外部文件
                world: "MAIN"
            });
            // await page.waitForFunction(() => typeof window.yourFunc === 'function');
            const domState = await page.evaluate(() => {

                const buildDomTree = window.buildDomTree({
                    doHighlightElements: true,
                    focusHighlightIndex: -1,
                    viewportExpansion: 0,
                    debugMode: false,
                })

                function findDomByXpath(xpath) {
                    try {
                        return document.evaluate('/' + xpath.replace(/^\/+/, ''), document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
                    } catch (e) {
                        return null;
                    }
                }
                function cleanHtmlTags(htmlStr, attrs = ['name', 'href']) {
                    const doc = new DOMParser().parseFromString(htmlStr.replace(/[\r\n]+/g, ''), 'text/html');
                    // 遍历所有元素
                    const walker = doc.createTreeWalker(doc.body, NodeFilter.SHOW_ELEMENT);
                    while (walker.nextNode()) {
                        const node = walker.currentNode;
                        // 移除指定属性
                        attrs.forEach(attr => node.removeAttribute(attr));
                        // 只保留当前节点和其文本内容
                        node.innerHTML = node.textContent;
                    }

                    return doc.body.innerHTML;
                }
                const domList = Object.values(buildDomTree.map)
                    .filter(n => typeof n === "object" && n.highlightIndex !== undefined)
                    .map(n => findDomByXpath(n.xpath))
                    .filter(x => !!x)
                    .map((item, index) => `[${index}] ${item.outerHTML}`)
                    .map((item)=>{
                        const attrs = ['name', 'href','src','class','id','target',]
                        return cleanHtmlTags(item, attrs);
                    })
                    .join("\n")
                return {buildDomTree:buildDomTree,domList:domList,body:document.body.innerText}
            });
            return {domState:domState, page:page,tabitem:tabitem};
        } finally {
            // if (browser) await browser.disconnect();

        }
    }
    async function browserautomate(domState,page,type,targetIndex,text) {
// console.log(domState,page,type,targetIndex,browser)
        // 查找目标元素
        const targetElement = findElementByIndex(domState.map, targetIndex);
        if (!targetElement) throw new Error('Element not found');

        // 生成增强选择器
        const selector = generateSelector(targetElement);
        // console.log(domState, targetElement, selector)
        // const [page] = await browser.pages()
        try {
            await page.waitForSelector(selector, {visible: true, timeout: 10000});
            switch (type) {
                case 'click':
                    await page.click(selector, {delay: 100});
                    break;
                case 'key_enter':
                    await page.click(selector, {delay: 100});
                    break;
                case 'navigate':
                    await page.goto('https://example.com', {waitUntil: 'networkidle2'});
                    break;
                case 'getcontent':
                    console.log(await page.content());
                    break;
                case 'get_text':
                    console.log(await page.content());
                    break;
                case 'input_text':
                    await page.type(selector, text, {delay: 600});
                    break;
                // 添加更多命令支持，如：
                case 'form':
                    const formData = [{
                        text: 'your-username',
                        index: 'your-password',
                        email: 'your-email@example.com',
                        question2: 'option2', // 假设有一个单选题，选项为 option2
                        question3: '我非常满意这次体验', // 假设有一个文本框，答案为 “我非常满意这次体验”
                    }];
                    await page.type(selector, 'your-username', {delay: 600});
                    await page.type(selector, 'your-password', {delay: 600});
                    await page.click(selector); // 或者其他登录按钮
                    break;
                default:
                    console.log('未知指令。');
            }

            // 验证点击结果
            // await page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 15000 });
            // console.log(`Successfully clicked element ${targetIndex} on attempt ${attempt}`);
            await page.evaluate(() => {

                // document.getElementById('playwright-highlight-container')?.remove();

            });
            return true;
        } catch (clickError) {
            // console.warn(`Attempt ${attempt} failed: ${clickError.message}`);
            // if (attempt === retries) throw clickError;
            // await page.waitForTimeout(delay);

            // await page.evaluate(() => {
            //     document.getElementById('playwright-highlight-container')?.remove();
            // });
            // if (browser) await browser.disconnect();
        }finally {
            console.log(123456789)
            await page.evaluate(() => {
                document.getElementById('playwright-highlight-container')?.remove();
            });
            if (browser) await browser.disconnect();
        }
    }
    function allTabs(){
        return new Promise( (resolve, reject) => {
            chrome.tabs.query({}, (tabs) => {
                resolve(tabs);
            });
        })
    }

    function getActiveTabId() {
        return new Promise((resolve) => {
            setTimeout(() => {
                chrome.tabs.query({active: true}, (tabs) => {
                    resolve(tabs[0]);
                });
            },500)
        });
    }
// 辅助函数：通过索引查找元素
    function findElementByIndex(domMap, targetIndex) {
        return Object.values(domMap).find(
            node => node.highlightIndex === targetIndex
        );
    }

// 辅助函数：生成 CSS 选择器
    function generateSelector(element) {
        const attrs = Object.entries(element.attributes)
            .map(([k, v]) => `[${k}="${v.replace(/"/g, '\\"')}"]`)
            .join('');
        return `${element.tagName}${attrs}`;
    }

    function gfe() {
        chrome.tabs.query({active: true, currentWindow: true}, async (tabs) => {
            const tabId = tabs[0].id;
            await smartClick(tabId, 1, {
                type: 'click',
                text: 3000
            });
        })
    }

// 背景脚本
    function switchTabOrOpenNew(url) {
        return new Promise(resolve => {
            // 查询是否存在指定的标签页
            // chrome.tabs.query({windowId: chrome.windows.WINDOW_ID_CURRENT}, async (tabs) => {
                chrome.tabs.query({}, async (tabs) => {
                let tabId = tabs.find(tabs => tabs.url.replace(/\/$/, '') === url);
                if (tabId) {
                    // 如果存在，则激活该标签页
                    await chrome.tabs.update(tabId.id, {active: true});
                    resolve()
                } else {
                    // 如果不存在，则新打开一个标签页
                    await chrome.tabs.create({url: url});
                    resolve()
                }
            });
        })
    }

    function getexecuteapi(params) {
       return  getexecute(params).then((res)=>{
           return res;
        })
    }

    async function agent(payload) {
        let isDOMType="element"
        for(let item of payload.steps){
            if(item.stepIndex === 0){
                let params = {
                    conversationId: payload.conversationId,
                    messageId: payload.messageId,
                    messageContent: payload.messageContent,
                    planId: payload.id,
                    currentPlanStepId: item.id,
                    envData: {
                        osName: "Google Chrome",
                        osVersion: "137.0.7151.69（正式版本）",
                        osArch: "arm64",
                        url: "",
                        title:"",
                        tabs: [],
                        interactiveElements: ""
                    },
                    resultSummary: null,
                    needSummary: true,
                    success: false
                }
                let add = await getexecuteapi(params)
                let toolExecution=JSON.parse(add.data.plan.steps[0].toolExecution)[0].toolParameters
                if (JSON.parse(toolExecution).action == 'navigate') {
                    await switchTabOrOpenNew(JSON.parse(toolExecution).url)
                }
            }else {
                let browdata=  await  browsercomm()
                console.log(browdata)
                const tabs = await allTabs()
                let params = {
                    conversationId: payload.conversationId,
                    messageId: payload.messageId,
                    messageContent: payload.messageContent,
                    planId: payload.id,
                    currentPlanStepId: item.id,
                    envData: {
                        osName: "Google Chrome",
                        osVersion: "137.0.7151.69（正式版本）",
                        osArch: "arm64",
                        url: browdata.tabitem.url,
                        title: browdata.tabitem.title,
                        tabs: tabs,
                        interactiveElements:isDOMType=='text'?browdata.domState.body: browdata.domState.domList
                        // interactiveElements: "[0] <a>新闻</a>\n        [1] <a>hao123</a>\n        [2] <a>地图</a>\n        [3] <a>贴吧</a>\n        [4] <a>视频</a>\n        [5] <a>图片</a>\n        [6] <a>网盘</a>\n        [7] <a>文库</a>\n        [8] <a id=\"csaitab\"></a>\n        [9] <a name=\"tj_briicon\">更多</a>\n        [10] <a></a>\n        [11] <a name=\"tj_login\" id=\"s-top-loginbtn\">登录</a>\n        [12] <input placeholder=\"梁靖崑称优势是有王楚钦\" name=\"wd\" id=\"kw\" value=\"\"></input>\n        [13] <input type=\"submit\" id=\"su\" value=\"百度一下\"></input>\n        [14] <a>AI搜索已支持DeepSeek R1最新版立即体验</a>\n        [16] <a id=\"hotsearch-refresh-btn\">换一换</a>\n        [17] <a>0让“干坡坡”变“金窝窝”</a>\n        [18] <a>5福建一楼房发生爆炸 有人员"
                    },
                    resultSummary: null,
                    needSummary: true,
                    success: false
                }
                let getexecutedaat = await getexecuteapi(params)
                let toolExecution=JSON.parse(getexecutedaat.data.plan.steps[item.stepIndex].toolExecution)
                for(let item of toolExecution){
                    if(JSON.parse(item.toolParameters).action=='get_text'){
                        isDOMType='text'
                    }else {
                        isDOMType='element'
                    }
                    await browserautomate(browdata.domState.buildDomTree, browdata.page, JSON.parse(item.toolParameters).action, JSON.parse(item.toolParameters).index||0,JSON.parse(item.toolParameters).text||'')
                }
            }
            chrome.runtime.sendMessage({
                type: "FROM_STEP",
                payload: item.stepIndex});
        }
    }

    chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
        console.log(request, sender, sendResponse);
        if (request.type == 'FROM_PLAN') {
            agent(request.payload)
        }
    })
}