upgrade dom package

.eslintignore

@@ -1,3 +1,4 @@
 dist
 node_modules
 tailwind.config.ts
+buildDomTree.js

.gitignore

@@ -23,4 +23,5 @@ chrome-extension/public/manifest.json
 **/tailwind-output.css
 
 .nanobrowser
-.vscode
+.vscode
+*.py

chrome-extension/src/background/browser/page.ts

@@ -169,6 +169,7 @@ export default class Page {
     }
     return _getClickableElements(
       this._tabId,
+      this.url(),
       this._config.highlightElements,
       focusElement,
       this._config.viewportExpansion,

chrome-extension/src/background/dom/clickable/service.ts

@@ -0,0 +1,130 @@
+import { DOMElementNode } from '../views';
+
+/**
+ * Get all clickable elements hashes in the DOM tree
+ */
+export async function getClickableElementsHashes(domElement: DOMElementNode): Promise<Set<string>> {
+  const clickableElements = getClickableElements(domElement);
+  const hashPromises = clickableElements.map(element => hashDomElement(element));
+  const hashes = await Promise.all(hashPromises);
+  return new Set(hashes);
+}
+
+/**
+ * Get all clickable elements in the DOM tree
+ */
+export function getClickableElements(domElement: DOMElementNode): DOMElementNode[] {
+  const clickableElements: DOMElementNode[] = [];
+
+  for (const child of domElement.children) {
+    if (child instanceof DOMElementNode) {
+      if (child.highlightIndex !== null) {
+        clickableElements.push(child);
+      }
+
+      clickableElements.push(...getClickableElements(child));
+    }
+  }
+
+  return clickableElements;
+}
+
+/**
+ * Hash a DOM element for identification
+ */
+export async function hashDomElement(domElement: DOMElementNode): Promise<string> {
+  const parentBranchPath = _getParentBranchPath(domElement);
+
+  // Run all hash operations in parallel
+  const [branchPathHash, attributesHash, xpathHash] = await Promise.all([
+    _parentBranchPathHash(parentBranchPath),
+    _attributesHash(domElement.attributes),
+    _xpathHash(domElement.xpath),
+    // _textHash(domElement) // Uncomment if needed
+  ]);
+
+  return _hashString(`${branchPathHash}-${attributesHash}-${xpathHash}`);
+}
+
+/**
+ * Get the branch path from parent elements
+ */
+function _getParentBranchPath(domElement: DOMElementNode): string[] {
+  const parents: DOMElementNode[] = [];
+  let currentElement: DOMElementNode | null = domElement;
+
+  while (currentElement?.parent !== null) {
+    parents.push(currentElement);
+    currentElement = currentElement.parent;
+  }
+
+  parents.reverse();
+
+  return parents.map(parent => parent.tagName || '');
+}
+
+/**
+ * Create a hash from the parent branch path
+ */
+async function _parentBranchPathHash(parentBranchPath: string[]): Promise<string> {
+  const parentBranchPathString = parentBranchPath.join('/');
+  return createSHA256Hash(parentBranchPathString);
+}
+
+/**
+ * Create a hash from the element attributes
+ */
+async function _attributesHash(attributes: Record<string, string>): Promise<string> {
+  const attributesString = Object.entries(attributes)
+    .map(([key, value]) => `${key}=${value}`)
+    .join('');
+  return createSHA256Hash(attributesString);
+}
+
+/**
+ * Create a hash from the element xpath
+ */
+async function _xpathHash(xpath: string | null): Promise<string> {
+  return createSHA256Hash(xpath || '');
+}
+
+/**
+ * Create a hash from the element text
+ * Currently unused but kept for potential future use
+ */
+/* 
+async function _textHash(domElement: DOMElementNode): Promise<string> {
+  const textString = domElement.getAllTextTillNextClickableElement();
+  return createSHA256Hash(textString);
+}
+*/
+
+/**
+ * Create a SHA-256 hash from a string using Web Crypto API
+ */
+async function createSHA256Hash(input: string): Promise<string> {
+  const encoder = new TextEncoder();
+  const data = encoder.encode(input);
+  const hashBuffer = await crypto.subtle.digest('SHA-256', data);
+  const hashArray = Array.from(new Uint8Array(hashBuffer));
+  return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
+}
+
+/**
+ * Create a hash from a string - synchronous version for combining hashes
+ * Used only for the final string combination
+ */
+function _hashString(string: string): string {
+  // Simple hash for combining existing hashes
+  // Real hashing is done in createSHA256Hash
+  return string;
+}
+
+/**
+ * ClickableElementProcessor namespace for backward compatibility
+ */
+export const ClickableElementProcessor = {
+  getClickableElementsHashes,
+  getClickableElements,
+  hashDomElement,
+};

chrome-extension/src/background/dom/history/service.ts

@@ -1,134 +1,175 @@
 import { DOMElementNode } from '../views';
 import { DOMHistoryElement, HashedDomElement } from './view';
 
-export namespace HistoryTreeProcessor {
-  /**
-   * Operations on the DOM elements
-   * @dev be careful - text nodes can change even if elements stay the same
-   */
-
-  export function convertDomElementToHistoryElement(domElement: DOMElementNode): DOMHistoryElement {
-    const parentBranchPath = getParentBranchPath(domElement);
-    const cssSelector = domElement.getAdvancedCssSelector();
-    return new DOMHistoryElement(
-      domElement.tagName ?? '', // Provide empty string as fallback
-      domElement.xpath ?? '', // Provide empty string as fallback
-      domElement.highlightIndex ?? null,
-      parentBranchPath,
-      domElement.attributes,
-      domElement.shadowRoot,
-      cssSelector,
-      domElement.pageCoordinates ?? null,
-      domElement.viewportCoordinates ?? null,
-      domElement.viewportInfo ?? null,
-    );
-  }
+/**
+ * Convert a DOM element to a history element
+ */
+export function convertDomElementToHistoryElement(domElement: DOMElementNode): DOMHistoryElement {
+  const parentBranchPath = _getParentBranchPath(domElement);
+  const cssSelector = domElement.getAdvancedCssSelector();
+  return new DOMHistoryElement(
+    domElement.tagName ?? '', // Provide empty string as fallback
+    domElement.xpath ?? '', // Provide empty string as fallback
+    domElement.highlightIndex ?? null,
+    parentBranchPath,
+    domElement.attributes,
+    domElement.shadowRoot,
+    cssSelector,
+    domElement.pageCoordinates ?? null,
+    domElement.viewportCoordinates ?? null,
+    domElement.viewportInfo ?? null,
+  );
+}
 
-  export async function findHistoryElementInTree(
-    domHistoryElement: DOMHistoryElement,
-    tree: DOMElementNode,
-  ): Promise<DOMElementNode | null> {
-    const hashedDomHistoryElement = await hashDomHistoryElement(domHistoryElement);
-
-    const processNode = async (node: DOMElementNode): Promise<DOMElementNode | null> => {
-      if (node.highlightIndex !== undefined) {
-        const hashedNode = await hashDomElement(node);
-        if (
-          hashedNode.branchPathHash === hashedDomHistoryElement.branchPathHash &&
-          hashedNode.attributesHash === hashedDomHistoryElement.attributesHash &&
-          hashedNode.xpathHash === hashedDomHistoryElement.xpathHash
-        ) {
-          return node;
-        }
+/**
+ * Find a history element in the DOM tree
+ */
+export async function findHistoryElementInTree(
+  domHistoryElement: DOMHistoryElement,
+  tree: DOMElementNode,
+): Promise<DOMElementNode | null> {
+  const hashedDomHistoryElement = await hashDomHistoryElement(domHistoryElement);
+
+  const processNode = async (node: DOMElementNode): Promise<DOMElementNode | null> => {
+    if (node.highlightIndex != null) {
+      const hashedNode = await hashDomElement(node);
+      if (
+        hashedNode.branchPathHash === hashedDomHistoryElement.branchPathHash &&
+        hashedNode.attributesHash === hashedDomHistoryElement.attributesHash &&
+        hashedNode.xpathHash === hashedDomHistoryElement.xpathHash
+      ) {
+        return node;
       }
-      for (const child of node.children) {
-        if (child instanceof DOMElementNode) {
-          const result = await processNode(child);
-          if (result !== null) {
-            return result;
-          }
+    }
+    for (const child of node.children) {
+      if (child instanceof DOMElementNode) {
+        const result = await processNode(child);
+        if (result !== null) {
+          return result;
         }
       }
-      return null;
-    };
+    }
+    return null;
+  };
 
-    return processNode(tree);
-  }
+  return processNode(tree);
+}
 
-  export async function compareHistoryElementAndDomElement(
-    domHistoryElement: DOMHistoryElement,
-    domElement: DOMElementNode,
-  ): Promise<boolean> {
-    const [hashedDomHistoryElement, hashedDomElement] = await Promise.all([
-      hashDomHistoryElement(domHistoryElement),
-      hashDomElement(domElement),
-    ]);
-
-    return (
-      hashedDomHistoryElement.branchPathHash === hashedDomElement.branchPathHash &&
-      hashedDomHistoryElement.attributesHash === hashedDomElement.attributesHash &&
-      hashedDomHistoryElement.xpathHash === hashedDomElement.xpathHash
-    );
-  }
+/**
+ * Compare a history element and a DOM element
+ */
+export async function compareHistoryElementAndDomElement(
+  domHistoryElement: DOMHistoryElement,
+  domElement: DOMElementNode,
+): Promise<boolean> {
+  const [hashedDomHistoryElement, hashedDomElement] = await Promise.all([
+    hashDomHistoryElement(domHistoryElement),
+    hashDomElement(domElement),
+  ]);
+
+  return (
+    hashedDomHistoryElement.branchPathHash === hashedDomElement.branchPathHash &&
+    hashedDomHistoryElement.attributesHash === hashedDomElement.attributesHash &&
+    hashedDomHistoryElement.xpathHash === hashedDomElement.xpathHash
+  );
+}
 
-  async function hashDomHistoryElement(domHistoryElement: DOMHistoryElement): Promise<HashedDomElement> {
-    const [branchPathHash, attributesHash, xpathHash] = await Promise.all([
-      parentBranchPathHash(domHistoryElement.entireParentBranchPath),
-      hashAttributes(domHistoryElement.attributes),
-      hashXPath(domHistoryElement.xpath ?? ''),
-    ]);
-    return new HashedDomElement(branchPathHash, attributesHash, xpathHash);
-  }
+/**
+ * Hash a DOM history element
+ */
+async function hashDomHistoryElement(domHistoryElement: DOMHistoryElement): Promise<HashedDomElement> {
+  const [branchPathHash, attributesHash, xpathHash] = await Promise.all([
+    _parentBranchPathHash(domHistoryElement.entireParentBranchPath),
+    _attributesHash(domHistoryElement.attributes),
+    _xpathHash(domHistoryElement.xpath ?? ''),
+  ]);
+  return new HashedDomElement(branchPathHash, attributesHash, xpathHash);
+}
 
-  export async function hashDomElement(domElement: DOMElementNode): Promise<HashedDomElement> {
-    const parentBranchPath = getParentBranchPath(domElement);
-    const [branchPathHash, attributesHash, xpathHash] = await Promise.all([
-      parentBranchPathHash(parentBranchPath),
-      hashAttributes(domElement.attributes),
-      hashXPath(domElement.xpath ?? ''),
-    ]);
-    return new HashedDomElement(branchPathHash, attributesHash, xpathHash);
-  }
+/**
+ * Hash a DOM element
+ */
+export async function hashDomElement(domElement: DOMElementNode): Promise<HashedDomElement> {
+  const parentBranchPath = _getParentBranchPath(domElement);
+  const [branchPathHash, attributesHash, xpathHash] = await Promise.all([
+    _parentBranchPathHash(parentBranchPath),
+    _attributesHash(domElement.attributes),
+    _xpathHash(domElement.xpath ?? ''),
+  ]);
+  return new HashedDomElement(branchPathHash, attributesHash, xpathHash);
+}
 
-  export function getParentBranchPath(domElement: DOMElementNode): string[] {
-    const parents: DOMElementNode[] = [];
-    let currentElement: DOMElementNode = domElement;
+/**
+ * Get the branch path from parent elements
+ */
+export function _getParentBranchPath(domElement: DOMElementNode): string[] {
+  const parents: DOMElementNode[] = [];
+  let currentElement: DOMElementNode = domElement;
 
-    while (currentElement.parent !== null && currentElement.parent !== undefined) {
-      parents.push(currentElement);
-      currentElement = currentElement.parent;
-    }
-
-    parents.reverse();
-    return parents.map(parent => parent.tagName ?? '');
+  while (currentElement.parent != null) {
+    parents.push(currentElement);
+    currentElement = currentElement.parent;
   }
 
-  export async function parentBranchPathHash(parentBranchPath: string[]): Promise<string> {
-    if (parentBranchPath.length === 0) return '';
-    return createSHA256Hash(parentBranchPath.join('/'));
-  }
+  parents.reverse();
+  return parents.map(parent => parent.tagName ?? '');
+}
 
-  export async function hashAttributes(attributes: Record<string, string>): Promise<string> {
-    const attributesString = Object.entries(attributes)
-      .map(([key, value]) => `${key}=${value}`)
-      .join('');
-    return createSHA256Hash(attributesString);
-  }
+/**
+ * Create a hash from the parent branch path
+ */
+async function _parentBranchPathHash(parentBranchPath: string[]): Promise<string> {
+  if (parentBranchPath.length === 0) return '';
+  return _createSHA256Hash(parentBranchPath.join('/'));
+}
 
-  export async function hashXPath(xpath: string): Promise<string> {
-    return createSHA256Hash(xpath);
-  }
+/**
+ * Create a hash from the element attributes
+ */
+async function _attributesHash(attributes: Record<string, string>): Promise<string> {
+  const attributesString = Object.entries(attributes)
+    .map(([key, value]) => `${key}=${value}`)
+    .join('');
+  return _createSHA256Hash(attributesString);
+}
 
-  // async function hashText(domElement: DOMElementNode): Promise<string> {
-  //     const textString = domElement.getAllTextTillNextClickableElement();
-  //     return createSHA256Hash(textString);
-  // }
-
-  export async function createSHA256Hash(input: string): Promise<string> {
-    const encoder = new TextEncoder();
-    const data = encoder.encode(input);
-    const hashBuffer = await crypto.subtle.digest('SHA-256', data);
-    const hashArray = Array.from(new Uint8Array(hashBuffer));
-    return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
-  }
+/**
+ * Create a hash from the element xpath
+ */
+async function _xpathHash(xpath: string): Promise<string> {
+  return _createSHA256Hash(xpath);
+}
+
+/**
+ * Create a hash from the element text
+ */
+async function _textHash(domElement: DOMElementNode): Promise<string> {
+  const textString = domElement.getAllTextTillNextClickableElement();
+  return _createSHA256Hash(textString);
 }
+
+/**
+ * Create a SHA-256 hash from a string using Web Crypto API
+ */
+async function _createSHA256Hash(input: string): Promise<string> {
+  const encoder = new TextEncoder();
+  const data = encoder.encode(input);
+  const hashBuffer = await crypto.subtle.digest('SHA-256', data);
+  const hashArray = Array.from(new Uint8Array(hashBuffer));
+  return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
+}
+
+/**
+ * HistoryTreeProcessor namespace to keep same pattern as in python
+ */
+export const HistoryTreeProcessor = {
+  convertDomElementToHistoryElement,
+  findHistoryElementInTree,
+  compareHistoryElementAndDomElement,
+  hashDomElement,
+  _getParentBranchPath,
+  _parentBranchPathHash,
+  _attributesHash,
+  _xpathHash,
+  _textHash,
+};

chrome-extension/src/background/dom/raw_types.ts

@@ -12,10 +12,11 @@ export type RawDomElementNode = {
   tagName: string | null;
   xpath: string | null;
   attributes: Record<string, string>;
-  children: (RawDomTreeNode | null)[];
+  children: string[]; // Array of node IDs
   isVisible?: boolean;
   isInteractive?: boolean;
   isTopElement?: boolean;
+  isInViewport?: boolean;
   highlightIndex?: number;
   viewportCoordinates?: CoordinateSet;
   pageCoordinates?: CoordinateSet;
@@ -29,4 +30,32 @@ export interface BuildDomTreeArgs {
   doHighlightElements: boolean;
   focusHighlightIndex: number;
   viewportExpansion: number;
+  debugMode?: boolean;
+}
+
+export interface PerfMetrics {
+  nodeMetrics: {
+    totalNodes: number;
+    processedNodes: number;
+    skippedNodes: number;
+  };
+  cacheMetrics: {
+    boundingRectCacheHits: number;
+    boundingRectCacheMisses: number;
+    computedStyleCacheHits: number;
+    computedStyleCacheMisses: number;
+    getBoundingClientRectTime: number;
+    getComputedStyleTime: number;
+    boundingRectHitRate: number;
+    computedStyleHitRate: number;
+    overallHitRate: number;
+  };
+  timings: Record<string, number>;
+  buildDomTreeBreakdown: Record<string, number | Record<string, number>>;
+}
+
+export interface BuildDomTreeResult {
+  rootId: string;
+  map: Record<string, RawDomTreeNode>;
+  perfMetrics?: PerfMetrics; // Only included when debugMode is true
 }

chrome-extension/src/background/dom/service.ts

@@ -1,6 +1,7 @@
 import { createLogger } from '@src/background/log';
-import type { BuildDomTreeArgs, RawDomTreeNode } from './raw_types';
+import type { BuildDomTreeArgs, RawDomTreeNode, BuildDomTreeResult } from './raw_types';
 import { type DOMState, type DOMBaseNode, DOMElementNode, DOMTextNode } from './views';
+import type { ViewportInfo } from './history/view';
 
 const logger = createLogger('DOMService');
 
@@ -92,10 +93,10 @@ export async function getReadabilityContent(tabId: number): Promise<ReadabilityR
   return result as ReadabilityResult;
 }
 
-/**
 /**
  * Get the clickable elements for the current page.
  * @param tabId - The ID of the tab to get the clickable elements for.
+ * @param url - The URL of the page.
  * @param highlightElements - Whether to highlight the clickable elements.
  * @param focusElement - The element to focus on.
  * @param viewportExpansion - The viewport expansion to use.
@@ -103,43 +104,45 @@ export async function getReadabilityContent(tabId: number): Promise<ReadabilityR
  */
 export async function getClickableElements(
   tabId: number,
+  url: string,
   highlightElements = true,
   focusElement = -1,
   viewportExpansion = 0,
-): Promise<DOMState | null> {
-  try {
-    const elementTree = await _buildDomTree(tabId, highlightElements, focusElement, viewportExpansion);
-    const selectorMap = createSelectorMap(elementTree);
-    return { elementTree, selectorMap };
-  } catch (error) {
-    logger.error('Failed to build DOM tree:', error);
-    return null;
-  }
-}
-
-function createSelectorMap(elementTree: DOMElementNode): Map<number, DOMElementNode> {
-  const selectorMap = new Map<number, DOMElementNode>();
-
-  function processNode(node: DOMBaseNode): void {
-    if (node instanceof DOMElementNode) {
-      if (node.highlightIndex != null) {
-        // console.log('createSelectorMap node.highlightIndex:', node.highlightIndex);
-        selectorMap.set(node.highlightIndex, node);
-      }
-      node.children.forEach(processNode);
-    }
-  }
-
-  processNode(elementTree);
-  return selectorMap;
+): Promise<DOMState> {
+  const [elementTree, selectorMap] = await _buildDomTree(
+    tabId,
+    url,
+    highlightElements,
+    focusElement,
+    viewportExpansion,
+  );
+  return { elementTree, selectorMap };
 }
 
 async function _buildDomTree(
   tabId: number,
+  url: string,
   highlightElements = true,
   focusElement = -1,
   viewportExpansion = 0,
-): Promise<DOMElementNode> {
+  debugMode = false,
+): Promise<[DOMElementNode, Map<number, DOMElementNode>]> {
+  // If URL is provided and it's about:blank, return a minimal DOM tree
+  if (url === 'about:blank') {
+    const elementTree = new DOMElementNode({
+      tagName: 'body',
+      xpath: '',
+      attributes: {},
+      children: [],
+      isVisible: false,
+      isInteractive: false,
+      isTopElement: false,
+      isInViewport: false,
+      parent: null,
+    });
+    return [elementTree, new Map<number, DOMElementNode>()];
+  }
+
   const results = await chrome.scripting.executeScript({
     target: { tabId },
     func: args => {
@@ -151,64 +154,130 @@ async function _buildDomTree(
         doHighlightElements: highlightElements,
         focusHighlightIndex: focusElement,
         viewportExpansion,
+        debugMode,
       },
     ],
   });
 
-  const rawDomTree = results[0].result;
-  if (rawDomTree !== null) {
-    const elementTree = parseNode(rawDomTree as RawDomTreeNode);
-    if (elementTree !== null && elementTree instanceof DOMElementNode) {
-      return elementTree;
+  // First cast to unknown, then to BuildDomTreeResult
+  const evalPage = results[0]?.result as unknown as BuildDomTreeResult;
+  if (!evalPage || !evalPage.map || !evalPage.rootId) {
+    throw new Error('Failed to build DOM tree: No result returned or invalid structure');
+  }
+
+  // Log performance metrics in debug mode
+  if (debugMode && evalPage.perfMetrics) {
+    logger.debug('DOM Tree Building Performance Metrics:', evalPage.perfMetrics);
+  }
+
+  return _constructDomTree(evalPage);
+}
+
+/**
+ * Constructs a DOM tree from the evaluated page data.
+ * @param evalPage - The result of building the DOM tree.
+ * @returns A tuple containing the DOM element tree and selector map.
+ */
+function _constructDomTree(evalPage: BuildDomTreeResult): [DOMElementNode, Map<number, DOMElementNode>] {
+  const jsNodeMap = evalPage.map;
+  const jsRootId = evalPage.rootId;
+
+  const selectorMap = new Map<number, DOMElementNode>();
+  const nodeMap: Record<string, DOMBaseNode> = {};
+
+  // First pass: create all nodes
+  for (const [id, nodeData] of Object.entries(jsNodeMap)) {
+    const [node] = _parse_node(nodeData);
+    if (node === null) {
+      continue;
+    }
+
+    nodeMap[id] = node;
+
+    // Add to selector map if it has a highlight index
+    if (node instanceof DOMElementNode && node.highlightIndex !== undefined && node.highlightIndex !== null) {
+      selectorMap.set(node.highlightIndex, node);
     }
   }
-  throw new Error('Failed to build DOM tree: Invalid or empty tree structure');
+
+  // Second pass: build the tree structure
+  for (const [id, node] of Object.entries(nodeMap)) {
+    if (node instanceof DOMElementNode) {
+      const nodeData = jsNodeMap[id];
+      const childrenIds = 'children' in nodeData ? nodeData.children : [];
+
+      for (const childId of childrenIds) {
+        if (!(childId in nodeMap)) {
+          continue;
+        }
+
+        const childNode = nodeMap[childId];
+
+        childNode.parent = node;
+        node.children.push(childNode);
+      }
+    }
+  }
+
+  const htmlToDict = nodeMap[jsRootId];
+
+  if (htmlToDict === undefined || !(htmlToDict instanceof DOMElementNode)) {
+    throw new Error('Failed to parse HTML to dictionary');
+  }
+
+  return [htmlToDict, selectorMap];
 }
 
-function parseNode(nodeData: RawDomTreeNode, parent: DOMElementNode | null = null): DOMBaseNode | null {
-  if (!nodeData) return null;
+/**
+ * Parse a raw DOM node and return the node object and its children IDs.
+ * @param nodeData - The raw DOM node data to parse.
+ * @returns A tuple containing the parsed node and an array of child IDs.
+ */
+export function _parse_node(nodeData: RawDomTreeNode): [DOMBaseNode | null, string[]] {
+  if (!nodeData) {
+    return [null, []];
+  }
 
-  if ('type' in nodeData) {
-    // && nodeData.type === 'TEXT_NODE'
-    return new DOMTextNode(nodeData.text, nodeData.isVisible, parent);
+  // Process text nodes immediately
+  if ('type' in nodeData && nodeData.type === 'TEXT_NODE') {
+    const textNode = new DOMTextNode(nodeData.text, nodeData.isVisible, null);
+    return [textNode, []];
   }
 
-  const tagName = nodeData.tagName;
+  // At this point, nodeData is RawDomElementNode (not a text node)
+  // TypeScript needs help to narrow the type
+  const elementData = nodeData as Exclude<RawDomTreeNode, { type: string }>;
 
-  // Parse coordinates if they exist
-  const viewportCoordinates = nodeData.viewportCoordinates;
-  const pageCoordinates = nodeData.pageCoordinates;
-  const viewportInfo = nodeData.viewportInfo;
+  // Process viewport info if it exists
+  let viewportInfo: ViewportInfo | undefined = undefined;
+  if ('viewport' in nodeData && typeof nodeData.viewport === 'object' && nodeData.viewport) {
+    const viewportObj = nodeData.viewport as { width: number; height: number };
+    viewportInfo = {
+      width: viewportObj.width,
+      height: viewportObj.height,
+      scrollX: 0,
+      scrollY: 0,
+    };
+  }
 
-  // Element node (possible other kinds of nodes, but we don't care about them for now)
   const elementNode = new DOMElementNode({
-    tagName: tagName,
-    xpath: nodeData.xpath,
-    attributes: nodeData.attributes ?? {},
+    tagName: elementData.tagName,
+    xpath: elementData.xpath,
+    attributes: elementData.attributes ?? {},
     children: [],
-    isVisible: nodeData.isVisible ?? false,
-    isInteractive: nodeData.isInteractive ?? false,
-    isTopElement: nodeData.isTopElement ?? false,
-    highlightIndex: nodeData.highlightIndex,
-    viewportCoordinates: viewportCoordinates ?? undefined,
-    pageCoordinates: pageCoordinates ?? undefined,
-    viewportInfo: viewportInfo ?? undefined,
-    shadowRoot: nodeData.shadowRoot ?? false,
-    parent,
+    isVisible: elementData.isVisible ?? false,
+    isInteractive: elementData.isInteractive ?? false,
+    isTopElement: elementData.isTopElement ?? false,
+    isInViewport: elementData.isInViewport ?? false,
+    highlightIndex: elementData.highlightIndex,
+    shadowRoot: elementData.shadowRoot ?? false,
+    parent: null,
+    viewportInfo: viewportInfo,
   });
 
-  const children: DOMBaseNode[] = [];
-  for (const child of nodeData.children || []) {
-    if (child !== null) {
-      const childNode = parseNode(child, elementNode);
-      if (childNode !== null) {
-        children.push(childNode);
-      }
-    }
-  }
+  const childrenIds = elementData.children || [];
 
-  elementNode.children = children;
-  return elementNode;
+  return [elementNode, childrenIds];
 }
 
 export async function removeHighlights(tabId: number): Promise<void> {

chrome-extension/src/background/dom/views.ts

@@ -1,15 +1,14 @@
-import type { ViewportInfo, CoordinateSet } from './history/view';
-import type { HashedDomElement } from './history/view';
+import type { CoordinateSet, HashedDomElement, ViewportInfo } from './history/view';
 import { HistoryTreeProcessor } from './history/service';
 
 export abstract class DOMBaseNode {
   isVisible: boolean;
-  parent?: DOMElementNode | null;
+  parent: DOMElementNode | null;
 
   constructor(isVisible: boolean, parent?: DOMElementNode | null) {
     this.isVisible = isVisible;
     // Use None as default and set parent later to avoid circular reference issues
-    this.parent = parent;
+    this.parent = parent ?? null;
   }
 }
 
@@ -32,6 +31,20 @@ export class DOMTextNode extends DOMBaseNode {
     }
     return false;
   }
+
+  isParentInViewport(): boolean {
+    if (this.parent === null) {
+      return false;
+    }
+    return this.parent.isInViewport;
+  }
+
+  isParentTopElement(): boolean {
+    if (this.parent === null) {
+      return false;
+    }
+    return this.parent.isTopElement;
+  }
 }
 
 export class DOMElementNode extends DOMBaseNode {
@@ -45,12 +58,20 @@ export class DOMElementNode extends DOMBaseNode {
   children: DOMBaseNode[];
   isInteractive: boolean;
   isTopElement: boolean;
+  isInViewport: boolean;
   shadowRoot: boolean;
-  highlightIndex?: number;
+  highlightIndex: number | null;
   viewportCoordinates?: CoordinateSet;
   pageCoordinates?: CoordinateSet;
   viewportInfo?: ViewportInfo;
 
+  /*
+	### State injected by the browser context.
+
+	The idea is that the clickable elements are sometimes persistent from the previous page -> tells the model which objects are new/_how_ the state has changed
+	*/
+  isNew: boolean | null;
+
   constructor(params: {
     tagName: string | null;
     xpath: string | null;
@@ -59,11 +80,13 @@ export class DOMElementNode extends DOMBaseNode {
     isVisible: boolean;
     isInteractive?: boolean;
     isTopElement?: boolean;
+    isInViewport?: boolean;
     shadowRoot?: boolean;
-    highlightIndex?: number;
+    highlightIndex?: number | null;
     viewportCoordinates?: CoordinateSet;
     pageCoordinates?: CoordinateSet;
     viewportInfo?: ViewportInfo;
+    isNew?: boolean | null;
     parent?: DOMElementNode | null;
   }) {
     super(params.isVisible, params.parent);
@@ -73,11 +96,13 @@ export class DOMElementNode extends DOMBaseNode {
     this.children = params.children;
     this.isInteractive = params.isInteractive ?? false;
     this.isTopElement = params.isTopElement ?? false;
+    this.isInViewport = params.isInViewport ?? false;
     this.shadowRoot = params.shadowRoot ?? false;
-    this.highlightIndex = params.highlightIndex;
+    this.highlightIndex = params.highlightIndex ?? null;
     this.viewportCoordinates = params.viewportCoordinates;
     this.pageCoordinates = params.pageCoordinates;
     this.viewportInfo = params.viewportInfo;
+    this.isNew = params.isNew ?? null;
   }
 
   // Cache for the hash value
@@ -100,12 +125,12 @@ export class DOMElementNode extends DOMBaseNode {
     // If a calculation is in progress, reuse that promise
     if (!this._hashPromise) {
       this._hashPromise = HistoryTreeProcessor.hashDomElement(this)
-        .then(result => {
+        .then((result: HashedDomElement) => {
           this._hashedValue = result;
           this._hashPromise = undefined; // Clean up
           return result;
         })
-        .catch(error => {
+        .catch((error: Error) => {
           // Clear the promise reference to allow retry on next call
           this._hashPromise = undefined;
 
@@ -147,7 +172,7 @@ export class DOMElementNode extends DOMBaseNode {
       }
 
       // Skip this branch if we hit a highlighted element (except for the current node)
-      if (node instanceof DOMElementNode && node !== this && node.highlightIndex !== undefined) {
+      if (node instanceof DOMElementNode && node !== this && node.highlightIndex != null) {
         return;
       }
 
@@ -168,29 +193,89 @@ export class DOMElementNode extends DOMBaseNode {
     const formattedText: string[] = [];
 
     const processNode = (node: DOMBaseNode, depth: number): void => {
+      let nextDepth = depth;
+      const depthStr = '\t'.repeat(depth);
+
       if (node instanceof DOMElementNode) {
         // Add element with highlight_index
-        if (node.highlightIndex !== undefined) {
-          let attributesStr = '';
+        if (node.highlightIndex !== null) {
+          nextDepth += 1;
+
+          const text = node.getAllTextTillNextClickableElement();
+          let attributesHtmlStr = '';
+
           if (includeAttributes.length) {
-            attributesStr = ` ${includeAttributes
-              .map(key => (node.attributes[key] ? `${key}="${node.attributes[key]}"` : ''))
-              .filter(Boolean)
-              .join(' ')}`;
+            // Create a new object to store attributes
+            const attributesToInclude: Record<string, string> = {};
+
+            for (const key of includeAttributes) {
+              // Include the attribute even if it's an empty string
+              if (key in node.attributes) {
+                attributesToInclude[key] = String(node.attributes[key]);
+              }
+            }
+
+            // Easy LLM optimizations
+            // if tag == role attribute, don't include it
+            if (node.tagName === attributesToInclude.role) {
+              // Use null instead of delete
+              attributesToInclude.role = null as unknown as string;
+            }
+
+            // if aria-label == text of the node, don't include it
+            if ('aria-label' in attributesToInclude && attributesToInclude['aria-label'].trim() === text.trim()) {
+              // Use null instead of delete
+              attributesToInclude['aria-label'] = null as unknown as string;
+            }
+
+            // if placeholder == text of the node, don't include it
+            if ('placeholder' in attributesToInclude && attributesToInclude.placeholder.trim() === text.trim()) {
+              // Use null instead of delete
+              attributesToInclude.placeholder = null as unknown as string;
+            }
+
+            if (Object.keys(attributesToInclude).length > 0) {
+              // Format as key1='value1' key2='value2'
+              attributesHtmlStr = Object.entries(attributesToInclude)
+                .filter(([, value]) => value !== null)
+                .map(([key, value]) => `${key}='${value}'`)
+                .join(' ');
+            }
           }
 
-          formattedText.push(
-            `[${node.highlightIndex}]<${node.tagName}${attributesStr}>${node.getAllTextTillNextClickableElement()}</${node.tagName}>`,
-          );
+          // Build the line
+          const highlightIndicator = node.isNew ? `*[${node.highlightIndex}]*` : `[${node.highlightIndex}]`;
+
+          let line = `${depthStr}${highlightIndicator}<${node.tagName}`;
+
+          if (attributesHtmlStr) {
+            line += ` ${attributesHtmlStr}`;
+          }
+
+          if (text) {
+            // Add space before >text only if there were NO attributes added before
+            if (!attributesHtmlStr) {
+              line += ' ';
+            }
+            line += `>${text}`;
+          }
+          // Add space before /> only if neither attributes NOR text were added
+          else if (!attributesHtmlStr) {
+            line += ' ';
+          }
+
+          line += ' />'; // 1 token
+          formattedText.push(line);
         }
+
         // Process children regardless
         for (const child of node.children) {
-          processNode(child, depth + 1);
+          processNode(child, nextDepth);
         }
       } else if (node instanceof DOMTextNode) {
-        // Add text node only if it doesn't have a highlighted parent
-        if (!node.hasParentWithHighlightIndex()) {
-          formattedText.push(`[]${node.text}`);
+        // Add text only if it doesn't have a highlighted parent
+        if (!node.hasParentWithHighlightIndex() && node.parent && node.parent.isVisible && node.parent.isTopElement) {
+          formattedText.push(`${depthStr}${node.text}`);
         }
       }
     };
@@ -200,11 +285,12 @@ export class DOMElementNode extends DOMBaseNode {
   }
 
   getFileUploadElement(checkSiblings = true): DOMElementNode | null {
-    // biome-ignore lint/complexity/useLiteralKeys: <explanation>
-    if (this.tagName === 'input' && this.attributes['type'] === 'file') {
+    // Check if current element is a file input
+    if (this.tagName === 'input' && this.attributes?.type === 'file') {
       return this;
     }
 
+    // Check children
     for (const child of this.children) {
       if (child instanceof DOMElementNode) {
         const result = child.getFileUploadElement(false);
@@ -212,6 +298,7 @@ export class DOMElementNode extends DOMBaseNode {
       }
     }
 
+    // Check siblings only for the initial call
     if (checkSiblings && this.parent) {
       for (const sibling of this.parent.children) {
         if (sibling !== this && sibling instanceof DOMElementNode) {
@@ -245,10 +332,23 @@ export class DOMElementNode extends DOMBaseNode {
         continue;
       }
 
+      // Handle custom elements with colons by escaping them
+      if (part.includes(':') && !part.includes('[')) {
+        const basePart = part.replace(/:/g, '\\:');
+        cssParts.push(basePart);
+        continue;
+      }
+
       // Handle index notation [n]
       if (part.includes('[')) {
         const bracketIndex = part.indexOf('[');
         let basePart = part.substring(0, bracketIndex);
+
+        // Handle custom elements with colons in the base part
+        if (basePart.includes(':')) {
+          basePart = basePart.replace(/:/g, '\\:');
+        }
+
         const indexPart = part.substring(bracketIndex);
 
         // Handle multiple indices
@@ -299,14 +399,13 @@ export class DOMElementNode extends DOMBaseNode {
       let cssSelector = this.convertSimpleXPathToCssSelector(this.xpath);
 
       // Handle class attributes
-      // biome-ignore lint/complexity/useLiteralKeys: <explanation>
-      if (this.attributes['class'] && includeDynamicAttributes) {
+      const classValue = this.attributes.class;
+      if (classValue && includeDynamicAttributes) {
         // Define a regex pattern for valid class names in CSS
         const validClassNamePattern = /^[a-zA-Z_][a-zA-Z0-9_-]*$/;
 
         // Iterate through the class attribute values
-        // biome-ignore lint/complexity/useLiteralKeys: <explanation>s
-        const classes = this.attributes['class'].split(/\s+/);
+        const classes = classValue.trim().split(/\s+/);
         for (const className of classes) {
           // Skip empty class names
           if (!className.trim()) {
@@ -328,7 +427,6 @@ export class DOMElementNode extends DOMBaseNode {
         // Standard HTML attributes
         'name',
         'type',
-        'value',
         'placeholder',
         // Accessibility attributes
         'aria-label',
@@ -394,7 +492,7 @@ export class DOMElementNode extends DOMBaseNode {
     } catch (error) {
       // Fallback to a more basic selector if something goes wrong
       const tagName = this.tagName || '*';
-      return `${tagName}[highlight-index='${this.highlightIndex}']`;
+      return `${tagName}[highlightIndex='${this.highlightIndex}']`;
     }
   }
 }
@@ -404,10 +502,8 @@ export interface DOMState {
   selectorMap: Map<number, DOMElementNode>;
 }
 
-// biome-ignore lint/suspicious/noExplicitAny: <explanation>
-export function domElementNodeToDict(elementTree: DOMBaseNode): any {
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  function nodeToDict(node: DOMBaseNode): any {
+export function domElementNodeToDict(elementTree: DOMBaseNode): unknown {
+  function nodeToDict(node: DOMBaseNode): unknown {
     if (node instanceof DOMTextNode) {
       return {
         type: 'text',
@@ -417,7 +513,7 @@ export function domElementNodeToDict(elementTree: DOMBaseNode): any {
     if (node instanceof DOMElementNode) {
       return {
         type: 'element',
-        tagName: node.tagName, // Note: using camelCase to match TypeScript conventions
+        tagName: node.tagName,
         attributes: node.attributes,
         highlightIndex: node.highlightIndex,
         children: node.children.map(child => nodeToDict(child)),