upgrade prompts and action schemas

chrome-extension/src/background/agent/actions/builder.ts

@@ -2,7 +2,6 @@ import { ActionResult, type AgentContext } from '@src/background/agent/types';
 import {
   clickElementActionSchema,
   doneActionSchema,
-  extractContentActionSchema,
   goBackActionSchema,
   goToUrlActionSchema,
   inputTextActionSchema,
@@ -17,11 +16,11 @@ import {
   cacheContentActionSchema,
   selectDropdownOptionActionSchema,
   getDropdownOptionsActionSchema,
+  closeTabActionSchema,
+  waitActionSchema,
 } from './schemas';
 import { z } from 'zod';
 import { createLogger } from '@src/background/log';
-import { PromptTemplate } from '@langchain/core/prompts';
-import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 import { ExecutionState, Actors } from '../event/types';
 
 const logger = createLogger('Action');
@@ -140,10 +139,10 @@ export class ActionBuilder {
     }, doneActionSchema);
     actions.push(done);
 
-    const searchGoogle = new Action(async (input: { query: string }) => {
+    const searchGoogle = new Action(async (input: z.infer<typeof searchGoogleActionSchema.schema>) => {
       const context = this.context;
-      const msg = `Searching for "${input.query}" in Google`;
-      context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, msg);
+      const intent = input.intent || `Searching for "${input.query}" in Google`;
+      context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
       await context.browserContext.navigateTo(`https://www.google.com/search?q=${input.query}`);
 
@@ -156,9 +155,9 @@ export class ActionBuilder {
     }, searchGoogleActionSchema);
     actions.push(searchGoogle);
 
-    const goToUrl = new Action(async (input: { url: string }) => {
-      const msg = `Navigating to ${input.url}`;
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, msg);
+    const goToUrl = new Action(async (input: z.infer<typeof goToUrlActionSchema.schema>) => {
+      const intent = input.intent || `Navigating to ${input.url}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
       await this.context.browserContext.navigateTo(input.url);
       const msg2 = `Navigated to ${input.url}`;
@@ -171,9 +170,9 @@ export class ActionBuilder {
     actions.push(goToUrl);
 
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const goBack = new Action(async (_input = {}) => {
-      const msg = 'Navigating back';
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, msg);
+    const goBack = new Action(async (input: z.infer<typeof goBackActionSchema.schema>) => {
+      const intent = input.intent || 'Navigating back';
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
       const page = await this.context.browserContext.getCurrentPage();
       await page.goBack();
@@ -186,11 +185,30 @@ export class ActionBuilder {
     }, goBackActionSchema);
     actions.push(goBack);
 
+    // # wait for x seconds
+    // @self.registry.action('Wait for x seconds default 3')
+    // async def wait(seconds: int = 3):
+    // 	msg = f'🕒  Waiting for {seconds} seconds'
+    // 	logger.info(msg)
+    // 	await asyncio.sleep(seconds)
+    // 	return ActionResult(extracted_content=msg, include_in_memory=True)
+
+    const wait = new Action(async (input: z.infer<typeof waitActionSchema.schema>) => {
+      const seconds = input.seconds || 3;
+      const intent = input.intent || `Waiting for ${seconds} seconds`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
+      await new Promise(resolve => setTimeout(resolve, seconds * 1000));
+      const msg = `${seconds} seconds elapsed`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_OK, msg);
+      return new ActionResult({ extractedContent: msg, includeInMemory: true });
+    }, waitActionSchema);
+    actions.push(wait);
+
     // Element Interaction Actions
     const clickElement = new Action(
       async (input: z.infer<typeof clickElementActionSchema.schema>) => {
-        const todo = input.desc || `Click element with index ${input.index}`;
-        this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, todo);
+        const intent = input.intent || `Click element with index ${input.index}`;
+        this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
         const page = await this.context.browserContext.getCurrentPage();
         const state = await page.getState();
@@ -245,8 +263,8 @@ export class ActionBuilder {
 
     const inputText = new Action(
       async (input: z.infer<typeof inputTextActionSchema.schema>) => {
-        const todo = input.desc || `Input text into index ${input.index}`;
-        this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, todo);
+        const intent = input.intent || `Input text into index ${input.index}`;
+        this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
         const page = await this.context.browserContext.getCurrentPage();
         const state = await page.getState();
@@ -268,7 +286,8 @@ export class ActionBuilder {
 
     // Tab Management Actions
     const switchTab = new Action(async (input: z.infer<typeof switchTabActionSchema.schema>) => {
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, `Switching to tab ${input.tab_id}`);
+      const intent = input.intent || `Switching to tab ${input.tab_id}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
       await this.context.browserContext.switchTab(input.tab_id);
       const msg = `Switched to tab ${input.tab_id}`;
       this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_OK, msg);
@@ -277,7 +296,8 @@ export class ActionBuilder {
     actions.push(switchTab);
 
     const openTab = new Action(async (input: z.infer<typeof openTabActionSchema.schema>) => {
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, `Opening ${input.url} in new tab`);
+      const intent = input.intent || `Opening ${input.url} in new tab`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
       await this.context.browserContext.openTab(input.url);
       const msg = `Opened ${input.url} in new tab`;
       this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_OK, msg);
@@ -285,39 +305,52 @@ export class ActionBuilder {
     }, openTabActionSchema);
     actions.push(openTab);
 
+    const closeTab = new Action(async (input: z.infer<typeof closeTabActionSchema.schema>) => {
+      const intent = input.intent || `Closing tab ${input.tab_id}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
+      await this.context.browserContext.closeTab(input.tab_id);
+      const msg = `Closed tab ${input.tab_id}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_OK, msg);
+      return new ActionResult({ extractedContent: msg, includeInMemory: true });
+    }, closeTabActionSchema);
+    actions.push(closeTab);
+
     // Content Actions
     // TODO: this is not used currently, need to improve on input size
-    const extractContent = new Action(async (input: z.infer<typeof extractContentActionSchema.schema>) => {
-      const goal = input.goal;
-      const page = await this.context.browserContext.getCurrentPage();
-      const content = await page.getReadabilityContent();
-      const promptTemplate = PromptTemplate.fromTemplate(
-        'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page}',
-      );
-      const prompt = await promptTemplate.invoke({ goal, page: content.content });
-
-      try {
-        const output = await this.extractorLLM.invoke(prompt);
-        const msg = `📄  Extracted from page\n: ${output.content}\n`;
-        return new ActionResult({
-          extractedContent: msg,
-          includeInMemory: true,
-        });
-      } catch (error) {
-        logger.error(`Error extracting content: ${error instanceof Error ? error.message : String(error)}`);
-        const msg =
-          'Failed to extract content from page, you need to extract content from the current state of the page and store it in the memory. Then scroll down if you still need more information.';
-        return new ActionResult({
-          extractedContent: msg,
-          includeInMemory: true,
-        });
-      }
-    }, extractContentActionSchema);
-    actions.push(extractContent);
+    // const extractContent = new Action(async (input: z.infer<typeof extractContentActionSchema.schema>) => {
+    //   const goal = input.goal;
+    //   const intent = input.intent || `Extracting content from page`;
+    //   this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
+    //   const page = await this.context.browserContext.getCurrentPage();
+    //   const content = await page.getReadabilityContent();
+    //   const promptTemplate = PromptTemplate.fromTemplate(
+    //     'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page}',
+    //   );
+    //   const prompt = await promptTemplate.invoke({ goal, page: content.content });
+
+    //   try {
+    //     const output = await this.extractorLLM.invoke(prompt);
+    //     const msg = `📄  Extracted from page\n: ${output.content}\n`;
+    //     return new ActionResult({
+    //       extractedContent: msg,
+    //       includeInMemory: true,
+    //     });
+    //   } catch (error) {
+    //     logger.error(`Error extracting content: ${error instanceof Error ? error.message : String(error)}`);
+    //     const msg =
+    //       'Failed to extract content from page, you need to extract content from the current state of the page and store it in the memory. Then scroll down if you still need more information.';
+    //     return new ActionResult({
+    //       extractedContent: msg,
+    //       includeInMemory: true,
+    //     });
+    //   }
+    // }, extractContentActionSchema);
+    // actions.push(extractContent);
 
     // cache content for future use
     const cacheContent = new Action(async (input: z.infer<typeof cacheContentActionSchema.schema>) => {
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, cacheContentActionSchema.name);
+      const intent = input.intent || `Caching findings: ${input.content}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
       const msg = `Cached findings: ${input.content}`;
       this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_OK, msg);
@@ -326,12 +359,13 @@ export class ActionBuilder {
     actions.push(cacheContent);
 
     const scrollDown = new Action(async (input: z.infer<typeof scrollDownActionSchema.schema>) => {
-      const todo = input.desc || 'Scroll down the page';
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, todo);
+      const amount = input.amount !== undefined && input.amount !== null ? `${input.amount} pixels` : 'one page';
+      const intent = input.intent || `Scroll down the page by ${amount}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
       const page = await this.context.browserContext.getCurrentPage();
       await page.scrollDown(input.amount);
-      const amount = input.amount !== undefined ? `${input.amount} pixels` : 'one page';
+
       const msg = `Scrolled down the page by ${amount}`;
       this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_OK, msg);
       return new ActionResult({ extractedContent: msg, includeInMemory: true });
@@ -339,12 +373,12 @@ export class ActionBuilder {
     actions.push(scrollDown);
 
     const scrollUp = new Action(async (input: z.infer<typeof scrollUpActionSchema.schema>) => {
-      const todo = input.desc || 'Scroll up the page';
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, todo);
+      const amount = input.amount !== undefined && input.amount !== null ? `${input.amount} pixels` : 'one page';
+      const intent = input.intent || `Scroll up the page by ${amount}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
       const page = await this.context.browserContext.getCurrentPage();
       await page.scrollUp(input.amount);
-      const amount = input.amount !== undefined ? `${input.amount} pixels` : 'one page';
       const msg = `Scrolled up the page by ${amount}`;
       this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_OK, msg);
       return new ActionResult({ extractedContent: msg, includeInMemory: true });
@@ -353,8 +387,8 @@ export class ActionBuilder {
 
     // Keyboard Actions
     const sendKeys = new Action(async (input: z.infer<typeof sendKeysActionSchema.schema>) => {
-      const todo = input.desc || `Send keys: ${input.keys}`;
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, todo);
+      const intent = input.intent || `Send keys: ${input.keys}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
       const page = await this.context.browserContext.getCurrentPage();
       await page.sendKeys(input.keys);
@@ -365,8 +399,8 @@ export class ActionBuilder {
     actions.push(sendKeys);
 
     const scrollToText = new Action(async (input: z.infer<typeof scrollToTextActionSchema.schema>) => {
-      const todo = input.desc || `Scroll to text: ${input.text}`;
-      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, todo);
+      const intent = input.intent || `Scroll to text: ${input.text}`;
+      this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
       const page = await this.context.browserContext.getCurrentPage();
       try {
@@ -387,8 +421,8 @@ export class ActionBuilder {
     // Get all options from a native dropdown
     const getDropdownOptions = new Action(
       async (input: z.infer<typeof getDropdownOptionsActionSchema.schema>) => {
-        const todo = `Getting options from dropdown with index ${input.index}`;
-        this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, todo);
+        const intent = input.intent || `Getting options from dropdown with index ${input.index}`;
+        this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
         const page = await this.context.browserContext.getCurrentPage();
         const state = await page.getState();
@@ -457,8 +491,8 @@ export class ActionBuilder {
     // Select dropdown option for interactive element index by the text of the option you want to select'
     const selectDropdownOption = new Action(
       async (input: z.infer<typeof selectDropdownOptionActionSchema.schema>) => {
-        const todo = `Select option "${input.text}" from dropdown with index ${input.index}`;
-        this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, todo);
+        const intent = input.intent || `Select option "${input.text}" from dropdown with index ${input.index}`;
+        this.context.emitEvent(Actors.NAVIGATOR, ExecutionState.ACT_START, intent);
 
         const page = await this.context.browserContext.getCurrentPage();
         const state = await page.getState();
@@ -514,8 +548,4 @@ export class ActionBuilder {
 
     return actions;
   }
-
-  // Get all options from a native dropdown
-
-  // Select dropdown option for interactive element index by the text of the option you want to select'
 }

chrome-extension/src/background/agent/actions/json_gemini.ts

@@ -6,9 +6,6 @@ export const geminiNavigatorOutputSchema = {
       type: 'object',
       description: 'Current state of the agent',
       properties: {
-        page_summary: {
-          type: 'string',
-        },
         evaluation_previous_goal: {
           type: 'string',
         },
@@ -19,7 +16,7 @@ export const geminiNavigatorOutputSchema = {
           type: 'string',
         },
       },
-      required: ['page_summary', 'evaluation_previous_goal', 'memory', 'next_goal'],
+      required: ['evaluation_previous_goal', 'memory', 'next_goal'],
     },
     action: {
       type: 'array',
@@ -32,42 +29,72 @@ export const geminiNavigatorOutputSchema = {
               text: {
                 type: 'string',
               },
+              success: {
+                type: 'boolean',
+              },
             },
-            required: ['text'],
+            required: ['text', 'success'],
             nullable: true,
           },
           search_google: {
             type: 'object',
             properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
               query: {
                 type: 'string',
               },
             },
-            required: ['query'],
+            required: ['intent', 'query'],
             nullable: true,
           },
           go_to_url: {
             type: 'object',
             properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
               url: {
                 type: 'string',
               },
             },
-            required: ['url'],
+            required: ['intent', 'url'],
             nullable: true,
           },
           go_back: {
-            type: 'string',
+            type: 'object',
+            properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
+            },
+            required: ['intent'],
+            nullable: true,
+          },
+          wait: {
+            type: 'object',
+            properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
+              seconds: {
+                type: 'integer',
+              },
+            },
+            required: ['intent', 'seconds'],
             nullable: true,
-            description:
-              'Accepts absolutely anything in the incoming data\nand discards it, so the final parsed model is empty.',
           },
           click_element: {
             type: 'object',
             properties: {
-              desc: {
+              intent: {
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               index: {
                 type: 'integer',
@@ -77,15 +104,15 @@ export const geminiNavigatorOutputSchema = {
                 nullable: true,
               },
             },
-            required: ['desc', 'index'],
+            required: ['intent', 'index'],
             nullable: true,
           },
           input_text: {
             type: 'object',
             properties: {
-              desc: {
+              intent: {
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               index: {
                 type: 'integer',
@@ -98,110 +125,144 @@ export const geminiNavigatorOutputSchema = {
                 nullable: true,
               },
             },
-            required: ['desc', 'index', 'text'],
+            required: ['intent', 'index', 'text'],
             nullable: true,
           },
           switch_tab: {
             type: 'object',
             properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
               tab_id: {
                 type: 'integer',
               },
             },
-            required: ['tab_id'],
+            required: ['intent', 'tab_id'],
             nullable: true,
           },
           open_tab: {
             type: 'object',
             properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
               url: {
                 type: 'string',
               },
             },
-            required: ['url'],
+            required: ['intent', 'url'],
+            nullable: true,
+          },
+          close_tab: {
+            type: 'object',
+            properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
+              tab_id: {
+                type: 'integer',
+              },
+            },
+            required: ['intent', 'tab_id'],
             nullable: true,
           },
           cache_content: {
             type: 'object',
             properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
               content: {
                 type: 'string',
               },
             },
-            required: ['content'],
+            required: ['intent', 'content'],
             nullable: true,
           },
           scroll_down: {
             type: 'object',
             properties: {
-              desc: {
+              intent: {
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               amount: {
                 type: 'integer',
                 nullable: true,
               },
             },
-            required: ['desc'],
+            required: ['intent', 'amount'],
             nullable: true,
           },
           scroll_up: {
             type: 'object',
             properties: {
-              desc: {
+              intent: {
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               amount: {
                 type: 'integer',
                 nullable: true,
               },
             },
-            required: ['desc'],
+            required: ['intent', 'amount'],
             nullable: true,
           },
           send_keys: {
             type: 'object',
             properties: {
-              desc: {
+              intent: {
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               keys: {
                 type: 'string',
               },
             },
-            required: ['desc', 'keys'],
+            required: ['intent', 'keys'],
             nullable: true,
           },
           scroll_to_text: {
             type: 'object',
             properties: {
-              desc: {
+              intent: {
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               text: {
                 type: 'string',
               },
             },
-            required: ['desc', 'text'],
+            required: ['intent', 'text'],
             nullable: true,
           },
           get_dropdown_options: {
             type: 'object',
             properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
               index: {
                 type: 'integer',
               },
             },
-            required: ['index'],
+            required: ['intent', 'index'],
             nullable: true,
           },
           select_dropdown_option: {
             type: 'object',
             properties: {
+              intent: {
+                type: 'string',
+                description: 'purpose of this action',
+              },
               index: {
                 type: 'integer',
               },
@@ -209,7 +270,7 @@ export const geminiNavigatorOutputSchema = {
                 type: 'string',
               },
             },
-            required: ['index', 'text'],
+            required: ['intent', 'index', 'text'],
             nullable: true,
           },
         },

chrome-extension/src/background/agent/actions/json_schema.ts

@@ -5,12 +5,8 @@ export const jsonNavigatorOutputSchema = {
     current_state: {
       description: 'Current state of the agent',
       properties: {
-        page_summary: {
-          title: 'Page Summary',
-          type: 'string',
-        },
         evaluation_previous_goal: {
-          title: 'Evaluation Previous Goal',
+          title: 'Evaluation of previous goal',
           type: 'string',
         },
         memory: {
@@ -22,7 +18,7 @@ export const jsonNavigatorOutputSchema = {
           type: 'string',
         },
       },
-      required: ['page_summary', 'evaluation_previous_goal', 'memory', 'next_goal'],
+      required: ['evaluation_previous_goal', 'memory', 'next_goal'],
       title: 'AgentBrain',
       type: 'object',
     },
@@ -35,73 +31,111 @@ export const jsonNavigatorOutputSchema = {
                 title: 'Text',
                 type: 'string',
               },
+              success: {
+                title: 'Success',
+                type: 'boolean',
+              },
             },
-            required: ['text'],
+            required: ['text', 'success'],
             title: 'DoneAction',
             type: 'object',
             nullable: true,
           },
           search_google: {
             properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
               query: {
                 title: 'Query',
                 type: 'string',
               },
             },
-            required: ['query'],
+            required: ['intent', 'query'],
             title: 'SearchGoogleAction',
             type: 'object',
             nullable: true,
           },
           go_to_url: {
             properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
               url: {
                 title: 'Url',
                 type: 'string',
               },
             },
-            required: ['url'],
+            required: ['intent', 'url'],
             title: 'GoToUrlAction',
             type: 'object',
             nullable: true,
           },
           go_back: {
-            additionalProperties: true,
-            description:
-              'Accepts absolutely anything in the incoming data\nand discards it, so the final parsed model is empty.',
-            properties: {},
-            title: 'NoParamsAction',
+            properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
+            },
+            required: ['intent'],
+            title: 'GoBackAction',
             type: 'object',
             nullable: true,
           },
+          // wait: {
+          //   properties: {
+          //     intent: {
+          //       title: 'Intent',
+          //       type: 'string',
+          //       description: 'purpose of this action'
+          //     },
+          //     seconds: {
+          //       title: 'Seconds',
+          //       type: 'integer',
+          //       default: 3
+          //     }
+          //   },
+          //   required: [
+          //     'intent',
+          //     'seconds'
+          //   ],
+          //   title: 'WaitAction',
+          //   type: 'object',
+          //   nullable: true
+          // },
           click_element: {
             properties: {
-              desc: {
+              intent: {
                 title: 'Intent',
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               index: {
                 title: 'Index',
                 type: 'integer',
               },
               xpath: {
-                title: 'XPath',
                 type: 'string',
                 nullable: true,
               },
             },
-            required: ['desc', 'index'],
+            required: ['intent', 'index'],
             title: 'ClickElementAction',
             type: 'object',
             nullable: true,
           },
           input_text: {
             properties: {
-              desc: {
+              intent: {
                 title: 'Intent',
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               index: {
                 title: 'Index',
@@ -112,136 +146,175 @@ export const jsonNavigatorOutputSchema = {
                 type: 'string',
               },
               xpath: {
-                title: 'XPath',
                 type: 'string',
                 nullable: true,
               },
             },
-            required: ['desc', 'index', 'text'],
+            required: ['intent', 'index', 'text'],
             title: 'InputTextAction',
             type: 'object',
             nullable: true,
           },
           switch_tab: {
             properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
               tab_id: {
-                title: 'Page Id',
+                title: 'Tab Id',
                 type: 'integer',
               },
             },
-            required: ['tab_id'],
+            required: ['intent', 'tab_id'],
             title: 'SwitchTabAction',
             type: 'object',
             nullable: true,
           },
           open_tab: {
             properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
               url: {
                 title: 'Url',
                 type: 'string',
               },
             },
-            required: ['url'],
+            required: ['intent', 'url'],
             title: 'OpenTabAction',
             type: 'object',
             nullable: true,
           },
+          close_tab: {
+            properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
+              tab_id: {
+                title: 'Tab Id',
+                type: 'integer',
+              },
+            },
+            required: ['intent', 'tab_id'],
+            title: 'CloseTabAction',
+            type: 'object',
+            nullable: true,
+          },
           cache_content: {
             properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
               content: {
                 title: 'Content',
                 type: 'string',
               },
             },
-            required: ['content'],
+            required: ['intent', 'content'],
             title: 'cache_content_parameters',
             type: 'object',
             nullable: true,
           },
           scroll_down: {
             properties: {
-              desc: {
+              intent: {
                 title: 'Intent',
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               amount: {
-                title: 'Amount',
                 type: 'integer',
                 nullable: true,
               },
             },
-            required: ['desc'],
+            required: ['intent', 'amount'],
             title: 'ScrollAction',
             type: 'object',
             nullable: true,
           },
           scroll_up: {
             properties: {
-              desc: {
+              intent: {
                 title: 'Intent',
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               amount: {
-                title: 'Amount',
                 type: 'integer',
                 nullable: true,
               },
             },
-            required: ['desc'],
+            required: ['intent', 'amount'],
             title: 'ScrollAction',
             type: 'object',
             nullable: true,
           },
           send_keys: {
             properties: {
-              desc: {
+              intent: {
                 title: 'Intent',
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               keys: {
                 title: 'Keys',
                 type: 'string',
               },
             },
-            required: ['desc', 'keys'],
+            required: ['intent', 'keys'],
             title: 'SendKeysAction',
             type: 'object',
             nullable: true,
           },
           scroll_to_text: {
             properties: {
-              desc: {
+              intent: {
                 title: 'Intent',
                 type: 'string',
-                description: 'Very short explanation of the intent or purpose for calling this action',
+                description: 'purpose of this action',
               },
               text: {
                 title: 'Text',
                 type: 'string',
               },
             },
-            required: ['desc', 'text'],
+            required: ['intent', 'text'],
             title: 'scroll_to_text_parameters',
             type: 'object',
             nullable: true,
           },
           get_dropdown_options: {
             properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
               index: {
                 title: 'Index',
                 type: 'integer',
               },
             },
-            required: ['index'],
+            required: ['intent', 'index'],
             title: 'get_dropdown_options_parameters',
             type: 'object',
             nullable: true,
           },
           select_dropdown_option: {
             properties: {
+              intent: {
+                title: 'Intent',
+                type: 'string',
+                description: 'purpose of this action',
+              },
               index: {
                 title: 'Index',
                 type: 'integer',
@@ -251,7 +324,7 @@ export const jsonNavigatorOutputSchema = {
                 type: 'string',
               },
             },
-            required: ['index', 'text'],
+            required: ['intent', 'index', 'text'],
             title: 'select_dropdown_option_parameters',
             type: 'object',
             nullable: true,

chrome-extension/src/background/agent/actions/schemas.ts

@@ -11,6 +11,7 @@ export const doneActionSchema: ActionSchema = {
   description: 'Complete task',
   schema: z.object({
     text: z.string(),
+    success: z.boolean(),
   }),
 };
 
@@ -19,6 +20,7 @@ export const searchGoogleActionSchema: ActionSchema = {
   name: 'search_google',
   description: 'Search Google in the current tab',
   schema: z.object({
+    intent: z.string().optional(),
     query: z.string(),
   }),
 };
@@ -27,6 +29,7 @@ export const goToUrlActionSchema: ActionSchema = {
   name: 'go_to_url',
   description: 'Navigate to URL in the current tab',
   schema: z.object({
+    intent: z.string().optional(),
     url: z.string(),
   }),
 };
@@ -34,14 +37,16 @@ export const goToUrlActionSchema: ActionSchema = {
 export const goBackActionSchema: ActionSchema = {
   name: 'go_back',
   description: 'Go back to the previous page',
-  schema: z.object({}),
+  schema: z.object({
+    intent: z.string().optional(),
+  }),
 };
 
 export const clickElementActionSchema: ActionSchema = {
   name: 'click_element',
-  description: 'Click element',
+  description: 'Click element by index',
   schema: z.object({
-    desc: z.string().optional(), // some small LLM can not generate a description, so let it be optional (but it's still makred as required in json schema)
+    intent: z.string().optional(), // some small LLM can not generate a intent, so let it be optional (but it's still makred as required in json schema)
     index: z.number(),
     xpath: z.string().nullable().optional(),
   }),
@@ -51,7 +56,7 @@ export const inputTextActionSchema: ActionSchema = {
   name: 'input_text',
   description: 'Input text into an interactive input element',
   schema: z.object({
-    desc: z.string().optional(),
+    intent: z.string().optional(),
     index: z.number(),
     text: z.string(),
     xpath: z.string().nullable().optional(),
@@ -63,6 +68,7 @@ export const switchTabActionSchema: ActionSchema = {
   name: 'switch_tab',
   description: 'Switch to tab by id',
   schema: z.object({
+    intent: z.string().optional(),
     tab_id: z.number(),
   }),
 };
@@ -71,25 +77,36 @@ export const openTabActionSchema: ActionSchema = {
   name: 'open_tab',
   description: 'Open URL in new tab',
   schema: z.object({
+    intent: z.string().optional(),
     url: z.string(),
   }),
 };
 
-// Content Actions
-export const extractContentActionSchema: ActionSchema = {
-  name: 'extract_content',
-  description:
-    'Extract page content to retrieve specific information from the page, e.g. all company names, a specific description, all information about, links with companies in structured format or simply links',
+export const closeTabActionSchema: ActionSchema = {
+  name: 'close_tab',
+  description: 'Close tab by id',
   schema: z.object({
-    goal: z.string(),
+    intent: z.string().optional(),
+    tab_id: z.number(),
   }),
 };
 
+// Content Actions, not used currently
+// export const extractContentActionSchema: ActionSchema = {
+//   name: 'extract_content',
+//   description:
+//     'Extract page content to retrieve specific information from the page, e.g. all company names, a specific description, all information about, links with companies in structured format or simply links',
+//   schema: z.object({
+//     goal: z.string(),
+//   }),
+// };
+
 // Cache Actions
 export const cacheContentActionSchema: ActionSchema = {
   name: 'cache_content',
-  description: 'Cache the extracted content of the page',
+  description: 'Cache what you have found so far from the current page for future use',
   schema: z.object({
+    intent: z.string().optional(),
     content: z.string(),
   }),
 };
@@ -98,7 +115,7 @@ export const scrollDownActionSchema: ActionSchema = {
   name: 'scroll_down',
   description: 'Scroll down the page by pixel amount - if no amount is specified, scroll down one page',
   schema: z.object({
-    desc: z.string().optional(),
+    intent: z.string().optional(),
     amount: z.number().nullable().optional(),
   }),
 };
@@ -107,7 +124,7 @@ export const scrollUpActionSchema: ActionSchema = {
   name: 'scroll_up',
   description: 'Scroll up the page by pixel amount - if no amount is specified, scroll up one page',
   schema: z.object({
-    desc: z.string().optional(),
+    intent: z.string().optional(),
     amount: z.number().nullable().optional(),
   }),
 };
@@ -117,7 +134,7 @@ export const sendKeysActionSchema: ActionSchema = {
   description:
     'Send strings of special keys like Backspace, Insert, PageDown, Delete, Enter. Shortcuts such as `Control+o`, `Control+Shift+T` are supported as well. This gets used in keyboard press. Be aware of different operating systems and their shortcuts',
   schema: z.object({
-    desc: z.string().optional(),
+    intent: z.string().optional(),
     keys: z.string(),
   }),
 };
@@ -126,7 +143,7 @@ export const scrollToTextActionSchema: ActionSchema = {
   name: 'scroll_to_text',
   description: 'If you dont find something which you want to interact with, scroll to it',
   schema: z.object({
-    desc: z.string().optional(),
+    intent: z.string().optional(),
     text: z.string(),
   }),
 };
@@ -135,6 +152,7 @@ export const getDropdownOptionsActionSchema: ActionSchema = {
   name: 'get_dropdown_options',
   description: 'Get all options from a native dropdown',
   schema: z.object({
+    intent: z.string().optional(),
     index: z.number(),
   }),
 };
@@ -143,7 +161,17 @@ export const selectDropdownOptionActionSchema: ActionSchema = {
   name: 'select_dropdown_option',
   description: 'Select dropdown option for interactive element index by the text of the option you want to select',
   schema: z.object({
+    intent: z.string().optional(),
     index: z.number(),
     text: z.string(),
   }),
 };
+
+export const waitActionSchema: ActionSchema = {
+  name: 'wait',
+  description: 'Wait for x seconds default 3',
+  schema: z.object({
+    intent: z.string().optional(),
+    seconds: z.number().nullable().optional(),
+  }),
+};

chrome-extension/src/background/agent/agents/navigator.ts

@@ -245,6 +245,11 @@ export class NavigatorAgent extends BaseAgent<z.ZodType, NavigatorResult> {
     this.context.stateMessageAdded = false;
   }
 
+  private async addModelOutputToMemory(modelOutput: this['ModelOutput']) {
+    const messageManager = this.context.messageManager;
+    messageManager.addModelOutput(modelOutput);
+  }
+
   private async doMultiAction(response: this['ModelOutput']): Promise<ActionResult[]> {
     const results: ActionResult[] = [];
     let errCount = 0;

chrome-extension/src/background/agent/prompts/base.ts

@@ -36,12 +36,14 @@ abstract class BasePrompt {
     let formattedElementsText = '';
     if (elementsText !== '') {
       if (hasContentAbove) {
+        // formattedElementsText = `... ${browserState.pixelsAbove} pixels above - scroll up or extract content to see more ...\n${elementsText}`;
         formattedElementsText = `... ${browserState.pixelsAbove} pixels above - scroll up to see more ...\n${elementsText}`;
       } else {
         formattedElementsText = `[Start of page]\n${elementsText}`;
       }
 
       if (hasContentBelow) {
+        // formattedElementsText = `${formattedElementsText}\n... ${browserState.pixelsBelow} pixels below - scroll down or extract content to see more ...`;
         formattedElementsText = `${formattedElementsText}\n... ${browserState.pixelsBelow} pixels below - scroll down to see more ...`;
       } else {
         formattedElementsText = `${formattedElementsText}\n[End of page]`;
@@ -66,24 +68,24 @@ abstract class BasePrompt {
           actionResultsDescription += `\nAction result ${i + 1}/${context.actionResults.length}: ${result.extractedContent}`;
         }
         if (result.error) {
-          // only use last 300 characters of error
-          const error = result.error.slice(-300);
+          // only use last line of error
+          const error = result.error.split('\n').pop();
           actionResultsDescription += `\nAction error ${i + 1}/${context.actionResults.length}: ...${error}`;
         }
       }
     }
 
     const stateDescription = `
-    [Task history memory ends here]
+    [Task history memory ends]
     [Current state starts here]
-    You will see the following only once - if you need to remember it and you dont know it yet, write it down in the memory:
+    The following is one-time information - if you need to remember it write it to memory:
     Current tab: {id: ${browserState.tabId}, url: ${browserState.url}, title: ${browserState.title}}
     Other available tabs:
     ${browserState.tabs
       .filter(tab => tab.id !== browserState.tabId)
       .map(tab => ` - {id: ${tab.id}, url: ${tab.url}, title: ${tab.title}}`)
       .join('\n')}
-    Interactive elements from current page:
+    Interactive elements from top layer of the current page inside the viewport:
     ${formattedElementsText}
     ${stepInfoDescription}
     ${actionResultsDescription}`;

chrome-extension/src/background/agent/prompts/navigator.ts

@@ -2,138 +2,21 @@
 import { BasePrompt } from './base';
 import { type HumanMessage, SystemMessage } from '@langchain/core/messages';
 import type { AgentContext } from '@src/background/agent/types';
+import { createLogger } from '@src/background/log';
+import { navigatorSystemPromptTemplate } from './templates/navigator';
+
+const logger = createLogger('agent/prompts/navigator');
 
 export class NavigatorPrompt extends BasePrompt {
-  private readonly default_action_description = 'A placeholder action description';
+  private systemMessage: SystemMessage;
 
   constructor(private readonly maxActionsPerStep = 10) {
     super();
-  }
-
-  importantRules(): string {
-    const text = `
-1. RESPONSE FORMAT: You must ALWAYS respond with valid JSON in this exact format:
-   {
-     "current_state": {
-		"page_summary": "Quick detailed summary of new information from the current page which is not yet in the task history memory. Be specific with details which are important for the task. This is not on the meta level, but should be facts. If all the information is already in the task history memory, leave this empty.",
-		"evaluation_previous_goal": "Success|Failed|Unknown - Analyze the current elements and the image to check if the previous goals/actions are successful like intended by the task. Ignore the action result. The website is the ground truth. Also mention if something unexpected happened like new suggestions in an input field. Shortly state why/why not",
-       "memory": "Description of what has been done and what you need to remember. Be very specific. Count here ALWAYS how many times you have done something and how many remain. E.g. 0 out of 10 websites analyzed. Continue with abc and xyz",
-       "next_goal": "What needs to be done with the next actions"
-     },
-     "action": [
-       {
-         "one_action_name": {
-           // action-specific parameter
-         }
-       },
-       // ... more actions in sequence
-     ]
-   }
-
-2. ACTIONS: You can specify multiple actions in the list to be executed in sequence. But always specify only one action name per item.
-
-   Common action sequences:
-   - Form filling: [
-       {"input_text": {"desc": "Fill title", "index": 1, "text": "example title"}},
-       {"input_text": {"desc": "Fill comment", "index": 2, "text": "example comment"}},
-       {"click_element": {"desc": "Click submit button", "index": 3}}
-     ]
-   - Navigation: [
-       {"open_tab": {"url": "https://example.com"}},
-       {"go_to_url": {"url": "https://example.com"}},
-     ]
-
-
-3. ELEMENT INTERACTION:
-   - Only use indexes that exist in the provided element list
-   - Each element has a unique index number (e.g., "[33]<button>")
-   - Elements marked with "[]Non-interactive text" are non-interactive (for context only)
-
-4. NAVIGATION & ERROR HANDLING:
-   - If you need to search in Google, use the search_google action. Don't need to input the search query manually, just use the action.
-   - If no suitable elements exist, use other functions to complete the task
-   - If stuck, try alternative approaches - like going back to a previous page, new search, new tab etc.
-   - Handle popups/cookies by accepting or closing them
-   - Use scroll to find elements you are looking for
-   - If you want to research something, open a new tab instead of using the current tab
-   - If captcha pops up, and you cant solve it, either ask for human help or try to continue the task on a different page.
-
-5. TASK COMPLETION:
-   - Use the done action as the last action as soon as the ultimate task is complete
-   - Dont use "done" before you are done with everything the user asked you. 
-   - If you have to do something repeatedly for example the task says for "each", or "for all", or "x times", count always inside "memory" how many times you have done it and how many remain. Don't stop until you have completed like the task asked you. Only call done after the last step.
-   - Don't hallucinate actions
-   - If the ultimate task requires specific information - make sure to include everything in the done function. This is what the user will see. Do not just say you are done, but include the requested information of the task.
-   - Include exact relevant urls if available, but do NOT make up any urls
-
-6. VISUAL CONTEXT:
-   - When an image is provided, use it to understand the page layout
-   - Bounding boxes with labels correspond to element indexes
-   - Each bounding box and its label have the same color
-   - Most often the label is inside the bounding box, on the top right
-   - Visual context helps verify element locations and relationships
-   - sometimes labels overlap, so use the context to verify the correct element
-
-7. Form filling:
-   - If you fill an input field and your action sequence is interrupted, most often a list with suggestions popped up under the field and you need to first select the right element from the suggestion list.
-
-8. ACTION SEQUENCING:
-   - Actions are executed in the order they appear in the list
-   - Each action should logically follow from the previous one
-   - If the page changes after an action, the sequence is interrupted and you get the new state.
-   - If content only disappears the sequence continues.
-   - Only provide the action sequence until you think the page will change.
-   - Try to be efficient, e.g. fill forms at once, or chain actions where nothing changes on the page like saving, extracting, checkboxes...
-   - only use multiple actions if it makes sense.
 
-9. Long tasks:
-- If the task is long keep track of the status in the memory. If the ultimate task requires multiple subinformation, keep track of the status in the memory.
-- If you get stuck, 
-
-10. Extraction:
-- When searching for information or conducting research:
-  1. First analyze and extract relevant content from the current visible state
-  2. If the needed information is incomplete:
-     - Use cache_content action to cache the current findings
-     - Scroll down EXACTLY ONE PAGE at a time using scroll_page action
-     - NEVER scroll more than one page at once as this will cause loss of information
-     - Repeat the analyze-cache-scroll cycle until either:
-       * All required information is found, or
-       * Maximum 5 page scrolls have been performed
-  3. Before completing the task:
-     - Combine all cached content with the current state
-     - Verify all required information is collected
-     - Present the complete findings in the done action
-- Important extraction guidelines:
-  - Be thorough and specific when extracting information
-  - Always cache findings before scrolling to avoid losing information
-  - Always verify source information before caching
-  - Scroll down EXACTLY ONE PAGE at a time
-  - Stop after maximum 5 page scrolls
-`;
-    return `${text}   - use maximum ${this.maxActionsPerStep} actions per sequence`;
-  }
-
-  inputFormat(): string {
-    return `
-INPUT STRUCTURE:
-1. Current URL: The webpage you're currently on
-2. Available Tabs: List of open browser tabs
-3. Interactive Elements: List in the format:
-   index[:]<element_type>element_text</element_type>
-   - index: Numeric identifier for interaction
-   - element_type: HTML element type (button, input, etc.)
-   - element_text: Visible text or element description
-
-Example:
-[33]<button>Submit Form</button>
-[] Non-interactive text
-
-
-Notes:
-- Only elements with numeric indexes inside [] are interactive
-- [] elements provide context but cannot be interacted with
-`;
+    const promptTemplate = navigatorSystemPromptTemplate;
+    // Format the template with the maxActionsPerStep
+    const formattedPrompt = promptTemplate.replace('{{max_actions}}', this.maxActionsPerStep.toString()).trim();
+    this.systemMessage = new SystemMessage(formattedPrompt);
   }
 
   getSystemMessage(): SystemMessage {
@@ -142,22 +25,7 @@ Notes:
      *
      * @returns SystemMessage containing the formatted system prompt
      */
-    const AGENT_PROMPT = `You are a precise browser automation agent that interacts with websites through structured commands. Your role is to:
-1. Analyze the provided webpage elements and structure
-2. Use the given information to accomplish the ultimate task
-3. Respond with valid JSON containing your next action sequence and state assessment
-4. If the webpage is asking for login credentials, never try to fill it by yourself. Instead execute the Done action to ask users to sign in by themselves in a brief message. Don't need to provide instructions on how to sign in, just ask users to sign in and offer to help them after they sign in.
-
-${this.inputFormat()}
-
-${this.importantRules()}
-
-Functions:
-${this.default_action_description}
-
-Remember: Your responses must be valid JSON matching the specified format. Each action in the sequence must be valid.`;
-
-    return new SystemMessage(AGENT_PROMPT);
+    return this.systemMessage;
   }
 
   async getUserMessage(context: AgentContext): Promise<HumanMessage> {

chrome-extension/src/background/agent/prompts/templates/navigator.ts

@@ -0,0 +1,84 @@
+export const navigatorSystemPromptTemplate = `
+You are an AI agent designed to automate browser tasks. Your goal is to accomplish the ultimate task following the rules.
+
+# Input Format
+
+Task
+Previous steps
+Current URL
+Open Tabs
+Interactive Elements
+[index]<type>text</type>
+
+- index: Numeric identifier for interaction
+- type: HTML element type (button, input, etc.)
+- text: Element description
+  Example:
+  [33]<div>User form</div>
+  \\t*[35]*<button aria-label='Submit form'>Submit</button>
+
+- Only elements with numeric indexes in [] are interactive
+- (stacked) indentation (with \\t) is important and means that the element is a (html) child of the element above (with a lower index)
+- Elements with * are new elements that were added after the previous step (if url has not changed)
+
+# Response Rules
+
+1. RESPONSE FORMAT: You must ALWAYS respond with valid JSON in this exact format:
+   {"current_state": {"evaluation_previous_goal": "Success|Failed|Unknown - Analyze the current elements and the image to check if the previous goals/actions are successful like intended by the task. Mention if something unexpected happened. Shortly state why/why not",
+   "memory": "Description of what has been done and what you need to remember. Be very specific. Count here ALWAYS how many times you have done something and how many remain. E.g. 0 out of 10 websites analyzed. Continue with abc and xyz",
+   "next_goal": "What needs to be done with the next immediate action"},
+   "action":[{"one_action_name": {// action-specific parameter}}, // ... more actions in sequence]}
+
+2. ACTIONS: You can specify multiple actions in the list to be executed in sequence. But always specify only one action name per item. Use maximum {{max_actions}} actions per sequence.
+Common action sequences:
+
+- Form filling: [{"input_text": {"index": 1, "text": "username"}}, {"input_text": {"index": 2, "text": "password"}}, {"click_element": {"index": 3}}]
+- Navigation and extraction: [{"go_to_url": {"url": "https://example.com"}}, {"extract_content": {"goal": "extract the names"}}]
+- Actions are executed in the given order
+- If the page changes after an action, the sequence is interrupted and you get the new state.
+- Only provide the action sequence until an action which changes the page state significantly.
+- Try to be efficient, e.g. fill forms at once, or chain actions where nothing changes on the page
+- only use multiple actions if it makes sense.
+
+3. ELEMENT INTERACTION:
+
+- Only use indexes of the interactive elements
+
+4. NAVIGATION & ERROR HANDLING:
+
+- If no suitable elements exist, use other functions to complete the task
+- If stuck, try alternative approaches - like going back to a previous page, new search, new tab etc.
+- Handle popups/cookies by accepting or closing them
+- Use scroll to find elements you are looking for
+- If you want to research something, open a new tab instead of using the current tab
+- If captcha pops up, try to solve it - else try a different approach
+- If the page is not fully loaded, use wait action
+
+5. TASK COMPLETION:
+
+- Use the done action as the last action as soon as the ultimate task is complete
+- Dont use "done" before you are done with everything the user asked you, except you reach the last step of max_steps.
+- If you reach your last step, use the done action even if the task is not fully finished. Provide all the information you have gathered so far. If the ultimate task is completely finished set success to true. If not everything the user asked for is completed set success in done to false!
+- If you have to do something repeatedly for example the task says for "each", or "for all", or "x times", count always inside "memory" how many times you have done it and how many remain. Don't stop until you have completed like the task asked you. Only call done after the last step.
+- Don't hallucinate actions
+- Make sure you include everything you found out for the ultimate task in the done text parameter. Do not just say you are done, but include the requested information of the task.
+
+6. VISUAL CONTEXT:
+
+- When an image is provided, use it to understand the page layout
+- Bounding boxes with labels on their top right corner correspond to element indexes
+
+7. Form filling:
+
+- If you fill an input field and your action sequence is interrupted, most often something changed e.g. suggestions popped up under the field.
+
+8. Long tasks:
+
+- Keep track of the status and subresults in the memory.
+- You are provided with procedural memory summaries that condense previous task history (every N steps). Use these summaries to maintain context about completed actions, current progress, and next steps. The summaries appear in chronological order and contain key information about navigation history, findings, errors encountered, and current state. Refer to these summaries to avoid repeating actions and to ensure consistent progress toward the task goal.
+
+9. Extraction:
+
+- If your task is to find information - call extract_content on the specific pages to get and store the information.
+  Your responses must be always JSON with the specified format.
+`;

chrome-extension/src/background/agent/types.ts

@@ -141,7 +141,6 @@ export type WrappedActionResult = ActionResult & {
 };
 
 export const agentBrainSchema = z.object({
-  page_summary: z.string(),
   evaluation_previous_goal: z.string(),
   memory: z.string(),
   next_goal: z.string(),

packages/schema-utils/examples/convert.ts

@@ -1,4 +1,4 @@
-import { convertOpenAISchemaToGemini } from '../lib/helper.js';
+import { convertOpenAISchemaToGemini, stringifyCustom } from '../lib/helper.js';
 import { jsonNavigatorOutputSchema } from '../lib/json_schema.js';
 
 // Convert the schema
@@ -6,4 +6,4 @@ console.log('Converting jsonNavigatorOutputSchema to Gemini format...');
 const geminiSchema = convertOpenAISchemaToGemini(jsonNavigatorOutputSchema);
 
 // pretty print the schema
-console.log(JSON.stringify(geminiSchema, null, 2));
+console.log(stringifyCustom(geminiSchema));

packages/schema-utils/examples/flatten.ts

@@ -1,4 +1,4 @@
-import { dereferenceJsonSchema } from '../lib/helper.js';
+import { dereferenceJsonSchema, stringifyCustom } from '../lib/helper.js';
 import { jsonNavigatorOutputSchema } from '../lib/json_schema.js';
 
 /**
@@ -10,9 +10,9 @@ import { jsonNavigatorOutputSchema } from '../lib/json_schema.js';
 console.log('Flattening jsonNavigatorOutputSchema...');
 const flattenedSchema = dereferenceJsonSchema(jsonNavigatorOutputSchema);
 
-// Pretty print the flattened schema
-console.log('Flattened Schema:');
-console.log(JSON.stringify(flattenedSchema, null, 2));
+// Pretty print the flattened schema using the custom function
+console.log('Flattened Schema (Custom Format):');
+console.log(stringifyCustom(flattenedSchema));
 
 // You can also see the size difference
 const originalSize = JSON.stringify(jsonNavigatorOutputSchema).length;

packages/schema-utils/lib/helper.ts

@@ -240,3 +240,43 @@ function processProperty(property: JsonSchemaObject, definitions: Record<string,
 
   return result;
 }
+
+export type JSONSchemaType = JsonSchemaObject | JSONSchemaType[];
+// Custom stringify function
+export function stringifyCustom(value: JSONSchemaType, indent = '', baseIndent = '  '): string {
+  const currentIndent = indent + baseIndent;
+  if (value === null) {
+    return 'null';
+  }
+  switch (typeof value) {
+    case 'string':
+      // Escape single quotes within the string if necessary
+      return `'${value.replace(/'/g, "\\\\'")}'`;
+    case 'number':
+    case 'boolean':
+      return String(value);
+    case 'object': {
+      if (Array.isArray(value)) {
+        if (value.length === 0) {
+          return '[]';
+        }
+        const items = value.map(item => `${currentIndent}${stringifyCustom(item, currentIndent, baseIndent)}`);
+        return `[\n${items.join(',\n')}\n${indent}]`;
+      }
+      const keys = Object.keys(value);
+      if (keys.length === 0) {
+        return '{}';
+      }
+      const properties = keys.map(key => {
+        // Assume keys are valid JS identifiers and don't need quotes
+        const formattedKey = key;
+        const formattedValue = stringifyCustom(value[key], currentIndent, baseIndent);
+        return `${currentIndent}${formattedKey}: ${formattedValue}`;
+      });
+      return `{\n${properties.join(',\n')}\n${indent}}`;
+    }
+    default:
+      // Handle undefined, etc.
+      return 'undefined';
+  }
+}

packages/schema-utils/lib/json_schema.ts

@@ -1,4 +1,11 @@
-// This is the json schema exported from browser-use, change page_id to tab_id
+// This is the json schema exported from browser-use v0.1.41 with minor changes,
+//  - change page_id to tab_id
+//  - add intent to some actions which is used to describe the action's purpose
+//  - remove extract_content action, because it usually submit very long content to LLM
+//  - remove DragDropAction, it's not supported yet
+//  - remove save_pdf action, it's not supported yet
+//  - remove Position, not needed
+//  - remove NoParamsAction, not needed
 // TODO: don't know why zod can not generate the same schema, need to fix it
 export const jsonNavigatorOutputSchema = {
   $defs: {
@@ -41,14 +48,25 @@ export const jsonNavigatorOutputSchema = {
         go_back: {
           anyOf: [
             {
-              $ref: '#/$defs/NoParamsAction',
+              $ref: '#/$defs/GoBackAction',
             },
             {
               type: 'null',
             },
           ],
-          description: 'Go back',
-        },
+          description: 'Go back to previous page',
+        },
+        // wait: {
+        //   anyOf: [
+        //     {
+        //       $ref: '#/$defs/WaitAction',
+        //     },
+        //     {
+        //       type: 'null',
+        //     },
+        //   ],
+        //   description: 'Wait for x seconds default 3',
+        // },
         click_element: {
           anyOf: [
             {
@@ -58,7 +76,7 @@ export const jsonNavigatorOutputSchema = {
               type: 'null',
             },
           ],
-          description: 'Click element',
+          description: 'Click element by index',
         },
         input_text: {
           anyOf: [
@@ -69,7 +87,7 @@ export const jsonNavigatorOutputSchema = {
               type: 'null',
             },
           ],
-          description: 'Input text into a input interactive element',
+          description: 'Input text into an interactive input element',
         },
         switch_tab: {
           anyOf: [
@@ -93,6 +111,18 @@ export const jsonNavigatorOutputSchema = {
           ],
           description: 'Open url in new tab',
         },
+        close_tab: {
+          anyOf: [
+            {
+              $ref: '#/$defs/CloseTabAction',
+            },
+            {
+              type: 'null',
+            },
+          ],
+          description: 'Close tab by tab_id',
+        },
+
         cache_content: {
           anyOf: [
             {
@@ -102,7 +132,7 @@ export const jsonNavigatorOutputSchema = {
               type: 'null',
             },
           ],
-          description: 'Cache what you have found so far from the current page so that it can be used in future steps',
+          description: 'Cache what you have found so far from the current page for future use',
         },
         scroll_down: {
           anyOf: [
@@ -136,7 +166,7 @@ export const jsonNavigatorOutputSchema = {
             },
           ],
           description:
-            'Send strings of special keys like Backspace, Insert, PageDown, Delete, Enter, Shortcuts such as `Control+o`, `Control+Shift+T` are supported as well. This gets used in keyboard.press. Be aware of different operating systems and their shortcuts',
+            'Send strings of special keys like Escape, Backspace, Insert, PageDown, Delete, Enter, Shortcuts such as `Control+o`, `Control+Shift+T` are supported as well. This gets used in keyboard.press.',
         },
         scroll_to_text: {
           anyOf: [
@@ -179,12 +209,8 @@ export const jsonNavigatorOutputSchema = {
     AgentBrain: {
       description: 'Current state of the agent',
       properties: {
-        page_summary: {
-          title: 'Page Summary',
-          type: 'string',
-        },
         evaluation_previous_goal: {
-          title: 'Evaluation Previous Goal',
+          title: 'Evaluation of previous goal',
           type: 'string',
         },
         memory: {
@@ -196,16 +222,16 @@ export const jsonNavigatorOutputSchema = {
           type: 'string',
         },
       },
-      required: ['page_summary', 'evaluation_previous_goal', 'memory', 'next_goal'],
+      required: ['evaluation_previous_goal', 'memory', 'next_goal'],
       title: 'AgentBrain',
       type: 'object',
     },
     ClickElementAction: {
       properties: {
-        desc: {
+        intent: {
           title: 'Intent',
           type: 'string',
-          description: 'Very short explanation of the intent or purpose for calling this action',
+          description: 'purpose of this action',
         },
         index: {
           title: 'Index',
@@ -223,38 +249,75 @@ export const jsonNavigatorOutputSchema = {
           title: 'Xpath',
         },
       },
-      required: ['desc', 'index'],
+      required: ['intent', 'index'],
       title: 'ClickElementAction',
       type: 'object',
     },
+    CloseTabAction: {
+      properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
+        tab_id: {
+          title: 'Tab Id',
+          type: 'integer',
+        },
+      },
+      required: ['intent', 'tab_id'],
+      title: 'CloseTabAction',
+      type: 'object',
+    },
     DoneAction: {
       properties: {
         text: {
           title: 'Text',
           type: 'string',
         },
+        success: {
+          title: 'Success',
+          type: 'boolean',
+        },
       },
-      required: ['text'],
+      required: ['text', 'success'],
       title: 'DoneAction',
       type: 'object',
     },
     GoToUrlAction: {
       properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
         url: {
           title: 'Url',
           type: 'string',
         },
       },
-      required: ['url'],
+      required: ['intent', 'url'],
       title: 'GoToUrlAction',
       type: 'object',
     },
+    GoBackAction: {
+      properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
+      },
+      required: ['intent'],
+      title: 'GoBackAction',
+      type: 'object',
+    },
     InputTextAction: {
       properties: {
-        desc: {
+        intent: {
           title: 'Intent',
           type: 'string',
-          description: 'Very short explanation of the intent or purpose for calling this action',
+          description: 'purpose of this action',
         },
         index: {
           title: 'Index',
@@ -276,35 +339,32 @@ export const jsonNavigatorOutputSchema = {
           title: 'Xpath',
         },
       },
-      required: ['desc', 'index', 'text'],
+      required: ['intent', 'index', 'text'],
       title: 'InputTextAction',
       type: 'object',
     },
-    NoParamsAction: {
-      additionalProperties: true,
-      description:
-        'Accepts absolutely anything in the incoming data\nand discards it, so the final parsed model is empty.',
-      properties: {},
-      title: 'NoParamsAction',
-      type: 'object',
-    },
     OpenTabAction: {
       properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
         url: {
           title: 'Url',
           type: 'string',
         },
       },
-      required: ['url'],
+      required: ['intent', 'url'],
       title: 'OpenTabAction',
       type: 'object',
     },
     ScrollAction: {
       properties: {
-        desc: {
+        intent: {
           title: 'Intent',
           type: 'string',
-          description: 'Very short explanation of the intent or purpose for calling this action',
+          description: 'purpose of this action',
         },
         amount: {
           anyOf: [
@@ -318,88 +378,113 @@ export const jsonNavigatorOutputSchema = {
           title: 'Amount',
         },
       },
-      required: ['desc'],
+      required: ['intent', 'amount'],
       title: 'ScrollAction',
       type: 'object',
     },
     SearchGoogleAction: {
       properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
         query: {
           title: 'Query',
           type: 'string',
         },
       },
-      required: ['query'],
+      required: ['intent', 'query'],
       title: 'SearchGoogleAction',
       type: 'object',
     },
     SendKeysAction: {
       properties: {
-        desc: {
+        intent: {
           title: 'Intent',
           type: 'string',
-          description: 'Very short explanation of the intent or purpose for calling this action',
+          description: 'purpose of this action',
         },
         keys: {
           title: 'Keys',
           type: 'string',
         },
       },
-      required: ['desc', 'keys'],
+      required: ['intent', 'keys'],
       title: 'SendKeysAction',
       type: 'object',
     },
     SwitchTabAction: {
       properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
         tab_id: {
-          title: 'Page Id',
+          title: 'Tab Id',
           type: 'integer',
         },
       },
-      required: ['tab_id'],
+      required: ['intent', 'tab_id'],
       title: 'SwitchTabAction',
       type: 'object',
     },
     cache_content_parameters: {
       properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
         content: {
           title: 'Content',
           type: 'string',
         },
       },
-      required: ['content'],
+      required: ['intent', 'content'],
       title: 'cache_content_parameters',
       type: 'object',
     },
     get_dropdown_options_parameters: {
       properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
         index: {
           title: 'Index',
           type: 'integer',
         },
       },
-      required: ['index'],
+      required: ['intent', 'index'],
       title: 'get_dropdown_options_parameters',
       type: 'object',
     },
     scroll_to_text_parameters: {
       properties: {
-        desc: {
+        intent: {
           title: 'Intent',
           type: 'string',
-          description: 'Very short explanation of the intent or purpose for calling this action',
+          description: 'purpose of this action',
         },
         text: {
           title: 'Text',
           type: 'string',
         },
       },
-      required: ['desc', 'text'],
+      required: ['intent', 'text'],
       title: 'scroll_to_text_parameters',
       type: 'object',
     },
     select_dropdown_option_parameters: {
       properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
         index: {
           title: 'Index',
           type: 'integer',
@@ -409,10 +494,27 @@ export const jsonNavigatorOutputSchema = {
           type: 'string',
         },
       },
-      required: ['index', 'text'],
+      required: ['intent', 'index', 'text'],
       title: 'select_dropdown_option_parameters',
       type: 'object',
     },
+    WaitAction: {
+      properties: {
+        intent: {
+          title: 'Intent',
+          type: 'string',
+          description: 'purpose of this action',
+        },
+        seconds: {
+          title: 'Seconds',
+          type: 'integer',
+          default: 3,
+        },
+      },
+      required: ['intent', 'seconds'],
+      title: 'WaitAction',
+      type: 'object',
+    },
   },
   properties: {
     current_state: {