mem0/embedchain/loaders/code_docs_page.py

import requests
from bs4 import BeautifulSoup

from embedchain.utils import clean_string


class CodeDocsPageLoader:
    def load_data(self, url):
        """Load data from a web page."""
        response = requests.get(url)
        data = response.content
        soup = BeautifulSoup(data, "html.parser")
        selectors = [
            "article.bd-article",
            'article[role="main"]',
            "div.md-content",
            'div[role="main"]',
            "div.container",
            "div.section",
            "article",
            "main",
        ]
        content = None
        for selector in selectors:
            element = soup.select_one(selector)
            if element is not None:
                content = element.prettify()
                break
        if not content:
            content = soup.get_text()
        soup = BeautifulSoup(content, "html.parser")
        for tag in soup(
            [
                "nav",
                "aside",
                "form",
                "header",
                "noscript",
                "svg",
                "canvas",
                "footer",
                "script",
                "style",
            ]
        ):
            tag.string = " "
        for div in soup.find_all("div", {"class": "cell_output"}):
            div.decompose()
        for div in soup.find_all("div", {"class": "output_wrapper"}):
            div.decompose()
        for div in soup.find_all("div", {"class": "output"}):
            div.decompose()
        content = clean_string(soup.get_text())
        output = []
        meta_data = {
            "url": url,
        }
        output.append(
            {
                "content": content,
                "meta_data": meta_data,
            }
        )
        return output