[Bugfix] fix poetry lock (#960)

embedchain/loaders/github.py

@@ -68,7 +68,7 @@ class GithubLoader(BaseLoader):
                             data.extend(results)
                             data_urls.extend([result.get("meta_data").get("url") for result in results])
                     except Exception as e:
-                        logging.error(f"Failed to process {file}: {e}")
+                        logging.warn(f"Failed to process {file}: {e}")
 
         source_hash = hashlib.sha256(repo_url.encode()).hexdigest()
         repo_path = f"/tmp/{source_hash}"

embedchain/loaders/unstructured_file.py

@@ -1,11 +1,5 @@
 import hashlib
 
-try:
-    from langchain.document_loaders import UnstructuredFileLoader
-except ImportError:
-    raise ImportError(
-        'Unstructured file requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'
-    ) from None
 from embedchain.helper.json_serializable import register_deserializable
 from embedchain.loaders.base_loader import BaseLoader
 from embedchain.utils import clean_string
@@ -15,6 +9,13 @@ from embedchain.utils import clean_string
 class UnstructuredLoader(BaseLoader):
     def load_data(self, url):
         """Load data from a Unstructured file."""
+        try:
+            from langchain.document_loaders import UnstructuredFileLoader
+        except ImportError:
+            raise ImportError(
+                'Unstructured file requires extra dependencies. Install with `pip install --upgrade "embedchain[dataloaders]"`'  # noqa: E501
+            ) from None
+
         loader = UnstructuredFileLoader(url)
         data = []
         all_content = []

poetry.lock

@@ -6892,30 +6892,30 @@ backoff = "*"
 beautifulsoup4 = "*"
 chardet = "*"
 dataclasses-json = "*"
-ebooklib = {version = "*", optional = true, markers = "extra == \"local-inference\""}
+ebooklib = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
 emoji = "*"
 filetype = "*"
 langdetect = "*"
 lxml = "*"
-markdown = {version = "*", optional = true, markers = "extra == \"local-inference\""}
-msg-parser = {version = "*", optional = true, markers = "extra == \"local-inference\""}
+markdown = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
+msg-parser = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
 nltk = "*"
 numpy = "*"
-openpyxl = {version = "*", optional = true, markers = "extra == \"local-inference\""}
-pandas = {version = "*", optional = true, markers = "extra == \"local-inference\""}
-pdf2image = {version = "*", optional = true, markers = "extra == \"local-inference\""}
-"pdfminer.six" = {version = "*", optional = true, markers = "extra == \"local-inference\""}
-pypandoc = {version = "*", optional = true, markers = "extra == \"local-inference\""}
-python-docx = {version = ">=1.0.1", optional = true, markers = "extra == \"local-inference\""}
+openpyxl = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
+pandas = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
+pdf2image = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
+"pdfminer.six" = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
+pypandoc = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
+python-docx = {version = ">=1.0.1", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
 python-iso639 = "*"
 python-magic = "*"
-python-pptx = {version = "<=0.6.21", optional = true, markers = "extra == \"local-inference\""}
+python-pptx = {version = "<=0.6.21", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
 rapidfuzz = "*"
 requests = "*"
 tabulate = "*"
-unstructured-inference = {version = "0.7.3", optional = true, markers = "extra == \"local-inference\""}
-"unstructured.pytesseract" = {version = ">=0.3.12", optional = true, markers = "extra == \"local-inference\""}
-xlrd = {version = "*", optional = true, markers = "extra == \"local-inference\""}
+unstructured-inference = {version = "0.7.3", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
+"unstructured.pytesseract" = {version = ">=0.3.12", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
+xlrd = {version = "*", optional = true, markers = "extra == \"all-docs\" or extra == \"local-inference\""}
 
 [package.extras]
 airtable = ["pyairtable"]
@@ -7641,7 +7641,7 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [extras]
 cohere = ["cohere"]
 community = ["llama-hub"]
-dataloaders = ["beautifulsoup4", "docx2txt", "duckduckgo-search", "pypdf", "pytube", "sentence-transformers", "unstructured"]
+dataloaders = ["beautifulsoup4", "docx2txt", "duckduckgo-search", "pypdf", "pytube", "sentence-transformers", "unstructured", "youtube-transcript-api"]
 discord = ["discord"]
 elasticsearch = ["elasticsearch"]
 git = ["gitpython"]
@@ -7663,9 +7663,9 @@ streamlit = []
 vertexai = ["google-cloud-aiplatform"]
 weaviate = ["weaviate-client"]
 whatsapp = ["flask", "twilio"]
-youtube-channel = ["yt_dlp"]
+youtube = ["youtube-transcript-api", "yt_dlp"]
 
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.12"
-content-hash = "a7282080c7a4379bdc6f33dfe9cae7eb20764aae0176137ba5c7af7cdcc58ede"
+content-hash = "58dce9bc5ef9c8d7e77d5fbc3176e24a8facbe89beddbf4c605b9c68e6617b5a"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "embedchain"
-version = "0.1.15"
+version = "0.1.16"
 description = "Data platform for LLMs - Load, index, retrieve and sync any unstructured data"
 authors = [
     "Taranjeet Singh <taranjeet@embedchain.ai>",
@@ -120,7 +120,7 @@ weaviate-client = { version = "^3.24.1", optional = true }
 docx2txt = { version = "^0.8", optional = true }
 pinecone-client = { version = "^2.2.4", optional = true }
 qdrant-client = { version = "1.6.3", optional = true }
-unstructured = {extras = ["local-inference"], version = "^0.10.18", optional = true}
+unstructured = {extras = ["local-inference", "all-docs"], version = "^0.10.18", optional = true}
 pillow = { version = "10.0.1", optional = true }
 torchvision = { version = ">=0.15.1, !=0.15.2", optional = true }
 ftfy = { version = "6.1.1", optional = true }
@@ -169,7 +169,7 @@ huggingface_hub=["huggingface_hub"]
 cohere = ["cohere"]
 milvus = ["pymilvus"]
 dataloaders=[
-    "youtube-transcripts-api",
+    "youtube-transcript-api",
     "beautifulsoup4",
     "docx2txt",
     "duckduckgo-search",
@@ -193,9 +193,9 @@ json = ["llama-hub"]
 postgres = ["psycopg", "psycopg-binary", "psycopg-pool"]
 mysql = ["mysql-connector-python"]
 git = ["gitpython"]
-youtube_channel = [
+youtube = [
     "yt_dlp",
-    "youtube-transcripts-api",
+    "youtube-transcript-api",
 ]
 
 [tool.poetry.group.docs.dependencies]