AddConfig.py 918 B

123456789101112131415161718192021222324252627282930313233343536373839404142
  1. from typing import Callable, Optional
  2. from embedchain.config.BaseConfig import BaseConfig
  3. class ChunkerConfig(BaseConfig):
  4. """
  5. Config for the chunker used in `add` method
  6. """
  7. def __init__(
  8. self,
  9. chunk_size: Optional[int] = 4000,
  10. chunk_overlap: Optional[int] = 200,
  11. length_function: Optional[Callable[[str], int]] = len,
  12. ):
  13. self.chunk_size = chunk_size
  14. self.chunk_overlap = chunk_overlap
  15. self.length_function = length_function
  16. class LoaderConfig(BaseConfig):
  17. """
  18. Config for the chunker used in `add` method
  19. """
  20. def __init__(self):
  21. pass
  22. class AddConfig(BaseConfig):
  23. """
  24. Config for the `add` method.
  25. """
  26. def __init__(
  27. self,
  28. chunker: Optional[ChunkerConfig] = None,
  29. loader: Optional[LoaderConfig] = None,
  30. ):
  31. self.loader = loader
  32. self.chunker = chunker