Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: added split by page to DocumentSplitter #6753

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
added test case and the suggested changes
  • Loading branch information
sahusiddharth committed Jan 17, 2024
commit a18939792ee02973618b0a904b82dfeed3f42436
4 changes: 2 additions & 2 deletions haystack/components/preprocessors/document_splitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def __init__(
):
"""
:param split_by: The unit by which the document should be split. Choose from "word" for splitting by " ",
"sentence" for splitting by ".", "page" for splittling by "\f" or "passage" for splitting by "\\n\\n".
"sentence" for splitting by ".", "page" for splitting by "\f" or "passage" for splitting by "\\n\\n".
:param split_length: The maximum number of units in each split.
:param split_overlap: The number of units that each split should overlap.
"""
Expand Down Expand Up @@ -74,7 +74,7 @@ def _split_into_units(self, text: str, split_by: Literal["word", "sentence", "pa
split_at = " "
else:
raise NotImplementedError(
"DocumentSplitter only supports 'page', 'passage', 'sentence' or 'word' split_by options."
"DocumentSplitter only supports 'word', 'sentence', 'passage' or 'page' split_by options."
)
units = text.split(split_at)
# Add the delimiter back to all units except the last one
Expand Down
14 changes: 14 additions & 0 deletions test/components/preprocessors/test_document_splitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,20 @@ def test_split_by_passage(self):
assert result["documents"][1].content == "And there is a third sentence.\n\n"
assert result["documents"][2].content == " And another passage."

def test_split_by_page(self):
splitter = DocumentSplitter(split_by="page", split_length=1)
result = splitter.run(
documents=[
Document(
content="This is a text with some words. There is a second sentence.\f And there is a third sentence.\f And another passage."
)
]
)
assert len(result["documents"]) == 3
assert result["documents"][0].content == "This is a text with some words. There is a second sentence.\x0c"
assert result["documents"][1].content == " And there is a third sentence.\x0c"
assert result["documents"][2].content == " And another passage."

def test_split_by_word_with_overlap(self):
splitter = DocumentSplitter(split_by="word", split_length=10, split_overlap=2)
result = splitter.run(
Expand Down