Add tests that reflect the desired behabiour. This behabiour is inffe…

…red from the one it had on Haystack 1.x

Solve some minor bugs spotted by tests.

haystack/components/preprocessors/document_splitter.py

@@ -73,7 +73,9 @@ def run(self, documents: List[Document]):
  text_splits, splits_pages = self._concatenate_units(units, self.split_length, self.split_overlap)
  metadata = deepcopy(doc.meta)
  metadata["source_id"] = doc.id
- split_docs += self._create_docs_from_splits(text_splits=text_splits, splits_pages=splits_pages)
+ split_docs += self._create_docs_from_splits(
+ text_splits=text_splits, splits_pages=splits_pages, meta=metadata
+ )
  return {"documents": split_docs}
 
  def _split_into_units(self, text: str, split_by: Literal["word", "sentence", "passage", "page"]) -> List[str]:
@@ -128,6 +130,6 @@ def _create_docs_from_splits(self, text_splits: List[str], splits_pages: List[in
  for i, txt in enumerate(text_splits):
  meta = deepcopy(meta)
  doc = Document(content=txt, meta=meta)
- doc.meta["page"] = splits_pages[i]
+ doc.meta["page_number"] = splits_pages[i]
  documents.append(doc)
  return documents

test/components/preprocessors/test_document_splitter.py

@@ -141,3 +141,100 @@ def test_copy_metadata(self):
  for doc, split_doc in zip(documents, result["documents"]):
  assert doc.meta.items() <= split_doc.meta.items()
  assert split_doc.content == "Text."
+
+ def test_add_page_number_to_metadata_with_no_overlap(self):
+ # Check for Word split
+ splitter = DocumentSplitter(split_by="word", split_length=2)
+ doc1 = Document(content="This is some text.\f This text is on another page.")
+ doc2 = Document(content="This content has two.\f\f page brakes.")
+ result = splitter.run(documents=[doc1, doc2])
+
+ expected_pages = [1, 1, 2, 2, 2, 1, 1, 3]
+ for doc, p in zip(result["documents"], expected_pages):
+ assert doc.meta["page_number"] == p
+
+ # Check for Sentence split
+ splitter = DocumentSplitter(split_by="sentence", split_length=1)
+ doc1 = Document(content="This is some text.\f This text is on another page.")
+ doc2 = Document(content="This content has two.\f\f page brakes.")
+ result = splitter.run(documents=[doc1, doc2])
+
+ expected_pages = [1, 1, 1, 1]
+ for doc, p in zip(result["documents"], expected_pages):
+ assert doc.meta["page_number"] == p
+
+ # Check for Passage Split
+ splitter = DocumentSplitter(split_by="passage", split_length=1)
+ doc1 = Document(
+ content="This is a text with some words.\f There is a second sentence.\n\nAnd there is a third sentence.\n\nAnd more passages.\n\n\f And another passage."
+ )
+ result = splitter.run(documents=[doc1])
+
+ expected_pages = [1, 2, 2, 2]
+ for doc, p in zip(result["documents"], expected_pages):
+ assert doc.meta["page_number"] == p
+
+ # Check for Page Split
+ splitter = DocumentSplitter(split_by="page", split_length=1)
+ doc1 = Document(
+ content="This is a text with some words. There is a second sentence.\f And there is a third sentence.\f And another passage."
+ )
+ result = splitter.run(documents=[doc1])
+ expected_pages = [1, 2, 3]
+ for doc, p in zip(result["documents"], expected_pages):
+ assert doc.meta["page_number"] == p
+
+ splitter = DocumentSplitter(split_by="page", split_length=2)
+ doc1 = Document(
+ content="This is a text with some words. There is a second sentence.\f And there is a third sentence.\f And another passage."
+ )
+ result = splitter.run(documents=[doc1])
+ expected_pages = [1, 3]
+
+ for doc, p in zip(result["documents"], expected_pages):
+ assert doc.meta["page_number"] == p
+
+ def test_add_page_number_to_metadata_with_overlap(self):
+ # Check for Word Split
+ splitter = DocumentSplitter(split_by="word", split_length=3, split_overlap=1)
+ doc1 = Document(content="This is some text. And\f this text is on another page.")
+ doc2 = Document(content="This content has two.\f\f page brakes.")
+ result = splitter.run(documents=[doc1, doc2])
+
+ expected_pages = [1, 1, 1, 2, 2, 1, 1, 3]
+ for doc, p in zip(result["documents"], expected_pages):
+ print(doc.content, doc.meta, p)
+ assert doc.meta["page_number"] == p
+
+ # Check for Sentence Split
+ splitter = DocumentSplitter(split_by="sentence", split_length=2, split_overlap=1)
+ doc1 = Document(content="This is some text. And this is more text.\f This text is on another page. End.")
+ doc2 = Document(content="This content has two.\f\f page brakes. More text.")
+ result = splitter.run(documents=[doc1, doc2])
+
+ expected_pages = [1, 1, 1, 2, 1, 1]
+ for doc, p in zip(result["documents"], expected_pages):
+ print(doc.content, doc.meta, p)
+ assert doc.meta["page_number"] == p
+
+ # Check for Passage Split
+ splitter = DocumentSplitter(split_by="passage", split_length=2, split_overlap=1)
+ doc1 = Document(
+ content="This is a text with some words.\f There is a second sentence.\n\nAnd there is a third sentence.\n\nAnd more passages.\n\n\f And another passage."
+ )
+ result = splitter.run(documents=[doc1])
+
+ expected_pages = [1, 2, 2]
+ for doc, p in zip(result["documents"], expected_pages):
+ assert doc.meta["page_number"] == p
+
+ # Check for Page Split
+ splitter = DocumentSplitter(split_by="page", split_length=2, split_overlap=1)
+ doc1 = Document(
+ content="This is a text with some words. There is a second sentence.\f And there is a third sentence.\f And another passage."
+ )
+ result = splitter.run(documents=[doc1])
+ expected_pages = [1, 2, 3]
+
+ for doc, p in zip(result["documents"], expected_pages):
+ assert doc.meta["page_number"] == p