Resolve Document unique id issue by using custom id calculation

haystack/components/converters/azure.py

@@ -1,4 +1,5 @@
 import copy
+import hashlib
 from collections import defaultdict
 from pathlib import Path
 from typing import Any, Dict, List, Literal, Optional, Union
@@ -284,7 +285,12 @@ def _convert_tables(self, result: AnalyzeResult, meta: Optional[Dict[str, Any]])
  table_meta["page"] = table.bounding_regions[0].page_number
 
  table_df = pd.DataFrame(columns=table_list[0], data=table_list[1:])
- converted_tables.append(Document(dataframe=table_df, meta=table_meta))
+
+ # Use custom ID for tables, as columns might not be unique and thus failing in the default ID generation
+ pd_hashes = pd.util.hash_pandas_object(table_df, index=True).values
+ data = f"{pd_hashes}{table_meta}"
+ doc_id = hashlib.sha256(data.encode()).hexdigest()
+ converted_tables.append(Document(id=doc_id, dataframe=table_df, meta=table_meta))
 
  return converted_tables
 

test/components/converters/test_azure_ocr_doc_converter.py

@@ -194,9 +194,6 @@ def result(self) -> AnalyzeResult:
  # assert docs[0].meta["preceding_context"] == ""
 
  @patch("haystack.utils.auth.EnvVarSecret.resolve_value")
- @pytest.mark.skip(
- reason="fails because of non-unique column names, azure_sample_pdf_3.json has duplicate column names"
- )
  def test_azure_converter_with_multicolumn_header_table(self, mock_resolve_value, test_files_path) -> None:
  mock_resolve_value.return_value = "test_api_key"
 
@@ -216,10 +213,10 @@ def result(self) -> AnalyzeResult:
  docs = out["documents"]
  assert len(docs) == 2
  assert docs[0].content_type == "table"
- assert docs[0].content.shape[0] == 1 # number of rows
- assert docs[0].content.shape[1] == 3 # number of columns
- assert list(docs[0].content.columns) == ["This is a subheader", "This is a subheader", "This is a subheader"]
- assert list(docs[0].content.iloc[0]) == ["Value 1", "Value 2", "Val 3"]
+ assert docs[0].dataframe.shape[0] == 1 # number of rows
+ assert docs[0].dataframe.shape[1] == 3 # number of columns
+ assert list(docs[0].dataframe.columns) == ["This is a subheader", "This is a subheader", "This is a subheader"]
+ assert list(docs[0].dataframe.iloc[0]) == ["Value 1", "Value 2", "Val 3"]
  assert (
  docs[0].meta["preceding_context"]
  == "Table 1. This is an example table with two multicolumn headers\nHeader 1"