Benchmarks (#30)

* benchmarks chart

* add charts

* adding benchmarks to header

Co-authored-by: Tuana Celik <[email protected]>

config.toml

@@ -71,18 +71,24 @@ disableKinds = ["taxonomy", "term"]
  name = 'Tutorials'
  url = '/tutorials'
  weight = 3
+
+[[menu.main]]
+ identifier = 'benchmarks'
+ name = 'Benchmarks'
+ url = '/benchmarks'
+ weight = 4
 
 [[menu.main]]
  identifier = 'community'
  name = 'Community'
  url = '/community'
- weight = 4
+ weight = 5
 
 [[menu.main]]
  identifier = 'learn'
  name = 'Learn'
  url = '/'
- weight = 5
+ weight = 6
 
 # Overview children
 [[menu.main]]

content/pages/benchmarks.md

@@ -0,0 +1,6 @@
+---
+layout: benchmarks
+header: dark
+footer: dark
+title: Haystack Benchmarks
+---

data/benchmarks/reader_performance.json

@@ -0,0 +1,40 @@
+{
+ "chart_type": "BarChart",
+ "title": "Reader Performance",
+ "subtitle": "Time and Accuracy Benchmarks",
+ "description": "Performance benchmarks of different Readers that can be used off-the-shelf in Haystack. Some models are geared towards speed, while others are more performance-focused. Accuracy is measured as F1 score and speed as passages/sec (with passages of 384 tokens). Each Reader is benchmarked using the SQuAD v2.0 development set, which contains 11866 question answer pairs. When tokenized using the BERT tokenizer and split using a sliding window approach, these become 12350 passages that are passed into the model. We set <i>max_seq_len=384</i> and <i>doc_stride=128</i>. These benchmarking tests are run using an AWS p3.2xlarge instance with a Nvidia V100 GPU with this <a href='https://github.com/deepset-ai/haystack/blob/master/test/benchmarks/reader.py'>script</a>. Please note that we are using the FARMReader class rather than the TransformersReader class. Also, the F1 measure that is reported here is in fact calculated on token level, rather than word level as is done in the official SQuAD script.",
+ "bars": "horizontal",
+ "columns": ["Model", "F1", "Speed (passages/sec)"],
+ "data": [
+ {
+ "F1": 78.21542223321491,
+ "Speed": 118.41880460779026,
+ "Model": "RoBERTa"
+ },
+ {
+ "F1": 79.6479775678566,
+ "Speed": 235.0263836060577,
+ "Model": "MiniLM"
+ },
+ {
+ "F1": 73.49806294434092,
+ "Speed": 116.99833085428466,
+ "Model": "BERT base"
+ },
+ {
+ "F1": 87.35038229973254,
+ "Speed": 43.45399164221017,
+ "Model": "RoBERTa large"
+ },
+ {
+ "F1": 82.29388851439239,
+ "Speed": 118.88131118898794,
+ "Model": "RoBERTa distilled"
+ },
+ {
+ "F1": 43.34028303155876,
+ "Speed": 198.4500591997352,
+ "Model": "distilBERT"
+ }
+ ]
+}

data/benchmarks/retriever_map.json

@@ -0,0 +1,185 @@
+{
+ "chart_type": "LineChart",
+ "title": "Retriever Accuracy",
+ "subtitle": "mAP at different number of docs",
+ "description": "Here you can see how the mean avg. precision (mAP) of the retriever decays as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.",
+ "columns": [
+ "n_docs",
+ "BM25 / Elasticsearch",
+ "DPR / Elasticsearch",
+ "DPR / FAISS (flat)",
+ "DPR / FAISS (HNSW)",
+ "BM25 / OpenSearch (flat)",
+ "DPR / OpenSearch (flat)",
+ "DPR / OpenSearch (HNSW)",
+ "Sentence Transformers / Elasticsearch"
+ ],
+ "axis": [
+ {
+ "x": "Number of docs",
+ "y": "mAP"
+ }
+ ],
+ "data": [
+ {
+ "model": "Sentence Transformers / Elasticsearch",
+ "n_docs": 500000,
+ "map": 76.49564526892908
+ },
+ {
+ "model": "DPR / Elasticsearch",
+ "n_docs": 500000,
+ "map": 80.86137228234092
+ },
+ {
+ "model": "Sentence Transformers / Elasticsearch",
+ "n_docs": 1000,
+ "map": 90.06638620360428
+ },
+ {
+ "model": "Sentence Transformers / Elasticsearch",
+ "n_docs": 10000,
+ "map": 87.11255142468552
+ },
+ {
+ "model": "DPR / Elasticsearch",
+ "n_docs": 100000,
+ "map": 86.5456409043424
+ },
+ {
+ "model": "BM25 / Elasticsearch",
+ "n_docs": 1000,
+ "map": 74.2044471297291
+ },
+ {
+ "model": "BM25 / Elasticsearch",
+ "n_docs": 500000,
+ "map": 45.603960228760656
+ },
+ {
+ "model": "BM25 / Elasticsearch",
+ "n_docs": 100000,
+ "map": 56.26106985872818
+ },
+ {
+ "model": "BM25 / Elasticsearch",
+ "n_docs": 10000,
+ "map": 66.32941083712781
+ },
+ {
+ "model": "DPR / Elasticsearch",
+ "n_docs": 1000,
+ "map": 92.9510532283089
+ },
+ {
+ "model": "DPR / Elasticsearch",
+ "n_docs": 10000,
+ "map": 89.87097014904354
+ },
+ {
+ "model": "Sentence Transformers / Elasticsearch",
+ "n_docs": 100000,
+ "map": 82.74686664920834
+ },
+ {
+ "model": "DPR / FAISS (flat)",
+ "n_docs": 500000,
+ "map": 80.86137228234091
+ },
+ {
+ "model": "DPR / FAISS (flat)",
+ "n_docs": 10000,
+ "map": 89.87097014904354
+ },
+ {
+ "model": "DPR / FAISS (flat)",
+ "n_docs": 1000,
+ "map": 92.9510532283089
+ },
+ {
+ "model": "DPR / FAISS (flat)",
+ "n_docs": 100000,
+ "map": 86.54606328368972
+ },
+ {
+ "model": "DPR / FAISS (HNSW)",
+ "n_docs": 100000,
+ "map": 84.33419639513305
+ },
+ {
+ "model": "DPR / FAISS (HNSW)",
+ "n_docs": 10000,
+ "map": 89.51337675393017
+ },
+ {
+ "model": "DPR / FAISS (HNSW)",
+ "n_docs": 1000,
+ "map": 92.9510532283089
+ },
+ {
+ "model": "DPR / FAISS (HNSW)",
+ "n_docs": 500000,
+ "map": 75.73315903145605
+ },
+ {
+ "model": "BM25 / OpenSearch (flat)",
+ "n_docs": 10000,
+ "map": 66.32941083712774
+ },
+ {
+ "model": "DPR / OpenSearch (flat)",
+ "n_docs": 10000,
+ "map": 89.87097014904354
+ },
+ {
+ "model": "BM25 / OpenSearch (flat)",
+ "n_docs": 100000,
+ "map": 56.26106985872818
+ },
+ {
+ "model": "BM25 / OpenSearch (flat)",
+ "n_docs": 1000,
+ "map": 74.2044471297291
+ },
+ {
+ "model": "DPR / OpenSearch (flat)",
+ "n_docs": 100000,
+ "map": 86.51945338480733
+ },
+ {
+ "model": "BM25 / OpenSearch (flat)",
+ "n_docs": 500000,
+ "map": 45.59509026246651
+ },
+ {
+ "model": "DPR / OpenSearch (flat)",
+ "n_docs": 500000,
+ "map": 80.861372282341
+ },
+ {
+ "model": "DPR / OpenSearch (flat)",
+ "n_docs": 1000,
+ "map": 92.9510532283089
+ },
+ {
+ "model": "DPR / OpenSearch (HNSW)",
+ "n_docs": 10000,
+ "map": 89.87097014904357
+ },
+ {
+ "model": "DPR / OpenSearch (HNSW)",
+ "n_docs": 1000,
+ "map": 92.9510532283089
+ },
+ {
+ "model": "DPR / OpenSearch (HNSW)",
+ "n_docs": 100000,
+ "map": 86.53719331739565
+ },
+ {
+ "model": "DPR / OpenSearch (HNSW)",
+ "n_docs": 500000,
+ "map": 80.861372282341
+ }
+ ]
+}

data/benchmarks/retriever_performance.json

@@ -0,0 +1,81 @@
+{
+ "chart_type": "BarChart",
+ "title": "Retriever Performance",
+ "subtitle": "Time and Accuracy Benchmarks",
+ "description": "Comparison of the speed and accuracy of different DocumentStore / Retriever combinations on 100k documents. <b>Indexing speed</b> (in docs/sec) refers to how quickly Documents can be inserted into a DocumentStore. <b>Querying speed</b> (in queries/sec) refers to the speed at which the system returns relevant Documents when presented with a query.\n\nThe dataset used is Wikipedia, split into 100 word passages (from <a href='https://github.com/facebookresearch/DPR/blob/master/dpr/data/download_data.py'>here</a>)). \n\nFor querying, we use the Natural Questions development set in combination with the wiki passages. The Document Store is populated with the 100 word passages in which the answer spans occur (i.e. gold passages) as well as a random selection of 100 word passages in which the answer spans do not occur (i.e. negative passages). We take a total of 100k gold and negative passages. Query and document embedding are generated by the <i>\"facebook/dpr-question_encoder-single-nq-base\"</i> and <i>\"facebook/dpr-ctx_encoder-single-nq-base\"</i> models. The retriever returns 10 candidates and both the recall and mAP scores are calculated on these 10.\n\nFor FAISS HNSW, we use <i>n_links=128</i>, <i>efSearch=20</i> and <i>efConstruction=80</i>. We use a cosine similarity function with BM25 retrievers, and dot product with DPR. Both index and query benchmarks are performed on an AWS P3.2xlarge instance which is accelerated by an Nvidia V100 GPU.",
+ "bars": "horizontal",
+ "columns": [
+ "Model",
+ "mAP",
+ "Index Speed (docs/sec)",
+ "Query Speed (queries/sec)"
+ ],
+ "series": {
+ "s0": "map",
+ "s1": "time",
+ "s2": "time"
+ },
+ "axes": {
+ "label": "map",
+ "time_side": "top",
+ "time_label": "seconds"
+ },
+ "data": [
+ {
+ "model": "DPR / Elasticsearch",
+ "n_docs": 100000,
+ "index_speed": 81.2504988059173,
+ "query_speed": 6.239008635095737,
+ "map": 86.5456409043424
+ },
+ {
+ "model": "BM25 / Elasticsearch",
+ "n_docs": 100000,
+ "index_speed": 483.4698964792531,
+ "query_speed": 132.02605052193084,
+ "map": 56.26106985872818
+ },
+ {
+ "model": "Sentence Transformers / Elasticsearch",
+ "n_docs": 100000,
+ "index_speed": 120.24128813537725,
+ "query_speed": 6.358106794217024,
+ "map": 82.74686664920834
+ },
+ {
+ "model": "DPR / FAISS (flat)",
+ "n_docs": 100000,
+ "index_speed": 93.59054930302146,
+ "query_speed": 6.700916380795566,
+ "map": 86.54606328368972
+ },
+ {
+ "model": "DPR / FAISS (HNSW)",
+ "n_docs": 100000,
+ "index_speed": 83.61910261947709,
+ "query_speed": 40.6580803190388,
+ "map": 84.33419639513305
+ },
+ {
+ "model": "BM25 / OpenSearch (flat)",
+ "n_docs": 100000,
+ "index_speed": 478.7131568904452,
+ "query_speed": 120.12526583899732,
+ "map": 56.26106985872818
+ },
+ {
+ "model": "DPR / OpenSearch (flat)",
+ "n_docs": 100000,
+ "index_speed": 80.32081169498916,
+ "query_speed": 17.604366772353018,
+ "map": 86.51945338480733
+ },
+ {
+ "model": "DPR / OpenSearch (HNSW)",
+ "n_docs": 100000,
+ "index_speed": 80.45309468360983,
+ "query_speed": 17.226750993345046,
+ "map": 86.53719331739565
+ }
+ ]
+}