forked from deepset-ai/haystack-website
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* new docs version * add version manually for testing * add more information to read.me * documentstores as dropdown * dorpdown for pros and cons documentstore * fix link * missing ) * point again to haystack master
- Loading branch information
Showing
44 changed files
with
5,381 additions
and
455 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,204 @@ | ||
| { | ||
| "chart_type": "LineChart", | ||
| "title": "Retriever Accuracy", | ||
| "subtitle": "mAP at different number of docs", | ||
| "description": "Here you can see how the mean avg. precision (mAP) of the retriever decays as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.", | ||
| "columns": [ | ||
| "n_docs", | ||
| "BM25 / Elasticsearch", | ||
| "DPR / Elasticsearch", | ||
| "DPR / FAISS (flat)", | ||
| "DPR / FAISS (HNSW)", | ||
| "DPR / Milvus (flat)", | ||
| "DPR / Milvus (HNSW)", | ||
| "Sentence Transformers / Elasticsearch" | ||
| ], | ||
| "axis": [ | ||
| { | ||
| "x": "Number of docs", | ||
| "y": "mAP" | ||
| } | ||
| ], | ||
| "data": [ | ||
| { | ||
| "model": "DPR / Elasticsearch", | ||
| "n_docs": 1000, | ||
| "map": 92.95105322830891 | ||
| }, | ||
| { | ||
| "model": "DPR / Elasticsearch", | ||
| "n_docs": 10000, | ||
| "map": 89.87097014904354 | ||
| }, | ||
| { | ||
| "model": "BM25 / Elasticsearch", | ||
| "n_docs": 10000, | ||
| "map": 66.26543444531747 | ||
| }, | ||
| { | ||
| "model": "Sentence Transformers / Elasticsearch", | ||
| "n_docs": 1000, | ||
| "map": 90.06638620360428 | ||
| }, | ||
| { | ||
| "model": "Sentence Transformers / Elasticsearch", | ||
| "n_docs": 10000, | ||
| "map": 87.11255142468549 | ||
| }, | ||
| { | ||
| "model": "DPR / FAISS (flat)", | ||
| "n_docs": 1000, | ||
| "map": 92.95105322830891 | ||
| }, | ||
| { | ||
| "model": "DPR / FAISS (flat)", | ||
| "n_docs": 10000, | ||
| "map": 89.87097014904354 | ||
| }, | ||
| { | ||
| "model": "DPR / FAISS (HNSW)", | ||
| "n_docs": 1000, | ||
| "map": 92.95105322830891 | ||
| }, | ||
| { | ||
| "model": "DPR / FAISS (HNSW)", | ||
| "n_docs": 10000, | ||
| "map": 89.51337675393017 | ||
| }, | ||
| { | ||
| "model": "DPR / Milvus (flat)", | ||
| "n_docs": 1000, | ||
| "map": 92.95105322830891 | ||
| }, | ||
| { | ||
| "model": "DPR / Milvus (flat)", | ||
| "n_docs": 10000, | ||
| "map": 89.87097014904354 | ||
| }, | ||
| { | ||
| "model": "DPR / Milvus (HNSW)", | ||
| "n_docs": 1000, | ||
| "map": 92.95105322830891 | ||
| }, | ||
| { | ||
| "model": "DPR / Milvus (HNSW)", | ||
| "n_docs": 10000, | ||
| "map": 88.24421129104469 | ||
| }, | ||
| { | ||
| "model": "DPR / Elasticsearch", | ||
| "n_docs": 100000, | ||
| "map": 86.54606328368976 | ||
| }, | ||
| { | ||
| "model": "DPR / Elasticsearch", | ||
| "n_docs": 500000, | ||
| "map": 80.86137228234091 | ||
| }, | ||
| { | ||
| "model": "BM25 / Elasticsearch", | ||
| "n_docs": 100000, | ||
| "map": 56.25299537353825 | ||
| }, | ||
| { | ||
| "model": "BM25 / Elasticsearch", | ||
| "n_docs": 500000, | ||
| "map": 45.595090262466535 | ||
| }, | ||
| { | ||
| "model": "Sentence Transformers / Elasticsearch", | ||
| "n_docs": 100000, | ||
| "map": 82.74686664920836 | ||
| }, | ||
| { | ||
| "model": "Sentence Transformers / Elasticsearch", | ||
| "n_docs": 500000, | ||
| "map": 76.49564526892904 | ||
| }, | ||
| { | ||
| "model": "DPR / FAISS (flat)", | ||
| "n_docs": 100000, | ||
| "map": 86.54606328368973 | ||
| }, | ||
| { | ||
| "model": "DPR / FAISS (flat)", | ||
| "n_docs": 500000, | ||
| "map": 80.86137228234091 | ||
| }, | ||
| { | ||
| "model": "DPR / FAISS (HNSW)", | ||
| "n_docs": 100000, | ||
| "map": 84.33419639513305 | ||
| }, | ||
| { | ||
| "model": "DPR / FAISS (HNSW)", | ||
| "n_docs": 500000, | ||
| "map": 75.73062475537202 | ||
| }, | ||
| { | ||
| "model": "DPR / Milvus (flat)", | ||
| "n_docs": 100000, | ||
| "map": 86.54606328368973 | ||
| }, | ||
| { | ||
| "model": "DPR / Milvus (flat)", | ||
| "n_docs": 500000, | ||
| "map": 80.86137228234091 | ||
| }, | ||
| { | ||
| "model": "DPR / Milvus (HNSW)", | ||
| "n_docs": 100000, | ||
| "map": 81.63864883662649 | ||
| }, | ||
| { | ||
| "model": "DPR / Milvus (HNSW)", | ||
| "n_docs": 500000, | ||
| "map": 73.57986207906387 | ||
| }, | ||
| { | ||
| "model": "BM25 / Elasticsearch", | ||
| "n_docs": 1000, | ||
| "map": 74.20444712972909 | ||
| }, | ||
| { | ||
| "model": "DPR / OpenSearch (flat)", | ||
| "n_docs": 1000, | ||
| "map": 92.95105322830891 | ||
| }, | ||
| { | ||
| "model": "DPR / OpenSearch (flat)", | ||
| "n_docs": 10000, | ||
| "map": 89.8709701490436 | ||
| }, | ||
| { | ||
| "model": "DPR / OpenSearch (flat)", | ||
| "n_docs": 100000, | ||
| "map": 86.54014997282701 | ||
| }, | ||
| { | ||
| "model": "DPR / OpenSearch (HNSW)", | ||
| "n_docs": 1000, | ||
| "map": 92.76308330349686 | ||
| }, | ||
| { | ||
| "model": "DPR / OpenSearch (HNSW)", | ||
| "n_docs": 10000, | ||
| "map": 89.00403653862938 | ||
| }, | ||
| { | ||
| "model": "DPR / OpenSearch (HNSW)", | ||
| "n_docs": 100000, | ||
| "map": 85.7342431384476 | ||
| }, | ||
| { | ||
| "model": "DPR / OpenSearch (flat)", | ||
| "n_docs": 500000, | ||
| "map": 80.85588135082547 | ||
| }, | ||
| { | ||
| "model": "DPR / OpenSearch (HNSW)", | ||
| "n_docs": 500000, | ||
| "map": 77.5426462347698 | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| { | ||
| "chart_type": "BarChart", | ||
| "title": "Reader Performance", | ||
| "subtitle": "Time and Accuracy Benchmarks", | ||
| "description": "Performance benchmarks of different Readers that can be used off-the-shelf in Haystack. Some models are geared towards speed, while others are more performance-focused. Accuracy is measured as F1 score and speed as passages/sec (with passages of 384 tokens). Each Reader is benchmarked using the SQuAD v2.0 development set, which contains 11866 question answer pairs. When tokenized using the BERT tokenizer and split using a sliding window approach, these become 12350 passages that are passed into the model. We set <i>max_seq_len=384</i> and <i>doc_stride=128</i>. These benchmarking tests are run using an AWS p3.2xlarge instance with a Nvidia V100 GPU with this <a href='https://github.com/deepset-ai/haystack/blob/master/test/benchmarks/reader.py'>script</a>. Please note that we are using the FARMReader class rather than the TransformersReader class. Also, the F1 measure that is reported here is in fact calculated on token level, rather than word level as is done in the official SQuAD script.", | ||
| "bars": "horizontal", | ||
| "columns": [ | ||
| "Model", | ||
| "F1", | ||
| "Speed (passages/sec)" | ||
| ], | ||
| "data": [ | ||
| { | ||
| "F1": 82.58860575299658, | ||
| "Speed": 125.81040525892848, | ||
| "Model": "RoBERTa" | ||
| }, | ||
| { | ||
| "F1": 78.87858491007042, | ||
| "Speed": 260.6443097981493, | ||
| "Model": "MiniLM" | ||
| }, | ||
| { | ||
| "F1": 74.31182400443286, | ||
| "Speed": 121.08066567525722, | ||
| "Model": "BERT base" | ||
| }, | ||
| { | ||
| "F1": 83.26306774734308, | ||
| "Speed": 42.21949937744112, | ||
| "Model": "BERT large" | ||
| }, | ||
| { | ||
| "F1": 84.50422699207468, | ||
| "Speed": 42.07400844838985, | ||
| "Model": "XLM-RoBERTa" | ||
| }, | ||
| { | ||
| "F1": 42.31925844723574, | ||
| "Speed": 222.91207128366702, | ||
| "Model": "DistilBERT" | ||
| } | ||
| ] | ||
| } |
Oops, something went wrong.