latest versions of docs via script

src/pages/benchmarks/versions/latest/site/en/map/retriever_map.json

@@ -20,82 +20,82 @@
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 1000,
- "map": 0.929
+ "map": 92.95105322830888
  },
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 10000,
- "map": 0.881
+ "map": 89.87097014904354
  },
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 100000,
- "map": 0.821
+ "map": 86.54564090434241
  },
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 500000,
- "map": 0.730
+ "map": 80.86137228234089
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 1000,
- "map": 0.929
+ "map": 74.20444712972909
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 10000,
- "map": 0.898
+ "map": 66.20627317806674
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 100000,
- "map": 0.863
+ "map": 56.25959153101251
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 500000,
- "map": 0.805
+ "map": 45.59452709000341
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 1000,
- "map": 0.748
+ "map": 92.95105322830888
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 10000,
- "map": 0.6609999999999999
+ "map": 89.87097014904354
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 100000,
- "map": 0.56
+ "map": 86.54606328368972
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 500000,
- "map": 0.452
+ "map": 80.8613722823409
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 1000,
- "map": 0.929
+ "map": 92.95105322830888
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 10000,
- "map": 0.896
+ "map": 89.69941373746582
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 100000,
- "map": 0.849
+ "map": 85.07984377595874
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 500000,
- "map": 0.766
+ "map": 76.91475821598232
  }
  ]
 }

src/pages/benchmarks/versions/latest/site/en/performance/reader_performance.json

@@ -11,33 +11,33 @@
  ],
  "data": [
  {
- "F1": 82.62983412843887,
- "Speed": 98.86638639776464,
+ "F1": 82.58860575299658,
+ "Speed": 125.81040525892848,
  "Model": "RoBERTa"
  },
  {
- "F1": 78.90026641413856,
- "Speed": 181.96379531485616,
+ "F1": 78.87858491007042,
+ "Speed": 260.6443097981493,
  "Model": "MiniLM"
  },
  {
- "F1": 74.32668866064459,
- "Speed": 106.04748306200683,
+ "F1": 74.31182400443286,
+ "Speed": 121.08066567525722,
  "Model": "BERT base"
  },
  {
- "F1": 83.29492827667042,
- "Speed": 40.408497243719076,
+ "F1": 83.26306774734308,
+ "Speed": 42.21949937744112,
  "Model": "BERT large"
  },
  {
- "F1": 84.62174414643722,
- "Speed": 40.483264542292716,
+ "F1": 84.50422699207468,
+ "Speed": 42.07400844838985,
  "Model": "XLM-RoBERTa"
  },
  {
- "F1": 42.342513261953935,
- "Speed": 160.41712955027901,
+ "F1": 42.31925844723574,
+ "Speed": 222.91207128366702,
  "Model": "DistilBERT"
  }
  ]

src/pages/benchmarks/versions/latest/site/en/performance/retriever_performance.json

@@ -2,7 +2,7 @@
  "chart_type": "BarChart",
  "title": "Retriever Performance",
  "subtitle": "Time and Accuracy Benchmarks",
- "description": "Comparison of the speed and accuracy of different DocumentStore / Retriever combinations on 100k documents. <b>Indexing speed</b> (in docs/sec) refers to how quickly Documents can be inserted into a DocumentStore. <b>Querying speed</b> (in queries/sec) refers to the speed at which the system returns relevant Documents when presented with a query.\n\nThe dataset used is Wikipedia, split into 100 word passages (from <a href='https://github.com/facebookresearch/DPR/blob/master/data/download_data.py'>here</a>)). \n\nFor querying, we use the Natural Questions development set in combination with the wiki passages. The Document Store is populated with the 100 word passages in which the answer spans occur (i.e. gold passages) as well as a random selection of 100 word passages in which the answer spans do not occur (i.e. negative passages). We take a total of 100k gold and negative passages. Query and document embedding are generated by the <i>\"facebook/dpr-question_encoder-single-nq-base\"</i> and <i>\"facebook/dpr-ctx_encoder-single-nq-base\"</i> models. The retriever returns 10 candidates and both the recall and mAP scores are calculated on these 10.\n\nFor FAISS HNSW, we use <i>n_links=128</i>, <i>efSearch=20</i> and <i>efConstruction=80</i>. Both index and query benchmarks are performed on an AWS P3.2xlarge instance which is accelerated by an Nvidia V100 GPU.",
+ "description": "Comparison of the speed and accuracy of different DocumentStore / Retriever combinations on 100k documents. <b>Indexing speed</b> (in docs/sec) refers to how quickly Documents can be inserted into a DocumentStore. <b>Querying speed</b> (in queries/sec) refers to the speed at which the system returns relevant Documents when presented with a query.\n\nThe dataset used is Wikipedia, split into 100 word passages (from <a href='https://github.com/facebookresearch/DPR/blob/master/data/download_data.py'>here</a>)). \n\nFor querying, we use the Natural Questions development set in combination with the wiki passages. The Document Store is populated with the 100 word passages in which the answer spans occur (i.e. gold passages) as well as a random selection of 100 word passages in which the answer spans do not occur (i.e. negative passages). We take a total of 100k gold and negative passages. Query and document embedding are generated by the <i>\"facebook/dpr-question_encoder-single-nq-base\"</i> and <i>\"facebook/dpr-ctx_encoder-single-nq-base\"</i> models. The retriever returns 10 candidates and both the recall and mAP scores are calculated on these 10.\n\nFor FAISS HNSW, we use <i>n_links=128</i>, <i>efSearch=20</i> and <i>efConstruction=80</i>. We use a cosine similarity function with BM25 retrievers, and dot product with DPR. Both index and query benchmarks are performed on an AWS P3.2xlarge instance which is accelerated by an Nvidia V100 GPU.",
  "bars": "horizontal",
  "columns": [
  "Model",
@@ -24,30 +24,30 @@
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 100000,
- "index_speed": 73.93635160290218,
- "query_speed": 6.23,
- "map": 82
+ "index_speed": 69.75508852811794,
+ "query_speed": 4.5992769354707805,
+ "map": 86.54564090434241
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 100000,
- "index_speed": 104.77116699738369,
- "query_speed": 4.89,
- "map": 86.3
+ "index_speed": 482.9993330442806,
+ "query_speed": 162.42378943468643,
+ "map": 56.25959153101251
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 100000,
- "index_speed": 484.32931514144724,
- "query_speed": 162.59,
- "map": 56
+ "index_speed": 95.52108545730724,
+ "query_speed": 6.511162294559942,
+ "map": 86.54606328368972
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 100000,
- "index_speed": 91.41086878008392,
- "query_speed": 12.85,
- "map": 84.9
+ "index_speed": 84.11829911061136,
+ "query_speed": 33.65729082116796,
+ "map": 85.07984377595874
  }
  ]
 }

src/pages/benchmarks/versions/latest/site/en/speed/retriever_speed.json

@@ -20,82 +20,82 @@
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 1000,
- "query_speed": 40.802
+ "query_speed": 30.68451185154913
  },
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 10000,
- "query_speed": 24.8
+ "query_speed": 19.568754413737462
  },
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 100000,
- "query_speed": 6.23
+ "query_speed": 4.5992769354707805
  },
  {
  "model": "DPR / ElasticSearch",
  "n_docs": 500000,
- "query_speed": 1.45
+ "query_speed": 1.0558140319761546
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 1000,
- "query_speed": 40.048
+ "query_speed": 262.9405144288997
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 10000,
- "query_speed": 22.47
+ "query_speed": 183.6070813438718
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 100000,
- "query_speed": 4.90
+ "query_speed": 162.42378943468643
  },
  {
- "model": "DPR / FAISS (flat)",
+ "model": "BM25 / ElasticSearch",
  "n_docs": 500000,
- "query_speed": 1.08
+ "query_speed": 82.43179203331141
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 1000,
- "query_speed": 232.97799999999998
+ "query_speed": 35.40380445859966
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 10000,
- "query_speed": 248.97
+ "query_speed": 25.78749025294445
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 100000,
- "query_speed": 162.59
+ "query_speed": 6.511162294559942
  },
  {
- "model": "BM25 / ElasticSearch",
+ "model": "DPR / FAISS (flat)",
  "n_docs": 500000,
- "query_speed": 91.39
+ "query_speed": 1.5161593755666505
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 1000,
- "query_speed": 37.884
+ "query_speed": 39.16414272911727
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 10000,
- "query_speed": 31.34
+ "query_speed": 33.6432023480111
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 100000,
- "query_speed": 12.85
+ "query_speed": 33.65729082116796
  },
  {
  "model": "DPR / FAISS (HSNW)",
  "n_docs": 500000,
- "query_speed": 3.32
+ "query_speed": 34.27671486454735
  }
  ]
 }