Merge pull request fkie-cad#746 from fkie-cad/tag-search

Add tag searching to basic search

src/install/frontend.py

@@ -128,6 +128,7 @@ def _install_css_and_js_files():
  'https://cdnjs.cloudflare.com/ajax/libs/bootstrap-datepicker/1.8.0/css/bootstrap-datepicker.standalone.css',
  'https://unpkg.com/[email protected]/styles/vis-network.min.css',
  'https://cdn.jsdelivr.net/npm/diff2html/bundles/css/diff2html.min.css',
+ 'https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap-select.min.css',
  ]:
  wget_static_web_content(css_url, 'web_css', [])
 
@@ -139,6 +140,7 @@ def _install_css_and_js_files():
  'https://raw.githubusercontent.com/moment/moment/develop/moment.js',
  'https://unpkg.com/[email protected]/standalone/umd/vis-network.min.js',
  'https://cdn.jsdelivr.net/npm/diff2html/bundles/js/diff2html-ui.min.js',
+ 'https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap-select.min.js',
  ]:
  wget_static_web_content(js_url, 'web_js', [])
 

src/storage/db_interface_frontend.py

@@ -92,6 +92,16 @@ def get_device_class_list(self):
  def get_vendor_list(self):
  return self.get_firmware_attribute_list('vendor')
 
+ def get_tag_list(self) -> List[str]:
+ query = self.firmwares.aggregate([
+ {'$project': {'tags': {'$objectToArray': '$tags'}}},
+ {'$unwind': '$tags'},
+ {'$group': {'_id': None, 'tag_set': {'$addToSet': '$tags.k'}}},
+ ], allowDiskUse=True)
+ for entry in query: # there should be only one result in this query
+ return sorted(entry['tag_set'])
+ return [] # no tags exist (yet)
+
  def get_device_name_dict(self):
  device_name_dict = {}
  query = self.firmwares.find()

src/test/integration/storage/test_db_interface_frontend.py

@@ -90,6 +90,20 @@ def test_get_firmware_attribute_list(self):
  self.assertEqual(self.db_frontend_interface.get_firmware_attribute_list('version'), ['0.1'])
  self.assertEqual(self.db_frontend_interface.get_device_name_dict(), {'Router': {'test_vendor': ['test_router']}})
 
+ def test_get_tag_list(self):
+ assert self.db_frontend_interface.get_tag_list() == []
+
+ fw_1 = create_test_firmware()
+ fw_1.uid = 'fw_1'
+ fw_1.tags = {'foo': 'some_color', 'bar': 'some_color'}
+ self.db_backend_interface.add_firmware(fw_1)
+ fw_2 = create_test_firmware()
+ fw_1.uid = 'fw_2'
+ fw_2.tags = {'foo': 'some_color', 'test': 'some_color'}
+ self.db_backend_interface.add_firmware(fw_2)
+
+ assert self.db_frontend_interface.get_tag_list() == ['bar', 'foo', 'test']
+
  def test_get_data_for_nice_list(self):
  uid_list = [self.test_firmware.uid]
  self.db_backend_interface.add_firmware(self.test_firmware)

src/web_interface/components/database_routes.py

@@ -142,15 +142,27 @@ def _search_database(self, query, skip=0, limit=0, only_firmwares=False, inverte
 
  def _build_search_query(self):
  query = {}
- if request.form['device_class_dropdown']:
- query.update({'device_class': request.form['device_class_dropdown']})
- for item in ['file_name', 'vendor', 'device_name', 'version', 'release_date']:
+ for item in ['device_class', 'vendor']:
+ if item in request.form and request.form[item]:
+ self._add_multiple_choice(query, item)
+ for item in ['file_name', 'device_name', 'version', 'release_date']:
  if request.form[item]:
  query.update({item: {'$options': 'si', '$regex': request.form[item]}})
  if request.form['hash_value']:
  self._add_hash_query_to_query(query, request.form['hash_value'])
+ if 'tags' in request.form and request.form['tags']:
+ self._add_tag_query(query)
  return json.dumps(query)
 
+ @staticmethod
+ def _add_multiple_choice(query, key):
+ query[key] = {'$in': list(dict(request.form.lists())[key])}
+
+ @staticmethod
+ def _add_tag_query(query):
+ tag_query = [{f'tags.{tag}': 'secondary'} for tag in dict(request.form.lists())['tags']]
+ query.setdefault('$or', []).extend(tag_query)
+
  def _add_hash_query_to_query(self, query, value):
  hash_types = read_list_from_config(self._config, 'file_hashes', 'hashes')
  hash_query = [{f'processed_analysis.file_hashes.{hash_type}': value} for hash_type in hash_types]
@@ -168,7 +180,8 @@ def show_basic_search(self):
  with ConnectTo(FrontEndDbInterface, self._config) as connection:
  device_classes = connection.get_device_class_list()
  vendors = connection.get_vendor_list()
- return render_template('database/database_search.html', device_classes=device_classes, vendors=vendors)
+ tags = connection.get_tag_list()
+ return render_template('database/database_search.html', device_classes=device_classes, vendors=vendors, tag_list=tags)
 
  @roles_accepted(*PRIVILEGES['advanced_search'])
  @AppRoute('/database/advanced_search', POST)
@@ -266,7 +279,8 @@ def start_quick_search(self):
  query['$or'].extend([
  {'device_name': {'$options': 'si', '$regex': search_term}},
  {'vendor': {'$options': 'si', '$regex': search_term}},
- {'file_name': {'$options': 'si', '$regex': search_term}}
+ {'file_name': {'$options': 'si', '$regex': search_term}},
+ {f'tags.{search_term}': 'secondary'}
  ])
  query = json.dumps(query)
  return redirect(url_for('browse_database', query=query))

src/web_interface/templates/base.html

@@ -281,6 +281,5 @@ <h5 class="modal-title" id="feedbackModalLabel">Feedback Options</h5>
  </div>
  </div>
 </div>
-
 </body>
 </html>

src/web_interface/templates/database/database_search.html

@@ -2,8 +2,10 @@
 
 {% set active_page = "Database" %}
 
-
 {% block head %}
+ {# bootstrap-select import #}
+ <link rel="stylesheet" href="{{ url_for('static', filename='web_css/bootstrap-select.min.css') }}" />
+ <script type="text/javascript" src="{{ url_for('static', filename='web_js/bootstrap-select.min.js') }}"></script>
  <script type="text/javascript" src="{{ url_for('static', filename='js/loading.js') }}"></script>
 {% endblock %}
 
@@ -18,20 +20,18 @@ <h3 class="mb-3">Search Firmware Database</h3>
  <div class="form-row">
  <div class="form-group col-md-6">
  <label class="control-label" for="device_class">Device Class:</label>
- <select class="form-control" name='device_class_dropdown' id="device_class">
- <option value=''></option>
+ <select class="selectpicker" name='device_class' id="device_class" multiple>
  {% for class in device_classes | sort %}
-  <option value='{{ class }}'>{{ class }}</option>
+ <option value='{{ class }}'>{{ class }}</option>
  {% endfor %}
  </select>
  </div>
 
  <div class="form-group col-md-6">
- <label class="control-label" for="device_class">Vendor:</label>
- <select class="form-control" name='vendor' id="vendor">
- <option value=''></option>
+ <label class="control-label" for="vendor">Vendor:</label>
+ <select class="selectpicker" name='vendor' id="vendor" multiple>
  {% for vendor in vendors | sort %}
-  <option value='{{ vendor }}'>{{ vendor }}</option>
+ <option value='{{ vendor }}'>{{ vendor }}</option>
  {% endfor %}
  </select>
  </div>
@@ -51,7 +51,17 @@ <h3 class="mb-3">Search Firmware Database</h3>
  </div>
  {% endfor %}
 
- <button type="submit" value=submit class="btn btn-primary" id="input_submit" onclick='showImg()'>
+ <div class="form-group">
+ <label class="control-label" for="tags">Tags:</label>
+ <select class="selectpicker" name='tags' id="tags" multiple>
+ {% for tag in tag_list | sort %}
+ <option value='{{ tag }}'>{{ tag }}</option>
+ {% endfor %}
+ </select>
+ </div>
+
+
+ <button type="submit" value=submit class="btn btn-primary" id="input_submit" onclick="showImg()">
  <i class="fas fa-search"></i> Search
  </button>