made cve_lookup installation more robust and verbose

src/plugins/analysis/cve_lookup/install.sh

@@ -6,7 +6,7 @@ echo "------------------------------------"
 echo " install cve lookup dependencies "
 echo "------------------------------------"
 
-sudo -EH pip3 install pyxdameraulevenshtein || exit 1
+sudo -EH pip3 install pyxdameraulevenshtein retry || exit 1
 
 #
 # setup_repository.py can be called with arguments specified by the user (call setup_repository.py -h for more info)
@@ -19,7 +19,7 @@ cd internal || exit
 if [ -e "cve_cpe.db" ]
 then
  echo "Updating existing database"
- python3 setup_repository.py --update true
+ python3 setup_repository.py --update
 else
  echo "Setting up database"
  python3 setup_repository.py

src/plugins/analysis/cve_lookup/internal/data_parsing.py

@@ -8,6 +8,8 @@
 from zipfile import ZipFile
 
 import requests
+from requests.exceptions import RequestException
+from retry import retry
 
 try:
  from ..internal.helper_functions import CveEntry, CveSummaryEntry, CveLookupException
@@ -28,14 +30,19 @@ def get_cve_links(url: str, selected_years: Optional[List[int]] = None) -> List[
 
 def process_url(download_url: str, path: str):
  try:
- request = requests.get(download_url, allow_redirects=True)
+ request = _retrieve_url(download_url)
  except requests.exceptions.RequestException:
- raise CveLookupException('URLs are invalid. URL format might have been changed or website might have moved.')
+ raise CveLookupException(f'URL {download_url} not found. URL might have changed.')
 
  zipped_data = ZipFile(BytesIO(request.content))
  zipped_data.extractall(path)
 
 
+@retry(RequestException, tries=3, delay=5, backoff=2)
+def _retrieve_url(download_url):
+ return requests.get(download_url, allow_redirects=True)
+
+
 def download_cve(download_path: str, years: Optional[List[int]] = None, update: bool = False):
  if update:
  process_url(CVE_URL.format('modified'), download_path)

src/plugins/analysis/cve_lookup/internal/helper_functions.py

@@ -2,7 +2,7 @@
 from typing import Dict, List, NamedTuple, Tuple
 
 CveEntry = NamedTuple('CveEntry', [('cve_id', str), ('impact', Dict[str, str]), ('cpe_list', List[Tuple[str, str, str, str, str]])])
-CveSummaryEntry = NamedTuple('CveSummaryEntry', [('cve_id', str), ('summary', str), ('impact', str)])
+CveSummaryEntry = NamedTuple('CveSummaryEntry', [('cve_id', str), ('summary', str), ('impact', dict)])
 
 
 def escape_special_characters(attribute: str) -> str:

src/plugins/analysis/cve_lookup/internal/setup_repository.py

@@ -12,14 +12,14 @@
 
 try:
  from ..internal import data_parsing as dp
- from ..internal.database_interface import DatabaseInterface, QUERIES
+ from ..internal.database_interface import DatabaseInterface, QUERIES, DB_PATH
  from ..internal.helper_functions import (
  CveEntry, CveSummaryEntry, replace_characters_and_wildcards, CveLookupException
  )
 except (ImportError, ValueError, SystemError):
  sys.path.append(str(Path(__file__).parent.parent / 'internal'))
  import data_parsing as dp
- from database_interface import DatabaseInterface, QUERIES
+ from database_interface import DatabaseInterface, QUERIES, DB_PATH
  from helper_functions import CveEntry, CveSummaryEntry, replace_characters_and_wildcards, CveLookupException
 
 CURRENT_YEAR = datetime.now().year
@@ -242,9 +242,8 @@ def setup_argparser():
  )
  parser.add_argument(
  '--update', '-u',
- help='Boolean which specifies if the DATABASE should be updated. Default: False',
- type=bool,
- default=False
+ help='specifies if the DATABASE should be updated. Default: False',
+ action='store_true'
  )
  parser.add_argument(
  '--years', '-y',
@@ -288,9 +287,11 @@ def main():
  init_repository(extraction_path, args.target, years=years)
  except CveLookupException as exception:
  logging.error(exception.message)
+ if not args.update:
+ Path(DB_PATH).unlink(missing_ok=True) # remove broken partial DB so that next install won't fail
  sys.exit(1)
-
- rmtree(extraction_path)
+ finally:
+  rmtree(extraction_path, ignore_errors=True)
 
 
 if __name__ == '__main__':