Merge branch 'master' into 521-fix-binary-service-paths

src/install/common.py

@@ -1,5 +1,4 @@
 import logging
-import os
 from contextlib import suppress
 from pathlib import Path
 
@@ -10,6 +9,8 @@
  dnf_install_packages, dnf_remove_packages, dnf_update_sources, install_github_project, pip3_install_packages
 )
 
+BIN_DIR = Path(__file__).parent.parent / 'bin'
+
 
 def install_pip(python_command):
  logging.info('Installing {} pip'.format(python_command))
@@ -39,8 +40,7 @@ def main(distribution): # pylint: disable=too-many-statements
  logging.warning('FACT is not set up using git. Note that *adding submodules* won\'t work!!')
 
  # make bin dir
- with suppress(FileExistsError):
- os.mkdir('../bin')
+ BIN_DIR.mkdir(exist_ok=True)
 
  if distribution == 'fedora':
  dnf_install_packages('python3')
@@ -70,15 +70,6 @@ def main(distribution): # pylint: disable=too-many-statements
  install_pip('python3')
  pip3_install_packages('setuptools==49.6.0')
 
- if distribution == 'fedora':
- pass
- else:
- # install python2
- apt_install_packages('python', 'python-dev')
- with suppress(InstallationError):
- apt_remove_packages('python-pip')
- install_pip('python2')
-
  if distribution == 'fedora':
  dnf_install_packages('file-devel')
  dnf_install_packages('libffi-devel')
@@ -95,15 +86,15 @@ def main(distribution): # pylint: disable=too-many-statements
  pip3_install_packages('psutil')
  pip3_install_packages('pytest==6.1.2', 'pytest-cov', 'pylint', 'python-magic', 'xmltodict', 'yara-python==3.7.0', 'appdirs')
 
- pip3_install_packages('lief')
+ pip3_install_packages('lief==0.10.1') # FIXME: unpin version when install bug is fixed
 
  pip3_install_packages('requests')
 
  # install python MongoDB bindings
  pip3_install_packages('pymongo', 'pyyaml')
 
  # VarietyJS (is executed by update_statistic.py)
- if Path('../bin/spec').exists():
+ if (BIN_DIR / 'spec').exists():
  logging.warning('variety spec not overwritten')
  else:
  install_github_project('variety/variety', ['git checkout 2f4d815', 'mv -f variety.js ../../bin', 'mv -f spec ../../bin'])

src/plugins/analysis/cwe_checker/code/cwe_checker.py

@@ -32,61 +32,50 @@ class AnalysisPlugin(AnalysisBasePlugin):
  NAME = 'cwe_checker'
  DESCRIPTION = 'This plugin checks ELF binaries for several CWEs (Common Weakness Enumeration) like'\
  'CWE-243 (Creation of chroot Jail Without Changing Working Directory) and'\
- 'CWE-676 (Use of Potentially Dangerous Function). Internally it uses BAP, which currently supports ARM, x86/x64, PPC and MIPS.'\
+ 'CWE-676 (Use of Potentially Dangerous Function).'\
  'Due to the nature of static analysis, this plugin may run for a long time.'
  DEPENDENCIES = ['cpu_architecture', 'file_type']
- VERSION = '0.4.0'
+ VERSION = '0.5.0'
  MIME_WHITELIST = ['application/x-executable', 'application/x-object', 'application/x-sharedlib']
  SUPPORTED_ARCHS = ['arm', 'x86', 'x64', 'mips', 'ppc']
 
  def __init__(self, plugin_administrator, config=None, recursive=True, timeout=TIMEOUT_IN_SECONDS + 30):
  self.config = config
  if not self._check_docker_installed():
  raise RuntimeError('Docker is not installed.')
- self._module_versions = self._get_module_versions()
- logging.info('Module versions are {}'.format(str(self._module_versions)))
+ self._log_version_string()
  super().__init__(plugin_administrator, config=config, plugin_path=__file__, recursive=recursive, timeout=timeout)
 
  @staticmethod
  def _check_docker_installed():
  _, return_code = execute_shell_command_get_return_code('docker -v')
  return return_code == 0
 
- def _get_module_versions(self):
- output = self._run_cwe_checker_to_get_module_versions()
+ def _log_version_string(self):
+ output = self._run_cwe_checker_to_get_version_string()
  if output is None:
- logging.error('Could not get module versions from Bap plugin.')
- return {}
- return self._parse_module_versions(output)
-
- @staticmethod
- def _parse_module_versions(bap_output):
- module_versions = {}
- for line in bap_output.splitlines():
- if 'module_versions:' in line:
- version_json = line.split('module_versions:')[-1].strip()
- module_versions = json.loads(version_json)
- return module_versions
+ logging.error('Could not get version string from cwe_checker.')
+ else:
+ logging.info('Version is {}'.format(str(output)))
+ return output
 
  @staticmethod
- def _run_cwe_checker_to_get_module_versions():
- # unfortunately, there must be a dummy file passed to BAP, I chose /bin/true because it is damn small
+ def _run_cwe_checker_to_get_version_string():
  return run_docker_container(DOCKER_IMAGE, timeout=60,
- command='bap /bin/true --pass=cwe-checker --cwe-checker-module-versions')
+ command='--version')
 
  @staticmethod
  def _run_cwe_checker_in_docker(file_object):
  return run_docker_container(DOCKER_IMAGE, timeout=TIMEOUT_IN_SECONDS,
- command='bap /tmp/input --pass=cwe-checker --cwe-checker-json --cwe-checker-no-logging',
- mount=('/tmp/input', file_object.file_path))
+ command='/input --json --quiet',
+ mount=('/input', file_object.file_path))
 
  @staticmethod
- def _parse_bap_output(output):
+ def _parse_cwe_checker_output(output):
  tmp = defaultdict(list)
  j_doc = json.loads(output)
- if 'warnings' in j_doc:
- for warning in j_doc['warnings']:
- tmp[warning['name']] = tmp[warning['name']] + [warning, ]
+ for warning in j_doc:
+ tmp[warning['name']] = tmp[warning['name']] + [warning, ]
 
  res = {}
  for key, values in tmp.items():
@@ -109,7 +98,7 @@ def _do_full_analysis(self, file_object):
  output = self._run_cwe_checker_in_docker(file_object)
  if output is not None:
  try:
- cwe_messages = self._parse_bap_output(output)
+ cwe_messages = self._parse_cwe_checker_output(output)
  file_object.processed_analysis[self.NAME] = {'full': cwe_messages, 'summary': list(cwe_messages.keys())}
  except json.JSONDecodeError:
  logging.error('cwe_checker execution failed: {}\nUID: {}'.format(output, file_object.uid))
@@ -121,8 +110,8 @@ def _do_full_analysis(self, file_object):
 
  def process_object(self, file_object):
  '''
- This function handles only ELF executable. Otherwise it returns an empty dictionary.
- It calls the external BAP plugin cwe_checker.
+ This function handles only ELF executables. Otherwise it returns an empty dictionary.
+ It calls the cwe_checker docker container.
  '''
  if not self._is_supported_arch(file_object):
  logging.debug('{}\'s arch is not supported ({})'.format(

src/plugins/analysis/cwe_checker/test/test_cwe_checker.py

@@ -10,47 +10,37 @@ class TestCweCheckerFunctions(AnalysisPluginTest):
  def setUp(self):
  super().setUp()
  config = self.init_basic_config()
- # TODO: Mock calls to BAP
- AnalysisPlugin._get_module_versions = lambda self: {}
+ # TODO: Mock calls to cwe_checker
  self.analysis_plugin = AnalysisPlugin(self, config=config)
 
- def test_parse_module_version(self):
- data = 'INFO: [cwe_checker] module_versions: {"CWE190": "0.1", "CWE215": "0.1", "CWE243": "0.1", "CWE248": "0.1", "CWE332": "0.1", "CWE367": "0.1", "CWE426": "0.1", "CWE457": "0.1", "CWE467": "0.1", "CWE476": "0.2", "CWE560": "0.1", "CWE676": "0.1", "CWE782": "0.1"}'
- expected_result = {'CWE190': '0.1',
- 'CWE215': '0.1',
- 'CWE243': '0.1',
- 'CWE248': '0.1',
- 'CWE332': '0.1',
- 'CWE367': '0.1',
- 'CWE426': '0.1',
- 'CWE457': '0.1',
- 'CWE467': '0.1',
- 'CWE476': '0.2',
- 'CWE560': '0.1',
- 'CWE676': '0.1',
- 'CWE782': '0.1'}
- res = self.analysis_plugin._parse_module_versions(data)
- self.assertEqual(res, expected_result)
-
- def test_parse_bap_output(self):
- test_data = """{
- "binary": "test/artificial_samples/build/cwe_190_x86_gcc.out",
- "time": 1564489060.0,
- "warnings": [
- {
- "name": "CWE190",
- "version": "0.1",
- "addresses": [ "0x6BC:32u" ],
- "symbols": [ "malloc" ],
- "other": [],
- "description":
- "(Integer Overflow or Wraparound) Potential overflow due to multiplication at 0x6BC:32u (malloc)"
- }]}"""
- result = self.analysis_plugin._parse_bap_output(test_data)
+ def test_parse_cwe_checker_output(self):
+ test_data = """[
+ {
+ "name": "CWE676",
+ "version": "0.1",
+ "addresses": [
+ "00103042"
+ ],
+ "tids": [
+ "instr_00103042_2"
+ ],
+ "symbols": [
+ "FUN_00102ef0"
+ ],
+ "other": [
+ [
+ "dangerous_function",
+ "strlen"
+ ]
+ ],
+ "description": "(Use of Potentially Dangerous Function) FUN_00102ef0 (00103042) -> strlen"
+ }
+ ]"""
+ result = self.analysis_plugin._parse_cwe_checker_output(test_data)
  print(result)
  assert isinstance(result, dict)
  assert len(result.keys()) == 1
- assert isinstance(result['CWE190'], dict)
+ assert isinstance(result['CWE676'], dict)
 
  def test_is_supported_arch(self):
  fo = FileObject()

src/plugins/analysis/software_components/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM fkiecad/ghidra_headless_base:9.1
+FROM fkiecad/ghidra_headless_base:9.2.1
 
 
 VOLUME ["/work"]
@@ -7,10 +7,9 @@ WORKDIR /work
 ADD ghidra_scripts /ghidra_scripts
 
 ENTRYPOINT [ \
- "/ghidra/ghidra_9.1_PUBLIC/support/analyzeHeadless", \
- "/work", "project_foo", \
- "-scriptPath", "/ghidra_scripts", \
- "-postScript", "format_string_version.py", \
+ "analyzeHeadless", \
+ "/tmp", "temporary_project", \
+ "-postScript", "/ghidra_scripts/format_string_version.py", \
  "-deleteProject", \
  "-import", "ghidra_input" \
 ]