added elf executable stats

src/statistic/update.py

@@ -47,6 +47,7 @@ def update_all_stats(self):
         self.db.update_statistic('exploit_mitigations', self.get_exploit_mitigations_stats())
         self.db.update_statistic('known_vulnerabilities', self.get_known_vulnerabilities_stats())
         self.db.update_statistic('software_components', self.get_software_components_stats())
+        self.db.update_statistic('elf_executable', self.get_executable_stats())
         # should always be the last, because of the benchmark
         self.db.update_statistic('general', self.get_general_stats())
 
@@ -279,6 +280,24 @@ def get_architecture_stats(self):
         ]
         return {'cpu_architecture': self._count_occurrences(result)}
 
+    def get_executable_stats(self):
+        total = self.db.file_objects.count_documents({'processed_analysis.file_type.full': {'$regex': 'ELF.*executable'}})
+        stats = []
+        for label, query_match in [
+            ('big endian', 'ELF.*MSB.*executable'),
+            ('little endian', 'ELF.*LSB.*executable'),
+            ('stripped', 'ELF.*executable.*, stripped'),
+            ('not stripped', 'ELF.*executable.*, not stripped'),
+            ('32-bit', 'ELF 32-bit.*executable'),
+            ('64-bit', 'ELF 64-bit.*executable'),
+            ('dynamically linked', 'ELF.*executable.*dynamically linked'),
+            ('statically linked', 'ELF.*executable.*statically linked'),
+            ('section info missing', 'ELF.*executable.*section header'),
+        ]:
+            count = self.db.file_objects.count_documents({'processed_analysis.file_type.full': {'$regex': query_match}})
+            stats.append((label, count, count / (total if total else 1), query_match))
+        return {'executable_stats': stats}
+
     def _find_most_frequent_architecture(self, arch_list):
         try:
             arch_frequency = sorted(self._count_occurrences(arch_list), key=lambda x: x[1], reverse=True)

src/test/integration/statistic/test_update.py

@@ -3,12 +3,13 @@
 import unittest
 
 from statistic.update import StatisticUpdater
+from storage.db_interface_backend import BackEndDbInterface
 from storage.db_interface_statistic import StatisticDbViewer
 from storage.MongoMgr import MongoMgr
-from test.common_helper import clean_test_database, get_config_for_testing, get_database_names
+from test.common_helper import clean_test_database, create_test_file_object, get_config_for_testing, get_database_names
 
 
-class TestStatistic(unittest.TestCase):
+class TestStatisticBase(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
@@ -29,6 +30,9 @@ def tearDown(self):
     def tearDownClass(cls):
         cls.mongo_server.shutdown()
 
+
+class TestStatistic(TestStatisticBase):
+
     def test_update_and_get_statistic(self):
         self.updater.db.update_statistic('test', {'test1': 1})
         result = self.frontend_db_interface.get_statistic('test')
@@ -185,3 +189,38 @@ def test_fetch_mitigations(self):
 
     def test_known_vulnerabilities_works(self):
         self.assertEqual(self.updater.get_known_vulnerabilities_stats(), {'known_vulnerabilities': []})
+
+
+class TestStatisticWithDb(TestStatisticBase):
+    def setUp(self):
+        super().setUp()
+        self.db_backend_interface = BackEndDbInterface(config=self.config)
+
+    def tearDown(self):
+        self.db_backend_interface.client.drop_database(self.config.get('data_storage', 'main_database'))
+        self.db_backend_interface.shutdown()
+        super().tearDown()
+
+    def test_get_executable_stats(self):
+        for i, file_str in enumerate([
+            'ELF 64-bit LSB executable, x86-64, dynamically linked, for GNU/Linux 2.6.32, not stripped',
+            'ELF 32-bit MSB executable, MIPS, MIPS32 rel2 version 1 (SYSV), statically linked, not stripped',
+            'ELF 64-bit LSB executable, x86-64, (SYSV), corrupted section header size',
+            'ELF 64-bit LSB executable, aarch64, dynamically linked, stripped',
+            'ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, stripped'
+        ]):
+            fo = create_test_file_object()
+            fo.processed_analysis['file_type'] = {'full': file_str}
+            fo.uid = str(i)
+            self.db_backend_interface.add_file_object(fo)
+
+        stats = self.updater.get_executable_stats().get('executable_stats')
+        expected = [
+            ('big endian', 1, 0.25), ('little endian', 3, 0.75), ('stripped', 1, 0.25), ('not stripped', 2, 0.5),
+            ('32-bit', 1, 0.25), ('64-bit', 3, 0.75), ('dynamically linked', 2, 0.5), ('statically linked', 1, 0.25),
+            ('section info missing', 1, 0.25)
+        ]
+        for (expected_label, expected_count, expected_percentage), (label, count, percentage, _) in zip(expected, stats):
+            assert label == expected_label
+            assert count == expected_count
+            assert percentage == expected_percentage

src/test/unit/web_interface/test_filter.py

@@ -37,7 +37,8 @@ def test_set_limit_for_data_to_chart():
                 'data': [1696, 207, 9],
                 'percentage': [0.89122, 0.10878, 0.00473],
                 'backgroundColor': ['#4062fa', '#f4c069', '#4062fa'],
-                'borderWidth': 0
+                'borderWidth': 0,
+                'links': 'null'
             }]
         }
     ),

src/web_interface/components/statistic_routes.py

@@ -13,26 +13,26 @@
 
 class StatisticRoutes(ComponentBase):
     def _init_component(self):
-        self._app.add_url_rule("/statistic", "statistic", self._show_statistic, methods=["GET"])
-        self._app.add_url_rule("/system_health", "system_health", self._show_system_health, methods=["GET"])
+        self._app.add_url_rule('/statistic', 'statistic', self._show_statistic, methods=['GET'])
+        self._app.add_url_rule('/system_health', 'system_health', self._show_system_health, methods=['GET'])
 
     @roles_accepted(*PRIVILEGES['status'])
     def _show_statistic(self):
-        filter_query = apply_filters_to_query(request, "{}")
+        filter_query = apply_filters_to_query(request, '{}')
         if filter_query == {}:
             stats = self._get_stats_from_db()
         else:
             stats = self._get_live_stats(filter_query)
         with ConnectTo(FrontEndDbInterface, self._config) as connection:
             device_classes = connection.get_device_class_list()
             vendors = connection.get_vendor_list()
-        return render_template("show_statistic.html", stats=stats, device_classes=device_classes,
-                               vendors=vendors, current_class=str(request.args.get("device_class")),
-                               current_vendor=str(request.args.get("vendor")))
+        return render_template('show_statistic.html', stats=stats, device_classes=device_classes,
+                               vendors=vendors, current_class=str(request.args.get('device_class')),
+                               current_vendor=str(request.args.get('vendor')))
 
     @roles_accepted(*PRIVILEGES['status'])
     def _show_system_health(self):
-        components = ["frontend", "database", "backend"]
+        components = ['frontend', 'database', 'backend']
         status = []
         with ConnectTo(StatisticDbViewer, self._config) as stats_db:
             for component in components:
@@ -41,41 +41,43 @@ def _show_system_health(self):
         with ConnectTo(InterComFrontEndBinding, self._config) as sc:
             plugin_dict = sc.get_available_analysis_plugins()
 
-        return render_template("system_health.html", status=status, analysis_plugin_info=plugin_dict)
+        return render_template('system_health.html', status=status, analysis_plugin_info=plugin_dict)
 
     def _get_stats_from_db(self):
         with ConnectTo(StatisticDbViewer, self._config) as stats_db:
             stats_dict = {
-                "general_stats": stats_db.get_statistic("general"),
-                "firmware_meta_stats": stats_db.get_statistic("firmware_meta"),
-                "file_type_stats": stats_db.get_statistic("file_type"),
-                "malware_stats": stats_db.get_statistic("malware"),
-                "crypto_material_stats": stats_db.get_statistic("crypto_material"),
-                "unpacker_stats": stats_db.get_statistic("unpacking"),
-                "ip_and_uri_stats": stats_db.get_statistic("ips_and_uris"),
-                "architecture_stats": stats_db.get_statistic("architecture"),
-                "release_date_stats": stats_db.get_statistic("release_date"),
-                "exploit_mitigations_stats": stats_db.get_statistic("exploit_mitigations"),
-                "known_vulnerabilities_stats": stats_db.get_statistic("known_vulnerabilities"),
-                "software_stats": stats_db.get_statistic("software_components"),
+                'general_stats': stats_db.get_statistic('general'),
+                'firmware_meta_stats': stats_db.get_statistic('firmware_meta'),
+                'file_type_stats': stats_db.get_statistic('file_type'),
+                'malware_stats': stats_db.get_statistic('malware'),
+                'crypto_material_stats': stats_db.get_statistic('crypto_material'),
+                'unpacker_stats': stats_db.get_statistic('unpacking'),
+                'ip_and_uri_stats': stats_db.get_statistic('ips_and_uris'),
+                'architecture_stats': stats_db.get_statistic('architecture'),
+                'release_date_stats': stats_db.get_statistic('release_date'),
+                'exploit_mitigations_stats': stats_db.get_statistic('exploit_mitigations'),
+                'known_vulnerabilities_stats': stats_db.get_statistic('known_vulnerabilities'),
+                'software_stats': stats_db.get_statistic('software_components'),
+                'elf_executable_stats': stats_db.get_statistic('elf_executable'),
             }
         return stats_dict
 
     def _get_live_stats(self, filter_query):
         with ConnectTo(StatisticUpdater, self._config) as stats_updater:
             stats_updater.set_match(filter_query)
             stats_dict = {
-                "firmware_meta_stats": stats_updater.get_firmware_meta_stats(),
-                "file_type_stats": stats_updater.get_file_type_stats(),
-                "malware_stats": stats_updater.get_malware_stats(),
-                "crypto_material_stats": stats_updater.get_crypto_material_stats(),
-                "unpacker_stats": stats_updater.get_unpacking_stats(),
-                "ip_and_uri_stats": stats_updater.get_ip_stats(),
-                "architecture_stats": stats_updater.get_architecture_stats(),
-                "release_date_stats": stats_updater.get_time_stats(),
-                "general_stats": stats_updater.get_general_stats(),
-                "exploit_mitigations_stats": stats_updater.get_exploit_mitigations_stats(),
-                "known_vulnerabilities_stats": stats_updater.get_known_vulnerabilities_stats(),
-                "software_stats": stats_updater.get_software_components_stats(),
+                'firmware_meta_stats': stats_updater.get_firmware_meta_stats(),
+                'file_type_stats': stats_updater.get_file_type_stats(),
+                'malware_stats': stats_updater.get_malware_stats(),
+                'crypto_material_stats': stats_updater.get_crypto_material_stats(),
+                'unpacker_stats': stats_updater.get_unpacking_stats(),
+                'ip_and_uri_stats': stats_updater.get_ip_stats(),
+                'architecture_stats': stats_updater.get_architecture_stats(),
+                'release_date_stats': stats_updater.get_time_stats(),
+                'general_stats': stats_updater.get_general_stats(),
+                'exploit_mitigations_stats': stats_updater.get_exploit_mitigations_stats(),
+                'known_vulnerabilities_stats': stats_updater.get_known_vulnerabilities_stats(),
+                'software_stats': stats_updater.get_software_components_stats(),
+                'elf_executable_stats': stats_updater.get_executable_stats(),
             }
         return stats_dict

src/web_interface/filter.py

@@ -238,7 +238,7 @@ def data_to_chart_limited(data, limit=10, color_list=None):
 
 def data_to_chart_with_value_percentage_pairs(data, limit=10):  # pylint: disable=invalid-name
     try:
-        label_list, value_list, percentage_list = [list(d) for d in zip(*data)]
+        label_list, value_list, percentage_list, *links = [list(d) for d in zip(*data)]
     except ValueError:
         return None
     label_list, value_list = set_limit_for_data_to_chart(label_list, limit, value_list)
@@ -249,7 +249,8 @@ def data_to_chart_with_value_percentage_pairs(data, limit=10):  # pylint: disabl
             'data': value_list,
             'percentage': percentage_list,
             'backgroundColor': color_list,
-            'borderWidth': 0
+            'borderWidth': 0,
+            'links': links[0] if links else 'null'
         }]
     }
     return result

src/web_interface/static/js/fact_statistics.js

@@ -70,7 +70,19 @@ function set_links(canvas_id, any_chart, link) {
 
 }
 
-function create_horizontal_bar_chart(canvas_id, chart_data, link, value_percentage_present_flag) {
+function set_links_from_data(canvas_id, chart, link) {
+
+    document.getElementById(canvas_id).onclick = function(evt){
+        var points = chart.getElementsAtEvent(evt);
+        if (chart.data.datasets[0].links !== undefined) {
+            var key = chart.data.datasets[0].links[points[0]._index];
+            window.location = link.replace("PLACEHOLDER", key);
+        }
+    };
+
+}
+
+function create_horizontal_bar_chart(canvas_id, chart_data, link, value_percentage_present_flag = false, links_in_data = false) {
     var ctx = document.getElementById(canvas_id);
 
     if (value_percentage_present_flag) {
@@ -90,7 +102,11 @@ function create_horizontal_bar_chart(canvas_id, chart_data, link, value_percenta
         }
     );
 
-    set_links(canvas_id, BarChart, link);
+    if (links_in_data) {
+        set_links_from_data(canvas_id, BarChart, link);
+    } else {
+        set_links(canvas_id, BarChart, link);
+    }
 
     return BarChart;
 }

src/web_interface/templates/show_statistic.html

@@ -183,6 +183,25 @@ <h3 class="mb-3">Firmware Statistics</h3>
 {% endif %}
 
 
+{# ------ ELF Executable Stats ------ #}
+
+{% if stats["elf_executable_stats"] %}
+    {% if stats["elf_executable_stats"]["executable_stats"] %}
+        {% call macros.stats_panel("ELF Executables") %}
+            <canvas id="elf_executable_canvas" width="100%" height="{{ stats['elf_executable_stats']['executable_stats'] | get_canvas_height }}"></canvas>
+            <script>
+                create_horizontal_bar_chart(
+                    canvas_id="elf_executable_canvas",
+                    chart_data={{ stats["elf_executable_stats"]["executable_stats"] | data_to_chart_with_value_percentage_pairs(limit=11) | safe }},
+                    link="{{ query_url + {'processed_analysis.file_type.full': {'$regex': 'PLACEHOLDER'}} | json_dumps | urlencode }}",
+                    value_percentage_present_flag=true, links_in_data=true
+                );
+            </script>
+        {% endcall %}
+    {% endif %}
+{% endif %}
+
+
 {# ------ Malware Stats ------ #}
 
 	{% if (stats["malware_stats"]["malware"] | length) > 0 %}