review changes + refactoring

src/intercom/back_end_binding.py

@@ -111,7 +111,8 @@ class InterComBackEndAnalysisTask(InterComListener):
 
  CONNECTION_TYPE = 'analysis_task'
 
- def additional_setup(self, config=None):
+ def __init__(self, config=None):
+ super().__init__(config)
  self.fs_organizer = FSOrganizer(config=config)
 
  def post_processing(self, task, task_id):
@@ -123,7 +124,8 @@ class InterComBackEndReAnalyzeTask(InterComListener):
 
  CONNECTION_TYPE = 're_analyze_task'
 
- def additional_setup(self, config=None):
+ def __init__(self, config=None):
+ super().__init__(config)
  self.fs_organizer = FSOrganizer(config=config)
 
  def post_processing(self, task, task_id):
@@ -152,7 +154,8 @@ class InterComBackEndRawDownloadTask(InterComListenerAndResponder):
  CONNECTION_TYPE = 'raw_download_task'
  OUTGOING_CONNECTION_TYPE = 'raw_download_task_resp'
 
- def additional_setup(self, config=None):
+ def __init__(self, config=None):
+ super().__init__(config)
  self.binary_service = BinaryService(config=self.config)
 
  def get_response(self, task):
@@ -164,7 +167,8 @@ class InterComBackEndTarRepackTask(InterComListenerAndResponder):
  CONNECTION_TYPE = 'tar_repack_task'
  OUTGOING_CONNECTION_TYPE = 'tar_repack_task_resp'
 
- def additional_setup(self, config=None):
+ def __init__(self, config=None):
+ super().__init__(config)
  self.binary_service = BinaryService(config=self.config)
 
  def get_response(self, task):
@@ -186,7 +190,8 @@ class InterComBackEndDeleteFile(InterComListener):
 
  CONNECTION_TYPE = 'file_delete_task'
 
- def additional_setup(self, config=None):
+ def __init__(self, config=None):
+ super().__init__(config)
  self.fs_organizer = FSOrganizer(config=config)
 
  def post_processing(self, task, task_id):

src/intercom/common_mongo_binding.py

@@ -51,10 +51,6 @@ class InterComListener(InterComMongoInterface):
 
  CONNECTION_TYPE = 'test' # unique for each listener
 
- def __init__(self, config=None):
- super().__init__(config=config)
- self.additional_setup(config=config)
-
  def get_next_task(self):
  try:
  task_obj = self.connections[self.CONNECTION_TYPE]['fs'].find_one()
@@ -64,18 +60,12 @@ def get_next_task(self):
  if task_obj is not None:
  task = pickle.loads(task_obj.read())
  task_id = task_obj.filename
- self.connections[self.CONNECTION_TYPE]['fs'].delete(task_obj._id)
+ self.connections[self.CONNECTION_TYPE]['fs'].delete(task_obj._id) # pylint: disable=protected-access
  task = self.post_processing(task, task_id)
  logging.debug('{}: New task received: {}'.format(self.CONNECTION_TYPE, task))
  return task
  return None
 
- def additional_setup(self, config=None):
- '''
- optional additional setup
- '''
- pass # pylint: disable=unnecessary-pass
-
  def post_processing(self, task, task_id): # pylint: disable=no-self-use,unused-argument
  '''
  optional post processing of a task

src/storage/binary_service.py

@@ -1,10 +1,11 @@
 import logging
-from pathlib import Path
+from typing import Optional, Tuple
 
 from common_helper_files.fail_safe_file_operations import get_binary_from_file
 
 from helperFunctions.database import ConnectTo
 from storage.db_interface_common import MongoInterfaceCommon
+from storage.fsorganizer import FSOrganizer
 from unpacker.tar_repack import TarRepack
 
 
@@ -15,40 +16,36 @@ class BinaryService:
 
  def __init__(self, config=None):
  self.config = config
- self.firmware_storage_directory = Path(self.config['data_storage']['firmware_file_storage_directory'])
+ self.fs_organizer = FSOrganizer(config=config)
  logging.info("binary service online")
 
- def get_binary_and_file_name(self, uid):
- query = self._get_file_name_from_db(uid)
- if query is None:
+ def get_binary_and_file_name(self, uid: str) -> Tuple[Optional[bytes], Optional[str]]:
+ file_name = self._get_file_name_from_db(uid)
+ if file_name is None:
  return None, None
- binary = get_binary_from_file(self._get_file_path(uid))
- return binary, query['file_name']
+ binary = get_binary_from_file(self.fs_organizer.generate_path_from_uid(uid))
+ return binary, file_name
 
- def get_repacked_binary_and_file_name(self, uid):
- query = self._get_file_name_from_db(uid)
- if query is None:
+ def get_repacked_binary_and_file_name(self, uid: str) -> Tuple[Optional[bytes], Optional[str]]:
+ file_name = self._get_file_name_from_db(uid)
+ if file_name is None:
  return None, None
  repack_service = TarRepack(config=self.config)
- tar = repack_service.tar_repack(self._get_file_path(uid))
- name = "{}.tar.gz".format(query['file_name'])
+ tar = repack_service.tar_repack(self.fs_organizer.generate_path_from_uid(uid))
+ name = "{}.tar.gz".format(file_name)
  return tar, name
 
- def _get_file_path(self, uid: str) -> str:
- return str(self.firmware_storage_directory / uid[:2] / uid)
-
- def _get_file_name_from_db(self, uid):
+ def _get_file_name_from_db(self, uid: str) -> Optional[str]:
  with ConnectTo(BinaryServiceDbInterface, self.config) as db_service:
- query = db_service.get_file_name(uid)
- return query
+ return db_service.get_file_name(uid)
 
 
 class BinaryServiceDbInterface(MongoInterfaceCommon):
 
  READ_ONLY = True
 
- def get_file_name(self, uid):
+ def get_file_name(self, uid: str) -> Optional[str]:
  result = self.firmwares.find_one({"_id": uid}, {'file_name': 1})
  if result is None:
  result = self.file_objects.find_one({"_id": uid}, {'file_name': 1})
- return result
+ return result['file_name'] if result is not None else None

src/test/common_helper.py

@@ -457,7 +457,7 @@ def get_config_for_testing(temp_dir=None):
  config.set('data_storage', 'mongo_port', '27018')
  config.set('data_storage', 'report_threshold', '2048')
  config.set('data_storage', 'password_salt', '1234')
- config.set('data_storage', 'firmware_file_storage_directory', '')
+ config.set('data_storage', 'firmware_file_storage_directory', '/tmp/fact_test_fs_directory')
  config.add_section('unpack')
  config.set('unpack', 'whitelist', '')
  config.set('unpack', 'max_depth', '10')
@@ -487,7 +487,7 @@ def load_users_from_main_config(config: ConfigParser):
  config.set('data_storage', 'db_readonly_pw', fact_config['data_storage']['db_readonly_pw'])
 
 
-def put_binary_for_binary_service(tmp_dir: str, test_object: Union[FileObject, Firmware]):
+def store_binary_on_file_system(tmp_dir: str, test_object: Union[FileObject, Firmware]):
  binary_dir = Path(tmp_dir) / test_object.uid[:2]
  binary_dir.mkdir(parents=True)
  (binary_dir / test_object.uid).write_bytes(test_object.binary)

src/test/integration/storage/test_binary_service.py

@@ -8,7 +8,7 @@
 from storage.binary_service import BinaryService
 from storage.db_interface_backend import BackEndDbInterface
 from storage.MongoMgr import MongoMgr
-from test.common_helper import create_test_firmware, get_config_for_testing, put_binary_for_binary_service
+from test.common_helper import create_test_firmware, get_config_for_testing, store_binary_on_file_system
 
 TEST_FW = create_test_firmware()
 
@@ -25,7 +25,7 @@ def setup(self):
  def _init_test_data(self):
  self.backend_db_interface = BackEndDbInterface(config=self.config)
  self.backend_db_interface.add_firmware(TEST_FW)
- put_binary_for_binary_service(self.tmp_dir.name, TEST_FW)
+ store_binary_on_file_system(self.tmp_dir.name, TEST_FW)
  self.backend_db_interface.shutdown()
 
  def teardown(self):

src/test/integration/web_interface/rest/test_rest_binary.py

@@ -5,7 +5,7 @@
 
 from intercom.back_end_binding import InterComBackEndBinding
 from storage.db_interface_backend import BackEndDbInterface
-from test.common_helper import create_test_firmware, put_binary_for_binary_service
+from test.common_helper import create_test_firmware, store_binary_on_file_system
 from test.integration.intercom import test_backend_scheduler
 from test.integration.web_interface.rest.base import RestTestBase
 
@@ -24,12 +24,13 @@ def teardown(self):
 
  def test_rest_download_valid(self):
  backend_binding = InterComBackEndBinding(
- self.config, analysis_service=test_backend_scheduler.AnalysisServiceMock(),
+ config=self.config,
+ analysis_service=test_backend_scheduler.AnalysisServiceMock(),
  compare_service=test_backend_scheduler.ServiceMock(self.test_queue),
  unpacking_service=test_backend_scheduler.ServiceMock(self.test_queue)
  )
  test_firmware = create_test_firmware(device_class='test class', device_name='test device', vendor='test vendor')
- put_binary_for_binary_service(self.tmp_dir.name, test_firmware)
+ store_binary_on_file_system(self.tmp_dir.name, test_firmware)
  self.db_interface.add_firmware(test_firmware)
 
  try: