Dynamic REST API port

doc/restapi/introduction.rst

@@ -16,11 +16,11 @@ Some requests require one or more parameters. These parameters are passed using
 
 .. code-block:: none
 
- curl -X PUT -H "X-Api-Key: <YOUR API KEY>" http:https://localhost:20100/mychannel/rssfeeds/http%3A%2F%2Frssfeed.com%2Frss.xml
+ curl -X PUT -H "X-Api-Key: <YOUR API KEY>" http:https://localhost:<port>/mychannel/rssfeeds/http%3A%2F%2Frssfeed.com%2Frss.xml
 
-Alternatively, requests can be made using Swagger UI by starting Tribler and opening `http:https://localhost:20100/docs` in a browser.
+Alternatively, requests can be made using Swagger UI by starting Tribler and opening `http:https://localhost:<port>/docs` in a browser.
 
-Note: 20100 is a default port value. It can be changed by setting up "CORE_API_PORT" environment variable.
+The port can be specified by setting up "CORE_API_PORT" environment variable.
 
 Error handling
 ==============

src/tribler/core/components/restapi/rest/rest_manager.py

@@ -20,12 +20,12 @@
 )
 from tribler.core.components.restapi.rest.root_endpoint import RootEndpoint
 from tribler.core.components.restapi.rest.settings import APISettings
+from tribler.core.utilities.network_utils import default_network_utils
 from tribler.core.utilities.process_manager import get_global_process_manager
 from tribler.core.version import version_id
 
 
 SITE_START_TIMEOUT = 5.0 # seconds
-BIND_ATTEMPTS = 10
 
 
 logger = logging.getLogger(__name__)
@@ -107,8 +107,11 @@ def get_endpoint(self, name):
  return self.root_endpoint.endpoints.get('/' + name)
 
  def set_api_port(self, api_port: int):
+ default_network_utils.remember(api_port)
+
  if self.config.http_port != api_port:
  self.config.http_port = api_port
+
  process_manager = get_global_process_manager()
  if process_manager:
  process_manager.current_process.set_api_port(api_port)
@@ -117,7 +120,7 @@ async def start(self):
  """
  Starts the HTTP API with the listen port as specified in the session configuration.
  """
- self._logger.info(f'An attempt to start REST API on {self.http_host}:{self.config.http_port}')
+ self._logger.info('Starting RESTManager...')
 
  # Not using setup_aiohttp_apispec here, as we need access to the APISpec to set the security scheme
  aiohttp_apispec = AiohttpApiSpec(
@@ -131,12 +134,8 @@ async def start(self):
  self._logger.info('Set security scheme and apply to all endpoints')
 
  aiohttp_apispec.spec.options['security'] = [{'apiKey': []}]
- aiohttp_apispec.spec.components.security_scheme('apiKey', {'type': 'apiKey',
- 'in': 'header',
- 'name': 'X-Api-Key'})
-
- self._logger.info(f'Swagger docs: http:https://{self.http_host}:{self.config.http_port}/docs')
- self._logger.info(f'Swagger JSON: http:https://{self.http_host}:{self.config.http_port}/docs/swagger.json')
+ api_key_scheme = {'type': 'apiKey', 'in': 'header', 'name': 'X-Api-Key'}
+ aiohttp_apispec.spec.components.security_scheme('apiKey', api_key_scheme)
 
  if 'head' in VALID_METHODS_OPENAPI_V2:
  self._logger.info('Remove head')
@@ -147,60 +146,47 @@ async def start(self):
 
  if self.config.http_enabled:
  self._logger.info('Http enabled')
+ await self.start_http_site()
 
- api_port = self.config.http_port
- if not self.config.retry_port:
- self.site = web.TCPSite(self.runner, self.http_host, api_port, shutdown_timeout=self.shutdown_timeout)
- self.set_api_port(api_port)
- await self.site.start()
- else:
- self._logger.info(f"Searching for a free port starting from {api_port}")
- for port in range(api_port, api_port + BIND_ATTEMPTS):
- try:
- await self.start_http_site(port)
- break
-
- except asyncio.TimeoutError:
- self._logger.warning(f"Timeout when starting HTTP REST API server on port {port}")
-
- except OSError as e:
- self._logger.warning(f"{e.__class__.__name__}: {e}")
-
- except BaseException as e:
- self._logger.error(f"{e.__class__.__name__}: {e}")
- raise # an unexpected exception; propagate it
+ if self.config.https_enabled:
+ self._logger.info('Https enabled')
+ await self.start_https_site()
 
- else:
- raise RuntimeError("Can't start HTTP REST API on any port in range "
- f"{api_port}..{api_port + BIND_ATTEMPTS}")
+ self._logger.info(f'Swagger docs: http:https://{self.http_host}:{self.config.http_port}/docs')
+ self._logger.info(f'Swagger JSON: http:https://{self.http_host}:{self.config.http_port}/docs/swagger.json')
 
- self._logger.info("Started HTTP REST API: %s", self.site.name)
+ async def start_http_site(self):
+ api_port = max(self.config.http_port, 0) # if the value in config is -1 we convert it to 0
 
- if self.config.https_enabled:
-  self._logger.info('Https enabled')
+ self.site = web.TCPSite(self.runner, self.http_host, api_port, shutdown_timeout=self.shutdown_timeout)
+ self._logger.info(f"Starting HTTP REST API server on port {api_port}...")
 
- ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+ try:
+ # The self.site.start() is expected to start immediately. It looks like on some machines, it hangs.
+ # The timeout is added to prevent the hypothetical hanging.
+ await asyncio.wait_for(self.site.start(), timeout=SITE_START_TIMEOUT)
 
- cert = self.config.get_path_as_absolute('https_certfile', self.state_dir)
- ssl_context.load_cert_chain(cert)
+ except BaseException as e:
+ self._logger.exception(f"Can't start HTTP REST API on port {api_port}: {e.__class__.__name__}: {e}")
+ raise
 
-  port = self.config.https_port
- self.site_https = web.TCPSite(self.runner, self.https_host, port, ssl_context=ssl_context)
+ if not api_port:
+ api_port = self.site._server.sockets[0].getsockname()[1] # pylint: disable=protected-access
 
-  await self.site_https.start()
-  self._logger.info("Started HTTPS REST API: %s", self.site_https.name)
+ self.set_api_port(api_port)
+ self._logger.info(f"HTTP REST API server started on port {api_port}")
 
- async def start_http_site(self, port):
- self.site = web.TCPSite(self.runner, self.http_host, port, shutdown_timeout=self.shutdown_timeout)
- self._logger.info(f"Starting HTTP REST API server on port {port}...")
+ async def start_https_site(self):
+ ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
 
- # The self.site.start() is expected to start immediately. It looks like on some machines,
- # it hangs. The timeout is added to prevent the hypothetical hanging.
- await asyncio.wait_for(self.site.start(), timeout=SITE_START_TIMEOUT)
+ cert = self.config.get_path_as_absolute('https_certfile', self.state_dir)
+ ssl_context.load_cert_chain(cert)
 
- self._logger.info(f"HTTP REST API server started on port {port}")
- self.set_api_port(port)
+ port = self.config.https_port
+ self.site_https = web.TCPSite(self.runner, self.https_host, port, ssl_context=ssl_context)
 
+ await self.site_https.start()
+ self._logger.info("Started HTTPS REST API: %s", self.site_https.name)
 
  async def stop(self):
  self._logger.info('Stopping...')

src/tribler/core/start_core.py

@@ -5,7 +5,7 @@
 import signal
 import sys
 from pathlib import Path
-from typing import List
+from typing import List, Optional
 
 from tribler.core import notifications
 from tribler.core.check_os import (
@@ -122,15 +122,16 @@ async def core_session(config: TriblerConfig, components: List[Component]) -> in
  return session.exit_code
 
 
-def run_tribler_core_session(api_port: int, api_key: str, state_dir: Path, gui_test_mode: bool = False) -> int:
+def run_tribler_core_session(api_port: Optional[int], api_key: str,
+ state_dir: Path, gui_test_mode: bool = False) -> int:
  """
  This method will start a new Tribler session.
  Note that there is no direct communication between the GUI process and the core: all communication is performed
  through the HTTP API.
 
  Returns an exit code value, which is non-zero if the Tribler session finished with an error.
  """
- logger.info(f'Start tribler core. API port: "{api_port}". '
+ logger.info(f'Start tribler core. API port: "{api_port or "<not specified>"}". '
  f'API key: "{api_key}". State dir: "{state_dir}". '
  f'Core test mode: "{gui_test_mode}"')
 
@@ -143,7 +144,10 @@ def run_tribler_core_session(api_port: int, api_key: str, state_dir: Path, gui_t
  if SentryReporter.is_in_test_mode():
  default_core_exception_handler.sentry_reporter.global_strategy = SentryStrategy.SEND_ALLOWED
 
- config.api.http_port = api_port
+ # The -1 value is assigned for backward compatibility reasons when the port is not specified.
+ # When RESTManager actually uses the value, it converts -1 to zero.
+ # It is possible that later we can directly store zero to config.api.http_port, but I prefer to be safe now.
+ config.api.http_port = api_port or -1
  # If the API key is set to an empty string, it will remain disabled
  if config.api.key not in ('', api_key):
  config.api.key = api_key
@@ -179,7 +183,7 @@ def run_tribler_core_session(api_port: int, api_key: str, state_dir: Path, gui_t
  return exit_code
 
 
-def run_core(api_port, api_key, root_state_dir, parsed_args):
+def run_core(api_port: Optional[int], api_key: Optional[str], root_state_dir, parsed_args):
  logger.info(f"Running Core in {'gui_test_mode' if parsed_args.gui_test_mode else 'normal mode'}")
 
  gui_pid = GuiProcessWatcher.get_gui_pid()
@@ -195,11 +199,6 @@ def run_core(api_port, api_key, root_state_dir, parsed_args):
  logger.warning(msg)
  process_manager.sys_exit(1, msg)
 
- if api_port is None:
- msg = 'api_port is not specified for a core process'
- logger.error(msg)
- process_manager.sys_exit(1, msg)
-
  version_history = VersionHistory(root_state_dir)
  state_dir = version_history.code_version.directory
  exit_code = run_tribler_core_session(api_port, api_key, state_dir, gui_test_mode=parsed_args.gui_test_mode)

src/tribler/gui/core_manager.py

@@ -30,7 +30,7 @@ class CoreManager(QObject):
  a fake API will be started.
  """
 
- def __init__(self, root_state_dir: Path, api_port: int, api_key: str,
+ def __init__(self, root_state_dir: Path, api_port: Optional[int], api_key: str,
  app_manager: AppManager, process_manager: ProcessManager, events_manager: EventRequestManager):
  QObject.__init__(self, None)
 
@@ -83,16 +83,21 @@ def start(self, core_args=None, core_env=None, upgrade_manager=None, run_core=Tr
  First test whether we already have a Tribler process listening on port <CORE_API_PORT>.
  If so, use that one and don't start a new, fresh Core.
  """
- # Connect to the events manager
- self.events_manager.connect_to_core(reschedule_on_err=False) # do not retry if tribler Core is not running yet
-
  if run_core:
  self.core_args = core_args
  self.core_env = core_env
  self.upgrade_manager = upgrade_manager
- connect(self.events_manager.reply.error, self.on_event_manager_initial_error)
 
- def on_event_manager_initial_error(self, _):
+ # Connect to the events manager
+ if self.events_manager.api_port:
+ self.events_manager.connect_to_core(
+ reschedule_on_err=False # do not retry if tribler Core is not running yet
+ )
+ connect(self.events_manager.reply.error, self.do_upgrade_and_start_core)
+ else:
+ self.do_upgrade_and_start_core()
+
+ def do_upgrade_and_start_core(self, _=None):
  if self.upgrade_manager:
  # Start Tribler Upgrader. When it finishes, start Tribler Core
  connect(self.upgrade_manager.upgrader_finished, self.start_tribler_core)
@@ -106,7 +111,6 @@ def start_tribler_core(self):
  core_env = self.core_env
  if not core_env:
  core_env = QProcessEnvironment.systemEnvironment()
- core_env.insert("CORE_API_PORT", f"{self.api_port}")
  core_env.insert("CORE_API_KEY", self.api_key)
  core_env.insert("TSTATEDIR", str(self.root_state_dir))
  core_env.insert("TRIBLER_GUI_PID", str(os.getpid()))
@@ -156,7 +160,7 @@ def check_core_api_port(self, *args):
  self._logger.info(f"Got REST API port value from the Core process: {api_port}")
  if api_port != self.api_port:
  self.api_port = api_port
- request_manager.port = api_port
+ request_manager.set_api_port(api_port)
  self.events_manager.set_api_port(api_port)
 
  # Previously it was necessary to reschedule on error because `events_manager.connect_to_core()` was executed

src/tribler/gui/defs.py

@@ -10,7 +10,6 @@
 
 DEFAULT_API_PROTOCOL = "http"
 DEFAULT_API_HOST = "localhost"
-DEFAULT_API_PORT = 20100
 
 # Define stacked widget page indices
 PAGE_SEARCH_RESULTS = 0

src/tribler/gui/event_request_manager.py

@@ -35,11 +35,11 @@ class EventRequestManager(QNetworkAccessManager):
  change_loading_text = pyqtSignal(str)
  config_error_signal = pyqtSignal(str)
 
- def __init__(self, api_port, api_key, error_handler):
+ def __init__(self, api_port: Optional[int], api_key, error_handler):
  QNetworkAccessManager.__init__(self)
  self.api_port = api_port
  self.api_key = api_key
- self.request = self.create_request()
+ self.request: Optional[QNetworkRequest] = None
  self.start_time = time.time()
  self.connect_timer = QTimer()
  self.current_event_string = ""
@@ -66,6 +66,9 @@ def __init__(self, api_port, api_key, error_handler):
  notifier.add_observer(notifications.report_config_error, self.on_report_config_error)
 
  def create_request(self) -> QNetworkRequest:
+ if not self.api_port:
+ raise RuntimeError("Can't create a request: api_port is not set")
+
  url = QUrl(f"http:https://localhost:{self.api_port}/events")
  request = QNetworkRequest(url)
  request.setRawHeader(b'X-Api-Key', self.api_key.encode('ascii'))
@@ -186,6 +189,9 @@ def on_finished(self):
  self.connect_timer.start(RECONNECT_INTERVAL_MS)
 
  def connect_to_core(self, reschedule_on_err=True):
+ if not self.api_port:
+ raise RuntimeError("Can't connect to core: api_port is not set")
+
  if reschedule_on_err:
  self._logger.info(f"Set event request manager timeout to {CORE_CONNECTION_TIMEOUT} seconds")
  self.start_time = time.time()
@@ -202,6 +208,9 @@ def _connect_to_core(self, reschedule_on_err):
  # A workaround for Qt5 bug. See https://github.com/Tribler/tribler/issues/7018
  self.setNetworkAccessible(QNetworkAccessManager.Accessible)
 
+ if not self.request:
+ self.request = self.create_request()
+
  self.reply = self.get(self.request)
 
  connect(self.reply.readyRead, self.on_read_data)

src/tribler/gui/network/request_manager.py

@@ -9,7 +9,7 @@
 from PyQt5.QtCore import QBuffer, QIODevice, QUrl
 from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest
 
-from tribler.gui.defs import BUTTON_TYPE_NORMAL, DEFAULT_API_HOST, DEFAULT_API_PORT, DEFAULT_API_PROTOCOL
+from tribler.gui.defs import BUTTON_TYPE_NORMAL, DEFAULT_API_HOST, DEFAULT_API_PROTOCOL
 from tribler.gui.dialogs.confirmationdialog import ConfirmationDialog
 from tribler.gui.network.request import DATA_TYPE, Request
 from tribler.gui.utilities import connect
@@ -35,12 +35,18 @@ def __init__(self, limit: int = 50, timeout_interval: int = 15):
 
  self.protocol = DEFAULT_API_PROTOCOL
  self.host = DEFAULT_API_HOST
- self.port = DEFAULT_API_PORT
+ self.port: Optional[int] = None
  self.key = ''
  self.limit = limit
  self.timeout_interval = timeout_interval
  self.last_request_id = 0
 
+ def set_api_key(self, key: str):
+ self.key = key
+
+ def set_api_port(self, api_port: int):
+ self.port = api_port
+
  def get(self,
  endpoint: str,
  on_success: Callable = lambda _: None,
@@ -167,6 +173,8 @@ def on_close(_):
  return text
 
  def get_base_url(self) -> str:
+ if not self.port:
+ raise RuntimeError("API port is not set")
  return f'{self.protocol}:https://{self.host}:{self.port}/'
 
  @staticmethod

src/tribler/gui/network/tests/test_request_manager.py

@@ -2,19 +2,27 @@
 
 import pytest
 
+from tribler.core.utilities.network_utils import default_network_utils
 from tribler.gui.network.request_manager import RequestManager
 
 
 # pylint: disable=protected-access, redefined-outer-name
 
 
 @pytest.fixture
-def request_manager():
- return RequestManager()
+def free_port():
+ return default_network_utils.get_random_free_port()
 
 
-def test_get_base_string(request_manager: RequestManager):
- assert request_manager.get_base_url() == 'http:https://localhost:20100/'
+@pytest.fixture
+def request_manager(free_port: int):
+ request_manager = RequestManager()
+ request_manager.set_api_port(free_port)
+ return request_manager
+
+
+def test_get_base_string(free_port: int, request_manager: RequestManager):
+ assert request_manager.get_base_url() == f'http:https://localhost:{free_port}/'
 
 
 def test_get_message_from_error_string(request_manager: RequestManager):