From 74337b237951f0002c13f9550c7941db8366544f Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Fri, 4 Aug 2023 13:30:53 +0200 Subject: [PATCH] Dynamic REST API port --- doc/restapi/introduction.rst | 6 +- .../components/restapi/rest/rest_manager.py | 86 ++++++++----------- src/tribler/core/start_core.py | 19 ++-- src/tribler/gui/core_manager.py | 20 +++-- src/tribler/gui/defs.py | 1 - src/tribler/gui/event_request_manager.py | 13 ++- src/tribler/gui/network/request_manager.py | 12 ++- .../gui/network/tests/test_request_manager.py | 16 +++- src/tribler/gui/start_gui.py | 3 +- src/tribler/gui/tests/test_core_manager.py | 4 +- src/tribler/gui/tribler_window.py | 26 +++--- src/tribler/gui/widgets/settingspage.py | 3 +- 12 files changed, 111 insertions(+), 98 deletions(-) diff --git a/doc/restapi/introduction.rst b/doc/restapi/introduction.rst index 89d8d2406f5..9c9d4e2992b 100644 --- a/doc/restapi/introduction.rst +++ b/doc/restapi/introduction.rst @@ -16,11 +16,11 @@ Some requests require one or more parameters. These parameters are passed using .. code-block:: none - curl -X PUT -H "X-Api-Key: " http://localhost:20100/mychannel/rssfeeds/http%3A%2F%2Frssfeed.com%2Frss.xml + curl -X PUT -H "X-Api-Key: " http://localhost:/mychannel/rssfeeds/http%3A%2F%2Frssfeed.com%2Frss.xml -Alternatively, requests can be made using Swagger UI by starting Tribler and opening `http://localhost:20100/docs` in a browser. +Alternatively, requests can be made using Swagger UI by starting Tribler and opening `http://localhost:/docs` in a browser. -Note: 20100 is a default port value. It can be changed by setting up "CORE_API_PORT" environment variable. +The port can be specified by setting up "CORE_API_PORT" environment variable. Error handling ============== diff --git a/src/tribler/core/components/restapi/rest/rest_manager.py b/src/tribler/core/components/restapi/rest/rest_manager.py index 962be0c85d5..460b75aeee2 100644 --- a/src/tribler/core/components/restapi/rest/rest_manager.py +++ b/src/tribler/core/components/restapi/rest/rest_manager.py @@ -20,12 +20,12 @@ ) from tribler.core.components.restapi.rest.root_endpoint import RootEndpoint from tribler.core.components.restapi.rest.settings import APISettings +from tribler.core.utilities.network_utils import default_network_utils from tribler.core.utilities.process_manager import get_global_process_manager from tribler.core.version import version_id SITE_START_TIMEOUT = 5.0 # seconds -BIND_ATTEMPTS = 10 logger = logging.getLogger(__name__) @@ -107,8 +107,11 @@ def get_endpoint(self, name): return self.root_endpoint.endpoints.get('/' + name) def set_api_port(self, api_port: int): + default_network_utils.remember(api_port) + if self.config.http_port != api_port: self.config.http_port = api_port + process_manager = get_global_process_manager() if process_manager: process_manager.current_process.set_api_port(api_port) @@ -117,7 +120,7 @@ async def start(self): """ Starts the HTTP API with the listen port as specified in the session configuration. """ - self._logger.info(f'An attempt to start REST API on {self.http_host}:{self.config.http_port}') + self._logger.info('Starting RESTManager...') # Not using setup_aiohttp_apispec here, as we need access to the APISpec to set the security scheme aiohttp_apispec = AiohttpApiSpec( @@ -131,12 +134,8 @@ async def start(self): self._logger.info('Set security scheme and apply to all endpoints') aiohttp_apispec.spec.options['security'] = [{'apiKey': []}] - aiohttp_apispec.spec.components.security_scheme('apiKey', {'type': 'apiKey', - 'in': 'header', - 'name': 'X-Api-Key'}) - - self._logger.info(f'Swagger docs: http://{self.http_host}:{self.config.http_port}/docs') - self._logger.info(f'Swagger JSON: http://{self.http_host}:{self.config.http_port}/docs/swagger.json') + api_key_scheme = {'type': 'apiKey', 'in': 'header', 'name': 'X-Api-Key'} + aiohttp_apispec.spec.components.security_scheme('apiKey', api_key_scheme) if 'head' in VALID_METHODS_OPENAPI_V2: self._logger.info('Remove head') @@ -147,60 +146,47 @@ async def start(self): if self.config.http_enabled: self._logger.info('Http enabled') + await self.start_http_site() - api_port = self.config.http_port - if not self.config.retry_port: - self.site = web.TCPSite(self.runner, self.http_host, api_port, shutdown_timeout=self.shutdown_timeout) - self.set_api_port(api_port) - await self.site.start() - else: - self._logger.info(f"Searching for a free port starting from {api_port}") - for port in range(api_port, api_port + BIND_ATTEMPTS): - try: - await self.start_http_site(port) - break - - except asyncio.TimeoutError: - self._logger.warning(f"Timeout when starting HTTP REST API server on port {port}") - - except OSError as e: - self._logger.warning(f"{e.__class__.__name__}: {e}") - - except BaseException as e: - self._logger.error(f"{e.__class__.__name__}: {e}") - raise # an unexpected exception; propagate it + if self.config.https_enabled: + self._logger.info('Https enabled') + await self.start_https_site() - else: - raise RuntimeError("Can't start HTTP REST API on any port in range " - f"{api_port}..{api_port + BIND_ATTEMPTS}") + self._logger.info(f'Swagger docs: http://{self.http_host}:{self.config.http_port}/docs') + self._logger.info(f'Swagger JSON: http://{self.http_host}:{self.config.http_port}/docs/swagger.json') - self._logger.info("Started HTTP REST API: %s", self.site.name) + async def start_http_site(self): + api_port = max(self.config.http_port, 0) # if the value in config is -1 we convert it to 0 - if self.config.https_enabled: - self._logger.info('Https enabled') + self.site = web.TCPSite(self.runner, self.http_host, api_port, shutdown_timeout=self.shutdown_timeout) + self._logger.info(f"Starting HTTP REST API server on port {api_port}...") - ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + try: + # The self.site.start() is expected to start immediately. It looks like on some machines, it hangs. + # The timeout is added to prevent the hypothetical hanging. + await asyncio.wait_for(self.site.start(), timeout=SITE_START_TIMEOUT) - cert = self.config.get_path_as_absolute('https_certfile', self.state_dir) - ssl_context.load_cert_chain(cert) + except BaseException as e: + self._logger.exception(f"Can't start HTTP REST API on port {api_port}: {e.__class__.__name__}: {e}") + raise - port = self.config.https_port - self.site_https = web.TCPSite(self.runner, self.https_host, port, ssl_context=ssl_context) + if not api_port: + api_port = self.site._server.sockets[0].getsockname()[1] # pylint: disable=protected-access - await self.site_https.start() - self._logger.info("Started HTTPS REST API: %s", self.site_https.name) + self.set_api_port(api_port) + self._logger.info(f"HTTP REST API server started on port {api_port}") - async def start_http_site(self, port): - self.site = web.TCPSite(self.runner, self.http_host, port, shutdown_timeout=self.shutdown_timeout) - self._logger.info(f"Starting HTTP REST API server on port {port}...") + async def start_https_site(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - # The self.site.start() is expected to start immediately. It looks like on some machines, - # it hangs. The timeout is added to prevent the hypothetical hanging. - await asyncio.wait_for(self.site.start(), timeout=SITE_START_TIMEOUT) + cert = self.config.get_path_as_absolute('https_certfile', self.state_dir) + ssl_context.load_cert_chain(cert) - self._logger.info(f"HTTP REST API server started on port {port}") - self.set_api_port(port) + port = self.config.https_port + self.site_https = web.TCPSite(self.runner, self.https_host, port, ssl_context=ssl_context) + await self.site_https.start() + self._logger.info("Started HTTPS REST API: %s", self.site_https.name) async def stop(self): self._logger.info('Stopping...') diff --git a/src/tribler/core/start_core.py b/src/tribler/core/start_core.py index 21fb0dc01b8..746a1cc577e 100644 --- a/src/tribler/core/start_core.py +++ b/src/tribler/core/start_core.py @@ -5,7 +5,7 @@ import signal import sys from pathlib import Path -from typing import List +from typing import List, Optional from tribler.core import notifications from tribler.core.check_os import ( @@ -122,7 +122,8 @@ async def core_session(config: TriblerConfig, components: List[Component]) -> in return session.exit_code -def run_tribler_core_session(api_port: int, api_key: str, state_dir: Path, gui_test_mode: bool = False) -> int: +def run_tribler_core_session(api_port: Optional[int], api_key: str, + state_dir: Path, gui_test_mode: bool = False) -> int: """ This method will start a new Tribler session. Note that there is no direct communication between the GUI process and the core: all communication is performed @@ -130,7 +131,7 @@ def run_tribler_core_session(api_port: int, api_key: str, state_dir: Path, gui_t Returns an exit code value, which is non-zero if the Tribler session finished with an error. """ - logger.info(f'Start tribler core. API port: "{api_port}". ' + logger.info(f'Start tribler core. API port: "{api_port or ""}". ' f'API key: "{api_key}". State dir: "{state_dir}". ' f'Core test mode: "{gui_test_mode}"') @@ -143,7 +144,10 @@ def run_tribler_core_session(api_port: int, api_key: str, state_dir: Path, gui_t if SentryReporter.is_in_test_mode(): default_core_exception_handler.sentry_reporter.global_strategy = SentryStrategy.SEND_ALLOWED - config.api.http_port = api_port + # The -1 value is assigned for backward compatibility reasons when the port is not specified. + # When RESTManager actually uses the value, it converts -1 to zero. + # It is possible that later we can directly store zero to config.api.http_port, but I prefer to be safe now. + config.api.http_port = api_port or -1 # If the API key is set to an empty string, it will remain disabled if config.api.key not in ('', api_key): config.api.key = api_key @@ -179,7 +183,7 @@ def run_tribler_core_session(api_port: int, api_key: str, state_dir: Path, gui_t return exit_code -def run_core(api_port, api_key, root_state_dir, parsed_args): +def run_core(api_port: Optional[int], api_key: Optional[str], root_state_dir, parsed_args): logger.info(f"Running Core in {'gui_test_mode' if parsed_args.gui_test_mode else 'normal mode'}") gui_pid = GuiProcessWatcher.get_gui_pid() @@ -195,11 +199,6 @@ def run_core(api_port, api_key, root_state_dir, parsed_args): logger.warning(msg) process_manager.sys_exit(1, msg) - if api_port is None: - msg = 'api_port is not specified for a core process' - logger.error(msg) - process_manager.sys_exit(1, msg) - version_history = VersionHistory(root_state_dir) state_dir = version_history.code_version.directory exit_code = run_tribler_core_session(api_port, api_key, state_dir, gui_test_mode=parsed_args.gui_test_mode) diff --git a/src/tribler/gui/core_manager.py b/src/tribler/gui/core_manager.py index a114a04cea2..80baf5e5ec8 100644 --- a/src/tribler/gui/core_manager.py +++ b/src/tribler/gui/core_manager.py @@ -30,7 +30,7 @@ class CoreManager(QObject): a fake API will be started. """ - def __init__(self, root_state_dir: Path, api_port: int, api_key: str, + def __init__(self, root_state_dir: Path, api_port: Optional[int], api_key: str, app_manager: AppManager, process_manager: ProcessManager, events_manager: EventRequestManager): QObject.__init__(self, None) @@ -83,16 +83,21 @@ def start(self, core_args=None, core_env=None, upgrade_manager=None, run_core=Tr First test whether we already have a Tribler process listening on port . If so, use that one and don't start a new, fresh Core. """ - # Connect to the events manager - self.events_manager.connect_to_core(reschedule_on_err=False) # do not retry if tribler Core is not running yet - if run_core: self.core_args = core_args self.core_env = core_env self.upgrade_manager = upgrade_manager - connect(self.events_manager.reply.error, self.on_event_manager_initial_error) - def on_event_manager_initial_error(self, _): + # Connect to the events manager + if self.events_manager.api_port: + self.events_manager.connect_to_core( + reschedule_on_err=False # do not retry if tribler Core is not running yet + ) + connect(self.events_manager.reply.error, self.do_upgrade_and_start_core) + else: + self.do_upgrade_and_start_core() + + def do_upgrade_and_start_core(self, _=None): if self.upgrade_manager: # Start Tribler Upgrader. When it finishes, start Tribler Core connect(self.upgrade_manager.upgrader_finished, self.start_tribler_core) @@ -106,7 +111,6 @@ def start_tribler_core(self): core_env = self.core_env if not core_env: core_env = QProcessEnvironment.systemEnvironment() - core_env.insert("CORE_API_PORT", f"{self.api_port}") core_env.insert("CORE_API_KEY", self.api_key) core_env.insert("TSTATEDIR", str(self.root_state_dir)) core_env.insert("TRIBLER_GUI_PID", str(os.getpid())) @@ -156,7 +160,7 @@ def check_core_api_port(self, *args): self._logger.info(f"Got REST API port value from the Core process: {api_port}") if api_port != self.api_port: self.api_port = api_port - request_manager.port = api_port + request_manager.set_api_port(api_port) self.events_manager.set_api_port(api_port) # Previously it was necessary to reschedule on error because `events_manager.connect_to_core()` was executed diff --git a/src/tribler/gui/defs.py b/src/tribler/gui/defs.py index ff8309656d5..40cc4a8bd63 100644 --- a/src/tribler/gui/defs.py +++ b/src/tribler/gui/defs.py @@ -10,7 +10,6 @@ DEFAULT_API_PROTOCOL = "http" DEFAULT_API_HOST = "localhost" -DEFAULT_API_PORT = 20100 # Define stacked widget page indices PAGE_SEARCH_RESULTS = 0 diff --git a/src/tribler/gui/event_request_manager.py b/src/tribler/gui/event_request_manager.py index 5a629b53f7e..8b26321de6a 100644 --- a/src/tribler/gui/event_request_manager.py +++ b/src/tribler/gui/event_request_manager.py @@ -35,11 +35,11 @@ class EventRequestManager(QNetworkAccessManager): change_loading_text = pyqtSignal(str) config_error_signal = pyqtSignal(str) - def __init__(self, api_port, api_key, error_handler): + def __init__(self, api_port: Optional[int], api_key, error_handler): QNetworkAccessManager.__init__(self) self.api_port = api_port self.api_key = api_key - self.request = self.create_request() + self.request: Optional[QNetworkRequest] = None self.start_time = time.time() self.connect_timer = QTimer() self.current_event_string = "" @@ -66,6 +66,9 @@ def __init__(self, api_port, api_key, error_handler): notifier.add_observer(notifications.report_config_error, self.on_report_config_error) def create_request(self) -> QNetworkRequest: + if not self.api_port: + raise RuntimeError("Can't create a request: api_port is not set") + url = QUrl(f"http://localhost:{self.api_port}/events") request = QNetworkRequest(url) request.setRawHeader(b'X-Api-Key', self.api_key.encode('ascii')) @@ -186,6 +189,9 @@ def on_finished(self): self.connect_timer.start(RECONNECT_INTERVAL_MS) def connect_to_core(self, reschedule_on_err=True): + if not self.api_port: + raise RuntimeError("Can't connect to core: api_port is not set") + if reschedule_on_err: self._logger.info(f"Set event request manager timeout to {CORE_CONNECTION_TIMEOUT} seconds") self.start_time = time.time() @@ -202,6 +208,9 @@ def _connect_to_core(self, reschedule_on_err): # A workaround for Qt5 bug. See https://github.com/Tribler/tribler/issues/7018 self.setNetworkAccessible(QNetworkAccessManager.Accessible) + if not self.request: + self.request = self.create_request() + self.reply = self.get(self.request) connect(self.reply.readyRead, self.on_read_data) diff --git a/src/tribler/gui/network/request_manager.py b/src/tribler/gui/network/request_manager.py index 4ab55182acd..855ba3683b7 100644 --- a/src/tribler/gui/network/request_manager.py +++ b/src/tribler/gui/network/request_manager.py @@ -9,7 +9,7 @@ from PyQt5.QtCore import QBuffer, QIODevice, QUrl from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkRequest -from tribler.gui.defs import BUTTON_TYPE_NORMAL, DEFAULT_API_HOST, DEFAULT_API_PORT, DEFAULT_API_PROTOCOL +from tribler.gui.defs import BUTTON_TYPE_NORMAL, DEFAULT_API_HOST, DEFAULT_API_PROTOCOL from tribler.gui.dialogs.confirmationdialog import ConfirmationDialog from tribler.gui.network.request import DATA_TYPE, Request from tribler.gui.utilities import connect @@ -35,12 +35,18 @@ def __init__(self, limit: int = 50, timeout_interval: int = 15): self.protocol = DEFAULT_API_PROTOCOL self.host = DEFAULT_API_HOST - self.port = DEFAULT_API_PORT + self.port: Optional[int] = None self.key = '' self.limit = limit self.timeout_interval = timeout_interval self.last_request_id = 0 + def set_api_key(self, key: str): + self.key = key + + def set_api_port(self, api_port: int): + self.port = api_port + def get(self, endpoint: str, on_success: Callable = lambda _: None, @@ -167,6 +173,8 @@ def on_close(_): return text def get_base_url(self) -> str: + if not self.port: + raise RuntimeError("API port is not set") return f'{self.protocol}://{self.host}:{self.port}/' @staticmethod diff --git a/src/tribler/gui/network/tests/test_request_manager.py b/src/tribler/gui/network/tests/test_request_manager.py index f123ab6346f..b9193b47a85 100644 --- a/src/tribler/gui/network/tests/test_request_manager.py +++ b/src/tribler/gui/network/tests/test_request_manager.py @@ -2,6 +2,7 @@ import pytest +from tribler.core.utilities.network_utils import default_network_utils from tribler.gui.network.request_manager import RequestManager @@ -9,12 +10,19 @@ @pytest.fixture -def request_manager(): - return RequestManager() +def free_port(): + return default_network_utils.get_random_free_port() -def test_get_base_string(request_manager: RequestManager): - assert request_manager.get_base_url() == 'http://localhost:20100/' +@pytest.fixture +def request_manager(free_port: int): + request_manager = RequestManager() + request_manager.set_api_port(free_port) + return request_manager + + +def test_get_base_string(free_port: int, request_manager: RequestManager): + assert request_manager.get_base_url() == f'http://localhost:{free_port}/' def test_get_message_from_error_string(request_manager: RequestManager): diff --git a/src/tribler/gui/start_gui.py b/src/tribler/gui/start_gui.py index 55d9668de27..76f9b221723 100644 --- a/src/tribler/gui/start_gui.py +++ b/src/tribler/gui/start_gui.py @@ -1,6 +1,7 @@ import logging import os import sys +from typing import Optional from PyQt5.QtCore import QSettings @@ -25,7 +26,7 @@ logger = logging.getLogger(__name__) -def run_gui(api_port, api_key, root_state_dir, parsed_args): +def run_gui(api_port: Optional[int], api_key: Optional[str], root_state_dir, parsed_args): logger.info(f"Running GUI in {'gui_test_mode' if parsed_args.gui_test_mode else 'normal mode'}") # Workaround for macOS Big Sur, see https://github.com/Tribler/tribler/issues/5728 diff --git a/src/tribler/gui/tests/test_core_manager.py b/src/tribler/gui/tests/test_core_manager.py index f5d9ddfb924..ba60f89f093 100644 --- a/src/tribler/gui/tests/test_core_manager.py +++ b/src/tribler/gui/tests/test_core_manager.py @@ -67,7 +67,7 @@ def test_check_core_api_port_not_set(core_manager): @patch('tribler.gui.core_manager.request_manager') -def test_check_core_api_port(request_manager: MagicMock, core_manager): +def test_check_core_api_port(request_manager: MagicMock, core_manager: CoreManager): core_manager.core_running = True core_manager.core_started_at = time.time() api_port = core_manager.process_manager.current_process.get_core_process().api_port @@ -75,7 +75,7 @@ def test_check_core_api_port(request_manager: MagicMock, core_manager): assert core_manager.process_manager.current_process.get_core_process.called assert not core_manager.check_core_api_port_timer.start.called assert core_manager.api_port == api_port - assert request_manager.port == api_port + assert request_manager.set_api_port.called_once_with(api_port) def test_check_core_api_port_timeout(core_manager): diff --git a/src/tribler/gui/tribler_window.py b/src/tribler/gui/tribler_window.py index bab3d28da22..6ad98da23a6 100644 --- a/src/tribler/gui/tribler_window.py +++ b/src/tribler/gui/tribler_window.py @@ -5,6 +5,7 @@ import time from base64 import b64encode from pathlib import Path +from typing import Optional from PyQt5 import QtCore, uic from PyQt5.QtCore import ( @@ -63,7 +64,6 @@ BUTTON_TYPE_NORMAL, CATEGORY_SELECTOR_FOR_POPULAR_ITEMS, DARWIN, - DEFAULT_API_PORT, PAGE_CHANNEL_CONTENTS, PAGE_DISCOVERED, PAGE_DISCOVERING, @@ -158,8 +158,8 @@ def __init__( root_state_dir: Path, core_args=None, core_env=None, - api_port=None, - api_key=None, + api_port: Optional[int] = None, + api_key: Optional[str] = None, run_core=True, ): QMainWindow.__init__(self) @@ -175,20 +175,20 @@ def __init__( self.root_state_dir = root_state_dir self.gui_settings = settings - api_port = api_port or default_network_utils.get_first_free_port( - start=int(get_gui_setting(self.gui_settings, "api_port", DEFAULT_API_PORT)) - ) - if not default_network_utils.is_port_free(api_port): - raise RuntimeError( - "Tribler configuration conflicts with the current OS state: " - "REST API port %i already in use" % api_port - ) - process_manager.current_process.set_api_port(api_port) + + if api_port: + if not default_network_utils.is_port_free(api_port): + raise RuntimeError( + "Tribler configuration conflicts with the current OS state: " + "REST API port %i already in use" % api_port + ) + process_manager.current_process.set_api_port(api_port) api_key = format_api_key(api_key or get_gui_setting(self.gui_settings, "api_key", None) or create_api_key()) set_api_key(self.gui_settings, api_key) - request_manager.port, request_manager.key = api_port, api_key + request_manager.set_api_key(api_key) + request_manager.set_api_port(api_port) self.tribler_started = False self.tribler_settings = None diff --git a/src/tribler/gui/widgets/settingspage.py b/src/tribler/gui/widgets/settingspage.py index 1cc9792b682..d7c8afd0e31 100644 --- a/src/tribler/gui/widgets/settingspage.py +++ b/src/tribler/gui/widgets/settingspage.py @@ -9,7 +9,6 @@ from tribler.core.utilities.simpledefs import MAX_LIBTORRENT_RATE_LIMIT from tribler.gui.defs import ( DARWIN, - DEFAULT_API_PORT, PAGE_SETTINGS_ANONYMITY, PAGE_SETTINGS_BANDWIDTH, PAGE_SETTINGS_CONNECTION, @@ -208,7 +207,7 @@ def initialize_with_settings(self, settings): max_conn_download = 0 self.window().max_connections_download_input.setText(str(max_conn_download)) - self.window().api_port_input.setText(f"{get_gui_setting(gui_settings, 'api_port', DEFAULT_API_PORT)}") + self.window().api_port_input.setText(f"{get_gui_setting(gui_settings, 'api_port', 0)}") # Bandwidth settings self.window().upload_rate_limit_input.setText(str(settings['libtorrent']['max_upload_rate'] // 1024))