diff --git a/src/run_tribler.py b/src/run_tribler.py index e43b791a464..b0a2a20411f 100644 --- a/src/run_tribler.py +++ b/src/run_tribler.py @@ -11,6 +11,7 @@ from tribler.core.sentry_reporter.sentry_reporter import SentryReporter, SentryStrategy from tribler.core.sentry_reporter.sentry_scrubber import SentryScrubber +from tribler.core.utilities.asyncio_fixes.finish_accept_patch import apply_finish_accept_patch from tribler.core.utilities.slow_coro_detection.main_thread_stack_tracking import start_main_thread_stack_tracing from tribler.core.utilities.osutils import get_root_state_directory from tribler.core.utilities.utilities import is_frozen @@ -92,6 +93,9 @@ def init_boot_logger(): # Check whether we need to start the core or the user interface if parsed_args.core: + if sys.platform == 'win32': + apply_finish_accept_patch() + from tribler.core.utilities.pony_utils import track_slow_db_sessions track_slow_db_sessions() diff --git a/src/tribler/core/check_os.py b/src/tribler/core/check_os.py index 832273306c8..87bca4423fa 100644 --- a/src/tribler/core/check_os.py +++ b/src/tribler/core/check_os.py @@ -7,6 +7,7 @@ import psutil +from tribler.core.utilities.path_util import Path from tribler.core.utilities.utilities import show_system_popup logger = logging.getLogger(__name__) @@ -45,17 +46,22 @@ def check_environment(): check_read_write() -def check_free_space(): - logger.info('Check free space') +def check_free_space(state_dir: Path): + """ Check if there is enough free space on the disk.""" + logger.info(f'Checking free space for the folder: {state_dir}') try: - free_space = psutil.disk_usage(".").free / (1024 * 1024.0) - if free_space < 100: + usage = psutil.disk_usage(str(state_dir.absolute())) + usage_mb = usage.free / (1024 * 1024) + if usage_mb < 100: error_and_exit("Insufficient disk space", "You have less than 100MB of usable disk space. " + "Please free up some space and run Tribler again.") + logger.info(f'There is enough free space: {usage_mb:.0f} MB') except ImportError as ie: logger.error(ie) error_and_exit("Import Error", f"Import error: {ie}") + except OSError as e: + logger.exception(e) def set_process_priority(pid=None, priority_order=1): @@ -66,22 +72,28 @@ def set_process_priority(pid=None, priority_order=1): """ if priority_order < 0 or priority_order > 5: return - - process = psutil.Process(pid if pid else os.getpid()) + if sys.platform not in {'win32', 'darwin', 'linux'}: + return if sys.platform == 'win32': - priority_classes = [psutil.IDLE_PRIORITY_CLASS, - psutil.BELOW_NORMAL_PRIORITY_CLASS, - psutil.NORMAL_PRIORITY_CLASS, - psutil.ABOVE_NORMAL_PRIORITY_CLASS, - psutil.HIGH_PRIORITY_CLASS, - psutil.REALTIME_PRIORITY_CLASS] - process.nice(priority_classes[priority_order]) - elif sys.platform == 'darwin' or sys.platform == 'linux2': + priority_classes = [ + psutil.IDLE_PRIORITY_CLASS, + psutil.BELOW_NORMAL_PRIORITY_CLASS, + psutil.NORMAL_PRIORITY_CLASS, + psutil.ABOVE_NORMAL_PRIORITY_CLASS, + psutil.HIGH_PRIORITY_CLASS, + psutil.REALTIME_PRIORITY_CLASS + ] + else: # On Unix, priority can be -20 to 20, but usually not allowed to set below 0, we set our classes somewhat in # that range. priority_classes = [5, 4, 3, 2, 1, 0] + + try: + process = psutil.Process(pid if pid else os.getpid()) process.nice(priority_classes[priority_order]) + except psutil.Error as e: + logger.exception(e) def enable_fault_handler(log_dir): diff --git a/src/tribler/core/tests/test_check_os.py b/src/tribler/core/tests/test_check_os.py index 97082b1a292..0b19fbad5ed 100644 --- a/src/tribler/core/tests/test_check_os.py +++ b/src/tribler/core/tests/test_check_os.py @@ -1,8 +1,14 @@ from logging import Logger -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch -from tribler.core.check_os import enable_fault_handler, error_and_exit +import psutil +import pytest + +from tribler.core.check_os import check_free_space, enable_fault_handler, error_and_exit, set_process_priority from tribler.core.utilities.patch_import import patch_import +from tribler.core.utilities.path_util import Path + +DISK_USAGE = 'tribler.core.check_os.psutil.disk_usage' # pylint: disable=import-outside-toplevel @@ -40,3 +46,58 @@ def test_enable_fault_handler_log_dir_not_exists(): enable_fault_handler(log_dir=log_dir) log_dir.mkdir.assert_called_once() + + +@patch.object(psutil.Process, 'nice') +def test_set_process_priority_supported_platform(mocked_nice: Mock): + """ Test that the process priority is set on supported platforms.""" + set_process_priority() + assert mocked_nice.called + + +@patch('sys.platform', 'freebsd7') +@patch.object(psutil.Process, 'nice') +def test_set_process_priority_unsupported_platform(mocked_nice: Mock): + """ Test that the process priority is not set on unsupported platforms.""" + set_process_priority() + assert not mocked_nice.called + + +def test_set_process_exception(): + """ Test that the set_process_priority does not re-raise an exception derived from `psutil.Error` + but re-raise all other exceptions""" + + # psutil.Error + with patch.object(psutil.Process, 'nice', new=Mock(side_effect=psutil.AccessDenied)): + set_process_priority() + + # any other error + with patch.object(psutil.Process, 'nice', new=Mock(side_effect=FileNotFoundError)): + with pytest.raises(FileNotFoundError): + set_process_priority() + + +def test_check_free_space_sufficient(): + # Test to ensure the function works correctly when there's sufficient disk space. + with patch(DISK_USAGE) as mock_usage: + mock_usage.return_value = MagicMock(free=1024 * 1024 * 200) # Simulating 200MB of free space + check_free_space(Path("/path/to/dir")) + + +def test_check_free_space_insufficient(): + # Test to ensure the function raises an exception when there's insufficient disk space. + with patch(DISK_USAGE) as mock_usage, pytest.raises(SystemExit): + mock_usage.return_value = MagicMock(free=1024 * 1024 * 50) # Simulating 50MB of free space + check_free_space(Path("/path/to/dir")) + + +def test_check_free_space_import_error(): + # Test to check the behavior when there's an ImportError. + with patch(DISK_USAGE, side_effect=ImportError("mock import error")), pytest.raises(SystemExit): + check_free_space(Path("/path/to/dir")) + + +def test_check_free_space_os_error(): + # Test to check the behavior when there's an OSError. + with patch(DISK_USAGE, side_effect=OSError("mock os error")): + check_free_space(Path("/path/to/dir")) diff --git a/src/tribler/core/utilities/asyncio_fixes/finish_accept_patch.py b/src/tribler/core/utilities/asyncio_fixes/finish_accept_patch.py new file mode 100644 index 00000000000..501d49d7fbc --- /dev/null +++ b/src/tribler/core/utilities/asyncio_fixes/finish_accept_patch.py @@ -0,0 +1,144 @@ +import socket +import struct +from asyncio import exceptions, tasks, trsock +from asyncio.log import logger + +try: + import _overlapped +except ImportError: + _overlapped = None + + +NULL = 0 +patch_applied = False + + +# pylint: disable=protected-access + + +def apply_finish_accept_patch(): # pragma: no cover + """ + The patch fixes the following issue with the IocpProactor._accept() method on Windows: + + OSError: [WinError 64] The specified network name is no longer available + File "asyncio\windows_events.py", line 571, in accept_coro + await future + File "asyncio\windows_events.py", line 817, in _poll + value = callback(transferred, key, ov) + File "asyncio\windows_events.py", line 560, in finish_accept + ov.getresult() + OSError: [WinError 64] The specified network name is no longer available. + + See: + * https://github.com/Tribler/tribler/issues/7759 + * https://github.com/python/cpython/issues/93821 + """ + + global patch_applied # pylint: disable=global-statement + if patch_applied: + return + + # pylint: disable=import-outside-toplevel + from asyncio.proactor_events import BaseProactorEventLoop + from asyncio.windows_events import IocpProactor + + BaseProactorEventLoop._start_serving = patched_proactor_event_loop_start_serving + IocpProactor.accept = patched_iocp_proacor_accept + + patch_applied = True + logger.info("Patched asyncio to fix accept() issues on Windows") + + +def patched_iocp_proacor_accept(self, listener, *, _overlapped=_overlapped): + self._register_with_iocp(listener) + conn = self._get_accept_socket(listener.family) + ov = _overlapped.Overlapped(NULL) + ov.AcceptEx(listener.fileno(), conn.fileno()) + + def finish_accept(trans, key, ov): # pylint: disable=unused-argument + # ov.getresult() + # start of the patched code + try: + ov.getresult() + except OSError as exc: + if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED, + _overlapped.ERROR_OPERATION_ABORTED): + logger.debug("Connection reset error occurred, continuing to accept connections") + conn.close() + raise ConnectionResetError(*exc.args) from exc + raise + # end of the patched code + + # Use SO_UPDATE_ACCEPT_CONTEXT so getsockname() etc work. + buf = struct.pack('@P', listener.fileno()) + conn.setsockopt(socket.SOL_SOCKET, + _overlapped.SO_UPDATE_ACCEPT_CONTEXT, buf) + conn.settimeout(listener.gettimeout()) + return conn, conn.getpeername() + + async def accept_coro(future, conn): + # Coroutine closing the accept socket if the future is cancelled + try: + await future + except exceptions.CancelledError: + conn.close() + raise + + future = self._register(ov, listener, finish_accept) + coro = accept_coro(future, conn) + tasks.ensure_future(coro, loop=self._loop) + return future + + +def patched_proactor_event_loop_start_serving(self, protocol_factory, sock, + sslcontext=None, server=None, backlog=100, + ssl_handshake_timeout=None, + ssl_shutdown_timeout=None): # pragma: no cover + # pylint: disable=unused-argument + + def loop(f=None): + try: + if f is not None: + conn, addr = f.result() + if self._debug: + logger.debug("%r got a new connection from %r: %r", + server, addr, conn) + protocol = protocol_factory() + if sslcontext is not None: + self._make_ssl_transport( + conn, protocol, sslcontext, server_side=True, + extra={'peername': addr}, server=server, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout) + else: + self._make_socket_transport( + conn, protocol, + extra={'peername': addr}, server=server) + if self.is_closed(): + return + f = self._proactor.accept(sock) + + # start of the patched code + except ConnectionResetError: + logger.debug("Connection reset error occurred, continuing to accept connections") + self.call_soon(loop) + # end of the patched code + + except OSError as exc: + if sock.fileno() != -1: + self.call_exception_handler({ + 'message': 'Accept failed on a socket', + 'exception': exc, + 'socket': trsock.TransportSocket(sock), + }) + sock.close() + elif self._debug: + logger.debug("Accept failed on socket %r", + sock, exc_info=True) + except exceptions.CancelledError: + sock.close() + else: + self._accept_futures[sock.fileno()] = f + f.add_done_callback(loop) + + self.call_soon(loop) diff --git a/src/tribler/core/utilities/asyncio_fixes/tests/test_finish_accept_patch.py b/src/tribler/core/utilities/asyncio_fixes/tests/test_finish_accept_patch.py new file mode 100644 index 00000000000..042bacd254f --- /dev/null +++ b/src/tribler/core/utilities/asyncio_fixes/tests/test_finish_accept_patch.py @@ -0,0 +1,107 @@ +from asyncio import exceptions +from dataclasses import dataclass +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from tribler.core.utilities.asyncio_fixes.finish_accept_patch import patched_iocp_proacor_accept + + +# pylint: disable=protected-access + + +@dataclass +class accept_mocks_dataclass: + proactor: MagicMock + future: AsyncMock + conn: MagicMock + listener: MagicMock + overlapped: MagicMock + + +@pytest.fixture(name='accept_mocks') +def accept_mocks_fixture(): + proactor = MagicMock() + + future = AsyncMock(side_effect=exceptions.CancelledError)() + proactor._register.return_value = future + + conn = MagicMock() + proactor._get_accept_socket.return_value = conn + + listener = MagicMock() + overlapped = MagicMock() + + return accept_mocks_dataclass(proactor, future, conn, listener, overlapped) + + +async def test_accept_coro(accept_mocks): + + with patch('asyncio.tasks.ensure_future') as ensure_future_mock: + f = patched_iocp_proacor_accept(accept_mocks.proactor, accept_mocks.listener, + _overlapped=accept_mocks.overlapped) + assert f is accept_mocks.future + + ensure_future_mock.assert_called_once() + coro = ensure_future_mock.call_args[0][0] + + assert not accept_mocks.conn.close.called + + with pytest.raises(exceptions.CancelledError): + await coro + + assert accept_mocks.conn.close.called + + finish_accept = accept_mocks.proactor._register.call_args[0][2] + finish_accept(None, None, accept_mocks.overlapped) + + assert accept_mocks.overlapped.getresult.called + assert accept_mocks.conn.getpeername.called + + +async def test_finish_accept_error_netname_deleted(accept_mocks): + with patch('asyncio.tasks.ensure_future') as ensure_future_mock: + patched_iocp_proacor_accept(accept_mocks.proactor, accept_mocks.listener, + _overlapped=accept_mocks.overlapped) + finish_accept = accept_mocks.proactor._register.call_args[0][2] + + # to avoid RuntimeWarning "coroutine 'accept_coro' was never awaited + coro = ensure_future_mock.call_args[0][0] + with pytest.raises(exceptions.CancelledError): + await coro + + exc = OSError() + exc.winerror = accept_mocks.overlapped.ERROR_NETNAME_DELETED + accept_mocks.overlapped.getresult.side_effect = exc + + accept_mocks.conn.close.reset_mock() + assert not accept_mocks.conn.close.called + + with pytest.raises(ConnectionResetError): + await finish_accept(None, None, accept_mocks.overlapped) + + assert accept_mocks.conn.close.called + + +async def test_finish_accept_other_os_error(accept_mocks): + with patch('asyncio.tasks.ensure_future') as ensure_future_mock: + patched_iocp_proacor_accept(accept_mocks.proactor, accept_mocks.listener, + _overlapped=accept_mocks.overlapped) + finish_accept = accept_mocks.proactor._register.call_args[0][2] + + # to avoid RuntimeWarning "coroutine 'accept_coro' was never awaited + coro = ensure_future_mock.call_args[0][0] + with pytest.raises(exceptions.CancelledError): + await coro + + exc = OSError() + exc.winerror = MagicMock() + accept_mocks.overlapped.getresult.side_effect = exc + + accept_mocks.conn.close.reset_mock() + assert not accept_mocks.conn.close.called + + with pytest.raises(OSError): + await finish_accept(None, None, accept_mocks.overlapped) + + assert not accept_mocks.conn.close.called diff --git a/src/tribler/core/version.py b/src/tribler/core/version.py index c18e3788406..be6def9445f 100644 --- a/src/tribler/core/version.py +++ b/src/tribler/core/version.py @@ -1,4 +1,4 @@ -version_id = "7.13.1-GIT" +version_id = "7.13.3-GIT" build_date = "Mon Jan 01 00:00:01 1970" commit_id = "none" sentry_url = "" diff --git a/src/tribler/gui/start_gui.py b/src/tribler/gui/start_gui.py index f84c88b4c3f..cba15b37cbc 100644 --- a/src/tribler/gui/start_gui.py +++ b/src/tribler/gui/start_gui.py @@ -53,7 +53,7 @@ def run_gui(api_port: Optional[int], api_key: Optional[str], root_state_dir: Pat enable_fault_handler(root_state_dir) # Exit if we cant read/write files, etc. check_environment() - check_free_space() + check_free_space(root_state_dir) try: app_name = os.environ.get('TRIBLER_APP_NAME', 'triblerapp')