From 7226455e37581ced2d6c7e7db7813c69dc1faf4a Mon Sep 17 00:00:00 2001 From: Elric Milon Date: Wed, 9 Oct 2013 16:28:00 +0200 Subject: [PATCH 01/24] Channel boosting implementation Some refactor, fixes and the following squashed commits: Squashed commits: - Add 'Credit Mining' panel - Add header to credit mining panel - Set investment yield text - Also show speed on credit mining overview - Added some additional counters - Text changes - Add share_mode param to Session.start_download - Add get_share_mode/set_share_mode to LibtorrentDownloadImpl - Initial channel boosting code - Fix for SeederRatioPolicy - Added RandomPolicy - Start ChannelBooster in tribler.py + delete share_mode downloads from - previous run - Add more parameters to command line tool - Initial channel boosting GUI - Added RSS feed + tracker scraping + bug fixes - Make sure that byte stats won't get reset after restarting a download - Small fix - Credit mining gui update - Rename - Fix for slow loading - Fixed some issues with downloading from RSS feeds - Fix for command line tool - Added DirectorySource - Run BoostingManager tasks on separate thread - Preload TorrentDefs - Set share_mode_target to 1 - Fix credit mining totals in GUI - Improvements to credit mining GUI - Added load/save functions to BoostingManager - Fixes - Added archive mode + changed tracker scraping - Fix - Give downloads in archive mode a higher priority - Update seeders and leechers numbers consistently - Added duplicate filter to BoostingManager - Disable tunneling and reduce collected torrents - Add directory to boosting dialog - Ignore Vim backup files and Pylint config - Use logging, Etree scrape - Log UTC time - Add boosting config - Add configuration file - Tidy up BoostingManager - Move Swift pointer - Update Swift pointer - Synchronize download stop - Do not log share mode status regularly - Call set_share_mode to 0 piece priorities - Use libtorrent max swarms, add configs for params - Remove default config - Do not save config - Fix GUI torrent is None bug - Add TaskQueue logging - Swarm replacement interval setup - Fix libtorrent info hash printing, add test - Set up experiments for swarm selection policy - Disable get_metainfo for debugging - Remove torlock from LibtorrentMgr.process_alert - Test piece priorities after setting share mode - Set share mode during adding of torrent - Reverse creation date sorting - Dump core - Also set local speed limit - Set speed limit to 50 Mbit/s for experiments - Config file without Etree RSS - Swarm selection interval setup - Sharing ratio target setup - Update RSS feed fast when 503 on adding --- .../Core/APIImplementation/LaunchManyCore.py | 4 +- .../Core/Libtorrent/LibtorrentDownloadImpl.py | 43 +- Tribler/Core/Libtorrent/LibtorrentMgr.py | 23 + Tribler/Core/defaults.py | 1 + Tribler/Core/simpledefs.py | 1 + Tribler/Main/Dialogs/BoostingDialogs.py | 147 ++++ Tribler/Main/vwxGUI/GuiUtility.py | 32 +- Tribler/Main/vwxGUI/list_body.py | 15 + Tribler/Main/vwxGUI/list_details.py | 2 +- Tribler/Policies/BoostingManager.py | 680 ++++++++++++++++++ Tribler/Test/test_BoostingManager.py | 53 ++ Tribler/Tools/boostchannel.py | 123 ++++ Tribler/Utilities/scraper.py | 50 ++ boosting.ini.1 | 11 + boosting.ini.2 | 11 + boosting.ini.3 | 11 + boosting.ini.one | 11 + 17 files changed, 1206 insertions(+), 12 deletions(-) create mode 100644 Tribler/Main/Dialogs/BoostingDialogs.py create mode 100644 Tribler/Policies/BoostingManager.py create mode 100644 Tribler/Test/test_BoostingManager.py create mode 100644 Tribler/Tools/boostchannel.py create mode 100644 Tribler/Utilities/scraper.py create mode 100644 boosting.ini.1 create mode 100644 boosting.ini.2 create mode 100644 boosting.ini.3 create mode 100644 boosting.ini.one diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index 24e2e1fc973..f19094c69df 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -319,7 +319,7 @@ def load_communities(): self.initComplete = True - def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidden=False): + def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidden=False, share_mode=False): """ Called by any thread """ d = None with self.sesslock: @@ -343,7 +343,7 @@ def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidd # Store in list of Downloads, always. self.downloads[infohash] = d - setup_deferred = d.setup(dscfg, pstate, initialdlstatus, wrapperDelay=setupDelay) + setup_deferred = d.setup(dscfg, pstate, initialdlstatus, wrapperDelay=setupDelay, share_mode=share_mode) setup_deferred.addCallback(self.on_download_wrapper_created) if d and not hidden and self.session.get_megacache(): diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index 4b678c15ef7..64b8066c548 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -157,7 +157,7 @@ def __repr__(self): def get_def(self): return self.tdef - def setup(self, dcfg=None, pstate=None, initialdlstatus=None, wrapperDelay=0): + def setup(self, dcfg=None, pstate=None, initialdlstatus=None, lm_network_engine_wrapper_created_callback=None, wrapperDelay=0, share_mode=False): """ Create a Download object. Used internally by Session. @param dcfg DownloadStartupConfig or None (in which case @@ -190,7 +190,7 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None, wrapperDelay=0): def schedule_create_engine(): self.cew_scheduled = True - create_engine_wrapper_deferred = self.network_create_engine_wrapper(self.pstate_for_restart, initialdlstatus) + create_engine_wrapper_deferred = self.network_create_engine_wrapper(self.pstate_for_restart, initialdlstatus, share_mode=share_mode) create_engine_wrapper_deferred.chainDeferred(deferred) @@ -242,7 +242,7 @@ def do_check(): do_check() return can_create_deferred - def network_create_engine_wrapper(self, pstate, initialdlstatus=None): + def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode=False): with self.dllock: self._logger.debug("LibtorrentDownloadImpl: network_create_engine_wrapper()") @@ -254,6 +254,9 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None): atp["duplicate_is_error"] = True atp["hops"] = self.get_hops() + if share_mode: + atp["flags"] = lt.add_torrent_params_flags_t.flag_share_mode + resume_data = pstate.get('state', 'engineresumedata') if pstate else None if not isinstance(self.tdef, TorrentDefNoMetainfo): metainfo = self.tdef.get_metainfo() @@ -290,9 +293,16 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None): if self.handle: self.set_selected_files() - # If we lost resume_data always resume download in order to force checking - if initialdlstatus != DLSTATUS_STOPPED or not resume_data: - self.handle.resume() + # set_selected_files sets priorities to 1, so we must set + # share_mode again, but first we must unset it, otherwise + # set_share_mode doesn't do anything + if share_mode: + self.handle.set_share_mode(not share_mode) + self.handle.set_share_mode(share_mode) + + # If we lost resume_data always resume download in order to force checking + if initialdlstatus != DLSTATUS_STOPPED or not resume_data: + self.handle.resume() # If we only needed to perform checking, pause download after it is complete self.pause_after_next_hashcheck = initialdlstatus == DLSTATUS_STOPPED @@ -344,6 +354,10 @@ def get_vod_fileindex(self): return self.vod_index return -1 + @checkHandleAndSynchronize(None) + def write_resume_data(self): + return self.handle.write_resume_data() + @checkHandleAndSynchronize(0) def get_vod_filesize(self): fileindex = self.get_vod_fileindex() @@ -1081,7 +1095,10 @@ def network_get_persistent_state(self): else: pstate.set('state', 'metainfo', self.tdef.get_metainfo()) - ds = self.network_get_state(None, False) + if self.get_share_mode(): + pstate.set('state', 'share_mode', True) + + ds = self.network_get_state(None, False, sessioncalling=True) dlstate = {'status': ds.get_status(), 'progress': ds.get_progress(), 'swarmcache': None} pstate.set('state', 'dlstate', dlstate) @@ -1115,6 +1132,10 @@ def add_peer(self, addr): """ self.handle.connect_peer(addr, 0) + @waitForHandleAndSynchronize() + def set_priority(self, prio): + self.handle.set_priority(prio) + @waitForHandleAndSynchronize(True) def dlconfig_changed_callback(self, section, name, new_value, old_value): if section == 'downloadconfig' and name == 'max_upload_rate': @@ -1125,6 +1146,14 @@ def dlconfig_changed_callback(self, section, name, new_value, old_value): return False return True + def get_share_mode(self): + if self.handle: + return self.handle.status().share_mode + + @checkHandleAndSynchronize + def set_share_mode(self, share_mode): + self.handle.set_share_mode(share_mode) + class LibtorrentStatisticsResponse: diff --git a/Tribler/Core/Libtorrent/LibtorrentMgr.py b/Tribler/Core/Libtorrent/LibtorrentMgr.py index 3fc0c1c3347..3540216a252 100644 --- a/Tribler/Core/Libtorrent/LibtorrentMgr.py +++ b/Tribler/Core/Libtorrent/LibtorrentMgr.py @@ -319,11 +319,34 @@ def process_alert(self, alert): elif infohash in self.metainfo_requests: if isinstance(alert, lt.metadata_received_alert): self.got_metainfo(infohash) + else: + self._logger.debug("LibtorrentMgr: could not find torrent %s", infohash) + else: + self._logger.debug("LibtorrentMgr: alert for invalid torrent") + + def reachability_check(self): + if self.ltsession and self.ltsession.status().has_incoming_connections: + self.trsession.lm.threadpool.add_task(self.trsession.lm.dialback_reachable_callback, 3) + else: + self.trsession.lm.threadpool.add_task(self.reachability_check, 10) + + def monitor_dht(self, chances_remaining=1): + # Sometimes the dht fails to start. To workaround this issue we monitor the #dht_nodes, and restart if needed. + if self.ltsession: + if self.get_dht_nodes() <= 25: + if self.get_dht_nodes() >= 5 and chances_remaining: + self._logger.info("LibtorrentMgr: giving the dht a chance (%d, %d)", self.ltsession.status().dht_nodes, chances_remaining) + self.trsession.lm.threadpool.add_task(lambda: self.monitor_dht(chances_remaining - 1), 5) else: self._logger.debug("could not find torrent %s", infohash) else: self._logger.debug("alert for invalid torrent") + def get_peers(self, infohash, callback, timeout=30): + def on_metainfo_retrieved(metainfo, infohash=infohash, callback=callback): + callback(infohash, metainfo.get('initial peers', [])) + self.get_metainfo(infohash, on_metainfo_retrieved, timeout, notify=False) + def get_metainfo(self, infohash_or_magnet, callback, timeout=30, timeout_callback=None, notify=True): if not self.is_dht_ready() and timeout > 5: self._logger.info("DHT not ready, rescheduling get_metainfo") diff --git a/Tribler/Core/defaults.py b/Tribler/Core/defaults.py index 0c1d68c3a1b..f82ea19de4b 100644 --- a/Tribler/Core/defaults.py +++ b/Tribler/Core/defaults.py @@ -252,4 +252,5 @@ tribler_defaults['Tribler']['mintray'] = 2 if sys.platform == 'win32' else 0 tribler_defaults['Tribler']['free_space_threshold'] = 100 * 1024 * 1024 tribler_defaults['Tribler']['version_info'] = {} +tribler_defaults['Tribler']['boosting_sources'] = "" tribler_defaults['Tribler']['last_reported_version'] = None diff --git a/Tribler/Core/simpledefs.py b/Tribler/Core/simpledefs.py index 7fcd13e2e20..d6dfc0ff793 100644 --- a/Tribler/Core/simpledefs.py +++ b/Tribler/Core/simpledefs.py @@ -98,6 +98,7 @@ NTFY_INSERT = 'insert' # new data is inserted NTFY_DELETE = 'delete' # data is deleted NTFY_CREATE = 'create' # new data is created, meaning in the case of Channels your own channel is created +NTFY_SCRAPE = 'scrape' NTFY_STARTED = 'started' NTFY_STATE = 'state' NTFY_MODIFIED = 'modified' diff --git a/Tribler/Main/Dialogs/BoostingDialogs.py b/Tribler/Main/Dialogs/BoostingDialogs.py new file mode 100644 index 00000000000..005c0154bab --- /dev/null +++ b/Tribler/Main/Dialogs/BoostingDialogs.py @@ -0,0 +1,147 @@ +# Written by Egbert Bouman + +import wx + +from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY +from Tribler.Main.vwxGUI.GuiUtility import GUIUtility +from Tribler.Policies.BoostingManager import BoostingManager + + +class AddBoostingSource(wx.Dialog): + + def __init__(self, parent): + wx.Dialog.__init__(self, parent, -1, 'Add boosting source', size=(475, 475), name="AddBoostingSourceDialog") + + self.guiutility = GUIUtility.getInstance() + self.channels = [] + self.source = '' + + text = wx.StaticText(self, -1, 'Please enter a RSS feed URL or select a channel to start boosting swarms:') + self.channel_radio = wx.RadioButton(self, -1, 'Channel:', style=wx.RB_GROUP) + self.channel_choice = wx.Choice(self, -1) + self.channel_choice.Bind(wx.EVT_CHOICE, lambda evt: self.channel_radio.SetValue(True)) + + self.rss_feed_radio = wx.RadioButton(self, -1, 'RSS feed:') + self.rss_feed_edit = wx.TextCtrl(self, -1) + self.rss_feed_edit.Bind(wx.EVT_TEXT, lambda evt: self.rss_feed_radio.SetValue(True)) + + self.rss_dir_radio = wx.RadioButton(self, -1, 'RSS dir:') + self.rss_dir_edit = wx.TextCtrl(self, -1) + self.rss_dir_edit.Bind(wx.EVT_TEXT, lambda evt: self.rss_dir_radio.SetValue(True)) + + self.archive_check = wx.CheckBox(self, -1, "Archive mode") + ok_btn = wx.Button(self, -1, "OK") + ok_btn.Bind(wx.EVT_BUTTON, self.OnOK) + cancel_btn = wx.Button(self, -1, "Cancel") + cancel_btn.Bind(wx.EVT_BUTTON, self.OnCancel) + + sourceGrid = wx.FlexGridSizer(2, 2, 0, 0) + sourceGrid.AddGrowableCol(1) + sourceGrid.Add(self.channel_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) + sourceGrid.Add(self.channel_choice, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + sourceGrid.Add(self.rss_feed_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) + sourceGrid.Add(self.rss_feed_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + sourceGrid.Add(self.rss_dir_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) + sourceGrid.Add(self.rss_dir_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + btnSizer = wx.BoxSizer(wx.HORIZONTAL) + btnSizer.Add(ok_btn, 0, wx.RIGHT | wx.TOP | wx.BOTTOM, 5) + btnSizer.Add(cancel_btn, 0, wx.ALL, 5) + vSizer = wx.BoxSizer(wx.VERTICAL) + vSizer.Add(text, 0, wx.EXPAND | wx.ALL, 5) + vSizer.Add(sourceGrid, 0, wx.EXPAND | wx.ALL, 5) + vSizer.AddSpacer((-1, 5)) + vSizer.Add(self.archive_check, 0, wx.LEFT | wx.RIGHT, 10) + vSizer.AddStretchSpacer() + vSizer.Add(btnSizer, 0, wx.EXPAND | wx.ALL, 5) + self.SetSizer(vSizer) + + def do_db(): + return self.guiutility.channelsearch_manager.getAllChannels() + + def do_gui(delayedResult): + _, channels = delayedResult.get() + self.channels = sorted([(channel.name, channel.dispersy_cid) for channel in channels]) + self.channel_choice.SetItems([channel[0] for channel in self.channels]) + + startWorker(do_gui, do_db, retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def OnOK(self, event): + if self.channel_radio.GetValue(): + selection = self.channel_choice.GetSelection() + if selection < len(self.channels): + self.source = self.channels[selection][1] + elif self.rss_feed_radio.GetValue(): + self.source = self.rss_feed_edit.GetValue() + else: + self.source = self.rss_dir_edit.GetValue() + self.EndModal(wx.ID_OK) + + def OnCancel(self, event): + self.EndModal(wx.ID_CANCEL) + + def GetValue(self): + return self.source, self.archive_check.GetValue() + + +class RemoveBoostingSource(wx.Dialog): + + def __init__(self, parent): + wx.Dialog.__init__(self, parent, -1, 'Remove boosting source', size=(475, 135), name="RemoveBoostingSourceDialog") + + self.guiutility = GUIUtility.getInstance() + self.boosting_manager = BoostingManager.get_instance() + self.sources = [] + self.source = '' + + text = wx.StaticText(self, -1, 'Please select the boosting source you wish to remove:') + self.source_label = wx.StaticText(self, -1, 'Source:', style=wx.RB_GROUP) + self.source_choice = wx.Choice(self, -1) + ok_btn = wx.Button(self, -1, "OK") + ok_btn.Bind(wx.EVT_BUTTON, self.OnOK) + cancel_btn = wx.Button(self, -1, "Cancel") + cancel_btn.Bind(wx.EVT_BUTTON, self.OnCancel) + + sourceSizer = wx.BoxSizer(wx.HORIZONTAL) + sourceSizer.Add(self.source_label, 0, wx.ALIGN_CENTER_VERTICAL | wx.RIGHT | wx.TOP, 5) + sourceSizer.Add(self.source_choice, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + btnSizer = wx.BoxSizer(wx.HORIZONTAL) + btnSizer.Add(ok_btn, 0, wx.RIGHT | wx.TOP | wx.BOTTOM, 5) + btnSizer.Add(cancel_btn, 0, wx.ALL, 5) + vSizer = wx.BoxSizer(wx.VERTICAL) + vSizer.Add(text, 0, wx.EXPAND | wx.ALL, 5) + vSizer.Add(sourceSizer, 0, wx.EXPAND | wx.ALL, 5) + vSizer.AddStretchSpacer() + vSizer.Add(btnSizer, 0, wx.EXPAND | wx.ALL, 5) + self.SetSizer(vSizer) + + channels = [] + for source in self.boosting_manager.boosting_sources.keys(): + if source.startswith('http://'): + self.sources.append(source) + elif len(source) == 20: + channels.append(source) + + def do_db(): + return self.guiutility.channelsearch_manager.getChannelsByCID(channels) + + def do_gui(delayedResult): + _, channels = delayedResult.get() + channels = sorted([(channel.name, channel.dispersy_cid) for channel in channels]) + self.sources = self.sources + [channel[1] for channel in channels] + self.source_choice.AppendItems([channel[0] for channel in channels]) + + startWorker(do_gui, do_db, retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + self.source_choice.SetItems(self.sources) + + def OnOK(self, event): + selection = self.source_choice.GetSelection() + if selection < len(self.sources): + self.source = self.sources[selection] + self.EndModal(wx.ID_OK) + + def OnCancel(self, event): + self.EndModal(wx.ID_CANCEL) + + def GetValue(self): + return self.source diff --git a/Tribler/Main/vwxGUI/GuiUtility.py b/Tribler/Main/vwxGUI/GuiUtility.py index 1e3dbd28db7..4fcf19f5c4a 100644 --- a/Tribler/Main/vwxGUI/GuiUtility.py +++ b/Tribler/Main/vwxGUI/GuiUtility.py @@ -250,6 +250,27 @@ def ShowPage(self, page, *args): # Hide list self.frame.librarylist.Show(False) + if page == 'creditmining': + # Show list + self.frame.creditmininglist.Show(True) + + # Open infohash + if args: + self.frame.creditmininglist.GetManager().refresh_or_expand(args[0]) + else: + items = self.frame.creditmininglist.GetExpandedItems() + if items: + items[0][1].expanded = False + self.frame.creditmininglist.Select(items[0][0]) + + # Open infohash + if args: + self.frame.creditmininglist.GetManager().refresh_or_expand(args[0]) + + elif self.guiPage == 'creditmining': + # Hide list + self.frame.creditmininglist.Show(False) + if page == 'home': self.frame.home.ResetSearchBox() self.frame.home.Show() @@ -290,6 +311,8 @@ def ShowPage(self, page, *args): self.frame.selectedchannellist.Focus() elif page == 'my_files': self.frame.librarylist.Focus() + elif page == 'creditmining': + self.frame.creditmininglist.Focus() @forceWxThread def on_show_startup_splash(self, subject, changetype, objectID, *args): @@ -356,6 +379,9 @@ def GetSelectedPage(self): if self.guiPage == 'my_files': return self.frame.librarylist + if self.guiPage == 'creditmining': + return self.frame.creditmininglist + def SetTopSplitterWindow(self, window=None, show=True): while self.frame.splitter_top.GetChildren(): self.frame.splitter_top.Detach(0) @@ -595,7 +621,8 @@ def OnList(self, goto_end, event=None): 'selectedchannel': self.frame.selectedchannellist, 'mychannel': self.frame.managechannel, 'search_results': self.frame.searchlist, - 'my_files': self.frame.librarylist} + 'my_files': self.frame.librarylist, + 'creditmining': self.frame.creditmininglist} if self.guiPage in lists and lists[self.guiPage].HasFocus(): lists[self.guiPage].ScrollToEnd(goto_end) elif event: @@ -607,7 +634,8 @@ def ScrollTo(self, id): 'selectedchannel': self.frame.selectedchannellist, 'mychannel': self.frame.managechannel, 'search_results': self.frame.searchlist, - 'my_files': self.frame.librarylist} + 'my_files': self.frame.librarylist, + 'creditmining': self.frame.creditmininglist} if self.guiPage in lists: lists[self.guiPage].ScrollToId(id) diff --git a/Tribler/Main/vwxGUI/list_body.py b/Tribler/Main/vwxGUI/list_body.py index 3c1ae7fed69..533fa03e36b 100644 --- a/Tribler/Main/vwxGUI/list_body.py +++ b/Tribler/Main/vwxGUI/list_body.py @@ -371,11 +371,26 @@ def IsSelected(control): else: self.BackgroundColor(self.list_deselected) + def SetSelectedColour(self, selected): + if selected.Get() != self.list_selected.Get(): + self.list_selected = selected + self.ShowSelected() + def SetDeselectedColour(self, deselected): if deselected.Get() != self.list_deselected.Get(): self.list_deselected = deselected self.ShowSelected() + def SetExpandedColour(self, expanded): + if expanded.Get() != self.list_expanded.Get(): + self.list_expanded = expanded + self.ShowSelected() + + def SetExpandedAndSelectedColour(self, selected_and_expanded): + if selected_and_expanded.Get() != self.list_selected_and_expanded.Get(): + self.list_selected_and_expanded = selected_and_expanded + self.ShowSelected() + @warnWxThread def BackgroundColor(self, color): if self.GetBackgroundColour() != color: diff --git a/Tribler/Main/vwxGUI/list_details.py b/Tribler/Main/vwxGUI/list_details.py index 333df91b974..7b8091f9eae 100644 --- a/Tribler/Main/vwxGUI/list_details.py +++ b/Tribler/Main/vwxGUI/list_details.py @@ -2376,7 +2376,7 @@ def RemoveFileindex(self, fileindex): self.library_manager.last_vod_torrent = None def SetNrFiles(self, nr): - videoplayer_item = self.guiutility.frame.actlist.GetItem(5) + videoplayer_item = self.guiutility.frame.actlist.GetItem(6) num_items = getattr(videoplayer_item, 'num_items', None) if num_items and self.guiutility.frame.videoparentpanel: num_items.SetValue(str(nr)) diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py new file mode 100644 index 00000000000..59dfc485f31 --- /dev/null +++ b/Tribler/Policies/BoostingManager.py @@ -0,0 +1,680 @@ +# -*- coding: utf-8 -*- +# Written by Egbert Bouman, Mihai Capotă, Elric Milon +# pylint: disable=too-few-public-methods, too-many-instance-attributes +# pylint: disable=too-many-arguments, too-many-branches +"""Manage boosting of swarms""" + +import ConfigParser +import HTMLParser +import glob +import json +import logging +import os +import random +import time +import urllib +from binascii import hexlify, unhexlify +from collections import defaultdict +from hashlib import sha1 + +import libtorrent as lt + +from Tribler.Core.DownloadConfig import DownloadStartupConfig +from Tribler.Core.TorrentChecker.session import MAX_TRACKER_MULTI_SCRAPE +from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo +from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_INSERT, NTFY_SCRAPE, NTFY_TORRENTS, NTFY_UPDATE +from Tribler.Main.globals import DefaultDownloadStartupConfig +from Tribler.Utilities.scraper import scrape_tcp, scrape_udp +from Tribler.community.allchannel.community import AllChannelCommunity +from Tribler.community.channel.community import ChannelCommunity +from Tribler.dispersy.taskmanager import TaskManager +from Tribler.dispersy.util import call_on_reactor_thread + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +formatter = logging.Formatter( + "%(asctime)s.%(msecs).03dZ-%(levelname)s-%(message)s", + datefmt="%Y%m%dT%H%M%S") +formatter.converter = time.gmtime +handler = logging.FileHandler("boosting.log", mode="w") +handler.setFormatter(formatter) +logger.addHandler(handler) + +# logging.getLogger(TimedTaskQueue.__name__+"BoostingManager").setLevel( +# logging.DEBUG) + +number_types = (int, long, float) + +CONFIG_FILE = os.path.abspath("boosting.ini") + + +def lev(a, b): + "Calculates the Levenshtein distance between a and b." + n, m = len(a), len(b) + if n > m: + # Make sure n <= m, to use O(min(n,m)) space + a, b = b, a + n, m = m, n + + current = range(n + 1) + for i in range(1, m + 1): + previous, current = current, [i] + [0] * n + for j in range(1, n + 1): + add, delete = previous[j] + 1, current[j - 1] + 1 + change = previous[j - 1] + if a[j - 1] != b[i - 1]: + change = change + 1 + current[j] = min(add, delete, change) + + return current[n] + + +class BoostingPolicy(object): + + def __init__(self, session): + self.session = session + self.key = lambda x: None + # function that checks if key can be applied to torrent + self.key_check = lambda x: None + self.reverse = None + + def apply(self, torrents, max_active): + sorted_torrents = sorted([torrent for torrent in torrents.itervalues() + if self.key_check(torrent)], + key=self.key, reverse=self.reverse) + torrents_start = [] + for torrent in sorted_torrents[:max_active]: + if not self.session.get_download(torrent["metainfo"].get_infohash()): + torrents_start.append(torrent) + torrents_stop = [] + for torrent in sorted_torrents[max_active:]: + if self.session.get_download(torrent["metainfo"].get_infohash()): + torrents_stop.append(torrent) + return (torrents_start, torrents_stop) + + +class RandomPolicy(BoostingPolicy): + + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.key = lambda v: random.random() + self.key_check = lambda v: True + self.reverse = False + + +class CreationDatePolicy(BoostingPolicy): + + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.key = lambda v: v['creation_date'] + self.key_check = lambda v: v['creation_date'] > 0 + self.reverse = True + + +class SeederRatioPolicy(BoostingPolicy): + + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.key = lambda v: v['num_seeders'] / float(v['num_seeders'] + + v['num_leechers']) + self.key_check = lambda v: isinstance(v['num_seeders'], number_types) and isinstance(v['num_leechers'], number_types) and v['num_seeders'] + v['num_leechers'] > 0 + self.reverse = False + + +class BoostingManager(TaskManager): + + __single = None + + def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval=20, sw_interval=20, max_per_source=100, max_active=2): + super(BoostingManager, self).__init__() + + BoostingManager.__single = self + + self._saved_attributes = ["max_torrents_per_source", + "max_torrents_active", "source_interval", + "swarm_interval", "share_mode_target", + "tracker_interval", "logging_interval"] + + self.session = session + self.utility = utility + self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), "credit_mining") + if not os.path.exists(self.credit_mining_path): + os.mkdir(self.credit_mining_path) + + self.boosting_sources = {} + self.torrents = {} + self.policy = None + self.share_mode_target = 3 + + self.max_torrents_per_source = max_per_source + self.max_torrents_active = max_active + self.source_interval = src_interval + self.swarm_interval = sw_interval + self.initial_swarm_interval = 30 + self.policy = policy(self.session) + self.tracker_interval = 300 + self.initial_tracker_interval = 25 + self.logging_interval = 60 + self.initial_logging_interval = 35 + + self.load_config() + + self.set_share_mode_params(share_mode_target=self.share_mode_target) + + if os.path.exists(CONFIG_FILE): + logger.info("Config file %s", open(CONFIG_FILE).read()) + else: + logger.info("Config file missing") + + # TODO(emilon): Refactor this to use taskmanager + self.session.lm.threadpool.add_task(self._select_torrent, self.initial_swarm_interval) + self.session.lm.threadpool.add_task(self.scrape_trackers, + self.initial_tracker_interval) + self.session.lm.threadpool.add_task(self.log_statistics, + self.initial_logging_interval) + + def get_instance(*args, **kw): + if BoostingManager.__single is None: + BoostingManager(*args, **kw) + return BoostingManager.__single + get_instance = staticmethod(get_instance) + + def del_instance(): + BoostingManager.__single = None + del_instance = staticmethod(del_instance) + + def shutdown(self): + for torrent in self.torrents.itervalues(): + try: + self.stop_download(torrent) + except: + continue + + def load(self): + if self.utility: + try: + string_to_source = lambda s: s.decode('hex') if len(s) == 40 and not (os.path.isdir(s) or s.startswith('http://')) else s + for source in json.loads(self.utility.config.Read('boosting_sources')): + self.add_source(string_to_source(source)) + logger.info("Initial boosting sources %s", + self.boosting_sources.keys()) + except: + logger.info("No initial boosting sources") + + def save(self): + if self.utility: + try: + source_to_string = lambda s: s.encode('hex') if len(s) == 20 and not (os.path.isdir(s) or s.startswith('http://')) else s + self.utility.write_config( + 'boosting_sources', + json.dumps([source_to_string(source) for + source in self.boosting_sources.keys()]), + flush=True) + logger.info("Saved sources %s", self.boosting_sources.keys()) + except: + logger.exception("Could not save state") + + def set_share_mode_params(self, share_mode_target=None, share_mode_bandwidth=None, share_mode_download=None, share_mode_seeders=None): + settings = self.session.lm.ltmgr.get_session().settings() + if share_mode_target is not None: + settings.share_mode_target = share_mode_target + if share_mode_bandwidth is not None: + settings.share_mode_bandwidth = share_mode_bandwidth + if share_mode_download is not None: + settings.share_mode_download = share_mode_download + if share_mode_seeders is not None: + settings.share_mode_seeders = share_mode_seeders + self.session.lm.ltmgr.get_session().set_settings(settings) + + def add_source(self, source): + if source not in self.boosting_sources: + args = (self.session, self.session.lm.threadpool, source, self.source_interval, self.max_torrents_per_source, self.on_torrent_insert) + # pylint: disable=star-args + if os.path.isdir(source): + self.boosting_sources[source] = DirectorySource(*args) + elif source.startswith('http://'): + self.boosting_sources[source] = RSSFeedSource(*args) + elif len(source) == 20: + self.boosting_sources[source] = ChannelSource(*args) + else: + logger.error("Cannot add unknown source %s", source) + else: + logger.info("Already have source %s", source) + + def remove_source(self, source_key): + if source_key in self.boosting_sources: + source = self.boosting_sources.pop(source_key) + source.kill_tasks() + + def compare_torrents(self, t1, t2): + # pylint: disable=no-self-use, bad-builtin + ff = lambda ft: ft[1] > 1024 * 1024 + files1 = filter(ff, t1['metainfo'].get_files_with_length()) + files2 = filter(ff, t2['metainfo'].get_files_with_length()) + + if len(files1) == len(files2): + for ft1 in files1: + for ft2 in files2: + if ft1[1] != ft2[1] or lev(ft1[0], ft2[0]) > 5: + return False + return True + return False + + def on_torrent_insert(self, source, infohash, torrent): + # Remember where we got this torrent from + if os.path.isdir(source) or source.startswith('http://'): + source_str = source + elif len(source) == 20: + source_str = source.encode('hex') + else: + source_str = 'unknown source' + torrent['source'] = source_str + + if self.boosting_sources[source].archive: + torrent['preload'] = True + torrent['prio'] = 100 + + # Preload the TorrentDef. + if torrent['metainfo']: + if not isinstance(torrent['metainfo'], TorrentDef): + torrent['metainfo'] = TorrentDef.load(torrent['metainfo']) + else: + torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) + + # If duplicates exist, set is_duplicate to True, except for the one with the most seeders. + duplicates = [other for other in self.torrents.values() if self.compare_torrents(torrent, other)] + if duplicates: + duplicates += [torrent] + healthiest_torrent = max([(torrent['num_seeders'], torrent) for torrent in duplicates])[1] + for duplicate in duplicates: + is_duplicate = healthiest_torrent != duplicate + duplicate['is_duplicate'] = is_duplicate + if is_duplicate and duplicate.get('download', None): + self.stop_download(duplicate) + + self.torrents[infohash] = torrent + logger.info("Got new torrent %s from %s", infohash.encode('hex'), + source_str) + + def scrape_trackers(self): + num_requests = 0 + trackers = defaultdict(list) + + for infohash, torrent in self.torrents.iteritems(): + if isinstance(torrent['metainfo'], TorrentDef): + tdef = torrent['metainfo'] + for tracker in tdef.get_trackers_as_single_tuple(): + trackers[tracker].append(infohash) + num_requests += 1 + + results = defaultdict(lambda: [0, 0]) + for tracker, infohashes in trackers.iteritems(): + try: + reply = {} + if tracker.startswith("http://tracker.etree.org"): + for infohash in infohashes: + reply.update(scrape_tcp(tracker, (infohash,))) + elif tracker.startswith("udp://"): + for group in range(len(infohashes) // + MAX_TRACKER_MULTI_SCRAPE): + reply.update(scrape_udp(tracker, infohashes[ + group * MAX_TRACKER_MULTI_SCRAPE: + (group + 1) * MAX_TRACKER_MULTI_SCRAPE])) + reply.update(scrape_udp(tracker, infohashes[ + -(len(infohashes) % MAX_TRACKER_MULTI_SCRAPE):])) + else: + reply = scrape_tcp(tracker, infohashes) + logger.debug("Got reply from tracker %s : %s", tracker, reply) + except: + logger.exception("Did not get reply from tracker %s", tracker) + else: + for infohash, info in reply.iteritems(): + if info['complete'] > results[infohash][0]: + results[infohash][0] = info['complete'] + results[infohash][1] = info['incomplete'] + + for infohash, num_peers in results.iteritems(): + self.torrents[infohash]['num_seeders'] = num_peers[0] + self.torrents[infohash]['num_leechers'] = num_peers[1] + self.session.notifier.notify(NTFY_TORRENTS, NTFY_SCRAPE, infohash) + + logger.debug("Finished tracker scraping for %s torrents", num_requests) + + self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) + + def set_archive(self, source, enable): + if source in self.boosting_sources: + self.boosting_sources[source].archive = enable + logger.info("Set archive mode for %s to %s", source, enable) + else: + logger.error("Could not set archive mode for unknown source %s", source) + + def start_download(self, torrent): + def do_start(): + dscfg = DownloadStartupConfig() + dscfg.set_dest_dir(self.credit_mining_path) + dscfg.set_safe_seeding(False) + + preload = torrent.get('preload', False) + logger.info("Starting %s preload %s", hexlify(torrent["metainfo"].get_infohash()), preload) + torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=torrent.get('pstate', None), + hidden=True, share_mode=not preload, checkpoint_disabled=True) + torrent['download'].set_priority(torrent.get('prio', 1)) + self.session.lm.threadpool.add_task_in_thread(do_start, 0) + + def stop_download(self, torrent): + def do_stop(): + ihash = lt.big_number(torrent["metainfo"].get_infohash()) + logger.info("Stopping %s", str(ihash)) + download = torrent.pop('download', False) + lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(ihash) + if download and lt_torrent.is_valid(): + logger.debug("Writing resume data") + torrent['pstate'] = {'engineresumedata': download.write_resume_data()} + self.session.remove_download(download) + + self.session.lm.threadpool.add_task_in_thread(do_stop, 0) + + def _select_torrent(self): + torrents = {} + for infohash, torrent in self.torrents.iteritems(): + if torrent.get('preload', False): + if 'download' not in torrent: + self.start_download(torrent) + elif torrent['download'].get_status() == DLSTATUS_SEEDING: + self.stop_download(torrent) + elif not torrent.get('is_duplicate', False): + torrents[infohash] = torrent + + if self.policy is not None and torrents: + + logger.debug("Selecting from %s torrents", len(torrents)) + + # Determine which torrent to start and which to stop. + torrents_start, torrents_stop = self.policy.apply( + torrents, self.max_torrents_active) + for torrent in torrents_stop: + self.stop_download(torrent) + for torrent in torrents_start: + self.start_download(torrent) + + self.session.lm.threadpool.add_task(self._select_torrent, self.swarm_interval) + + def load_config(self): + config = ConfigParser.ConfigParser() + config.read(CONFIG_FILE) + for k, v in config.items(__name__): + if k in self._saved_attributes: + object.__setattr__(self, k, int(v)) + elif k == "policy": + if v == "random": + self.policy = RandomPolicy(self.session) + elif v == "creation": + self.policy = CreationDatePolicy(self.session) + elif v == "seederratio": + self.policy = SeederRatioPolicy(self.session) + elif k == "boosting_sources": + for boosting_source in json.loads(v): + self.add_source(boosting_source) + elif k == "archive_sources": + for archive_source in json.loads(v): + self.set_archive(archive_source, True) + + def save_config(self): + config = ConfigParser.ConfigParser() + config.add_section(__name__) + for k in self._saved_attributes: + config.set(__name__, k, BoostingManager.__getattribute__(self, k)) + config.set(__name__, "boosting_sources", + json.dumps(self.boosting_sources.keys())) + archive_sources = [] + for boosting_source_name, boosting_source in \ + self.boosting_sources.iteritems(): + if boosting_source.archive: + archive_sources.append(boosting_source_name) + if archive_sources: + config.set(__name__, "archive_sources", + json.dumps(archive_sources)) + if isinstance(self.policy, RandomPolicy): + policy = "random" + elif isinstance(self.policy, CreationDatePolicy): + policy = "creation" + elif isinstance(self.policy, SeederRatioPolicy): + policy = "seederratio" + config.set(__name__, "policy", policy) + with open(CONFIG_FILE, "w") as configf: + config.write(configf) + + def log_statistics(self): + """Log transfer statistics""" + lt_torrents = self.session.lm.ltmgr.get_session().get_torrents() + for lt_torrent in lt_torrents: + status = lt_torrent.status() + if unhexlify(str(status.info_hash)) in self.torrents: + logger.debug("Status for %s : %s %s", status.info_hash, + status.all_time_download, + status.all_time_upload) + non_zero_values = [] + for piece_priority in lt_torrent.piece_priorities(): + if piece_priority != 0: + non_zero_values.append(piece_priority) + if non_zero_values: + logger.debug("Non zero priorities for %s : %s", + status.info_hash, non_zero_values) + self.session.lm.threadpool.add_task(self.log_statistics, self.logging_interval) + + +class BoostingSource(object): + + def __init__(self, session, tqueue, source, interval, max_torrents, callback): + self.session = session + self.session.lm.threadpool = tqueue + + self.torrents = {} + self.source = source + self.interval = interval + self.max_torrents = max_torrents + self.callback = callback + self.archive = False + + def kill_tasks(self): + self.session.lm.threadpool.remove_task(self.source) + + def _load(self, source): + pass + + def _update(self): + pass + + +class ChannelSource(BoostingSource): + + def __init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback): + BoostingSource.__init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback) + + self.channel_id = None + + self.channelcast_db = self.session.lm.channelcast_db + + self.community = None + self.database_updated = True + + self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) + self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load(cid), 0, task_name=self.source) + + def kill_tasks(self): + BoostingSource.kill_tasks(self) + self.session.remove_observer(self._on_database_updated) + + def _load(self, dispersy_cid): + dispersy = self.session.get_dispersy_instance() + + @call_on_reactor_thread + def join_community(): + try: + self.community = dispersy.get_community(dispersy_cid, True) + self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=self.source) + + except KeyError: + + allchannelcommunity = None + for community in dispersy.get_communities(): + if isinstance(community, AllChannelCommunity): + allchannelcommunity = community + break + + if allchannelcommunity: + # pylint: disable=protected-access + self.community = ChannelCommunity.init_community(dispersy, dispersy.get_member(mid=dispersy_cid), allchannelcommunity._my_member, True) + logger.info("Joined channel community %s", + dispersy_cid.encode("HEX")) + self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=self.source) + else: + logger.error("Could not find AllChannelCommunity") + + def get_channel_id(): + # pylint: disable=protected-access + if self.community and self.community._channel_id: + self.channel_id = self.community._channel_id + self.session.lm.threadpool.add_task(self._update, 0, task_name=self.source) + logger.info("Got channel id %s", self.channel_id) + else: + logger.warning("Could not get channel id, retrying in 10 s") + self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=self.source) + + join_community() + + def _update(self): + if len(self.torrents) < self.max_torrents: + + if self.database_updated: + infohashes_old = set(self.torrents.keys()) + + torrent_keys_db = ['infohash', 'Torrent.name', 'torrent_file_name', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + torrent_keys_dict = ['infohash', 'name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, torrent_keys_db, self.max_torrents) + self.torrents = dict((torrent[0], dict(zip(torrent_keys_dict[1:], torrent[1:]))) for torrent in torrent_values) + + infohashes_new = set(self.torrents.keys()) + for infohash in infohashes_new - infohashes_old: + if self.callback: + self.callback(self.source, infohash, self.torrents[infohash]) + + self.database_updated = False + + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) + + def _on_database_updated(self, subject, change_type, infohash): + if (subject, change_type, infohash) is None: + # Unused arguments + pass + self.database_updated = True + + +class RSSFeedSource(BoostingSource): + + def __init__(self, session, tqueue, rss_feed, interval, max_torrents, callback): + BoostingSource.__init__(self, session, tqueue, rss_feed, interval, max_torrents, callback) + + self.unescape = HTMLParser.HTMLParser().unescape + + self.feed_handle = None + + self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load(feed), 0, task_name=self.source) + + def _load(self, rss_feed): + self.feed_handle = self.session.lm.ltmgr.get_session().add_feed({'url': rss_feed, 'auto_download': False, 'auto_map_handles': False}) + + def wait_for_feed(): + # Wait until the RSS feed is longer updating. + feed_status = self.feed_handle.get_feed_status() + if feed_status['updating']: + self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=self.source) + elif len(feed_status['error']) > 0: + logger.error("Got error for RSS feed %s : %s", + feed_status['url'], feed_status['error']) + if "503" in feed_status["error"]: + time.sleep(5 * random.random()) + self.feed_handle.update_feed() + self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=self.source) + else: + # The feed is done updating. Now periodically start retrieving torrents. + self.session.lm.threadpool.add_task(self._update, 0, task_name=self.source) + logger.info("Got RSS feed %s", feed_status['url']) + + wait_for_feed() + + def _update(self): + if len(self.torrents) < self.max_torrents: + + feed_status = self.feed_handle.get_feed_status() + + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + + for item in feed_status['items']: + # Not all RSS feeds provide us with the infohash, so we use a fake infohash based on the URL to identify the torrents. + infohash = sha1(item['url']).digest() + if infohash not in self.torrents: + # Store the torrents as rss-infohash_as_hex.torrent. + torrent_filename = os.path.join(BoostingManager.get_instance().credit_mining_path, 'rss-%s.torrent' % infohash.encode('hex')) + if not os.path.exists(torrent_filename): + try: + # Download the torrent and create a TorrentDef. + f = urllib.urlopen(self.unescape(item['url'])) + metainfo = lt.bdecode(f.read()) + tdef = TorrentDef.load_from_dict(metainfo) + tdef.save(torrent_filename) + except: + logger.error("Could not get torrent, skipping %s", + item['url']) + continue + else: + tdef = TorrentDef.load(torrent_filename) + # Create a torrent dict. + torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1] + self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) + # Notify the BoostingManager and provide the real infohash. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) + + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) + + +class DirectorySource(BoostingSource): + + def __init__(self, session, tqueue, directory, interval, max_torrents, callback): + BoostingSource.__init__(self, session, tqueue, directory, interval, max_torrents, callback) + + self._load(directory) + + def _load(self, directory): + if os.path.isdir(directory): + # Wait for __init__ to finish so the source is registered with the + # BoostinManager, otherwise adding torrents won't work + self.session.lm.threadpool.add_task(self._update, 1, task_name=self.source) + logger.info("Got directory %s", directory) + else: + logger.error("Could not find directory %s", directory) + + def _update(self): + if len(self.torrents) < self.max_torrents: + + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + + for torrent_filename in glob.glob(self.source + '/*.torrent'): + if torrent_filename not in self.torrents: + try: + tdef = TorrentDef.load(torrent_filename) + except: + logger.error("Could not load torrent, skipping %s", + torrent_filename) + continue + # Create a torrent dict. + torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1] + self.torrents[torrent_filename] = dict(zip(torrent_keys, torrent_values)) + # Notify the BoostingManager. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[torrent_filename]) + + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) diff --git a/Tribler/Test/test_BoostingManager.py b/Tribler/Test/test_BoostingManager.py new file mode 100644 index 00000000000..9c507522c36 --- /dev/null +++ b/Tribler/Test/test_BoostingManager.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Written by Mihai Capotă +# pylint: disable=too-many-public-methods +"""Test Tribler.Policies.BoostingManager""" + +import mock +import random +import unittest + +import Tribler.Policies.BoostingManager as bm + +class TestBoostingManagerPolicies(unittest.TestCase): + + def setUp(self): + random.seed(0) + self.session = mock.Mock() + self.session.get_download = lambda i: i % 2 + self.torrents = dict() + for i in range(1, 11): + mock_metainfo = mock.Mock() + mock_metainfo.get_id.return_value = i + self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": i, + "num_leechers": i-1, "creation_date": i} + + def test_RandomPolicy(self): + policy = bm.RandomPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 2) + ids_start = [torrent["metainfo"].get_id() for torrent in + torrents_start] + self.assertEqual(ids_start, [4, 8]) + ids_stop = [torrent["metainfo"].get_id() for torrent in torrents_stop] + self.assertEqual(ids_stop, [3, 9, 5, 7, 1]) + + def test_SeederRatioPolicy(self): + policy = bm.SeederRatioPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6) + ids_start = [torrent["metainfo"].get_id() for torrent in + torrents_start] + self.assertEqual(ids_start, [10, 8, 6]) + ids_stop = [torrent["metainfo"].get_id() for torrent in torrents_stop] + self.assertEqual(ids_stop, [3, 1]) + + def test_CreationDatePolicy(self): + policy = bm.CreationDatePolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 5) + ids_start = [torrent["metainfo"].get_id() for torrent in + torrents_start] + self.assertEqual(ids_start, [10, 8, 6]) + ids_stop = [torrent["metainfo"].get_id() for torrent in torrents_stop] + self.assertEqual(ids_stop, [5, 3, 1]) + +if __name__ == "__main__": + unittest.main() diff --git a/Tribler/Tools/boostchannel.py b/Tribler/Tools/boostchannel.py new file mode 100644 index 00000000000..082010c2d89 --- /dev/null +++ b/Tribler/Tools/boostchannel.py @@ -0,0 +1,123 @@ +# Written by Egbert Bouman + +import sys +import time +import shutil +import random +import getopt +import tempfile +from traceback import print_exc + +from Tribler.Core.Session import Session +from Tribler.Core.SessionConfig import SessionStartupConfig +from Tribler.Policies.BoostingManager import BoostingManager, RandomPolicy, CreationDatePolicy, SeederRatioPolicy +from Tribler.community.channel.community import ChannelCommunity +from Tribler.community.channel.preview import PreviewChannelCommunity +from Tribler.community.allchannel.community import AllChannelCommunity + + +def usage(): + print "Usage: python boostchannel.py [options] dispersy_cid" + print "Options:" + print " --db_interval \tnumber of seconds between database refreshes" + print " --sw_interval \tnumber of seconds between swarm selection" + print " --max_per_source \tmaximum number of swarms per source" + print " \t\t\t\tthat should be taken into consideration" + print " --max_active \t\tmaximum number of swarms that should be" + print " \t\t\t\tactive simultaneously" + print " --policy \t\tpolicy for swarm selection" + print " \t\t\t\tpossible values: RandomPolicy" + print " \t\t\t\t CreationDatePolicy" + print " \t\t\t\t SeederRatioPolicy (default)" + print " --help\t\t\tprint this help screen" + print + print "Example:" + print " python boostchannel.py --max_active=5 3c8378fc3493b5772b1e6a25672d3889367cb7c3" + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "hd:s:m:a:p:", ["help", "db_interval=", "sw_interval=", "max_per_source=", "max_active=", "policy="]) + except getopt.GetoptError as err: + print str(err) + usage() + sys.exit(2) + + kwargs = {} + for o, a in opts: + if o in ("-h", "--help"): + usage() + sys.exit(0) + elif o in ("-d", "--db_interval"): + kwargs['src_interval'] = int(a) + elif o in ("-s", "--sw_interval"): + kwargs['sw_interval'] = int(a) + elif o in ("-m", "--max_per_source"): + kwargs['max_eligible'] = int(a) + elif o in ("-a", "--max_active"): + kwargs['max_active'] = int(a) + elif o in ("-p", "--policy"): + if a == 'RandomPolicy': + kwargs['policy'] = RandomPolicy + elif a == 'CreationDatePolicy': + kwargs['policy'] = CreationDatePolicy + elif a == 'SeederRatioPolicy': + kwargs['policy'] = SeederRatioPolicy + else: + assert False, "Unknown policy" + else: + assert False, "Unhandled option" + + if len(args[0]) != 40: + print "Incorrect dispersy_cid" + sys.exit(2) + else: + dispersy_cid = args[0].decode('hex') + + print "Press Ctrl-C to stop boosting this channel" + + statedir = tempfile.mkdtemp() + + config = SessionStartupConfig() + config.set_state_dir(statedir) + config.set_listen_port(random.randint(10000, 60000)) + config.set_torrent_checking(False) + config.set_multicast_local_peer_discovery(False) + config.set_megacache(True) + config.set_dispersy(True) + config.set_swift_proc(False) + config.set_mainline_dht(False) + config.set_torrent_collecting(False) + config.set_libtorrent(True) + config.set_dht_torrent_collecting(False) + + s = Session(config) + s.start() + + while not s.lm.initComplete: + time.sleep(1) + + def load_communities(): + dispersy.define_auto_load(AllChannelCommunity, (s.dispersy_member,), load=True) + dispersy.define_auto_load(ChannelCommunity, load=True) + dispersy.define_auto_load(PreviewChannelCommunity) + print >> sys.stderr, "Dispersy communities are ready" + + dispersy = s.get_dispersy_instance() + dispersy.callback.call(load_communities) + + bm = BoostingManager.get_instance(s, None, **kwargs) + bm.add_source(dispersy_cid) + + try: + while True: + time.sleep(sys.maxsize / 2048) + except: + print_exc() + + s.shutdown() + time.sleep(3) + shutil.rmtree(statedir) + + +if __name__ == "__main__": + main() diff --git a/Tribler/Utilities/scraper.py b/Tribler/Utilities/scraper.py new file mode 100644 index 00000000000..bad6dfafde6 --- /dev/null +++ b/Tribler/Utilities/scraper.py @@ -0,0 +1,50 @@ +import urllib +import socket, random, struct + +from libtorrent import bdecode + +def scrape_udp(tracker, infohashes): + tracker = tracker.lower() + host, port = tracker[6:].split(':') + addr = (host, int(port.split('/')[0])) + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.settimeout(6) + + # Connection request + conn_req = struct.pack("!QII", 0x41727101980, 0, random.getrandbits(32)) + sock.sendto(conn_req, addr) + + # Process connection response + conn_resp, _ = sock.recvfrom(1024) + action, _, connection_id = struct.unpack_from("!IIQ", conn_resp) + if action != 0: + return + + # Scrape request + transaction_id = random.getrandbits(32) + scrp_req = struct.pack("!QII", connection_id, 2, transaction_id) + ''.join([struct.pack("!20s", i) for i in infohashes]) + sock.sendto(scrp_req, addr) + + # Process scrape response + scrp_resp, _ = sock.recvfrom(1024) + action, scrp_transaction_id = struct.unpack_from("!II", scrp_resp) + if action != 2 or scrp_transaction_id != transaction_id: + return + + result = {} + for index, infohash in enumerate(infohashes): + t = struct.unpack_from("!III", scrp_resp, 8 + (index * 12)) + result[infohash] = dict(zip(['complete', 'downloaded', 'incomplete'], t)) + return result + +def scrape_tcp(tracker, infohashes): + infohashes_quoted = [urllib.quote(infohash) for infohash in infohashes] + tracker = tracker.replace("announce", "scrape") + tracker = tracker + ('?' if tracker.find('?') == -1 else '&') + 'info_hash=' + '&info_hash='.join(infohashes_quoted) + response = urllib.urlopen(tracker).read() + response = bdecode(response) + + result = {} + for h, i in response['files'].iteritems(): + result[h] = {"complete" : i["complete"], "downloaded" : i["downloaded"], "incomplete" : i["incomplete"]} + return result diff --git a/boosting.ini.1 b/boosting.ini.1 new file mode 100644 index 00000000000..8f0dd95ee73 --- /dev/null +++ b/boosting.ini.1 @@ -0,0 +1,11 @@ +[Tribler.Policies.BoostingManager] +max_torrents_per_source = 100 +max_torrents_active = 13 +source_interval = 300 +swarm_interval = 300 +share_mode_target = 3 +tracker_interval = 300 +logging_interval = 60 +boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +policy = seederratio + diff --git a/boosting.ini.2 b/boosting.ini.2 new file mode 100644 index 00000000000..d4e2539bd4e --- /dev/null +++ b/boosting.ini.2 @@ -0,0 +1,11 @@ +[Tribler.Policies.BoostingManager] +max_torrents_per_source = 100 +max_torrents_active = 13 +source_interval = 300 +swarm_interval = 300 +share_mode_target = 5 +tracker_interval = 300 +logging_interval = 60 +boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +policy = seederratio + diff --git a/boosting.ini.3 b/boosting.ini.3 new file mode 100644 index 00000000000..637ec58653f --- /dev/null +++ b/boosting.ini.3 @@ -0,0 +1,11 @@ +[Tribler.Policies.BoostingManager] +max_torrents_per_source = 100 +max_torrents_active = 13 +source_interval = 300 +swarm_interval = 300 +share_mode_target = 1 +tracker_interval = 300 +logging_interval = 60 +boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +policy = seederratio + diff --git a/boosting.ini.one b/boosting.ini.one new file mode 100644 index 00000000000..90575fd95fd --- /dev/null +++ b/boosting.ini.one @@ -0,0 +1,11 @@ +[Tribler.Policies.BoostingManager] +max_torrents_per_source = 100 +max_torrents_active = 13 +source_interval = 300 +swarm_interval = 300 +share_mode_target = 3 +tracker_interval = 300 +logging_interval = 60 +boosting_sources = ["/media/hdd250/Experiments/last/torrents-one"] +policy = seederratio + From 168f7ccc29c88b76f6218be04b7a3eb41cb6b353 Mon Sep 17 00:00:00 2001 From: Elric Milon Date: Tue, 18 Aug 2015 14:18:00 +0200 Subject: [PATCH 02/24] sessioncalling arg was removed a while ago. --- Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index 64b8066c548..5c624626c18 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -1098,7 +1098,7 @@ def network_get_persistent_state(self): if self.get_share_mode(): pstate.set('state', 'share_mode', True) - ds = self.network_get_state(None, False, sessioncalling=True) + ds = self.network_get_state(None, False) dlstate = {'status': ds.get_status(), 'progress': ds.get_progress(), 'swarmcache': None} pstate.set('state', 'dlstate', dlstate) From bd87ab16086d9696c2af49b734f76d616d6f5ab4 Mon Sep 17 00:00:00 2001 From: Elric Milon Date: Tue, 18 Aug 2015 14:39:32 +0200 Subject: [PATCH 03/24] Updates for API changes --- Tribler/Test/test_BoostingManager.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tribler/Test/test_BoostingManager.py b/Tribler/Test/test_BoostingManager.py index 9c507522c36..48f47fc41c8 100644 --- a/Tribler/Test/test_BoostingManager.py +++ b/Tribler/Test/test_BoostingManager.py @@ -25,28 +25,28 @@ def setUp(self): def test_RandomPolicy(self): policy = bm.RandomPolicy(self.session) torrents_start, torrents_stop = policy.apply(self.torrents, 2) - ids_start = [torrent["metainfo"].get_id() for torrent in + ids_start = [torrent["metainfo"].get_infohash() for torrent in torrents_start] self.assertEqual(ids_start, [4, 8]) - ids_stop = [torrent["metainfo"].get_id() for torrent in torrents_stop] + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] self.assertEqual(ids_stop, [3, 9, 5, 7, 1]) def test_SeederRatioPolicy(self): policy = bm.SeederRatioPolicy(self.session) torrents_start, torrents_stop = policy.apply(self.torrents, 6) - ids_start = [torrent["metainfo"].get_id() for torrent in + ids_start = [torrent["metainfo"].get_infohash() for torrent in torrents_start] self.assertEqual(ids_start, [10, 8, 6]) - ids_stop = [torrent["metainfo"].get_id() for torrent in torrents_stop] + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] self.assertEqual(ids_stop, [3, 1]) def test_CreationDatePolicy(self): policy = bm.CreationDatePolicy(self.session) torrents_start, torrents_stop = policy.apply(self.torrents, 5) - ids_start = [torrent["metainfo"].get_id() for torrent in + ids_start = [torrent["metainfo"].get_infohash() for torrent in torrents_start] self.assertEqual(ids_start, [10, 8, 6]) - ids_stop = [torrent["metainfo"].get_id() for torrent in torrents_stop] + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] self.assertEqual(ids_stop, [5, 3, 1]) if __name__ == "__main__": From 897cdffa8eefab9d09f4cef0a8361bf8f264c065 Mon Sep 17 00:00:00 2001 From: Elric Milon Date: Tue, 18 Aug 2015 17:10:12 +0200 Subject: [PATCH 04/24] Add missing __init__.py --- Tribler/Policies/__init__.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 Tribler/Policies/__init__.py diff --git a/Tribler/Policies/__init__.py b/Tribler/Policies/__init__.py new file mode 100644 index 00000000000..07aca525373 --- /dev/null +++ b/Tribler/Policies/__init__.py @@ -0,0 +1,40 @@ +# __init__.py --- +# +# Filename: __init__.py +# Description: +# Author: Elric Milon +# Maintainer: +# Created: Thu Aug 13 17:17:18 2015 (+0200) + +# Commentary: +# +# +# +# + +# Change Log: +# +# +# +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNU Emacs. If not, see . +# +# + +# Code: + + + +# +# __init__.py ends here From d7138950af9670b40de5e6c74075cc688f08268a Mon Sep 17 00:00:00 2001 From: Elric Milon Date: Tue, 18 Aug 2015 19:17:52 +0200 Subject: [PATCH 05/24] Don't fail when a torrent not in the download list finishes. --- Tribler/Main/tribler_main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Tribler/Main/tribler_main.py b/Tribler/Main/tribler_main.py index f013d254338..5db8cc94dcf 100755 --- a/Tribler/Main/tribler_main.py +++ b/Tribler/Main/tribler_main.py @@ -695,15 +695,17 @@ def sesscb_ntfy_torrentupdates(self, events): torrent = t.torrent if isinstance(t, CollectedTorrent) else t self.frame.librarydetailspanel.setTorrent(torrent) - def sesscb_ntfy_torrentfinished(self, subject, changeType, objectID, *args): + def sesscb_ntfy_torrentfinished(self, subject, changeType, infohash, *args): self.guiUtility.Notify( "Download Completed", "Torrent '%s' has finished downloading. Now seeding." % args[0], icon='seed') if self._frame_and_ready(): - infohash = objectID torrent = self.guiUtility.torrentsearch_manager.getTorrentByInfohash(infohash) - self.guiUtility.library_manager.addDownloadState(torrent) + # Check if we got the actual torrent as the bandwith investor + # downloads aren't going to be there. + if torrent: + self.guiUtility.library_manager.addDownloadState(torrent) @forceWxThread def sesscb_ntfy_newversion(self, subject, changeType, objectID, *args): From 9d3201bb44d2b42f5d92d3e33e2208d906b9fe71 Mon Sep 17 00:00:00 2001 From: Elric Milon Date: Tue, 18 Aug 2015 20:16:52 +0200 Subject: [PATCH 06/24] Remove DOS newlines from BoostingManager modifications. --- Tribler/Policies/BoostingManager.py | 1360 +++++++++++++-------------- Tribler/Tools/boostchannel.py | 246 ++--- 2 files changed, 803 insertions(+), 803 deletions(-) diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 59dfc485f31..07b410166fa 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -1,680 +1,680 @@ -# -*- coding: utf-8 -*- -# Written by Egbert Bouman, Mihai Capotă, Elric Milon -# pylint: disable=too-few-public-methods, too-many-instance-attributes -# pylint: disable=too-many-arguments, too-many-branches -"""Manage boosting of swarms""" - -import ConfigParser -import HTMLParser -import glob -import json -import logging -import os -import random -import time -import urllib -from binascii import hexlify, unhexlify -from collections import defaultdict -from hashlib import sha1 - -import libtorrent as lt - -from Tribler.Core.DownloadConfig import DownloadStartupConfig -from Tribler.Core.TorrentChecker.session import MAX_TRACKER_MULTI_SCRAPE -from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo -from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_INSERT, NTFY_SCRAPE, NTFY_TORRENTS, NTFY_UPDATE -from Tribler.Main.globals import DefaultDownloadStartupConfig -from Tribler.Utilities.scraper import scrape_tcp, scrape_udp -from Tribler.community.allchannel.community import AllChannelCommunity -from Tribler.community.channel.community import ChannelCommunity -from Tribler.dispersy.taskmanager import TaskManager -from Tribler.dispersy.util import call_on_reactor_thread - - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -formatter = logging.Formatter( - "%(asctime)s.%(msecs).03dZ-%(levelname)s-%(message)s", - datefmt="%Y%m%dT%H%M%S") -formatter.converter = time.gmtime -handler = logging.FileHandler("boosting.log", mode="w") -handler.setFormatter(formatter) -logger.addHandler(handler) - -# logging.getLogger(TimedTaskQueue.__name__+"BoostingManager").setLevel( -# logging.DEBUG) - -number_types = (int, long, float) - -CONFIG_FILE = os.path.abspath("boosting.ini") - - -def lev(a, b): - "Calculates the Levenshtein distance between a and b." - n, m = len(a), len(b) - if n > m: - # Make sure n <= m, to use O(min(n,m)) space - a, b = b, a - n, m = m, n - - current = range(n + 1) - for i in range(1, m + 1): - previous, current = current, [i] + [0] * n - for j in range(1, n + 1): - add, delete = previous[j] + 1, current[j - 1] + 1 - change = previous[j - 1] - if a[j - 1] != b[i - 1]: - change = change + 1 - current[j] = min(add, delete, change) - - return current[n] - - -class BoostingPolicy(object): - - def __init__(self, session): - self.session = session - self.key = lambda x: None - # function that checks if key can be applied to torrent - self.key_check = lambda x: None - self.reverse = None - - def apply(self, torrents, max_active): - sorted_torrents = sorted([torrent for torrent in torrents.itervalues() - if self.key_check(torrent)], - key=self.key, reverse=self.reverse) - torrents_start = [] - for torrent in sorted_torrents[:max_active]: - if not self.session.get_download(torrent["metainfo"].get_infohash()): - torrents_start.append(torrent) - torrents_stop = [] - for torrent in sorted_torrents[max_active:]: - if self.session.get_download(torrent["metainfo"].get_infohash()): - torrents_stop.append(torrent) - return (torrents_start, torrents_stop) - - -class RandomPolicy(BoostingPolicy): - - def __init__(self, session): - BoostingPolicy.__init__(self, session) - self.key = lambda v: random.random() - self.key_check = lambda v: True - self.reverse = False - - -class CreationDatePolicy(BoostingPolicy): - - def __init__(self, session): - BoostingPolicy.__init__(self, session) - self.key = lambda v: v['creation_date'] - self.key_check = lambda v: v['creation_date'] > 0 - self.reverse = True - - -class SeederRatioPolicy(BoostingPolicy): - - def __init__(self, session): - BoostingPolicy.__init__(self, session) - self.key = lambda v: v['num_seeders'] / float(v['num_seeders'] + - v['num_leechers']) - self.key_check = lambda v: isinstance(v['num_seeders'], number_types) and isinstance(v['num_leechers'], number_types) and v['num_seeders'] + v['num_leechers'] > 0 - self.reverse = False - - -class BoostingManager(TaskManager): - - __single = None - - def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval=20, sw_interval=20, max_per_source=100, max_active=2): - super(BoostingManager, self).__init__() - - BoostingManager.__single = self - - self._saved_attributes = ["max_torrents_per_source", - "max_torrents_active", "source_interval", - "swarm_interval", "share_mode_target", - "tracker_interval", "logging_interval"] - - self.session = session - self.utility = utility - self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), "credit_mining") - if not os.path.exists(self.credit_mining_path): - os.mkdir(self.credit_mining_path) - - self.boosting_sources = {} - self.torrents = {} - self.policy = None - self.share_mode_target = 3 - - self.max_torrents_per_source = max_per_source - self.max_torrents_active = max_active - self.source_interval = src_interval - self.swarm_interval = sw_interval - self.initial_swarm_interval = 30 - self.policy = policy(self.session) - self.tracker_interval = 300 - self.initial_tracker_interval = 25 - self.logging_interval = 60 - self.initial_logging_interval = 35 - - self.load_config() - - self.set_share_mode_params(share_mode_target=self.share_mode_target) - - if os.path.exists(CONFIG_FILE): - logger.info("Config file %s", open(CONFIG_FILE).read()) - else: - logger.info("Config file missing") - - # TODO(emilon): Refactor this to use taskmanager - self.session.lm.threadpool.add_task(self._select_torrent, self.initial_swarm_interval) - self.session.lm.threadpool.add_task(self.scrape_trackers, - self.initial_tracker_interval) - self.session.lm.threadpool.add_task(self.log_statistics, - self.initial_logging_interval) - - def get_instance(*args, **kw): - if BoostingManager.__single is None: - BoostingManager(*args, **kw) - return BoostingManager.__single - get_instance = staticmethod(get_instance) - - def del_instance(): - BoostingManager.__single = None - del_instance = staticmethod(del_instance) - - def shutdown(self): - for torrent in self.torrents.itervalues(): - try: - self.stop_download(torrent) - except: - continue - - def load(self): - if self.utility: - try: - string_to_source = lambda s: s.decode('hex') if len(s) == 40 and not (os.path.isdir(s) or s.startswith('http://')) else s - for source in json.loads(self.utility.config.Read('boosting_sources')): - self.add_source(string_to_source(source)) - logger.info("Initial boosting sources %s", - self.boosting_sources.keys()) - except: - logger.info("No initial boosting sources") - - def save(self): - if self.utility: - try: - source_to_string = lambda s: s.encode('hex') if len(s) == 20 and not (os.path.isdir(s) or s.startswith('http://')) else s - self.utility.write_config( - 'boosting_sources', - json.dumps([source_to_string(source) for - source in self.boosting_sources.keys()]), - flush=True) - logger.info("Saved sources %s", self.boosting_sources.keys()) - except: - logger.exception("Could not save state") - - def set_share_mode_params(self, share_mode_target=None, share_mode_bandwidth=None, share_mode_download=None, share_mode_seeders=None): - settings = self.session.lm.ltmgr.get_session().settings() - if share_mode_target is not None: - settings.share_mode_target = share_mode_target - if share_mode_bandwidth is not None: - settings.share_mode_bandwidth = share_mode_bandwidth - if share_mode_download is not None: - settings.share_mode_download = share_mode_download - if share_mode_seeders is not None: - settings.share_mode_seeders = share_mode_seeders - self.session.lm.ltmgr.get_session().set_settings(settings) - - def add_source(self, source): - if source not in self.boosting_sources: - args = (self.session, self.session.lm.threadpool, source, self.source_interval, self.max_torrents_per_source, self.on_torrent_insert) - # pylint: disable=star-args - if os.path.isdir(source): - self.boosting_sources[source] = DirectorySource(*args) - elif source.startswith('http://'): - self.boosting_sources[source] = RSSFeedSource(*args) - elif len(source) == 20: - self.boosting_sources[source] = ChannelSource(*args) - else: - logger.error("Cannot add unknown source %s", source) - else: - logger.info("Already have source %s", source) - - def remove_source(self, source_key): - if source_key in self.boosting_sources: - source = self.boosting_sources.pop(source_key) - source.kill_tasks() - - def compare_torrents(self, t1, t2): - # pylint: disable=no-self-use, bad-builtin - ff = lambda ft: ft[1] > 1024 * 1024 - files1 = filter(ff, t1['metainfo'].get_files_with_length()) - files2 = filter(ff, t2['metainfo'].get_files_with_length()) - - if len(files1) == len(files2): - for ft1 in files1: - for ft2 in files2: - if ft1[1] != ft2[1] or lev(ft1[0], ft2[0]) > 5: - return False - return True - return False - - def on_torrent_insert(self, source, infohash, torrent): - # Remember where we got this torrent from - if os.path.isdir(source) or source.startswith('http://'): - source_str = source - elif len(source) == 20: - source_str = source.encode('hex') - else: - source_str = 'unknown source' - torrent['source'] = source_str - - if self.boosting_sources[source].archive: - torrent['preload'] = True - torrent['prio'] = 100 - - # Preload the TorrentDef. - if torrent['metainfo']: - if not isinstance(torrent['metainfo'], TorrentDef): - torrent['metainfo'] = TorrentDef.load(torrent['metainfo']) - else: - torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) - - # If duplicates exist, set is_duplicate to True, except for the one with the most seeders. - duplicates = [other for other in self.torrents.values() if self.compare_torrents(torrent, other)] - if duplicates: - duplicates += [torrent] - healthiest_torrent = max([(torrent['num_seeders'], torrent) for torrent in duplicates])[1] - for duplicate in duplicates: - is_duplicate = healthiest_torrent != duplicate - duplicate['is_duplicate'] = is_duplicate - if is_duplicate and duplicate.get('download', None): - self.stop_download(duplicate) - - self.torrents[infohash] = torrent - logger.info("Got new torrent %s from %s", infohash.encode('hex'), - source_str) - - def scrape_trackers(self): - num_requests = 0 - trackers = defaultdict(list) - - for infohash, torrent in self.torrents.iteritems(): - if isinstance(torrent['metainfo'], TorrentDef): - tdef = torrent['metainfo'] - for tracker in tdef.get_trackers_as_single_tuple(): - trackers[tracker].append(infohash) - num_requests += 1 - - results = defaultdict(lambda: [0, 0]) - for tracker, infohashes in trackers.iteritems(): - try: - reply = {} - if tracker.startswith("http://tracker.etree.org"): - for infohash in infohashes: - reply.update(scrape_tcp(tracker, (infohash,))) - elif tracker.startswith("udp://"): - for group in range(len(infohashes) // - MAX_TRACKER_MULTI_SCRAPE): - reply.update(scrape_udp(tracker, infohashes[ - group * MAX_TRACKER_MULTI_SCRAPE: - (group + 1) * MAX_TRACKER_MULTI_SCRAPE])) - reply.update(scrape_udp(tracker, infohashes[ - -(len(infohashes) % MAX_TRACKER_MULTI_SCRAPE):])) - else: - reply = scrape_tcp(tracker, infohashes) - logger.debug("Got reply from tracker %s : %s", tracker, reply) - except: - logger.exception("Did not get reply from tracker %s", tracker) - else: - for infohash, info in reply.iteritems(): - if info['complete'] > results[infohash][0]: - results[infohash][0] = info['complete'] - results[infohash][1] = info['incomplete'] - - for infohash, num_peers in results.iteritems(): - self.torrents[infohash]['num_seeders'] = num_peers[0] - self.torrents[infohash]['num_leechers'] = num_peers[1] - self.session.notifier.notify(NTFY_TORRENTS, NTFY_SCRAPE, infohash) - - logger.debug("Finished tracker scraping for %s torrents", num_requests) - - self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) - - def set_archive(self, source, enable): - if source in self.boosting_sources: - self.boosting_sources[source].archive = enable - logger.info("Set archive mode for %s to %s", source, enable) - else: - logger.error("Could not set archive mode for unknown source %s", source) - - def start_download(self, torrent): - def do_start(): - dscfg = DownloadStartupConfig() - dscfg.set_dest_dir(self.credit_mining_path) - dscfg.set_safe_seeding(False) - - preload = torrent.get('preload', False) - logger.info("Starting %s preload %s", hexlify(torrent["metainfo"].get_infohash()), preload) - torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=torrent.get('pstate', None), - hidden=True, share_mode=not preload, checkpoint_disabled=True) - torrent['download'].set_priority(torrent.get('prio', 1)) - self.session.lm.threadpool.add_task_in_thread(do_start, 0) - - def stop_download(self, torrent): - def do_stop(): - ihash = lt.big_number(torrent["metainfo"].get_infohash()) - logger.info("Stopping %s", str(ihash)) - download = torrent.pop('download', False) - lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(ihash) - if download and lt_torrent.is_valid(): - logger.debug("Writing resume data") - torrent['pstate'] = {'engineresumedata': download.write_resume_data()} - self.session.remove_download(download) - - self.session.lm.threadpool.add_task_in_thread(do_stop, 0) - - def _select_torrent(self): - torrents = {} - for infohash, torrent in self.torrents.iteritems(): - if torrent.get('preload', False): - if 'download' not in torrent: - self.start_download(torrent) - elif torrent['download'].get_status() == DLSTATUS_SEEDING: - self.stop_download(torrent) - elif not torrent.get('is_duplicate', False): - torrents[infohash] = torrent - - if self.policy is not None and torrents: - - logger.debug("Selecting from %s torrents", len(torrents)) - - # Determine which torrent to start and which to stop. - torrents_start, torrents_stop = self.policy.apply( - torrents, self.max_torrents_active) - for torrent in torrents_stop: - self.stop_download(torrent) - for torrent in torrents_start: - self.start_download(torrent) - - self.session.lm.threadpool.add_task(self._select_torrent, self.swarm_interval) - - def load_config(self): - config = ConfigParser.ConfigParser() - config.read(CONFIG_FILE) - for k, v in config.items(__name__): - if k in self._saved_attributes: - object.__setattr__(self, k, int(v)) - elif k == "policy": - if v == "random": - self.policy = RandomPolicy(self.session) - elif v == "creation": - self.policy = CreationDatePolicy(self.session) - elif v == "seederratio": - self.policy = SeederRatioPolicy(self.session) - elif k == "boosting_sources": - for boosting_source in json.loads(v): - self.add_source(boosting_source) - elif k == "archive_sources": - for archive_source in json.loads(v): - self.set_archive(archive_source, True) - - def save_config(self): - config = ConfigParser.ConfigParser() - config.add_section(__name__) - for k in self._saved_attributes: - config.set(__name__, k, BoostingManager.__getattribute__(self, k)) - config.set(__name__, "boosting_sources", - json.dumps(self.boosting_sources.keys())) - archive_sources = [] - for boosting_source_name, boosting_source in \ - self.boosting_sources.iteritems(): - if boosting_source.archive: - archive_sources.append(boosting_source_name) - if archive_sources: - config.set(__name__, "archive_sources", - json.dumps(archive_sources)) - if isinstance(self.policy, RandomPolicy): - policy = "random" - elif isinstance(self.policy, CreationDatePolicy): - policy = "creation" - elif isinstance(self.policy, SeederRatioPolicy): - policy = "seederratio" - config.set(__name__, "policy", policy) - with open(CONFIG_FILE, "w") as configf: - config.write(configf) - - def log_statistics(self): - """Log transfer statistics""" - lt_torrents = self.session.lm.ltmgr.get_session().get_torrents() - for lt_torrent in lt_torrents: - status = lt_torrent.status() - if unhexlify(str(status.info_hash)) in self.torrents: - logger.debug("Status for %s : %s %s", status.info_hash, - status.all_time_download, - status.all_time_upload) - non_zero_values = [] - for piece_priority in lt_torrent.piece_priorities(): - if piece_priority != 0: - non_zero_values.append(piece_priority) - if non_zero_values: - logger.debug("Non zero priorities for %s : %s", - status.info_hash, non_zero_values) - self.session.lm.threadpool.add_task(self.log_statistics, self.logging_interval) - - -class BoostingSource(object): - - def __init__(self, session, tqueue, source, interval, max_torrents, callback): - self.session = session - self.session.lm.threadpool = tqueue - - self.torrents = {} - self.source = source - self.interval = interval - self.max_torrents = max_torrents - self.callback = callback - self.archive = False - - def kill_tasks(self): - self.session.lm.threadpool.remove_task(self.source) - - def _load(self, source): - pass - - def _update(self): - pass - - -class ChannelSource(BoostingSource): - - def __init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback): - BoostingSource.__init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback) - - self.channel_id = None - - self.channelcast_db = self.session.lm.channelcast_db - - self.community = None - self.database_updated = True - - self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) - self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load(cid), 0, task_name=self.source) - - def kill_tasks(self): - BoostingSource.kill_tasks(self) - self.session.remove_observer(self._on_database_updated) - - def _load(self, dispersy_cid): - dispersy = self.session.get_dispersy_instance() - - @call_on_reactor_thread - def join_community(): - try: - self.community = dispersy.get_community(dispersy_cid, True) - self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=self.source) - - except KeyError: - - allchannelcommunity = None - for community in dispersy.get_communities(): - if isinstance(community, AllChannelCommunity): - allchannelcommunity = community - break - - if allchannelcommunity: - # pylint: disable=protected-access - self.community = ChannelCommunity.init_community(dispersy, dispersy.get_member(mid=dispersy_cid), allchannelcommunity._my_member, True) - logger.info("Joined channel community %s", - dispersy_cid.encode("HEX")) - self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=self.source) - else: - logger.error("Could not find AllChannelCommunity") - - def get_channel_id(): - # pylint: disable=protected-access - if self.community and self.community._channel_id: - self.channel_id = self.community._channel_id - self.session.lm.threadpool.add_task(self._update, 0, task_name=self.source) - logger.info("Got channel id %s", self.channel_id) - else: - logger.warning("Could not get channel id, retrying in 10 s") - self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=self.source) - - join_community() - - def _update(self): - if len(self.torrents) < self.max_torrents: - - if self.database_updated: - infohashes_old = set(self.torrents.keys()) - - torrent_keys_db = ['infohash', 'Torrent.name', 'torrent_file_name', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] - torrent_keys_dict = ['infohash', 'name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] - torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, torrent_keys_db, self.max_torrents) - self.torrents = dict((torrent[0], dict(zip(torrent_keys_dict[1:], torrent[1:]))) for torrent in torrent_values) - - infohashes_new = set(self.torrents.keys()) - for infohash in infohashes_new - infohashes_old: - if self.callback: - self.callback(self.source, infohash, self.torrents[infohash]) - - self.database_updated = False - - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) - - def _on_database_updated(self, subject, change_type, infohash): - if (subject, change_type, infohash) is None: - # Unused arguments - pass - self.database_updated = True - - -class RSSFeedSource(BoostingSource): - - def __init__(self, session, tqueue, rss_feed, interval, max_torrents, callback): - BoostingSource.__init__(self, session, tqueue, rss_feed, interval, max_torrents, callback) - - self.unescape = HTMLParser.HTMLParser().unescape - - self.feed_handle = None - - self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load(feed), 0, task_name=self.source) - - def _load(self, rss_feed): - self.feed_handle = self.session.lm.ltmgr.get_session().add_feed({'url': rss_feed, 'auto_download': False, 'auto_map_handles': False}) - - def wait_for_feed(): - # Wait until the RSS feed is longer updating. - feed_status = self.feed_handle.get_feed_status() - if feed_status['updating']: - self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=self.source) - elif len(feed_status['error']) > 0: - logger.error("Got error for RSS feed %s : %s", - feed_status['url'], feed_status['error']) - if "503" in feed_status["error"]: - time.sleep(5 * random.random()) - self.feed_handle.update_feed() - self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=self.source) - else: - # The feed is done updating. Now periodically start retrieving torrents. - self.session.lm.threadpool.add_task(self._update, 0, task_name=self.source) - logger.info("Got RSS feed %s", feed_status['url']) - - wait_for_feed() - - def _update(self): - if len(self.torrents) < self.max_torrents: - - feed_status = self.feed_handle.get_feed_status() - - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] - - for item in feed_status['items']: - # Not all RSS feeds provide us with the infohash, so we use a fake infohash based on the URL to identify the torrents. - infohash = sha1(item['url']).digest() - if infohash not in self.torrents: - # Store the torrents as rss-infohash_as_hex.torrent. - torrent_filename = os.path.join(BoostingManager.get_instance().credit_mining_path, 'rss-%s.torrent' % infohash.encode('hex')) - if not os.path.exists(torrent_filename): - try: - # Download the torrent and create a TorrentDef. - f = urllib.urlopen(self.unescape(item['url'])) - metainfo = lt.bdecode(f.read()) - tdef = TorrentDef.load_from_dict(metainfo) - tdef.save(torrent_filename) - except: - logger.error("Could not get torrent, skipping %s", - item['url']) - continue - else: - tdef = TorrentDef.load(torrent_filename) - # Create a torrent dict. - torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1] - self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) - # Notify the BoostingManager and provide the real infohash. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) - - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) - - -class DirectorySource(BoostingSource): - - def __init__(self, session, tqueue, directory, interval, max_torrents, callback): - BoostingSource.__init__(self, session, tqueue, directory, interval, max_torrents, callback) - - self._load(directory) - - def _load(self, directory): - if os.path.isdir(directory): - # Wait for __init__ to finish so the source is registered with the - # BoostinManager, otherwise adding torrents won't work - self.session.lm.threadpool.add_task(self._update, 1, task_name=self.source) - logger.info("Got directory %s", directory) - else: - logger.error("Could not find directory %s", directory) - - def _update(self): - if len(self.torrents) < self.max_torrents: - - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] - - for torrent_filename in glob.glob(self.source + '/*.torrent'): - if torrent_filename not in self.torrents: - try: - tdef = TorrentDef.load(torrent_filename) - except: - logger.error("Could not load torrent, skipping %s", - torrent_filename) - continue - # Create a torrent dict. - torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1] - self.torrents[torrent_filename] = dict(zip(torrent_keys, torrent_values)) - # Notify the BoostingManager. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[torrent_filename]) - - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) +# -*- coding: utf-8 -*- +# Written by Egbert Bouman, Mihai Capotă, Elric Milon +# pylint: disable=too-few-public-methods, too-many-instance-attributes +# pylint: disable=too-many-arguments, too-many-branches +"""Manage boosting of swarms""" + +import ConfigParser +import HTMLParser +import glob +import json +import logging +import os +import random +import time +import urllib +from binascii import hexlify, unhexlify +from collections import defaultdict +from hashlib import sha1 + +import libtorrent as lt + +from Tribler.Core.DownloadConfig import DownloadStartupConfig +from Tribler.Core.TorrentChecker.session import MAX_TRACKER_MULTI_SCRAPE +from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo +from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_INSERT, NTFY_SCRAPE, NTFY_TORRENTS, NTFY_UPDATE +from Tribler.Main.globals import DefaultDownloadStartupConfig +from Tribler.Utilities.scraper import scrape_tcp, scrape_udp +from Tribler.community.allchannel.community import AllChannelCommunity +from Tribler.community.channel.community import ChannelCommunity +from Tribler.dispersy.taskmanager import TaskManager +from Tribler.dispersy.util import call_on_reactor_thread + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +formatter = logging.Formatter( + "%(asctime)s.%(msecs).03dZ-%(levelname)s-%(message)s", + datefmt="%Y%m%dT%H%M%S") +formatter.converter = time.gmtime +handler = logging.FileHandler("boosting.log", mode="w") +handler.setFormatter(formatter) +logger.addHandler(handler) + +# logging.getLogger(TimedTaskQueue.__name__+"BoostingManager").setLevel( +# logging.DEBUG) + +number_types = (int, long, float) + +CONFIG_FILE = os.path.abspath("boosting.ini") + + +def lev(a, b): + "Calculates the Levenshtein distance between a and b." + n, m = len(a), len(b) + if n > m: + # Make sure n <= m, to use O(min(n,m)) space + a, b = b, a + n, m = m, n + + current = range(n + 1) + for i in range(1, m + 1): + previous, current = current, [i] + [0] * n + for j in range(1, n + 1): + add, delete = previous[j] + 1, current[j - 1] + 1 + change = previous[j - 1] + if a[j - 1] != b[i - 1]: + change = change + 1 + current[j] = min(add, delete, change) + + return current[n] + + +class BoostingPolicy(object): + + def __init__(self, session): + self.session = session + self.key = lambda x: None + # function that checks if key can be applied to torrent + self.key_check = lambda x: None + self.reverse = None + + def apply(self, torrents, max_active): + sorted_torrents = sorted([torrent for torrent in torrents.itervalues() + if self.key_check(torrent)], + key=self.key, reverse=self.reverse) + torrents_start = [] + for torrent in sorted_torrents[:max_active]: + if not self.session.get_download(torrent["metainfo"].get_infohash()): + torrents_start.append(torrent) + torrents_stop = [] + for torrent in sorted_torrents[max_active:]: + if self.session.get_download(torrent["metainfo"].get_infohash()): + torrents_stop.append(torrent) + return (torrents_start, torrents_stop) + + +class RandomPolicy(BoostingPolicy): + + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.key = lambda v: random.random() + self.key_check = lambda v: True + self.reverse = False + + +class CreationDatePolicy(BoostingPolicy): + + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.key = lambda v: v['creation_date'] + self.key_check = lambda v: v['creation_date'] > 0 + self.reverse = True + + +class SeederRatioPolicy(BoostingPolicy): + + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.key = lambda v: v['num_seeders'] / float(v['num_seeders'] + + v['num_leechers']) + self.key_check = lambda v: isinstance(v['num_seeders'], number_types) and isinstance(v['num_leechers'], number_types) and v['num_seeders'] + v['num_leechers'] > 0 + self.reverse = False + + +class BoostingManager(TaskManager): + + __single = None + + def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval=20, sw_interval=20, max_per_source=100, max_active=2): + super(BoostingManager, self).__init__() + + BoostingManager.__single = self + + self._saved_attributes = ["max_torrents_per_source", + "max_torrents_active", "source_interval", + "swarm_interval", "share_mode_target", + "tracker_interval", "logging_interval"] + + self.session = session + self.utility = utility + self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), "credit_mining") + if not os.path.exists(self.credit_mining_path): + os.mkdir(self.credit_mining_path) + + self.boosting_sources = {} + self.torrents = {} + self.policy = None + self.share_mode_target = 3 + + self.max_torrents_per_source = max_per_source + self.max_torrents_active = max_active + self.source_interval = src_interval + self.swarm_interval = sw_interval + self.initial_swarm_interval = 30 + self.policy = policy(self.session) + self.tracker_interval = 300 + self.initial_tracker_interval = 25 + self.logging_interval = 60 + self.initial_logging_interval = 35 + + self.load_config() + + self.set_share_mode_params(share_mode_target=self.share_mode_target) + + if os.path.exists(CONFIG_FILE): + logger.info("Config file %s", open(CONFIG_FILE).read()) + else: + logger.info("Config file missing") + + # TODO(emilon): Refactor this to use taskmanager + self.session.lm.threadpool.add_task(self._select_torrent, self.initial_swarm_interval) + self.session.lm.threadpool.add_task(self.scrape_trackers, + self.initial_tracker_interval) + self.session.lm.threadpool.add_task(self.log_statistics, + self.initial_logging_interval) + + def get_instance(*args, **kw): + if BoostingManager.__single is None: + BoostingManager(*args, **kw) + return BoostingManager.__single + get_instance = staticmethod(get_instance) + + def del_instance(): + BoostingManager.__single = None + del_instance = staticmethod(del_instance) + + def shutdown(self): + for torrent in self.torrents.itervalues(): + try: + self.stop_download(torrent) + except: + continue + + def load(self): + if self.utility: + try: + string_to_source = lambda s: s.decode('hex') if len(s) == 40 and not (os.path.isdir(s) or s.startswith('http://')) else s + for source in json.loads(self.utility.config.Read('boosting_sources')): + self.add_source(string_to_source(source)) + logger.info("Initial boosting sources %s", + self.boosting_sources.keys()) + except: + logger.info("No initial boosting sources") + + def save(self): + if self.utility: + try: + source_to_string = lambda s: s.encode('hex') if len(s) == 20 and not (os.path.isdir(s) or s.startswith('http://')) else s + self.utility.write_config( + 'boosting_sources', + json.dumps([source_to_string(source) for + source in self.boosting_sources.keys()]), + flush=True) + logger.info("Saved sources %s", self.boosting_sources.keys()) + except: + logger.exception("Could not save state") + + def set_share_mode_params(self, share_mode_target=None, share_mode_bandwidth=None, share_mode_download=None, share_mode_seeders=None): + settings = self.session.lm.ltmgr.get_session().settings() + if share_mode_target is not None: + settings.share_mode_target = share_mode_target + if share_mode_bandwidth is not None: + settings.share_mode_bandwidth = share_mode_bandwidth + if share_mode_download is not None: + settings.share_mode_download = share_mode_download + if share_mode_seeders is not None: + settings.share_mode_seeders = share_mode_seeders + self.session.lm.ltmgr.get_session().set_settings(settings) + + def add_source(self, source): + if source not in self.boosting_sources: + args = (self.session, self.session.lm.threadpool, source, self.source_interval, self.max_torrents_per_source, self.on_torrent_insert) + # pylint: disable=star-args + if os.path.isdir(source): + self.boosting_sources[source] = DirectorySource(*args) + elif source.startswith('http://'): + self.boosting_sources[source] = RSSFeedSource(*args) + elif len(source) == 20: + self.boosting_sources[source] = ChannelSource(*args) + else: + logger.error("Cannot add unknown source %s", source) + else: + logger.info("Already have source %s", source) + + def remove_source(self, source_key): + if source_key in self.boosting_sources: + source = self.boosting_sources.pop(source_key) + source.kill_tasks() + + def compare_torrents(self, t1, t2): + # pylint: disable=no-self-use, bad-builtin + ff = lambda ft: ft[1] > 1024 * 1024 + files1 = filter(ff, t1['metainfo'].get_files_with_length()) + files2 = filter(ff, t2['metainfo'].get_files_with_length()) + + if len(files1) == len(files2): + for ft1 in files1: + for ft2 in files2: + if ft1[1] != ft2[1] or lev(ft1[0], ft2[0]) > 5: + return False + return True + return False + + def on_torrent_insert(self, source, infohash, torrent): + # Remember where we got this torrent from + if os.path.isdir(source) or source.startswith('http://'): + source_str = source + elif len(source) == 20: + source_str = source.encode('hex') + else: + source_str = 'unknown source' + torrent['source'] = source_str + + if self.boosting_sources[source].archive: + torrent['preload'] = True + torrent['prio'] = 100 + + # Preload the TorrentDef. + if torrent['metainfo']: + if not isinstance(torrent['metainfo'], TorrentDef): + torrent['metainfo'] = TorrentDef.load(torrent['metainfo']) + else: + torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) + + # If duplicates exist, set is_duplicate to True, except for the one with the most seeders. + duplicates = [other for other in self.torrents.values() if self.compare_torrents(torrent, other)] + if duplicates: + duplicates += [torrent] + healthiest_torrent = max([(torrent['num_seeders'], torrent) for torrent in duplicates])[1] + for duplicate in duplicates: + is_duplicate = healthiest_torrent != duplicate + duplicate['is_duplicate'] = is_duplicate + if is_duplicate and duplicate.get('download', None): + self.stop_download(duplicate) + + self.torrents[infohash] = torrent + logger.info("Got new torrent %s from %s", infohash.encode('hex'), + source_str) + + def scrape_trackers(self): + num_requests = 0 + trackers = defaultdict(list) + + for infohash, torrent in self.torrents.iteritems(): + if isinstance(torrent['metainfo'], TorrentDef): + tdef = torrent['metainfo'] + for tracker in tdef.get_trackers_as_single_tuple(): + trackers[tracker].append(infohash) + num_requests += 1 + + results = defaultdict(lambda: [0, 0]) + for tracker, infohashes in trackers.iteritems(): + try: + reply = {} + if tracker.startswith("http://tracker.etree.org"): + for infohash in infohashes: + reply.update(scrape_tcp(tracker, (infohash,))) + elif tracker.startswith("udp://"): + for group in range(len(infohashes) // + MAX_TRACKER_MULTI_SCRAPE): + reply.update(scrape_udp(tracker, infohashes[ + group * MAX_TRACKER_MULTI_SCRAPE: + (group + 1) * MAX_TRACKER_MULTI_SCRAPE])) + reply.update(scrape_udp(tracker, infohashes[ + -(len(infohashes) % MAX_TRACKER_MULTI_SCRAPE):])) + else: + reply = scrape_tcp(tracker, infohashes) + logger.debug("Got reply from tracker %s : %s", tracker, reply) + except: + logger.exception("Did not get reply from tracker %s", tracker) + else: + for infohash, info in reply.iteritems(): + if info['complete'] > results[infohash][0]: + results[infohash][0] = info['complete'] + results[infohash][1] = info['incomplete'] + + for infohash, num_peers in results.iteritems(): + self.torrents[infohash]['num_seeders'] = num_peers[0] + self.torrents[infohash]['num_leechers'] = num_peers[1] + self.session.notifier.notify(NTFY_TORRENTS, NTFY_SCRAPE, infohash) + + logger.debug("Finished tracker scraping for %s torrents", num_requests) + + self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) + + def set_archive(self, source, enable): + if source in self.boosting_sources: + self.boosting_sources[source].archive = enable + logger.info("Set archive mode for %s to %s", source, enable) + else: + logger.error("Could not set archive mode for unknown source %s", source) + + def start_download(self, torrent): + def do_start(): + dscfg = DownloadStartupConfig() + dscfg.set_dest_dir(self.credit_mining_path) + dscfg.set_safe_seeding(False) + + preload = torrent.get('preload', False) + logger.info("Starting %s preload %s", hexlify(torrent["metainfo"].get_infohash()), preload) + torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=torrent.get('pstate', None), + hidden=True, share_mode=not preload, checkpoint_disabled=True) + torrent['download'].set_priority(torrent.get('prio', 1)) + self.session.lm.threadpool.add_task_in_thread(do_start, 0) + + def stop_download(self, torrent): + def do_stop(): + ihash = lt.big_number(torrent["metainfo"].get_infohash()) + logger.info("Stopping %s", str(ihash)) + download = torrent.pop('download', False) + lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(ihash) + if download and lt_torrent.is_valid(): + logger.debug("Writing resume data") + torrent['pstate'] = {'engineresumedata': download.write_resume_data()} + self.session.remove_download(download) + + self.session.lm.threadpool.add_task_in_thread(do_stop, 0) + + def _select_torrent(self): + torrents = {} + for infohash, torrent in self.torrents.iteritems(): + if torrent.get('preload', False): + if 'download' not in torrent: + self.start_download(torrent) + elif torrent['download'].get_status() == DLSTATUS_SEEDING: + self.stop_download(torrent) + elif not torrent.get('is_duplicate', False): + torrents[infohash] = torrent + + if self.policy is not None and torrents: + + logger.debug("Selecting from %s torrents", len(torrents)) + + # Determine which torrent to start and which to stop. + torrents_start, torrents_stop = self.policy.apply( + torrents, self.max_torrents_active) + for torrent in torrents_stop: + self.stop_download(torrent) + for torrent in torrents_start: + self.start_download(torrent) + + self.session.lm.threadpool.add_task(self._select_torrent, self.swarm_interval) + + def load_config(self): + config = ConfigParser.ConfigParser() + config.read(CONFIG_FILE) + for k, v in config.items(__name__): + if k in self._saved_attributes: + object.__setattr__(self, k, int(v)) + elif k == "policy": + if v == "random": + self.policy = RandomPolicy(self.session) + elif v == "creation": + self.policy = CreationDatePolicy(self.session) + elif v == "seederratio": + self.policy = SeederRatioPolicy(self.session) + elif k == "boosting_sources": + for boosting_source in json.loads(v): + self.add_source(boosting_source) + elif k == "archive_sources": + for archive_source in json.loads(v): + self.set_archive(archive_source, True) + + def save_config(self): + config = ConfigParser.ConfigParser() + config.add_section(__name__) + for k in self._saved_attributes: + config.set(__name__, k, BoostingManager.__getattribute__(self, k)) + config.set(__name__, "boosting_sources", + json.dumps(self.boosting_sources.keys())) + archive_sources = [] + for boosting_source_name, boosting_source in \ + self.boosting_sources.iteritems(): + if boosting_source.archive: + archive_sources.append(boosting_source_name) + if archive_sources: + config.set(__name__, "archive_sources", + json.dumps(archive_sources)) + if isinstance(self.policy, RandomPolicy): + policy = "random" + elif isinstance(self.policy, CreationDatePolicy): + policy = "creation" + elif isinstance(self.policy, SeederRatioPolicy): + policy = "seederratio" + config.set(__name__, "policy", policy) + with open(CONFIG_FILE, "w") as configf: + config.write(configf) + + def log_statistics(self): + """Log transfer statistics""" + lt_torrents = self.session.lm.ltmgr.get_session().get_torrents() + for lt_torrent in lt_torrents: + status = lt_torrent.status() + if unhexlify(str(status.info_hash)) in self.torrents: + logger.debug("Status for %s : %s %s", status.info_hash, + status.all_time_download, + status.all_time_upload) + non_zero_values = [] + for piece_priority in lt_torrent.piece_priorities(): + if piece_priority != 0: + non_zero_values.append(piece_priority) + if non_zero_values: + logger.debug("Non zero priorities for %s : %s", + status.info_hash, non_zero_values) + self.session.lm.threadpool.add_task(self.log_statistics, self.logging_interval) + + +class BoostingSource(object): + + def __init__(self, session, tqueue, source, interval, max_torrents, callback): + self.session = session + self.session.lm.threadpool = tqueue + + self.torrents = {} + self.source = source + self.interval = interval + self.max_torrents = max_torrents + self.callback = callback + self.archive = False + + def kill_tasks(self): + self.session.lm.threadpool.remove_task(self.source) + + def _load(self, source): + pass + + def _update(self): + pass + + +class ChannelSource(BoostingSource): + + def __init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback): + BoostingSource.__init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback) + + self.channel_id = None + + self.channelcast_db = self.session.lm.channelcast_db + + self.community = None + self.database_updated = True + + self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) + self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load(cid), 0, task_name=self.source) + + def kill_tasks(self): + BoostingSource.kill_tasks(self) + self.session.remove_observer(self._on_database_updated) + + def _load(self, dispersy_cid): + dispersy = self.session.get_dispersy_instance() + + @call_on_reactor_thread + def join_community(): + try: + self.community = dispersy.get_community(dispersy_cid, True) + self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=self.source) + + except KeyError: + + allchannelcommunity = None + for community in dispersy.get_communities(): + if isinstance(community, AllChannelCommunity): + allchannelcommunity = community + break + + if allchannelcommunity: + # pylint: disable=protected-access + self.community = ChannelCommunity.init_community(dispersy, dispersy.get_member(mid=dispersy_cid), allchannelcommunity._my_member, True) + logger.info("Joined channel community %s", + dispersy_cid.encode("HEX")) + self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=self.source) + else: + logger.error("Could not find AllChannelCommunity") + + def get_channel_id(): + # pylint: disable=protected-access + if self.community and self.community._channel_id: + self.channel_id = self.community._channel_id + self.session.lm.threadpool.add_task(self._update, 0, task_name=self.source) + logger.info("Got channel id %s", self.channel_id) + else: + logger.warning("Could not get channel id, retrying in 10 s") + self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=self.source) + + join_community() + + def _update(self): + if len(self.torrents) < self.max_torrents: + + if self.database_updated: + infohashes_old = set(self.torrents.keys()) + + torrent_keys_db = ['infohash', 'Torrent.name', 'torrent_file_name', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + torrent_keys_dict = ['infohash', 'name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, torrent_keys_db, self.max_torrents) + self.torrents = dict((torrent[0], dict(zip(torrent_keys_dict[1:], torrent[1:]))) for torrent in torrent_values) + + infohashes_new = set(self.torrents.keys()) + for infohash in infohashes_new - infohashes_old: + if self.callback: + self.callback(self.source, infohash, self.torrents[infohash]) + + self.database_updated = False + + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) + + def _on_database_updated(self, subject, change_type, infohash): + if (subject, change_type, infohash) is None: + # Unused arguments + pass + self.database_updated = True + + +class RSSFeedSource(BoostingSource): + + def __init__(self, session, tqueue, rss_feed, interval, max_torrents, callback): + BoostingSource.__init__(self, session, tqueue, rss_feed, interval, max_torrents, callback) + + self.unescape = HTMLParser.HTMLParser().unescape + + self.feed_handle = None + + self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load(feed), 0, task_name=self.source) + + def _load(self, rss_feed): + self.feed_handle = self.session.lm.ltmgr.get_session().add_feed({'url': rss_feed, 'auto_download': False, 'auto_map_handles': False}) + + def wait_for_feed(): + # Wait until the RSS feed is longer updating. + feed_status = self.feed_handle.get_feed_status() + if feed_status['updating']: + self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=self.source) + elif len(feed_status['error']) > 0: + logger.error("Got error for RSS feed %s : %s", + feed_status['url'], feed_status['error']) + if "503" in feed_status["error"]: + time.sleep(5 * random.random()) + self.feed_handle.update_feed() + self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=self.source) + else: + # The feed is done updating. Now periodically start retrieving torrents. + self.session.lm.threadpool.add_task(self._update, 0, task_name=self.source) + logger.info("Got RSS feed %s", feed_status['url']) + + wait_for_feed() + + def _update(self): + if len(self.torrents) < self.max_torrents: + + feed_status = self.feed_handle.get_feed_status() + + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + + for item in feed_status['items']: + # Not all RSS feeds provide us with the infohash, so we use a fake infohash based on the URL to identify the torrents. + infohash = sha1(item['url']).digest() + if infohash not in self.torrents: + # Store the torrents as rss-infohash_as_hex.torrent. + torrent_filename = os.path.join(BoostingManager.get_instance().credit_mining_path, 'rss-%s.torrent' % infohash.encode('hex')) + if not os.path.exists(torrent_filename): + try: + # Download the torrent and create a TorrentDef. + f = urllib.urlopen(self.unescape(item['url'])) + metainfo = lt.bdecode(f.read()) + tdef = TorrentDef.load_from_dict(metainfo) + tdef.save(torrent_filename) + except: + logger.error("Could not get torrent, skipping %s", + item['url']) + continue + else: + tdef = TorrentDef.load(torrent_filename) + # Create a torrent dict. + torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1] + self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) + # Notify the BoostingManager and provide the real infohash. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) + + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) + + +class DirectorySource(BoostingSource): + + def __init__(self, session, tqueue, directory, interval, max_torrents, callback): + BoostingSource.__init__(self, session, tqueue, directory, interval, max_torrents, callback) + + self._load(directory) + + def _load(self, directory): + if os.path.isdir(directory): + # Wait for __init__ to finish so the source is registered with the + # BoostinManager, otherwise adding torrents won't work + self.session.lm.threadpool.add_task(self._update, 1, task_name=self.source) + logger.info("Got directory %s", directory) + else: + logger.error("Could not find directory %s", directory) + + def _update(self): + if len(self.torrents) < self.max_torrents: + + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + + for torrent_filename in glob.glob(self.source + '/*.torrent'): + if torrent_filename not in self.torrents: + try: + tdef = TorrentDef.load(torrent_filename) + except: + logger.error("Could not load torrent, skipping %s", + torrent_filename) + continue + # Create a torrent dict. + torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1] + self.torrents[torrent_filename] = dict(zip(torrent_keys, torrent_values)) + # Notify the BoostingManager. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[torrent_filename]) + + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) diff --git a/Tribler/Tools/boostchannel.py b/Tribler/Tools/boostchannel.py index 082010c2d89..55c0b1ada8e 100644 --- a/Tribler/Tools/boostchannel.py +++ b/Tribler/Tools/boostchannel.py @@ -1,123 +1,123 @@ -# Written by Egbert Bouman - -import sys -import time -import shutil -import random -import getopt -import tempfile -from traceback import print_exc - -from Tribler.Core.Session import Session -from Tribler.Core.SessionConfig import SessionStartupConfig -from Tribler.Policies.BoostingManager import BoostingManager, RandomPolicy, CreationDatePolicy, SeederRatioPolicy -from Tribler.community.channel.community import ChannelCommunity -from Tribler.community.channel.preview import PreviewChannelCommunity -from Tribler.community.allchannel.community import AllChannelCommunity - - -def usage(): - print "Usage: python boostchannel.py [options] dispersy_cid" - print "Options:" - print " --db_interval \tnumber of seconds between database refreshes" - print " --sw_interval \tnumber of seconds between swarm selection" - print " --max_per_source \tmaximum number of swarms per source" - print " \t\t\t\tthat should be taken into consideration" - print " --max_active \t\tmaximum number of swarms that should be" - print " \t\t\t\tactive simultaneously" - print " --policy \t\tpolicy for swarm selection" - print " \t\t\t\tpossible values: RandomPolicy" - print " \t\t\t\t CreationDatePolicy" - print " \t\t\t\t SeederRatioPolicy (default)" - print " --help\t\t\tprint this help screen" - print - print "Example:" - print " python boostchannel.py --max_active=5 3c8378fc3493b5772b1e6a25672d3889367cb7c3" - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], "hd:s:m:a:p:", ["help", "db_interval=", "sw_interval=", "max_per_source=", "max_active=", "policy="]) - except getopt.GetoptError as err: - print str(err) - usage() - sys.exit(2) - - kwargs = {} - for o, a in opts: - if o in ("-h", "--help"): - usage() - sys.exit(0) - elif o in ("-d", "--db_interval"): - kwargs['src_interval'] = int(a) - elif o in ("-s", "--sw_interval"): - kwargs['sw_interval'] = int(a) - elif o in ("-m", "--max_per_source"): - kwargs['max_eligible'] = int(a) - elif o in ("-a", "--max_active"): - kwargs['max_active'] = int(a) - elif o in ("-p", "--policy"): - if a == 'RandomPolicy': - kwargs['policy'] = RandomPolicy - elif a == 'CreationDatePolicy': - kwargs['policy'] = CreationDatePolicy - elif a == 'SeederRatioPolicy': - kwargs['policy'] = SeederRatioPolicy - else: - assert False, "Unknown policy" - else: - assert False, "Unhandled option" - - if len(args[0]) != 40: - print "Incorrect dispersy_cid" - sys.exit(2) - else: - dispersy_cid = args[0].decode('hex') - - print "Press Ctrl-C to stop boosting this channel" - - statedir = tempfile.mkdtemp() - - config = SessionStartupConfig() - config.set_state_dir(statedir) - config.set_listen_port(random.randint(10000, 60000)) - config.set_torrent_checking(False) - config.set_multicast_local_peer_discovery(False) - config.set_megacache(True) - config.set_dispersy(True) - config.set_swift_proc(False) - config.set_mainline_dht(False) - config.set_torrent_collecting(False) - config.set_libtorrent(True) - config.set_dht_torrent_collecting(False) - - s = Session(config) - s.start() - - while not s.lm.initComplete: - time.sleep(1) - - def load_communities(): - dispersy.define_auto_load(AllChannelCommunity, (s.dispersy_member,), load=True) - dispersy.define_auto_load(ChannelCommunity, load=True) - dispersy.define_auto_load(PreviewChannelCommunity) - print >> sys.stderr, "Dispersy communities are ready" - - dispersy = s.get_dispersy_instance() - dispersy.callback.call(load_communities) - - bm = BoostingManager.get_instance(s, None, **kwargs) - bm.add_source(dispersy_cid) - - try: - while True: - time.sleep(sys.maxsize / 2048) - except: - print_exc() - - s.shutdown() - time.sleep(3) - shutil.rmtree(statedir) - - -if __name__ == "__main__": - main() +# Written by Egbert Bouman + +import sys +import time +import shutil +import random +import getopt +import tempfile +from traceback import print_exc + +from Tribler.Core.Session import Session +from Tribler.Core.SessionConfig import SessionStartupConfig +from Tribler.Policies.BoostingManager import BoostingManager, RandomPolicy, CreationDatePolicy, SeederRatioPolicy +from Tribler.community.channel.community import ChannelCommunity +from Tribler.community.channel.preview import PreviewChannelCommunity +from Tribler.community.allchannel.community import AllChannelCommunity + + +def usage(): + print "Usage: python boostchannel.py [options] dispersy_cid" + print "Options:" + print " --db_interval \tnumber of seconds between database refreshes" + print " --sw_interval \tnumber of seconds between swarm selection" + print " --max_per_source \tmaximum number of swarms per source" + print " \t\t\t\tthat should be taken into consideration" + print " --max_active \t\tmaximum number of swarms that should be" + print " \t\t\t\tactive simultaneously" + print " --policy \t\tpolicy for swarm selection" + print " \t\t\t\tpossible values: RandomPolicy" + print " \t\t\t\t CreationDatePolicy" + print " \t\t\t\t SeederRatioPolicy (default)" + print " --help\t\t\tprint this help screen" + print + print "Example:" + print " python boostchannel.py --max_active=5 3c8378fc3493b5772b1e6a25672d3889367cb7c3" + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], "hd:s:m:a:p:", ["help", "db_interval=", "sw_interval=", "max_per_source=", "max_active=", "policy="]) + except getopt.GetoptError as err: + print str(err) + usage() + sys.exit(2) + + kwargs = {} + for o, a in opts: + if o in ("-h", "--help"): + usage() + sys.exit(0) + elif o in ("-d", "--db_interval"): + kwargs['src_interval'] = int(a) + elif o in ("-s", "--sw_interval"): + kwargs['sw_interval'] = int(a) + elif o in ("-m", "--max_per_source"): + kwargs['max_eligible'] = int(a) + elif o in ("-a", "--max_active"): + kwargs['max_active'] = int(a) + elif o in ("-p", "--policy"): + if a == 'RandomPolicy': + kwargs['policy'] = RandomPolicy + elif a == 'CreationDatePolicy': + kwargs['policy'] = CreationDatePolicy + elif a == 'SeederRatioPolicy': + kwargs['policy'] = SeederRatioPolicy + else: + assert False, "Unknown policy" + else: + assert False, "Unhandled option" + + if len(args[0]) != 40: + print "Incorrect dispersy_cid" + sys.exit(2) + else: + dispersy_cid = args[0].decode('hex') + + print "Press Ctrl-C to stop boosting this channel" + + statedir = tempfile.mkdtemp() + + config = SessionStartupConfig() + config.set_state_dir(statedir) + config.set_listen_port(random.randint(10000, 60000)) + config.set_torrent_checking(False) + config.set_multicast_local_peer_discovery(False) + config.set_megacache(True) + config.set_dispersy(True) + config.set_swift_proc(False) + config.set_mainline_dht(False) + config.set_torrent_collecting(False) + config.set_libtorrent(True) + config.set_dht_torrent_collecting(False) + + s = Session(config) + s.start() + + while not s.lm.initComplete: + time.sleep(1) + + def load_communities(): + dispersy.define_auto_load(AllChannelCommunity, (s.dispersy_member,), load=True) + dispersy.define_auto_load(ChannelCommunity, load=True) + dispersy.define_auto_load(PreviewChannelCommunity) + print >> sys.stderr, "Dispersy communities are ready" + + dispersy = s.get_dispersy_instance() + dispersy.callback.call(load_communities) + + bm = BoostingManager.get_instance(s, None, **kwargs) + bm.add_source(dispersy_cid) + + try: + while True: + time.sleep(sys.maxsize / 2048) + except: + print_exc() + + s.shutdown() + time.sleep(3) + shutil.rmtree(statedir) + + +if __name__ == "__main__": + main() From 24ebb7b93808c7e595db9e4135059b43ec5124dd Mon Sep 17 00:00:00 2001 From: Elric Milon Date: Mon, 24 Aug 2015 14:11:31 +0200 Subject: [PATCH 07/24] WIP commit --- Tribler/Policies/BoostingManager.py | 41 +++++++++++++++-------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 07b410166fa..81863b59017 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -276,11 +276,11 @@ def on_torrent_insert(self, source, infohash, torrent): torrent['prio'] = 100 # Preload the TorrentDef. - if torrent['metainfo']: - if not isinstance(torrent['metainfo'], TorrentDef): - torrent['metainfo'] = TorrentDef.load(torrent['metainfo']) - else: - torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) + if not isinstance(torrent['metainfo'], TorrentDef): + print ">>>>>>>>>>>>>>>>>>>>>", torrent['infohash'] + torrent['metainfo'] = TorrentDef.load_from_memory(self.session.lm.torrent_store.get(torrent['infohash'])) + #else: + # torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) # If duplicates exist, set is_duplicate to True, except for the one with the most seeders. duplicates = [other for other in self.torrents.values() if self.compare_torrents(torrent, other)] @@ -479,7 +479,7 @@ def __init__(self, session, tqueue, source, interval, max_torrents, callback): self.archive = False def kill_tasks(self): - self.session.lm.threadpool.remove_task(self.source) + self.session.lm.threadpool.cancel_pending_task(self.source) def _load(self, source): pass @@ -514,7 +514,7 @@ def _load(self, dispersy_cid): def join_community(): try: self.community = dispersy.get_community(dispersy_cid, True) - self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=self.source) + self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=str(self.source)+"_get_channel_id") except KeyError: @@ -529,7 +529,7 @@ def join_community(): self.community = ChannelCommunity.init_community(dispersy, dispersy.get_member(mid=dispersy_cid), allchannelcommunity._my_member, True) logger.info("Joined channel community %s", dispersy_cid.encode("HEX")) - self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=self.source) + self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=str(self.source)+"_get_channel_id") else: logger.error("Could not find AllChannelCommunity") @@ -537,11 +537,12 @@ def get_channel_id(): # pylint: disable=protected-access if self.community and self.community._channel_id: self.channel_id = self.community._channel_id - self.session.lm.threadpool.add_task(self._update, 0, task_name=self.source) + + self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") logger.info("Got channel id %s", self.channel_id) else: logger.warning("Could not get channel id, retrying in 10 s") - self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=self.source) + self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=str(self.source)+"_get_channel_id") join_community() @@ -551,8 +552,8 @@ def _update(self): if self.database_updated: infohashes_old = set(self.torrents.keys()) - torrent_keys_db = ['infohash', 'Torrent.name', 'torrent_file_name', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] - torrent_keys_dict = ['infohash', 'name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + torrent_keys_db = ['infohash', 'Torrent.name', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + torrent_keys_dict = ['infohash', 'name', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, torrent_keys_db, self.max_torrents) self.torrents = dict((torrent[0], dict(zip(torrent_keys_dict[1:], torrent[1:]))) for torrent in torrent_values) @@ -563,7 +564,7 @@ def _update(self): self.database_updated = False - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") def _on_database_updated(self, subject, change_type, infohash): if (subject, change_type, infohash) is None: @@ -581,7 +582,7 @@ def __init__(self, session, tqueue, rss_feed, interval, max_torrents, callback): self.feed_handle = None - self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load(feed), 0, task_name=self.source) + self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load(feed), 0, task_name=str(self.source)+"_load") def _load(self, rss_feed): self.feed_handle = self.session.lm.ltmgr.get_session().add_feed({'url': rss_feed, 'auto_download': False, 'auto_map_handles': False}) @@ -590,17 +591,17 @@ def wait_for_feed(): # Wait until the RSS feed is longer updating. feed_status = self.feed_handle.get_feed_status() if feed_status['updating']: - self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=self.source) + self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=str(self.source)+"_wait_for_feed") elif len(feed_status['error']) > 0: logger.error("Got error for RSS feed %s : %s", feed_status['url'], feed_status['error']) if "503" in feed_status["error"]: time.sleep(5 * random.random()) self.feed_handle.update_feed() - self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=self.source) + self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=str(self.source)+"_wait_for_feed") else: # The feed is done updating. Now periodically start retrieving torrents. - self.session.lm.threadpool.add_task(self._update, 0, task_name=self.source) + self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") logger.info("Got RSS feed %s", feed_status['url']) wait_for_feed() @@ -638,7 +639,7 @@ def _update(self): if self.callback: self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") class DirectorySource(BoostingSource): @@ -652,7 +653,7 @@ def _load(self, directory): if os.path.isdir(directory): # Wait for __init__ to finish so the source is registered with the # BoostinManager, otherwise adding torrents won't work - self.session.lm.threadpool.add_task(self._update, 1, task_name=self.source) + self.session.lm.threadpool.add_task(self._update, 1, task_name=str(self.source)+"_update") logger.info("Got directory %s", directory) else: logger.error("Could not find directory %s", directory) @@ -677,4 +678,4 @@ def _update(self): if self.callback: self.callback(self.source, tdef.get_infohash(), self.torrents[torrent_filename]) - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=self.source) + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") From 2184356fb54619c009fb2de07cf3ccfee5fda827 Mon Sep 17 00:00:00 2001 From: Elric Milon Date: Fri, 28 Aug 2015 12:29:09 +0200 Subject: [PATCH 08/24] WIP commit --- Tribler/Main/Dialogs/BoostingDialogs.py | 4 ++-- Tribler/Policies/BoostingManager.py | 23 ++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/Tribler/Main/Dialogs/BoostingDialogs.py b/Tribler/Main/Dialogs/BoostingDialogs.py index 005c0154bab..ca5d27f97bf 100644 --- a/Tribler/Main/Dialogs/BoostingDialogs.py +++ b/Tribler/Main/Dialogs/BoostingDialogs.py @@ -17,7 +17,7 @@ def __init__(self, parent): self.source = '' text = wx.StaticText(self, -1, 'Please enter a RSS feed URL or select a channel to start boosting swarms:') - self.channel_radio = wx.RadioButton(self, -1, 'Channel:', style=wx.RB_GROUP) + self.channel_radio = wx.RadioButton(self, -1, 'Subscribed Channel:', style=wx.RB_GROUP) self.channel_choice = wx.Choice(self, -1) self.channel_choice.Bind(wx.EVT_CHOICE, lambda evt: self.channel_radio.SetValue(True)) @@ -56,7 +56,7 @@ def __init__(self, parent): self.SetSizer(vSizer) def do_db(): - return self.guiutility.channelsearch_manager.getAllChannels() + return self.guiutility.channelsearch_manager.getMySubscriptions() def do_gui(delayedResult): _, channels = delayedResult.get() diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 81863b59017..0891d026795 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -126,8 +126,10 @@ class BoostingManager(TaskManager): __single = None - def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval=20, sw_interval=20, max_per_source=100, max_active=2): + def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval=20, sw_interval=20, + max_per_source=100, max_active=2): super(BoostingManager, self).__init__() + self._logger = logging.getLogger(self.__class__.__name__) BoostingManager.__single = self @@ -271,16 +273,23 @@ def on_torrent_insert(self, source, infohash, torrent): source_str = 'unknown source' torrent['source'] = source_str - if self.boosting_sources[source].archive: + boost_source = self.boosting_sources.get(source, None) + if not boost_source: + self._logger.info("Dropping torrent insert from removed source: %s" % repr(torrent)) + return + elif boost_source.archive: torrent['preload'] = True torrent['prio'] = 100 # Preload the TorrentDef. - if not isinstance(torrent['metainfo'], TorrentDef): - print ">>>>>>>>>>>>>>>>>>>>>", torrent['infohash'] - torrent['metainfo'] = TorrentDef.load_from_memory(self.session.lm.torrent_store.get(torrent['infohash'])) - #else: - # torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) + if not isinstance(torrent.get('metainfo', None), TorrentDef): + torrent_data = self.session.lm.torrent_store.get(infohash) + if torrent_data: + torrent['metainfo'] = TorrentDef.load_from_memory(torrent_data) + else: + # TODO(emilon): Handle the case where the torrent hasn't been collected. (collected from the DHT) + # torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) + pass # If duplicates exist, set is_duplicate to True, except for the one with the most seeders. duplicates = [other for other in self.torrents.values() if self.compare_torrents(torrent, other)] From b7010d8f5eda662e2e158139d231c1f7b9cf26d0 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 27 Jun 2016 16:31:24 +0200 Subject: [PATCH 09/24] Fix initial work in BoostingManager that cannot work Squashed Commits: - Uniform channel source format - Properly remove credit mining data and sources - Several tweaks in BoostingManager This commit contains: - Add ability to enable/disable source - Read configuration file (boosting.ini) - Store DownloadState data in BoostingManager - Validate source before processing - Enable RSS Source (in CM) with local database This commit also put back checkpoint capabilities in LibTorrentDownloadImpl --- .../Core/APIImplementation/LaunchManyCore.py | 6 +- .../Core/Libtorrent/LibtorrentDownloadImpl.py | 27 +- Tribler/Policies/BoostingManager.py | 409 ++++++++++++++---- boosting.ini | 13 + 4 files changed, 371 insertions(+), 84 deletions(-) create mode 100644 boosting.ini diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index f19094c69df..78bdbef6ed1 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -368,7 +368,7 @@ def write_my_pref(): def on_download_wrapper_created(self, (d, pstate)): """ Called by network thread """ try: - if pstate is None: + if pstate is None and not d.get_checkpoint_disabled(): # Checkpoint at startup (infohash, pstate) = d.network_checkpoint() self.save_download_pstate(infohash, pstate) @@ -594,9 +594,7 @@ def checkpoint(self, stop=False, checkpoint=True, gracetime=2.0): # Download, and additions are no problem (just won't be included # in list of states returned via callback. # - dllist = self.downloads.values() - self._logger.debug("tlm: checkpointing %s stopping %s", len(dllist), stop) - + dllist = [dl for dl in self.downloads.values() if not dl.checkpoint_disabled] network_checkpoint_callback_lambda = lambda: self.network_checkpoint_callback(dllist, stop, checkpoint, gracetime) self.threadpool.add_task(network_checkpoint_callback_lambda, 0.0) diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index 5c624626c18..74811b38d08 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -145,11 +145,12 @@ def __init__(self, session, tdef): self.askmoreinfo = False self.correctedinfoname = u"" + self.checkpoint_disabled = False self.deferreds_resume = [] def __str__(self): - return "LibtorrentDownloadImpl " % (self.correctedinfoname, self.get_hops()) + return "LibtorrentDownloadImpl " % (self.correctedinfoname, self.get_hops(), self.checkpoint_disabled) def __repr__(self): return self.__str__() @@ -157,7 +158,14 @@ def __repr__(self): def get_def(self): return self.tdef - def setup(self, dcfg=None, pstate=None, initialdlstatus=None, lm_network_engine_wrapper_created_callback=None, wrapperDelay=0, share_mode=False): + def set_checkpoint_disabled(self, val=True): + self.checkpoint_disabled = val + + def get_checkpoint_disabled(self): + return self.checkpoint_disabled + + def setup(self, dcfg=None, pstate=None, initialdlstatus=None, lm_network_engine_wrapper_created_callback=None, + wrapperDelay=0, share_mode=False, checkpoint_disabled=False): """ Create a Download object. Used internally by Session. @param dcfg DownloadStartupConfig or None (in which case @@ -167,6 +175,8 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None, lm_network_engine_ network_create_engine_wrapper. """ # Called by any thread, assume sessionlock is held + self.set_checkpoint_disabled(checkpoint_disabled) + try: # The deferred to be returned deferred = Deferred() @@ -242,7 +252,7 @@ def do_check(): do_check() return can_create_deferred - def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode=False): + def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode=False,checkpoint_disabled=False): with self.dllock: self._logger.debug("LibtorrentDownloadImpl: network_create_engine_wrapper()") @@ -257,6 +267,8 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode if share_mode: atp["flags"] = lt.add_torrent_params_flags_t.flag_share_mode + self.set_checkpoint_disabled(checkpoint_disabled) + resume_data = pstate.get('state', 'engineresumedata') if pstate else None if not isinstance(self.tdef, TorrentDefNoMetainfo): metainfo = self.tdef.get_metainfo() @@ -1059,9 +1071,12 @@ def get_dest_files(self, exts=None): def checkpoint(self): """ Called by any thread """ - (infohash, pstate) = self.network_checkpoint() - checkpoint = lambda: self.session.lm.save_download_pstate(infohash, pstate) - self.session.lm.threadpool.add_task(checkpoint, 0) + if self.checkpoint_disabled: + self._logger.warning("Ignoring checkpoint() call as is checkpointing disabled for this download.") + else: + (infohash, pstate) = self.network_checkpoint() + checkpoint = lambda: self.session.lm.save_download_pstate(infohash, pstate) + self.session.lm.threadpool.add_task(checkpoint, 0) def network_checkpoint(self): """ Called by network thread """ diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 0891d026795..d354e79da57 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -21,18 +21,19 @@ from Tribler.Core.DownloadConfig import DownloadStartupConfig from Tribler.Core.TorrentChecker.session import MAX_TRACKER_MULTI_SCRAPE -from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo +from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_INSERT, NTFY_SCRAPE, NTFY_TORRENTS, NTFY_UPDATE +from Tribler.Main.Utility.GuiDBTuples import Torrent from Tribler.Main.globals import DefaultDownloadStartupConfig +from Tribler.Main.vwxGUI.GuiUtility import GUIUtility +from Tribler.Main.vwxGUI.SearchGridManager import ChannelManager from Tribler.Utilities.scraper import scrape_tcp, scrape_udp from Tribler.community.allchannel.community import AllChannelCommunity from Tribler.community.channel.community import ChannelCommunity -from Tribler.dispersy.taskmanager import TaskManager from Tribler.dispersy.util import call_on_reactor_thread - logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.INFO) formatter = logging.Formatter( "%(asctime)s.%(msecs).03dZ-%(levelname)s-%(message)s", datefmt="%Y%m%dT%H%M%S") @@ -46,8 +47,11 @@ number_types = (int, long, float) -CONFIG_FILE = os.path.abspath("boosting.ini") +# CONFIG_FILE = "boosting.ini" +from Tribler.Core.Utilities.install_dir import determine_install_dir +TRIBLER_ROOT = determine_install_dir() +CONFIG_FILE = os.path.join(TRIBLER_ROOT, "boosting.ini") def lev(a, b): "Calculates the Levenshtein distance between a and b." @@ -83,6 +87,7 @@ def apply(self, torrents, max_active): sorted_torrents = sorted([torrent for torrent in torrents.itervalues() if self.key_check(torrent)], key=self.key, reverse=self.reverse) + torrents_start = [] for torrent in sorted_torrents[:max_active]: if not self.session.get_download(torrent["metainfo"].get_infohash()): @@ -91,6 +96,10 @@ def apply(self, torrents, max_active): for torrent in sorted_torrents[max_active:]: if self.session.get_download(torrent["metainfo"].get_infohash()): torrents_stop.append(torrent) + + # if not torrents_start and not torrents_stop: + # torrents_start = torrents.copy() + return (torrents_start, torrents_stop) @@ -132,6 +141,7 @@ def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval self._logger = logging.getLogger(self.__class__.__name__) BoostingManager.__single = self + self.gui_util = GUIUtility.getInstance() self._saved_attributes = ["max_torrents_per_source", "max_torrents_active", "source_interval", @@ -141,6 +151,7 @@ def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval self.session = session self.utility = utility self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), "credit_mining") + if not os.path.exists(self.credit_mining_path): os.mkdir(self.credit_mining_path) @@ -187,22 +198,42 @@ def del_instance(): del_instance = staticmethod(del_instance) def shutdown(self): + # save configuration before stopping stuffs + self.save_config() + for torrent in self.torrents.itervalues(): try: self.stop_download(torrent) except: continue - def load(self): - if self.utility: - try: - string_to_source = lambda s: s.decode('hex') if len(s) == 40 and not (os.path.isdir(s) or s.startswith('http://')) else s - for source in json.loads(self.utility.config.Read('boosting_sources')): - self.add_source(string_to_source(source)) - logger.info("Initial boosting sources %s", - self.boosting_sources.keys()) - except: - logger.info("No initial boosting sources") + def get_source_object(self, sourcekey): + return self.boosting_sources.get(sourcekey, None) + + def set_enable_mining(self, source, mining_bool=True, force_restart=False): + tor_not_exist = True + + for ihash, tor in self.torrents.iteritems(): + if tor['source'] == source: + tor_not_exist = False + self.torrents[ihash]['enabled'] = mining_bool + + if (not mining_bool): + self.stop_download(tor) + + # this only happen via new channel boosting interface + if tor_not_exist and mining_bool and not (source in self.boosting_sources.keys()): + self.add_source(source) + self.set_archive(source, False) + self.set_enable_mining(source, mining_bool) + + string_to_source = lambda s: s.decode('hex') if len(s) == 40 and not (os.path.isdir(s) or s.startswith('http://')) else s + self.boosting_sources[string_to_source(source)].enabled = mining_bool + + logger.info("Set mining source %s %s", source, mining_bool) + + if force_restart: + self._select_torrent() def save(self): if self.utility: @@ -233,14 +264,22 @@ def add_source(self, source): if source not in self.boosting_sources: args = (self.session, self.session.lm.threadpool, source, self.source_interval, self.max_torrents_per_source, self.on_torrent_insert) # pylint: disable=star-args - if os.path.isdir(source): + + try: + isdir = os.path.isdir(source) + except TypeError: + isdir = False + + if isdir: self.boosting_sources[source] = DirectorySource(*args) - elif source.startswith('http://'): + elif source.startswith('http://') or source.startswith('https://'): self.boosting_sources[source] = RSSFeedSource(*args) elif len(source) == 20: self.boosting_sources[source] = ChannelSource(*args) else: logger.error("Cannot add unknown source %s", source) + + logger.info("Added source %s", source) else: logger.info("Already have source %s", source) @@ -248,24 +287,41 @@ def remove_source(self, source_key): if source_key in self.boosting_sources: source = self.boosting_sources.pop(source_key) source.kill_tasks() + logger.info("Removed source %s", source_key) + + rm_torrents = [torrent for _, torrent in self.torrents.items() if torrent['source'] == source_key] + map(self.stop_download,rm_torrents) + logger.info("Torrents download stopped") + + map(lambda x:self.torrents.pop(x["metainfo"].get_infohash(), None), rm_torrents) + logger.info("Removing from possible swarms") def compare_torrents(self, t1, t2): # pylint: disable=no-self-use, bad-builtin - ff = lambda ft: ft[1] > 1024 * 1024 - files1 = filter(ff, t1['metainfo'].get_files_with_length()) - files2 = filter(ff, t2['metainfo'].get_files_with_length()) - - if len(files1) == len(files2): - for ft1 in files1: - for ft2 in files2: - if ft1[1] != ft2[1] or lev(ft1[0], ft2[0]) > 5: - return False - return True - return False + try: + ff = lambda ft: ft[1] > 1024 * 1024 + files1 = filter(ff, t1['metainfo'].get_files_with_length()) + files2 = filter(ff, t2['metainfo'].get_files_with_length()) + + if len(files1) == len(files2): + for ft1 in files1: + for ft2 in files2: + if ft1[1] != ft2[1] or lev(ft1[0], ft2[0]) > 5: + return False + return True + return False + except: + return False def on_torrent_insert(self, source, infohash, torrent): # Remember where we got this torrent from - if os.path.isdir(source) or source.startswith('http://'): + + try: + isdir = os.path.isdir(source) + except TypeError: + isdir = False + + if isdir or source.startswith('http://') or source.startswith('https://'): source_str = source elif len(source) == 20: source_str = source.encode('hex') @@ -287,6 +343,7 @@ def on_torrent_insert(self, source, infohash, torrent): if torrent_data: torrent['metainfo'] = TorrentDef.load_from_memory(torrent_data) else: + self._logger.info("Not collected yet: %s %s ", infohash, torrent['name']) # TODO(emilon): Handle the case where the torrent hasn't been collected. (collected from the DHT) # torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) pass @@ -303,24 +360,36 @@ def on_torrent_insert(self, source, infohash, torrent): self.stop_download(duplicate) self.torrents[infohash] = torrent - logger.info("Got new torrent %s from %s", infohash.encode('hex'), - source_str) + # logger.info("Got new torrent %s from %s", infohash.encode('hex'), + # source_str) def scrape_trackers(self): + + for infohash, torrent in self.torrents.iteritems(): + tf = torrent['metainfo'] + + self.session.check_torrent_health(infohash) + + self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) + return None + num_requests = 0 trackers = defaultdict(list) for infohash, torrent in self.torrents.iteritems(): - if isinstance(torrent['metainfo'], TorrentDef): + if isinstance(torrent['metainfo'], TorrentDef) and torrent['enabled']: tdef = torrent['metainfo'] for tracker in tdef.get_trackers_as_single_tuple(): trackers[tracker].append(infohash) num_requests += 1 + logger.info("Start tracker scraping for %s torrents", num_requests) + results = defaultdict(lambda: [0, 0]) for tracker, infohashes in trackers.iteritems(): try: reply = {} + #TODO(ardhi) : specific tracker? make it more general if tracker.startswith("http://tracker.etree.org"): for infohash in infohashes: reply.update(scrape_tcp(tracker, (infohash,))) @@ -334,9 +403,19 @@ def scrape_trackers(self): -(len(infohashes) % MAX_TRACKER_MULTI_SCRAPE):])) else: reply = scrape_tcp(tracker, infohashes) - logger.debug("Got reply from tracker %s : %s", tracker, reply) - except: - logger.exception("Did not get reply from tracker %s", tracker) + + #RD : hack for readable infohash + logger.debug("Got reply from tracker %s : %s", tracker, + {hexlify(k): v for k, v in reply.items()}) + except Exception as e: + type, obj, _ = sys.exc_info() + + # pretty lazy hack for debugging purpose + ihash = infohashes[0] + + source = self.torrents[ihash]['source'] + logger.error("%s Did not get reply from tracker %s. Reason : %s,%s", + hexlify(source) if len(hexlify(source)) == 40 else source, tracker, obj, str(type)) else: for infohash, info in reply.iteritems(): if info['complete'] > results[infohash][0]: @@ -348,7 +427,7 @@ def scrape_trackers(self): self.torrents[infohash]['num_leechers'] = num_peers[1] self.session.notifier.notify(NTFY_TORRENTS, NTFY_SCRAPE, infohash) - logger.debug("Finished tracker scraping for %s torrents", num_requests) + logger.info("Finished tracker scraping for %s torrents", num_requests) self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) @@ -365,11 +444,21 @@ def do_start(): dscfg.set_dest_dir(self.credit_mining_path) dscfg.set_safe_seeding(False) - preload = torrent.get('preload', False) - logger.info("Starting %s preload %s", hexlify(torrent["metainfo"].get_infohash()), preload) - torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=torrent.get('pstate', None), + #TODO (ardhi) debug variable so I can change this later + tobj = torrent + + preload = tobj.get('preload', False) + logger.info("Starting %s preload %s", + hexlify(tobj["metainfo"].get_infohash()), + preload) + + # not using Session.start_download because we need to specify pstate + assert self.session.get_libtorrent() + + tobj['download'] = self.session.lm.add(tobj['metainfo'], dscfg, pstate=tobj.get('pstate', None), hidden=True, share_mode=not preload, checkpoint_disabled=True) - torrent['download'].set_priority(torrent.get('prio', 1)) + tobj['download'].set_priority(tobj.get('prio', 1)) + self.session.lm.threadpool.add_task_in_thread(do_start, 0) def stop_download(self, torrent): @@ -394,12 +483,10 @@ def _select_torrent(self): elif torrent['download'].get_status() == DLSTATUS_SEEDING: self.stop_download(torrent) elif not torrent.get('is_duplicate', False): - torrents[infohash] = torrent + if torrent.get('enabled', True): + torrents[infohash] = torrent if self.policy is not None and torrents: - - logger.debug("Selecting from %s torrents", len(torrents)) - # Determine which torrent to start and which to stop. torrents_start, torrents_stop = self.policy.apply( torrents, self.max_torrents_active) @@ -408,11 +495,14 @@ def _select_torrent(self): for torrent in torrents_start: self.start_download(torrent) + logger.info("Selecting from %s torrents %s start download", len(torrents), len(torrents_start)) + self.session.lm.threadpool.add_task(self._select_torrent, self.swarm_interval) def load_config(self): config = ConfigParser.ConfigParser() config.read(CONFIG_FILE) + validate_source = lambda s: unhexlify(s) if len(s) == 40 and not s.startswith("http") else s for k, v in config.items(__name__): if k in self._saved_attributes: object.__setattr__(self, k, int(v)) @@ -425,26 +515,58 @@ def load_config(self): self.policy = SeederRatioPolicy(self.session) elif k == "boosting_sources": for boosting_source in json.loads(v): + boosting_source = validate_source(boosting_source) self.add_source(boosting_source) elif k == "archive_sources": for archive_source in json.loads(v): + archive_source = validate_source(archive_source) self.set_archive(archive_source, True) + elif k == "boosting_enabled": + for boosting_source in json.loads(v): + boosting_source = validate_source(boosting_source) + if not self.boosting_sources[boosting_source]: + self.add_source(boosting_source) + self.boosting_sources[boosting_source].enabled = True + elif k == "boosting_disabled": + for boosting_source in json.loads(v): + boosting_source = validate_source(boosting_source) + if not self.boosting_sources[boosting_source]: + self.add_source(boosting_source) + self.boosting_sources[boosting_source].enabled = False + def save_config(self): config = ConfigParser.ConfigParser() config.add_section(__name__) for k in self._saved_attributes: config.set(__name__, k, BoostingManager.__getattribute__(self, k)) - config.set(__name__, "boosting_sources", - json.dumps(self.boosting_sources.keys())) + + source_to_string = lambda s: hexlify(s) if len(s) == 20 and not (s.startswith('http://') or s.startswith('https://')) else s + archive_sources = [] + lboosting_sources = [] + flag_enabled_sources = [] + flag_disabled_sources = [] for boosting_source_name, boosting_source in \ self.boosting_sources.iteritems(): + + bsname = source_to_string(boosting_source_name) + + lboosting_sources.append(bsname) + if boosting_source.enabled: + flag_enabled_sources.append(bsname) + else: + flag_disabled_sources.append(bsname) + if boosting_source.archive: - archive_sources.append(boosting_source_name) + archive_sources.append(bsname) + + config.set(__name__, "boosting_sources", json.dumps(lboosting_sources)) + config.set(__name__, "boosting_enabled", json.dumps(flag_enabled_sources)) + config.set(__name__, "boosting_disabled", json.dumps(flag_disabled_sources)) if archive_sources: - config.set(__name__, "archive_sources", - json.dumps(archive_sources)) + config.set(__name__, "archive_sources", json.dumps(archive_sources)) + if isinstance(self.policy, RandomPolicy): policy = "random" elif isinstance(self.policy, CreationDatePolicy): @@ -458,9 +580,24 @@ def save_config(self): def log_statistics(self): """Log transfer statistics""" lt_torrents = self.session.lm.ltmgr.get_session().get_torrents() + for lt_torrent in lt_torrents: status = lt_torrent.status() if unhexlify(str(status.info_hash)) in self.torrents: + t = self.torrents[unhexlify(str(status.info_hash))] + # print({str(status.info_hash):{t['name']:t['num_seeders']}}) + # + # print("numpeer %d lastscrape: %s #upload : %d #conn : %d" %(status.num_peers, status.last_scrape, status.num_uploads, + # status.num_connections)) + + # pprint.pprint(lt_torrent.get_peer_info()) + + # tz = self.gui_util.torrentsearch_manager.getTorrentByInfohash(str(status.info_hash)) + # pprint.pprint(tz) + # pprint.pprint({str(status.info_hash):pprint.pformat(self.torrents[unhexlify(str(status.info_hash))])}) + # .update( + #self.torrents[unhexlify(str(status.info_hash))]['download'].network_tracker_status()) + logger.debug("Status for %s : %s %s", status.info_hash, status.all_time_download, status.all_time_upload) @@ -487,6 +624,16 @@ def __init__(self, session, tqueue, source, interval, max_torrents, callback): self.callback = callback self.archive = False + self.enabled = True + + self.av_uprate = 0 + self.av_dwnrate = 0 + self.storage_used = 0 + self.ready = False + + self.gui_util = GUIUtility.getInstance() + + def kill_tasks(self): self.session.lm.threadpool.cancel_pending_task(self.source) @@ -496,6 +643,8 @@ def _load(self, source): def _update(self): pass + def getSource(self): + return self.source class ChannelSource(BoostingSource): @@ -504,6 +653,8 @@ def __init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callba self.channel_id = None + self.channel = None + self.channelcast_db = self.session.lm.channelcast_db self.community = None @@ -512,6 +663,8 @@ def __init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callba self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load(cid), 0, task_name=self.source) + self.unavail_torrent = {} + def kill_tasks(self): BoostingSource.kill_tasks(self) self.session.remove_observer(self._on_database_updated) @@ -544,34 +697,81 @@ def join_community(): def get_channel_id(): # pylint: disable=protected-access - if self.community and self.community._channel_id: + if self.community and self.community._channel_id and self.gui_util.registered: self.channel_id = self.community._channel_id + self.channel = self.gui_util.channelsearch_manager.getChannel(self.channel_id) + self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") logger.info("Got channel id %s", self.channel_id) else: logger.warning("Could not get channel id, retrying in 10 s") self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=str(self.source)+"_get_channel_id") - join_community() + try: + join_community() + self.ready = True + except: + logger.info("Channel %s was not ready, waits for next interval", hexlify(self.source)) + self.session.lm.threadpool.add_task(lambda cid=self.source: self._load(cid), 10, task_name=self.source) + + + def _check_tor(self): + from Tribler.Main.Utility.GuiDBHandler import startWorker + def doGui(delayedResult): + # wait here + requesttype = delayedResult.get(timeout=70) + + def showTorrent(torrent): + if (torrent.files): + infohash = torrent.infohash + self.torrents[infohash] = {} + self.torrents[infohash]['name'] = torrent.name + self.torrents[infohash]['metainfo'] = torrent.tdef + self.torrents[infohash]['creation_date'] = torrent.creation_date + self.torrents[infohash]['length'] = torrent.tdef.get_length() + self.torrents[infohash]['num_files'] = len(torrent.files) + self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] + self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] + self.torrents[infohash]['enabled'] = self.enabled + + del self.unavail_torrent[infohash] + + # logger.info("Torrent %s from %s ready to start", hexlify(infohash), hexlify(self.source)) + + if self.callback: + self.callback(self.source, infohash, self.torrents[infohash]) + self.database_updated = False + + logger.info("Unavailable #torrents : %d from %s", len(self.unavail_torrent), hexlify(self.source)) + + if len(self.unavail_torrent) and self.enabled: + for k,t in self.unavail_torrent.items(): + startWorker(doGui, self.gui_util.torrentsearch_manager.loadTorrent, + wargs=(t,), wkwargs={'callback': showTorrent}) + + if not self.session.lm.threadpool.is_pending_task_active(str(self.source)+"_checktor"): + self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=str(self.source)+"_checktor") def _update(self): if len(self.torrents) < self.max_torrents: if self.database_updated: - infohashes_old = set(self.torrents.keys()) - torrent_keys_db = ['infohash', 'Torrent.name', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] - torrent_keys_dict = ['infohash', 'name', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] - torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, torrent_keys_db, self.max_torrents) - self.torrents = dict((torrent[0], dict(zip(torrent_keys_dict[1:], torrent[1:]))) for torrent in torrent_values) + CHANTOR_DB = ['ChannelTorrents.channel_id', 'Torrent.torrent_id', 'infohash', '""', 'length', 'category', 'status', 'num_seeders', 'num_leechers', 'ChannelTorrents.id', 'ChannelTorrents.dispersy_id', 'ChannelTorrents.name', 'Torrent.name', 'ChannelTorrents.description', 'ChannelTorrents.time_stamp', 'ChannelTorrents.inserted'] - infohashes_new = set(self.torrents.keys()) - for infohash in infohashes_new - infohashes_old: - if self.callback: - self.callback(self.source, infohash, self.torrents[infohash]) + try: + torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, CHANTOR_DB, self.max_torrents) - self.database_updated = False + listtor = self.gui_util.channelsearch_manager._createTorrents(torrent_values, True, + {self.channel_id: self.channelcast_db.getChannel(self.channel_id)})[2] + + # dict {key_infohash(binary):Torrent(object-GUIDBTuple)} + self.unavail_torrent = {t.infohash:t for t in listtor} + + self.session.lm.threadpool.add_task(self._check_tor, 0, task_name=str(self.source)+"_init_checktor") + except: + logger.info("Channel %s was not ready, waits for next interval", hexlify(self.source)) self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") @@ -581,8 +781,18 @@ def _on_database_updated(self, subject, change_type, infohash): pass self.database_updated = True + def getSource(self): + return self.channel.name if self.channel else None + class RSSFeedSource(BoostingSource): + # supported list (tested) : + # http://bt.etree.org/rss/bt_etree_org.rdf + # http://www.mininova.org/rss.xml + # https://kat.cr (via torcache) + + # not supported (till now) + # https://eztv.ag/ezrss.xml (https link) def __init__(self, session, tqueue, rss_feed, interval, max_torrents, callback): BoostingSource.__init__(self, session, tqueue, rss_feed, interval, max_torrents, callback) @@ -593,6 +803,10 @@ def __init__(self, session, tqueue, rss_feed, interval, max_torrents, callback): self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load(feed), 0, task_name=str(self.source)+"_load") + self.title = "" + self.description = "" + self.total_torrents = 0 + def _load(self, rss_feed): self.feed_handle = self.session.lm.ltmgr.get_session().add_feed({'url': rss_feed, 'auto_download': False, 'auto_map_handles': False}) @@ -614,42 +828,88 @@ def wait_for_feed(): logger.info("Got RSS feed %s", feed_status['url']) wait_for_feed() + self.ready = True def _update(self): if len(self.torrents) < self.max_torrents: feed_status = self.feed_handle.get_feed_status() - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + self.title = feed_status['title'] + self.description = feed_status['description'] + + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', 'enabled'] + + self.total_torrents = len(feed_status['items']) for item in feed_status['items']: # Not all RSS feeds provide us with the infohash, so we use a fake infohash based on the URL to identify the torrents. infohash = sha1(item['url']).digest() + if infohash not in self.torrents: # Store the torrents as rss-infohash_as_hex.torrent. torrent_filename = os.path.join(BoostingManager.get_instance().credit_mining_path, 'rss-%s.torrent' % infohash.encode('hex')) + tdef = None if not os.path.exists(torrent_filename): try: # Download the torrent and create a TorrentDef. - f = urllib.urlopen(self.unescape(item['url'])) - metainfo = lt.bdecode(f.read()) - tdef = TorrentDef.load_from_dict(metainfo) - tdef.save(torrent_filename) + if "torcache" in item['url']: + import urllib2 + import StringIO + import gzip + req = urllib2.Request(self.unescape(item['url'])) + req.add_header('Referer', 'http://torcache.net/') + req.add_header('Accept-encoding', 'deflate,gzip') + req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36') + + res = urllib2.urlopen(req) + cfile = StringIO.StringIO() + cfile.write(res.read()) + cfile.seek(0) + + f = gzip.GzipFile(fileobj=cfile, mode='rb') + else: + f = urllib.urlopen(self.unescape(item['url'])) except: logger.error("Could not get torrent, skipping %s", item['url']) continue + + try: + metainfo = lt.bdecode(f.read()) + tdef = TorrentDef.load_from_dict(metainfo) + tdef.save(torrent_filename) + + except: + logger.error("Could not parse/save torrent, skipping %s", torrent_filename) + else: tdef = TorrentDef.load(torrent_filename) - # Create a torrent dict. - torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1] - self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) - # Notify the BoostingManager and provide the real infohash. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) + + if tdef: + # Create a torrent dict. + torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled] + self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) + + # manually generate an ID and put this into DB + self.session.lm.torrent_db.addOrGetTorrentID(infohash) + self.session.lm.torrent_db.addExternalTorrent(tdef) + + # create Torrent object and store it + self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) + + # Notify the BoostingManager and provide the real infohash. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") + def kill_tasks(self): + BoostingSource.kill_tasks(self) + + #stop updating + self.session.lm.threadpool.cancel_pending_task(str(self.source)+"_update") + class DirectorySource(BoostingSource): @@ -664,13 +924,14 @@ def _load(self, directory): # BoostinManager, otherwise adding torrents won't work self.session.lm.threadpool.add_task(self._update, 1, task_name=str(self.source)+"_update") logger.info("Got directory %s", directory) + self.ready = True else: logger.error("Could not find directory %s", directory) def _update(self): if len(self.torrents) < self.max_torrents: - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers'] + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', 'enabled'] for torrent_filename in glob.glob(self.source + '/*.torrent'): if torrent_filename not in self.torrents: @@ -681,7 +942,7 @@ def _update(self): torrent_filename) continue # Create a torrent dict. - torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1] + torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled] self.torrents[torrent_filename] = dict(zip(torrent_keys, torrent_values)) # Notify the BoostingManager. if self.callback: diff --git a/boosting.ini b/boosting.ini new file mode 100644 index 00000000000..db5e851479c --- /dev/null +++ b/boosting.ini @@ -0,0 +1,13 @@ +[Tribler.Policies.BoostingManager] +max_torrents_per_source = 100 +max_torrents_active = 20 +source_interval = 300 +swarm_interval = 300 +share_mode_target = 3 +tracker_interval = 300 +logging_interval = 60 +boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +boosting_enabled = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +boosting_disabled = [] +policy = seederratio + From a31b923eb0c161b1a6a81fd8dcd2d94daccd2d32 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 27 Jun 2016 16:06:14 +0200 Subject: [PATCH 10/24] Revamped creditmining GUI Squashed Commits: - Integrate uniform channel source into GUI - Add credit mining manager and list - Integrating gumby with Boostingmanager - Update info to summary panel - Fix bugs in credit mining panel and make prettier popular channel @home - Refreshing-connecting home and Credit Mining Panel - Beautify home channel list -need to expand horizontally- - Change credit mining order in GUI - Remove PostInit in creditmining list - Fix channel source interpretation What this commits do is : - Change BoostingDialog GUI as we don't need channel there - Create CreditMiningPanel as container for credit mining tab - Add Credit Mining option in GUI Utility (lists in tab) - Put CreditMiningPanel in MainFrame - Put popular channels in Home. Directly affected with credit mining - Make current credit mining list to adapt new GUI structure --- Tribler/Main/Dialogs/BoostingDialogs.py | 48 ++- Tribler/Main/Utility/GuiDBTuples.py | 7 +- Tribler/Main/tribler_main.py | 12 + Tribler/Main/vwxGUI/CreditMiningPanel.py | 493 +++++++++++++++++++++++ Tribler/Main/vwxGUI/GuiUtility.py | 50 ++- Tribler/Main/vwxGUI/MainFrame.py | 19 +- Tribler/Main/vwxGUI/SearchGridManager.py | 4 +- Tribler/Main/vwxGUI/home.py | 235 ++++++++++- Tribler/Main/vwxGUI/list.py | 386 +++++++++++++++++- Tribler/Main/vwxGUI/list_item.py | 9 +- 10 files changed, 1186 insertions(+), 77 deletions(-) create mode 100644 Tribler/Main/vwxGUI/CreditMiningPanel.py diff --git a/Tribler/Main/Dialogs/BoostingDialogs.py b/Tribler/Main/Dialogs/BoostingDialogs.py index ca5d27f97bf..36b613ec6b6 100644 --- a/Tribler/Main/Dialogs/BoostingDialogs.py +++ b/Tribler/Main/Dialogs/BoostingDialogs.py @@ -10,16 +10,16 @@ class AddBoostingSource(wx.Dialog): def __init__(self, parent): - wx.Dialog.__init__(self, parent, -1, 'Add boosting source', size=(475, 475), name="AddBoostingSourceDialog") + wx.Dialog.__init__(self, parent, -1, 'Add boosting source', size=(275, 275), name="AddBoostingSourceDialog") self.guiutility = GUIUtility.getInstance() self.channels = [] self.source = '' - text = wx.StaticText(self, -1, 'Please enter a RSS feed URL or select a channel to start boosting swarms:') - self.channel_radio = wx.RadioButton(self, -1, 'Subscribed Channel:', style=wx.RB_GROUP) - self.channel_choice = wx.Choice(self, -1) - self.channel_choice.Bind(wx.EVT_CHOICE, lambda evt: self.channel_radio.SetValue(True)) + text = wx.StaticText(self, -1, 'Please enter a RSS feed URL or directory to start boosting swarms:') + # self.channel_radio = wx.RadioButton(self, -1, 'Subscribed Channel:', style=wx.RB_GROUP) + # self.channel_choice = wx.Choice(self, -1) + # self.channel_choice.Bind(wx.EVT_CHOICE, lambda evt: self.channel_radio.SetValue(True)) self.rss_feed_radio = wx.RadioButton(self, -1, 'RSS feed:') self.rss_feed_edit = wx.TextCtrl(self, -1) @@ -37,8 +37,8 @@ def __init__(self, parent): sourceGrid = wx.FlexGridSizer(2, 2, 0, 0) sourceGrid.AddGrowableCol(1) - sourceGrid.Add(self.channel_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) - sourceGrid.Add(self.channel_choice, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + # sourceGrid.Add(self.channel_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) + # sourceGrid.Add(self.channel_choice, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) sourceGrid.Add(self.rss_feed_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) sourceGrid.Add(self.rss_feed_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) sourceGrid.Add(self.rss_dir_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) @@ -55,25 +55,31 @@ def __init__(self, parent): vSizer.Add(btnSizer, 0, wx.EXPAND | wx.ALL, 5) self.SetSizer(vSizer) - def do_db(): - return self.guiutility.channelsearch_manager.getMySubscriptions() - - def do_gui(delayedResult): - _, channels = delayedResult.get() - self.channels = sorted([(channel.name, channel.dispersy_cid) for channel in channels]) - self.channel_choice.SetItems([channel[0] for channel in self.channels]) - - startWorker(do_gui, do_db, retryOnBusy=True, priority=GUI_PRI_DISPERSY) + # def do_db(): + # return self.guiutility.channelsearch_manager.getAllChannels() + # + # def do_gui(delayedResult): + # _, channels = delayedResult.get() + # self.channels = sorted([(channel.name, channel.dispersy_cid) for channel in channels]) + # self.channel_choice.SetItems([channel[0] for channel in self.channels]) + # + # startWorker(do_gui, do_db, retryOnBusy=True, priority=GUI_PRI_DISPERSY) def OnOK(self, event): - if self.channel_radio.GetValue(): - selection = self.channel_choice.GetSelection() - if selection < len(self.channels): - self.source = self.channels[selection][1] - elif self.rss_feed_radio.GetValue(): + # if self.channel_radio.GetValue(): + # selection = self.channel_choice.GetSelection() + # if selection < len(self.channels): + # self.source = self.channels[selection][1] + # el + + if self.rss_feed_radio.GetValue(): self.source = self.rss_feed_edit.GetValue() else: self.source = self.rss_dir_edit.GetValue() + + self.guiutility.Notify( + "Successfully add source for credit mining %s" % self.source) + self.EndModal(wx.ID_OK) def OnCancel(self, event): diff --git a/Tribler/Main/Utility/GuiDBTuples.py b/Tribler/Main/Utility/GuiDBTuples.py index 74ccb304c42..26b779f4d8c 100644 --- a/Tribler/Main/Utility/GuiDBTuples.py +++ b/Tribler/Main/Utility/GuiDBTuples.py @@ -101,7 +101,7 @@ def __init__(self, torrent_id, infohash, name, length, category, status, num_see Helper.__init__(self) assert isinstance(infohash, str), type(infohash) - assert isinstance(name, basestring), type(name) + # assert isinstance(name, basestring), type(name) self.infohash = infohash self.name = name @@ -127,7 +127,7 @@ def __init__(self, torrent_id, infohash, name, length, category, status, num_see @cacheProperty def torrent_id(self): - self._logger.debug("Torrent: fetching getTorrentID from DB %s", self) + self._logger.error("Torrent: fetching getTorrentID from DB %s", self) return self.torrent_db.getTorrentID(self.infohash) def update_torrent_id(self, torrent_id): @@ -300,7 +300,7 @@ def __init__(self, torrent_id, infohash, name, length=0, category=None, status=N class CollectedTorrent(Helper): - __slots__ = ('comment', 'trackers', 'creation_date', 'files', 'last_check', 'torrent') + __slots__ = ('comment', 'trackers', 'creation_date', 'files', 'last_check', 'torrent', 'tdef') def __init__(self, torrent, torrentdef): assert isinstance(torrent, Torrent) @@ -314,6 +314,7 @@ def __init__(self, torrent, torrentdef): self.creation_date = min(long(time()), torrentdef.get_creation_date()) self.files = torrentdef.get_files_as_unicode_with_length() self.last_check = -1 + self.tdef = torrentdef def __getattr__(self, name): return getattr(self.torrent, name) diff --git a/Tribler/Main/tribler_main.py b/Tribler/Main/tribler_main.py index 5db8cc94dcf..764b2f27313 100755 --- a/Tribler/Main/tribler_main.py +++ b/Tribler/Main/tribler_main.py @@ -9,6 +9,7 @@ # see LICENSE.txt for license information # from Tribler.Core.Modules.process_checker import ProcessChecker +from Tribler.Policies.BoostingManager import BoostingManager from Tribler.Main.Dialogs.NewVersionDialog import NewVersionDialog try: @@ -263,6 +264,8 @@ def __init__(self, params, installdir, autoload_discovery=True, # gracefully closes Tribler after 120 seconds. # wx.CallLater(120*1000, wx.GetApp().Exit) + self.boosting_manager = BoostingManager.get_instance() + self.ready = True except Exception as e: @@ -815,6 +818,15 @@ def OnExit(self): self.webUI.stop() self.webUI.delInstance() + if self.boosting_manager: + + #remove credit mining data + #TODO(ardhi) : not persistent mode only + import shutil; shutil.rmtree(self.boosting_manager.credit_mining_path, ignore_errors=True) + + self.boosting_manager.shutdown() + self.boosting_manager.del_instance() + if self.frame: self.frame.Destroy() self.frame = None diff --git a/Tribler/Main/vwxGUI/CreditMiningPanel.py b/Tribler/Main/vwxGUI/CreditMiningPanel.py new file mode 100644 index 00000000000..502b852522a --- /dev/null +++ b/Tribler/Main/vwxGUI/CreditMiningPanel.py @@ -0,0 +1,493 @@ +# Written by Ardhi Putra Pratama Hartono + + +import os +import sys +import logging +import wx +from binascii import hexlify, unhexlify +from wx.lib.agw.ultimatelistctrl import ULC_VIRTUAL, EVT_LIST_ITEM_CHECKED + +from Tribler import LIBRARYNAME +from Tribler.Core.exceptions import NotYetImplementedException +from Tribler.Core.simpledefs import NTFY_TORRENTS +from Tribler.Main.Dialogs.BoostingDialogs import RemoveBoostingSource, AddBoostingSource + +from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent, Torrent, Channel +from Tribler.Main.Utility.GuiDBHandler import startWorker, cancelWorker, GUI_PRI_DISPERSY +from Tribler.Main.vwxGUI import forceWxThread, TRIBLER_RED, SEPARATOR_GREY, GRADIENT_LGREY, GRADIENT_DGREY, format_time +from Tribler.Main.vwxGUI.GuiUtility import GUIUtility +from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager +from Tribler.Main.vwxGUI.list import CreditMiningList +from Tribler.Main.vwxGUI.widgets import ActionButton, FancyPanel, TextCtrlAutoComplete, ProgressButton, LinkStaticText, \ + _set_font +from Tribler.Main.Dialogs.AddTorrent import AddTorrent +from Tribler.Main.Dialogs.RemoveTorrent import RemoveTorrent +from Tribler.Policies.BoostingManager import BoostingManager, RSSFeedSource, DirectorySource, ChannelSource, \ + BoostingSource + +try: + from agw import ultimatelistctrl as ULC +except ImportError: + from wx.lib.agw import ultimatelistctrl as ULC + + + +class CpanelCheckListCtrl(wx.ScrolledWindow, ULC.UltimateListCtrl): + def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, size=wx.DefaultSize, style=0, \ + agwStyle=0): + ULC.UltimateListCtrl.__init__(self, parent, id, pos, size, style, agwStyle) + self.boosting_manager = BoostingManager.get_instance() + self.guiutility = GUIUtility.getInstance() + self.utility = self.guiutility.utility + self.channel_list = {} + + self._logger = logging.getLogger(self.__class__.__name__) + + self.InsertColumn(0,'col') + self.SetColumnWidth(0, -3) + + self.label_rss_idx = 0 + self.InsertStringItem(self.label_rss_idx,"RSS") + it = self.GetItem(self.label_rss_idx) + it.Enable(False) + it.SetData("RSS") + self.SetItem(it) + + self.label_dir_idx = 1 + self.InsertStringItem(self.label_dir_idx,"Directory") + it = self.GetItem(self.label_dir_idx) + it.Enable(False) + it.SetData("Directory") + self.SetItem(it) + + self.label_channel_idx = 2 + self.InsertStringItem(self.label_channel_idx,"Channel") + it = self.GetItem(self.label_channel_idx) + it.Enable(False) + it.SetData("Channel") + self.SetItem(it) + + self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.OnItemActivated) + self.Bind(EVT_LIST_ITEM_CHECKED, self.OnGetItemCheck) + + self.getting_channels = False + self._mainWin.Bind(wx.EVT_SCROLLWIN, self.OnScroll) + + def OnItemActivated(self, evt): + # double click + pass + + def OnScroll(self, evt): + vpos = self._mainWin.GetScrollPos(wx.VERTICAL) + list_total = self.GetItemCount() + list_pp = self.GetCountPerPage() + topitem_idx, bottomitem_idx = self._mainWin.GetVisibleLinesRange() + + total_page = list_total/list_pp + + # print "vpos %d totlist %d topidx %d pp %d" "btmidx %s" %(vpos, total_page*list_pp, topitem_idx, list_pp, bottomitem_idx) + if (vpos >= list_total and total_page*list_pp < vpos and vpos > topitem_idx+list_pp)\ + or vpos == 0: + #not so accurate but this will do + + if self.getting_channels: + evt.Skip() + return + + self.LoadMore() + + evt.Skip() + + def LoadMore(self): + self._logger.info("getting new channels..") + self.getting_channels = True + + def do_query_channels(): + RETURNED_CHANNELS = 30 + + _, channels = self.guiutility.channelsearch_manager.getPopularChannels(20) + dict_channels = {channel.dispersy_cid: channel for channel in channels} + new_channels_ids = list(set(dict_channels.keys()) - set(self.channel_list.keys())) + + return_list = [dict_channels.get(new_channels_ids[i]) for i in range(0, + len(new_channels_ids) if len(new_channels_ids) < RETURNED_CHANNELS else RETURNED_CHANNELS)] + + if return_list: + return [l for l in sorted(return_list, key=lambda x: x.nr_favorites, reverse=True)]# if "tribler" in l.name.lower() or "linux" in l.name.lower()] + else: + None + + def do_update_gui(delayedResult): + channels = delayedResult.get() + + if channels: + for s in channels: + # s is channel object + self.channel_list[s.dispersy_cid] = s + self.CreateSourceItem(s) + + self.getting_channels = False + + startWorker(do_update_gui, do_query_channels, retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def CreateSourceItem(self, source): + item_count = self.GetItemCount() + + if isinstance(source, RSSFeedSource): + + self.InsertStringItem(self.label_dir_idx, source.getSource(), 1) + item = self.GetItem(self.label_dir_idx) + item.Check(source.enabled) + item.SetData(source) + self.SetItem(item) + self.label_dir_idx += 1 + self.label_channel_idx += 1 + elif isinstance(source, DirectorySource): + self.InsertStringItem(self.label_channel_idx, source.getSource(), 1) + item = self.GetItem(self.label_channel_idx) + item.Check(source.enabled) + item.SetData(source) + self.SetItem(item) + self.label_channel_idx += 1 + elif isinstance(source, ChannelSource): + self.InsertStringItem(self.label_channel_idx+1, source.getSource() or "Loading..", 1) + item = self.GetItem(self.label_channel_idx+1) + item.Check(source.enabled) + item.SetData(source) + self.SetItem(item) + + self.channel_list[source.source] = source + elif isinstance(source, Channel): + # channel can't be 'added'. Initialization only + self.InsertStringItem(item_count, source.name, 1) + self.SetItemData(item_count, source) + else: + raise NotYetImplementedException('Source type unknown') + + def OnGetItemCheck(self, evt): + item = evt.GetItem() + data = item.GetData() + flag = item.IsChecked() + + self.boosting_manager.set_enable_mining( + data.dispersy_cid if not isinstance(data, BoostingSource) + else data.source, flag, True) + + if isinstance(data, Channel): + # channel -> channel source + channel_src = self.boosting_manager.boosting_sources.get(data.dispersy_cid) + channel_src.channel = data + item.SetData(channel_src) + self.SetItem(item) + + + def RefreshData(self, rerun=True): + for i in range(0, self.GetItemCount()): + item = self.GetItem(i) + data = item.GetData() + + if isinstance(data, BoostingSource): + source = data.getSource() + if isinstance(data, RSSFeedSource): + sobj = self.boosting_manager.boosting_sources[source] + elif isinstance(data, DirectorySource): + sobj = self.boosting_manager.boosting_sources[source] + elif isinstance(data, ChannelSource): + source = data.source + sobj = self.boosting_manager.boosting_sources[source] + if item.GetText() == "Loading..": + item.SetText(data.getSource() or "Loading..") + self.SetItem(item) + + elif isinstance(data, Channel): + pass + + if rerun: + self.utility.session.lm.threadpool.add_task(self.RefreshData, 30, + task_name=str(self)+"_refresh_data_ULC") + + def FixChannelPos(self, source): + chn_source = self.boosting_manager.boosting_sources[source] + + chn = self.channel_list[source] + + idx = self.FindItemData(-1, chn) + self.DeleteItem(idx) + + self.InsertStringItem(self.label_channel_idx+1,chn_source.getSource() or "Loading..", 1) + item = self.GetItem(self.label_channel_idx+1) + item.Check(chn_source.enabled) + item.SetData(chn_source) + self.SetItem(item) + + self.channel_list[source] = chn_source + + +class CreditMiningPanel(FancyPanel): + def __init__(self, parent): + self._logger = logging.getLogger(self.__class__.__name__) + + self._logger.debug("CreditMiningPanel: __init__") + + self.guiutility = GUIUtility.getInstance() + self.utility = self.guiutility.utility + self.installdir = self.utility.getPath() + + self.boosting_manager = BoostingManager.get_instance() + + self.tdb = self.utility.session.open_dbhandler(NTFY_TORRENTS) + + FancyPanel.__init__(self, parent, border=wx.BOTTOM) + + self.SetBorderColour(SEPARATOR_GREY) + self.SetBackgroundColour(GRADIENT_LGREY, GRADIENT_DGREY) + + self.main_sizer = wx.BoxSizer(wx.VERTICAL) + + self.header = self.CreateHeader(self) + if self.header: + self.main_sizer.Add(self.header, 0, wx.EXPAND) + + self.main_splitter = wx.SplitterWindow(self, style=wx.SP_BORDER) + + self.sourcelist = CpanelCheckListCtrl(self.main_splitter, -1, + agwStyle=wx.LC_REPORT | wx.LC_NO_HEADER | wx.LC_VRULES + | wx.LC_HRULES | wx.LC_SINGLE_SEL | ULC.ULC_HAS_VARIABLE_ROW_HEIGHT) + + self.AddComponents(self.main_splitter) + self.SetSizer(self.main_sizer) + + + self.guiutility.utility.session.lm.threadpool.add_task(self._PostInit, 2, + task_name=str(self)+"_post_init") + + + def AddComponents(self,parent): + self.infoPanel = FancyPanel(parent, style=wx.BORDER_SUNKEN) + + if_sizer = wx.BoxSizer(wx.VERTICAL) + self.top_info_p = FancyPanel(self.infoPanel, border=wx.ALL, style=wx.BORDER_SUNKEN, name="top_info_p") + tinfo_sizer = wx.BoxSizer(wx.VERTICAL) + + self.tnfo_subpanel_top = FancyPanel(self.top_info_p, border=wx.ALL) + tinfo_spanel_sizer = wx.BoxSizer(wx.HORIZONTAL) + + stat_sizer = wx.BoxSizer(wx.VERTICAL) + + self.source_label = wx.StaticText(self.tnfo_subpanel_top, -1, 'Source : -') + stat_sizer.Add(self.source_label, 1) + self.source_name = wx.StaticText(self.tnfo_subpanel_top, -1, 'Name : -') + stat_sizer.Add(self.source_name, 1) + self.torrent_num = wx.StaticText(self.tnfo_subpanel_top, -1, '# Torrents : -') + stat_sizer.Add(self.torrent_num, 1) + + # channels only + self.last_updt = wx.StaticText(self.tnfo_subpanel_top, -1, 'Latest update : -') + stat_sizer.Add(self.last_updt, 1) + self.votes_num = wx.StaticText(self.tnfo_subpanel_top, -1, 'Favorite votes : -') + stat_sizer.Add(self.votes_num, 1) + + # rss only + self.rss_title = wx.StaticText(self.tnfo_subpanel_top, -1, 'Title : -') + stat_sizer.Add(self.rss_title, 1) + self.rss_desc = wx.StaticText(self.tnfo_subpanel_top, -1, 'Description : -') + stat_sizer.Add(self.rss_desc, 1) + + self.debug_info = wx.StaticText(self.tnfo_subpanel_top, -1, 'Debug Info : -') + stat_sizer.Add(self.debug_info) + + tinfo_spanel_sizer.Add(stat_sizer,-1) + tinfo_spanel_sizer.Add(wx.StaticText(self.tnfo_subpanel_top, -1, 'Credit Mining Status: ')) + self.status_cm = wx.StaticText(self.tnfo_subpanel_top, -1, '-') + tinfo_spanel_sizer.Add(self.status_cm) + self.tnfo_subpanel_top.SetSizer(tinfo_spanel_sizer) + + tinfo_sizer.Add(self.tnfo_subpanel_top, 1, wx.EXPAND) + tinfo_sizer.Add(wx.StaticLine(self.top_info_p), 0, wx.ALL|wx.EXPAND, 5) + + self.up_rate = wx.StaticText(self.top_info_p, -1, 'Upload rate : -', name="up_rate") + tinfo_sizer.Add(self.up_rate) + self.dwn_rate = wx.StaticText(self.top_info_p, -1, 'Download rate : -', name="dwn_rate") + tinfo_sizer.Add(self.dwn_rate) + self.storage_used = wx.StaticText(self.top_info_p, -1, 'Storage Used : -', name="storage_used") + tinfo_sizer.Add(self.storage_used) + + self.top_info_p.SetSizer(tinfo_sizer) + + if_sizer.Add(self.top_info_p, 1, wx.EXPAND) + + self.cmlist = CreditMiningList(self.infoPanel) + self.cmlist.do_or_schedule_refresh(True) + self.cmlist.library_manager.add_download_state_callback(self.cmlist.RefreshItems) + + if_sizer.Add(self.cmlist, 1, wx.EXPAND) + self.infoPanel.SetSizer(if_sizer) + + self.sourcelist.Hide() + self.loading_holder = wx.StaticText(self.main_splitter, -1, 'Loading..') + + parent.SplitVertically(self.loading_holder, self.infoPanel) + parent.SetMinimumPaneSize(100) + parent.SetSashGravity(0.25) + self.main_sizer.Add(parent, 1, wx.EXPAND) + + def OnItemSelected(self, event): + idx = event.m_itemIndex + data = self.sourcelist.GetItem(idx).GetData() + + if isinstance(data, ChannelSource): + self.cmlist.GotFilter(data.source) + else: + self.cmlist.GotFilter(data.getSource() if isinstance(data, BoostingSource) else '') + + self.ShowInfo(data) + + def ShowInfo(self, data): + + if isinstance(data, ChannelSource): + self.last_updt.Show() + self.votes_num.Show() + self.rss_title.Hide() + self.rss_desc.Hide() + + self.source_label.SetLabel("Source : Channel (stored)") + self.source_name.SetLabel("Name : "+data.getSource()) + self.torrent_num.SetLabel("# Torrents : "+str(data.channel.nr_torrents)) + self.last_updt.SetLabel("Latest update : "+format_time(data.channel.modified)) + self.votes_num.SetLabel('Favorite votes : '+str(data.channel.nr_favorites)) + self.status_cm.SetLabel("Active" if data.enabled else "Inactive") + + + debug_str = hexlify(data.source) + self.debug_info.SetLabel("Debug Info : \n"+debug_str) + + elif isinstance(data, Channel): + self.last_updt.Show() + self.votes_num.Show() + self.rss_title.Hide() + self.rss_desc.Hide() + + self.source_label.SetLabel("Source : Channel") + self.source_name.SetLabel("Name : "+data.name) + self.torrent_num.SetLabel("# Torrents : "+str(data.nr_torrents)) + self.last_updt.SetLabel("Latest update : "+format_time(data.modified)) + self.votes_num.SetLabel('Favorite votes : '+str(data.nr_favorites)) + self.status_cm.SetLabel("Inactive") + + debug_str = hexlify(data.dispersy_cid) + self.debug_info.SetLabel("Debug Info : \n"+debug_str) + + elif isinstance(data, RSSFeedSource): + self.last_updt.Hide() + self.votes_num.Hide() + self.rss_title.Show() + self.rss_desc.Show() + + self.source_label.SetLabel("Source : RSS Web Feed") + self.source_name.SetLabel("Source URL : "+data.getSource()) + self.torrent_num.SetLabel("# Torrents : "+str(data.total_torrents)) + self.rss_title.SetLabel("Title : "+data.title) + self.rss_desc.SetLabel("Description : "+data.description) + self.status_cm.SetLabel("Active" if data.enabled else "Inactive") + + debug_str = "-" + self.debug_info.SetLabel("Debug Info : \n"+debug_str) + + elif isinstance(data, DirectorySource): + self.last_updt.Hide() + self.votes_num.Hide() + self.rss_title.Hide() + self.rss_desc.Hide() + + self.source_label.SetLabel("Source : Directory") + self.source_name.SetLabel("Name : "+data.getSource()) + self.torrent_num.SetLabel("# Torrents : "+str(12345)) + self.status_cm.SetLabel("Active" if data.enabled else "Inactive") + + debug_str = "-" + self.debug_info.SetLabel("Debug Info : \n"+debug_str) + + else: + self._logger.debug("Not implemented yet") + pass + + # show/hide items + self.tnfo_subpanel_top.Layout() + + + def CreateHeader(self, parent): + if self.guiutility.frame.top_bg: + header = FancyPanel(parent, border=wx.BOTTOM, name="cm_header") + text = wx.StaticText(header, -1, 'Investment overview') + + def OnAddSource(event): + dlg = AddBoostingSource(None) + if dlg.ShowModal() == wx.ID_OK: + source, archive = dlg.GetValue() + if source: + self.boosting_manager.add_source(source) + self.boosting_manager.set_archive(source, archive) + + self.sourcelist.CreateSourceItem(self.boosting_manager.boosting_sources[source]) + + dlg.Destroy() + + def OnRemoveSource(event): + dlg = RemoveBoostingSource(None) + if dlg.ShowModal() == wx.ID_OK and dlg.GetValue(): + self.boosting_manager.remove_source(dlg.GetValue()) + self.GetManager().refresh() + dlg.Destroy() + + addsource = LinkStaticText(header, 'Add', icon=None) + addsource.Bind(wx.EVT_LEFT_UP, OnAddSource) + removesource = LinkStaticText(header, 'Remove', icon=None) + removesource.Bind(wx.EVT_LEFT_UP, OnRemoveSource) + self.b_up = wx.StaticText(header, -1, 'Total bytes up: -',name="b_up") + self.b_down = wx.StaticText(header, -1, 'Total bytes down: -',name="b_down") + self.s_up = wx.StaticText(header, -1, 'Total speed up: -',name="s_up") + self.s_down = wx.StaticText(header, -1, 'Total speed down: -',name="s_down") + self.iv_sum = wx.StaticText(header, -1, 'Investment summary: -',name="iv_sum") + _set_font(text, size_increment=2, fontweight=wx.FONTWEIGHT_BOLD) + sizer = wx.BoxSizer(wx.VERTICAL) + sizer.AddStretchSpacer() + titleSizer = wx.BoxSizer(wx.HORIZONTAL) + titleSizer.Add(text, 0, wx.ALIGN_BOTTOM | wx.RIGHT, 5) + titleSizer.Add(wx.StaticText(header, -1, '('), 0, wx.ALIGN_BOTTOM) + titleSizer.Add(addsource, 0, wx.ALIGN_BOTTOM) + titleSizer.Add(wx.StaticText(header, -1, '/'), 0, wx.ALIGN_BOTTOM) + titleSizer.Add(removesource, 0, wx.ALIGN_BOTTOM) + titleSizer.Add(wx.StaticText(header, -1, ' boosting source)'), 0, wx.ALIGN_BOTTOM) + sizer.Add(titleSizer, 0, wx.LEFT | wx.BOTTOM, 5) + sizer.Add(self.b_up, 0, wx.LEFT, 5) + sizer.Add(self.b_down, 0, wx.LEFT, 5) + sizer.Add(self.s_up, 0, wx.LEFT, 5) + sizer.Add(self.s_down, 0, wx.LEFT, 5) + sizer.Add(self.iv_sum, 0, wx.LEFT, 5) + sizer.AddStretchSpacer() + header.SetSizer(sizer) + header.SetMinSize((-1, 100)) + else: + raise NotYetImplementedException('') + + return header + + def _PostInit(self): + + for i in self.boosting_manager.boosting_sources: + if not self.boosting_manager.boosting_sources[i].ready: + self.guiutility.utility.session.lm.threadpool.add_task(self._PostInit, 2, task_name=str(self)+"_post_init") + return + + for source, source_obj in self.boosting_manager.boosting_sources.items(): + self.sourcelist.CreateSourceItem(source_obj) + + self.sourcelist.Show() + self.main_splitter.ReplaceWindow(self.loading_holder, self.sourcelist) + self.loading_holder.Close() + + self.Bind(ULC.EVT_LIST_ITEM_SELECTED, self.OnItemSelected, self.sourcelist) + + self.sourcelist.LoadMore() + self.sourcelist.RefreshData() diff --git a/Tribler/Main/vwxGUI/GuiUtility.py b/Tribler/Main/vwxGUI/GuiUtility.py index 4fcf19f5c4a..8121667af5e 100644 --- a/Tribler/Main/vwxGUI/GuiUtility.py +++ b/Tribler/Main/vwxGUI/GuiUtility.py @@ -252,24 +252,32 @@ def ShowPage(self, page, *args): if page == 'creditmining': # Show list - self.frame.creditmininglist.Show(True) - - # Open infohash - if args: - self.frame.creditmininglist.GetManager().refresh_or_expand(args[0]) - else: - items = self.frame.creditmininglist.GetExpandedItems() - if items: - items[0][1].expanded = False - self.frame.creditmininglist.Select(items[0][0]) - - # Open infohash - if args: - self.frame.creditmininglist.GetManager().refresh_or_expand(args[0]) + # self.frame.creditmininglist.Show(True) + # + # # Open infohash + # if args: + # self.frame.creditmininglist.GetManager().refresh_or_expand(args[0]) + # else: + # items = self.frame.creditmininglist.GetExpandedItems() + # if items: + # items[0][1].expanded = False + # self.frame.creditmininglist.Select(items[0][0]) + # + # # Open infohash + # if args: + # self.frame.creditmininglist.GetManager().refresh_or_expand(args[0]) + + self.frame.creditminingpanel.Show(True) elif self.guiPage == 'creditmining': + self.frame.creditminingpanel.Show(False) + + if page == 'cmbeta': + # Show list + self.frame.creditminingpanel.Show(True) + elif self.guiPage == 'cmbeta': # Hide list - self.frame.creditmininglist.Show(False) + self.frame.creditminingpanel.Show(False) if page == 'home': self.frame.home.ResetSearchBox() @@ -312,7 +320,8 @@ def ShowPage(self, page, *args): elif page == 'my_files': self.frame.librarylist.Focus() elif page == 'creditmining': - self.frame.creditmininglist.Focus() + pass + # self.frame.creditmininglist.Focus() @forceWxThread def on_show_startup_splash(self, subject, changetype, objectID, *args): @@ -380,7 +389,7 @@ def GetSelectedPage(self): return self.frame.librarylist if self.guiPage == 'creditmining': - return self.frame.creditmininglist + return self.frame.creditminingpanel def SetTopSplitterWindow(self, window=None, show=True): while self.frame.splitter_top.GetChildren(): @@ -622,7 +631,7 @@ def OnList(self, goto_end, event=None): 'mychannel': self.frame.managechannel, 'search_results': self.frame.searchlist, 'my_files': self.frame.librarylist, - 'creditmining': self.frame.creditmininglist} + 'creditmining': self.frame.creditminingpanel} if self.guiPage in lists and lists[self.guiPage].HasFocus(): lists[self.guiPage].ScrollToEnd(goto_end) elif event: @@ -635,7 +644,7 @@ def ScrollTo(self, id): 'mychannel': self.frame.managechannel, 'search_results': self.frame.searchlist, 'my_files': self.frame.librarylist, - 'creditmining': self.frame.creditmininglist} + 'creditmining': self.frame.creditminingpanel} if self.guiPage in lists: lists[self.guiPage].ScrollToId(id) @@ -673,7 +682,8 @@ def toggleFamilyFilter(self, newState=None, setCheck=False): if setCheck: self.frame.SRstatusbar.ff_checkbox.SetValue(newState) - self.frame.home.aw_panel.refreshNow() + # we don't use this panel in credit mining + # self.frame.home.aw_panel.refreshNow() if newState: self.utility.write_config('family_filter', 1) diff --git a/Tribler/Main/vwxGUI/MainFrame.py b/Tribler/Main/vwxGUI/MainFrame.py index 16a6d99d655..6dad4df652c 100644 --- a/Tribler/Main/vwxGUI/MainFrame.py +++ b/Tribler/Main/vwxGUI/MainFrame.py @@ -34,6 +34,7 @@ from Tribler.Core.exceptions import DuplicateDownloadException from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo from Tribler.Core.Utilities.utilities import parse_magnetlink, fix_torrent +from Tribler.Main.Dialogs.SaveAs import SaveAs from Tribler.Core.DownloadConfig import DefaultDownloadStartupConfig from Tribler.Main.Utility.GuiDBHandler import startWorker @@ -41,8 +42,10 @@ from Tribler.Main.Dialogs.ConfirmationDialog import ConfirmationDialog from Tribler.Main.Dialogs.FeedbackWindow import FeedbackWindow from Tribler.Main.Dialogs.systray import ABCTaskBarIcon -from Tribler.Main.Dialogs.SaveAs import SaveAs - +from Tribler.Main.Utility.GuiDBHandler import startWorker +from Tribler.Main.globals import DefaultDownloadStartupConfig +from Tribler.Main.vwxGUI import DEFAULT_BACKGROUND, SEPARATOR_GREY +from Tribler.Main.vwxGUI.CreditMiningPanel import CreditMiningPanel from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread from Tribler.Main.vwxGUI import DEFAULT_BACKGROUND, SEPARATOR_GREY from Tribler.Main.vwxGUI.list import SearchList, ChannelList, LibraryList, ActivitiesList @@ -187,6 +190,8 @@ def __init__(self, abc, parent, internalvideo): self.searchlist.Show(False) self.librarylist = LibraryList(self.splitter_top_window) self.librarylist.Show(False) + # self.creditmininglist = CreditMiningList(self) + # self.creditmininglist.Show(False) self.channellist = ChannelList(self.splitter_top_window) self.channellist.Show(False) self.selectedchannellist = SelectedChannelList(self.splitter_top_window) @@ -194,6 +199,10 @@ def __init__(self, abc, parent, internalvideo): self.playlist = Playlist(self.splitter_top_window) self.playlist.Show(False) + + self.creditminingpanel = CreditMiningPanel(self) + self.creditminingpanel.Show(False) + # Populate the bottom window self.splitter_bottom = wx.BoxSizer(wx.HORIZONTAL) self.torrentdetailspanel = TorrentDetails(self.splitter_bottom_window) @@ -236,6 +245,9 @@ def OnShowSplitter(event): event.Skip() self.splitter.Bind(wx.EVT_SHOW, OnShowSplitter) + # self.creditmininglist = CreditMiningList(self) + # self.creditmininglist.Show(False) + self.stats = Stats(self) self.stats.Show(False) self.managechannel = ManageChannel(self) @@ -259,12 +271,15 @@ def OnShowSplitter(event): hSizer.Add(self.stats, 1, wx.EXPAND) hSizer.Add(self.networkgraph, 1, wx.EXPAND) hSizer.Add(self.splitter, 1, wx.EXPAND) + # hSizer.Add(self.creditmininglist, 1, wx.EXPAND) hSizer.Add(self.managechannel, 1, wx.EXPAND) if self.videoparentpanel: hSizer.Add(self.videoparentpanel, 1, wx.EXPAND) + hSizer.Add(self.creditminingpanel, 1, wx.EXPAND) + self.SetSizer(vSizer) # set sizes diff --git a/Tribler/Main/vwxGUI/SearchGridManager.py b/Tribler/Main/vwxGUI/SearchGridManager.py index f7570926b74..e65e269b9e7 100644 --- a/Tribler/Main/vwxGUI/SearchGridManager.py +++ b/Tribler/Main/vwxGUI/SearchGridManager.py @@ -1003,8 +1003,8 @@ def getMySubscriptions(self): subscriptions = self.channelcast_db.getMySubscribedChannels(include_dispersy=True) return self._createChannels(subscriptions) - def getPopularChannels(self): - pchannels = self.channelcast_db.getMostPopularChannels() + def getPopularChannels(self, nr_top_popular=20): + pchannels = self.channelcast_db.getMostPopularChannels(nr_top_popular) return self._createChannels(pchannels) def getUpdatedChannels(self): diff --git a/Tribler/Main/vwxGUI/home.py b/Tribler/Main/vwxGUI/home.py index 7fc44d402e8..dc3b7320668 100644 --- a/Tribler/Main/vwxGUI/home.py +++ b/Tribler/Main/vwxGUI/home.py @@ -3,7 +3,12 @@ import sys import os import datetime +from binascii import hexlify +from wx.lib.scrolledpanel import ScrolledPanel +import math + +from Tribler.Policies.BoostingManager import BoostingManager from Tribler.community.tunnel.hidden_community import HiddenTunnelCommunity from Tribler.community.tunnel.routing import Hop from Tribler.community.multichain.community import MultiChainCommunity @@ -22,7 +27,7 @@ NTFY_ONCREATED_E2E, NTFY_IP_CREATED, NTFY_RP_CREATED, NTFY_REMOVE) from Tribler.Core.Session import Session -from Tribler.Main.vwxGUI import SEPARATOR_GREY, DEFAULT_BACKGROUND, LIST_BLUE, THUMBNAIL_FILETYPES +from Tribler.Main.vwxGUI import SEPARATOR_GREY, DEFAULT_BACKGROUND, LIST_BLUE, THUMBNAIL_FILETYPES, warnWxThread from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY from Tribler.Main.vwxGUI.list_header import DetailHeader @@ -30,7 +35,7 @@ from Tribler.Main.vwxGUI.list_item import ThumbnailListItemNoTorrent from Tribler.Main.vwxGUI.list_footer import ListFooter from Tribler.Main.vwxGUI.widgets import (SelectableListCtrl, TextCtrlAutoComplete, BetterText as StaticText, - LinkStaticText, ActionButton) + LinkStaticText, ActionButton, HorizontalGauge, TagText) from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager from Tribler.Core.CacheDB.sqlitecachedb import bin2str from Tribler.Core.Video.VideoUtility import considered_xxx @@ -40,11 +45,20 @@ class Home(wx.Panel): + COLUMN_SIZE = 3 + def __init__(self, parent): wx.Panel.__init__(self, parent) self.guiutility = GUIUtility.getInstance() + self.gui_image_manager = GuiImageManager.getInstance() + self.session = self.guiutility.utility.session + self.boosting_manager = BoostingManager.get_instance(self.session) - self.SetBackgroundColour(DEFAULT_BACKGROUND) + #dispersy_cid:Channel + self.channels = {None:None} + + #dispersy_cid:Popular Torrents + self.chn_torrents = {} vSizer = wx.BoxSizer(wx.VERTICAL) vSizer.AddStretchSpacer() @@ -78,7 +92,7 @@ def __init__(self, parent): search_button = ActionButton(self, -1, search_img) search_button.Bind(wx.EVT_LEFT_UP, self.OnClick) - scalingSizer.Add(self.searchBox) + scalingSizer.Add(self.searchBox, 0, wx.ALIGN_CENTER_VERTICAL) scalingSizer.AddSpacer(3, -1) scalingSizer.Add(search_button, 0, wx.ALIGN_CENTER_VERTICAL, 3) @@ -91,29 +105,64 @@ def __init__(self, parent): hSizer.Add(StaticText(self, -1, " to see what others are sharing.")) vSizer.Add(text, 0, wx.ALIGN_CENTER) - vSizer.AddSpacer(10) - vSizer.Add(scalingSizer, 0, wx.ALIGN_CENTER) - vSizer.AddSpacer(10) + vSizer.Add(scalingSizer, 1, wx.ALIGN_CENTER_HORIZONTAL | wx.ALIGN_TOP) vSizer.Add(hSizer, 0, wx.ALIGN_CENTER) vSizer.AddStretchSpacer() + # channel panel is popular channel + self.channel_panel = ScrolledPanel(self, 1) + self.channel_panel.SetBackgroundColour(wx.WHITE) + self.channel_panel.SetForegroundColour(parent.GetForegroundColour()) + + v_chn_Sizer = wx.BoxSizer(wx.VERTICAL) + v_chn_Sizer.Add( + DetailHeader(self.channel_panel, "Select popular channels to mine"), + 0, wx.EXPAND, 5) + + self.chn_sizer = wx.FlexGridSizer(0,self.COLUMN_SIZE,5,5) + + for i in range(0,self.COLUMN_SIZE): + if wx.MAJOR_VERSION > 2: + if self.chn_sizer.IsColGrowable(i): + self.chn_sizer.AddGrowableCol(i,1) + else: + self.chn_sizer.AddGrowableCol(i,1) + + v_chn_Sizer.Add(self.chn_sizer, 0, wx.EXPAND, 5) + + self.channel_panel.SetSizer(v_chn_Sizer) + self.channel_panel.SetupScrolling() + + vSizer.Add(self.channel_panel, 5, wx.EXPAND) + + # # TODO (ardhi) : enable this once the layout finishes + # video thumbnail panel self.aw_panel = ArtworkPanel(self) self.aw_panel.SetMinSize((-1, 275)) - self.aw_panel.Hide() + # TODO(lipu): enable this when metadata PR is merged + #self.aw_panel.Show(self.guiutility.ReadGuiSetting('show_artwork', True)) + self.aw_panel.Show(True) vSizer.Add(self.aw_panel, 0, wx.EXPAND) self.SetSizer(vSizer) self.Layout() - self.Bind(wx.EVT_RIGHT_UP, self.OnRightClick) - self.SearchFocus() + self.channel_list_ready = False + self.session.lm.threadpool.add_task(self.RefreshChannels, 10, + task_name=str(self.__class__)+"_refreshchannel") + + def OnRightClick(self, event): menu = wx.Menu() - itemid = wx.NewId() - menu.AppendCheckItem(itemid, 'Show recent videos') - menu.Check(itemid, self.aw_panel.IsShown()) + itemid_rcvid = wx.NewId() + itemid_popchn = wx.NewId() + menu.AppendCheckItem(itemid_rcvid, 'Show recent videos') + menu.AppendCheckItem(itemid_popchn, 'Show popular channels') + + menu.Check(itemid_rcvid, self.aw_panel.IsShown()) + menu.Check(itemid_popchn, self.channel_panel.IsShown()) def toggleArtwork(event): show = not self.aw_panel.IsShown() @@ -121,7 +170,13 @@ def toggleArtwork(event): self.guiutility.WriteGuiSetting("show_artwork", show) self.Layout() - menu.Bind(wx.EVT_MENU, toggleArtwork, id=itemid) + def toggleChannels(event): + show = not self.channel_panel.IsShown() + self.channel_panel.Show(show) + self.Layout() + + menu.Bind(wx.EVT_MENU, toggleChannels, id=itemid_popchn) + menu.Bind(wx.EVT_MENU, toggleArtwork, id=itemid_rcvid) if menu: self.PopupMenu(menu, self.ScreenToClient(wx.GetMousePosition())) @@ -144,6 +199,158 @@ def SearchFocus(self): self.searchBox.SetFocus() self.searchBox.SelectAll() + def CreateChannelItem(self, parent, channel, torrents, max_fav): + """ + Function to create channel (and it's torrents) checkbox on home panel + :param parent: where we put this element + :param channel: channel object + :param torrents: torrents of that particular channel + :param max_fav: max possible votes for ALL channel (to count relative popularity) + """ + + from Tribler.Main.Utility.GuiDBTuples import Channel as ChannelObj + assert isinstance(channel, ChannelObj), "Type channel should be ChannelObj %s" %channel + + STRING_LENGTH = 35 + + vsizer = wx.BoxSizer(wx.VERTICAL) + hsizer = wx.BoxSizer(wx.HORIZONTAL) + + chn_pn = wx.Panel(parent, -1, style=wx.SUNKEN_BORDER) + + cb_chn = wx.CheckBox(chn_pn, 1, '',name=hexlify(channel.dispersy_cid)) + obj = self.boosting_manager.get_source_object(channel.dispersy_cid) + + cb_chn.SetValue(False if not obj else obj.enabled) + normalministar = self.gui_image_manager.getImage(u"ministar.png") + ministar = self.gui_image_manager.getImage(u"ministarEnabled.png") + + control = HorizontalGauge(chn_pn, normalministar, ministar, 5) + + # count popularity + pop = channel.nr_favorites + if pop <= 0: + control.SetPercentage(0) + else: + control.SetPercentage(pop/float(max_fav)) + + control.SetToolTipString('%s users marked this channel as one of their favorites.' % pop) + hsizer.Add(cb_chn, 0, wx.ALIGN_LEFT) + hsizer.Add(TagText(chn_pn, -1, label='channel', fill_colour=wx.Colour(210, 252, 120)),0, + wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL) + hsizer.AddSpacer(5) + hsizer.Add(wx.StaticText(chn_pn, -1, channel.name.encode('utf-8')), 0, wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL) + hsizer.AddSpacer(30) + hsizer.AddStretchSpacer() + hsizer.Add(control, 0, wx.ALIGN_RIGHT | wx.ALIGN_CENTER_VERTICAL) + + vsizer.Add(hsizer, 0, wx.EXPAND) + + for t in torrents: + t = wx.StaticText(chn_pn, 1, t.name[:STRING_LENGTH] + (t.name[STRING_LENGTH:] and '...')) + vsizer.Add(t, 0, wx.EXPAND | wx.LEFT, 25) + + chn_pn.SetSizer(vsizer) + self.Bind(wx.EVT_CHECKBOX, self.OnCheckBox, cb_chn) + return chn_pn + + def RefreshChannels(self): + """ + This function will be called to get popular channel list in Home + + """ + # number of popular torrent fetched to know the 'content' of channels + TORRENT_FETCHED = 5 + + # max number of channel shown in the panel + MAX_CHANNEL_SHOW = 9 + + def do_query(): + _, channels = self.guiutility.channelsearch_manager.getPopularChannels(2*MAX_CHANNEL_SHOW) + + dict_channels = {channel.dispersy_cid:channel for channel in channels} + dict_torrents = {} + new_channels_ids = list(set(dict_channels.keys()) - + set(self.channels.keys() if not self.channel_list_ready else [])) + + for c in new_channels_ids: + channel = dict_channels.get(c) + torrents = self.guiutility.channelsearch_manager.getRecentReceivedTorrentsFromChannel\ + (channel, limit=TORRENT_FETCHED)[2] + dict_torrents[c] = torrents + return (dict_channels, dict_torrents, new_channels_ids) + + def do_gui(delayedResult): + (dict_channels,dict_torrents, new_channels_ids) = delayedResult.get() + count = 0 + + if self.channel_list_ready: + # reset it. Not reseting torrent_dict because it dynamically added anyway + self.channels = {} + + for c in new_channels_ids: + self.channels[c] = dict_channels.get(c) + self.chn_torrents.update(dict_torrents) + + self.chn_sizer.Clear(True) + self.chn_sizer.Layout() + for i in range(0,self.COLUMN_SIZE): + if wx.MAJOR_VERSION > 2: + if self.chn_sizer.IsColGrowable(i): + self.chn_sizer.AddGrowableCol(i,1) + else: + self.chn_sizer.AddGrowableCol(i,1) + + sortedchannels = sorted(self.channels.values(), + key=lambda x: x.nr_favorites if x else 0, reverse=True) + + max_favourite = sortedchannels[0].nr_favorites if sortedchannels else 0 + + for c in [x for x in sortedchannels]: + d = c.dispersy_cid + # if we can't find channel details, ignore it, or + # if no torrent available for that channel + if not dict_channels.get(d) or not len(self.chn_torrents.get(d)): + continue + + self.chn_sizer.Add(self.CreateChannelItem(self.channel_panel,dict_channels.get(d), + self.chn_torrents.get(d),max_favourite), + 0, wx.ALL|wx.EXPAND) + count += 1 + if count >= MAX_CHANNEL_SHOW: + break + + if new_channels_ids: + self.chn_sizer.Layout() + self.channel_panel.SetupScrolling() + + if self.guiutility.frame.ready and isinstance(self.guiutility.GetSelectedPage(), Home): + startWorker(do_gui, do_query, retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + repeat = len(self.channels) < MAX_CHANNEL_SHOW + self.channel_list_ready = not repeat + if repeat: + self.session.lm.threadpool.add_task_in_thread(self.RefreshChannels, 10, + task_name=str(self.__class__)+"_refreshchannel") + else : + # try to update the popular channel once in a while + self.session.lm.threadpool.add_task_in_thread(self.RefreshChannels, 10, + task_name=str(self.__class__)+"_refreshchannel") + + def OnCheckBox(self, evt): + cb = evt.GetEventObject() + self.boosting_manager.set_enable_mining(binascii.unhexlify(cb.GetName()), evt.IsChecked()) + + + if evt.IsChecked(): + chn_src = self.boosting_manager.boosting_sources[binascii.unhexlify(cb.GetName())] + sourcelist = self.guiutility.frame.creditminingpanel.sourcelist + + if binascii.unhexlify(cb.GetName()) in sourcelist.channel_list: + sourcelist.FixChannelPos(binascii.unhexlify(cb.GetName())) + else: + sourcelist.CreateSourceItem(chn_src) + class Stats(wx.Panel): diff --git a/Tribler/Main/vwxGUI/list.py b/Tribler/Main/vwxGUI/list.py index 775c893ec84..ef0c6b65aa5 100644 --- a/Tribler/Main/vwxGUI/list.py +++ b/Tribler/Main/vwxGUI/list.py @@ -1,13 +1,18 @@ # Written by Niels Zeilemaker import copy import logging +import os import re import sys +from binascii import hexlify, unhexlify from colorsys import hsv_to_rgb, rgb_to_hsv from math import log from time import time import wx +from wx._core import LayoutConstraints +from wx.lib.mixins.listctrl import CheckListCtrlMixin +from wx.lib.scrolledpanel import ScrolledPanel from wx.lib.wordwrap import wordwrap from Tribler.Category.Category import Category @@ -15,10 +20,11 @@ from Tribler.Core.simpledefs import (DLSTATUS_HASHCHECKING, DLSTATUS_STOPPED, DLSTATUS_STOPPED_ON_ERROR, DLSTATUS_WAITING4HASHCHECK, DLSTATUS_SEEDING, DLSTATUS_DOWNLOADING) from Tribler.Main.Utility.GuiDBHandler import GUI_PRI_DISPERSY, cancelWorker, startWorker -from Tribler.Main.Utility.GuiDBTuples import Channel, ChannelTorrent, CollectedTorrent, Torrent +from Tribler.Main.Utility.GuiDBTuples import Channel, ChannelTorrent, CollectedTorrent, Torrent, LibraryTorrent from Tribler.Main.Utility.utility import eta_value, size_format, speed_format from Tribler.Main.vwxGUI import (DEFAULT_BACKGROUND, GRADIENT_DGREY, GRADIENT_LGREY, LIST_DESELECTED, LIST_GREEN, - LIST_GREY, LIST_ORANGE, SEPARATOR_GREY, TRIBLER_RED, format_time, warnWxThread) + LIST_GREY, LIST_ORANGE, SEPARATOR_GREY, TRIBLER_RED, format_time, warnWxThread, + LIST_SELECTED, LIST_EXPANDED, LIST_DARKBLUE) from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread from Tribler.Main.vwxGUI.list_body import FixedListBody, ListBody @@ -27,10 +33,11 @@ from Tribler.Main.vwxGUI.list_footer import ListFooter from Tribler.Main.vwxGUI.list_header import ChannelFilter, DownloadFilter, ListHeader, TorrentFilter from Tribler.Main.vwxGUI.list_item import (ActivityListItem, ChannelListItem, ChannelListItemAssociatedTorrents, - ColumnsManager, DragItem, LibraryListItem, TorrentListItem) + ColumnsManager, DragItem, LibraryListItem, TorrentListItem, + CreditMiningListItem) from Tribler.Main.vwxGUI.widgets import (BetterText, FancyPanel, HorizontalGauge, LinkStaticText, SwarmHealth, TagText, TorrentStatus, TransparentStaticBitmap, TransparentText, _set_font) - +from Tribler.Policies.BoostingManager import BoostingManager, BoostingSource DEBUG_RELEVANCE = False MAX_REFRESH_PARTIAL = 5 @@ -301,6 +308,72 @@ def downloadStarted(self, _): self.refresh() +class CreditMiningSearchManager(BaseManager): + + def __init__(self, list): + BaseManager.__init__(self, list) + self.boosting_manager = BoostingManager.get_instance() + self.library_manager = self.guiutility.library_manager + + def refresh(self): + startWorker(self._on_data, self.getHitsInCategory, uId=u"CreditMiningSearchManager_refresh", retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def getTorrentFromInfohash(self, infohash): + torrent = self.boosting_manager.torrents.get(infohash, None) + if torrent: + t = LibraryTorrent('', infohash, name=torrent['name'], length=torrent['length'], category='', status='', + num_seeders=torrent['num_seeders'], num_leechers=torrent['num_leechers']) + t.torrent_db = self.library_manager.torrent_db + t.channelcast_db = self.library_manager.channelcast_db + + #touch channel instance + t.channel + self.library_manager.addDownloadState(t) + return t + + def getHitsInCategory(self): + hits = [self.getTorrentFromInfohash(infohash) for infohash in self.boosting_manager.torrents.keys()] + return [len(hits), hits] + + def refresh_partial(self, ids): + for infohash in ids: + startWorker(self.list.RefreshDelayedData, self.getTorrentFromInfohash, cargs=(infohash,), wargs=(infohash,), retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def refresh_if_exists(self, infohashes, force=False): + if any([self.boosting_manager.torrents.has_key(infohash) for infohash in infohashes]): + print >> sys.stderr, long(time()), "Scheduling a refresh, missing some infohashes in the Credit Mining overview" + self.refresh() + else: + print >> sys.stderr, long(time()), "Not scheduling a refresh" + + def refresh_or_expand(self, infohash): + if not self.list.InList(infohash): + def select(delayedResult): + delayedResult.get() + self.refresh_or_expand(infohash) + + startWorker(select, self.refresh_partial, wargs=([infohash],), priority=GUI_PRI_DISPERSY) + else: + self.list.Select(infohash) + + @forceWxThread + def _on_data(self, delayedResult): + total_items, data = delayedResult.get() + self.list.SetData(data) + self.list.Layout() + + def torrentUpdated(self, infohash): + if self.list.InList(infohash): + self.do_or_schedule_partial([infohash]) + + def torrentsUpdated(self, infohashes): + infohashes = [infohash for infohash in infohashes if self.list.InList(infohash)] + self.do_or_schedule_partial(infohashes) + + def downloadStarted(self, infohash): + self.refresh() + + class ChannelSearchManager(BaseManager): def __init__(self, list): @@ -342,7 +415,6 @@ def refreshDirty(self): def refresh(self, search_results=None): self._logger.debug("ChannelManager complete refresh") - if self.category != 'searchresults': category = self.category @@ -462,7 +534,7 @@ def joinChannel(self, cid): class List(wx.BoxSizer): def __init__(self, columns, background, spacers=[0, 0], singleSelect=False, - showChange=False, borders=True, parent=None): + showChange=False, borders=True, parent=None, list_item_max=None): """ Column alignment: @@ -491,6 +563,7 @@ def __init__(self, columns, background, spacers=[0, 0], singleSelect=False, self.singleSelect = singleSelect self.borders = borders self.showChange = showChange + self.list_item_max = list_item_max self.dirty = False self.hasData = False self.rawfilter = '' @@ -551,7 +624,8 @@ def CreateHeader(self, parent): def CreateList(self, parent=None, listRateLimit=1): if not parent: parent = self - return ListBody(parent, self, self.columns, self.spacers[0], self.spacers[1], self.singleSelect, self.showChange, listRateLimit=listRateLimit) + return ListBody(parent, self, self.columns, self.spacers[0], self.spacers[1], self.singleSelect, self.showChange, listRateLimit=listRateLimit, + list_item_max=self.list_item_max) def CreateFooter(self, parent): return ListFooter(parent) @@ -854,6 +928,7 @@ def MatchFilter(self, item): return re.search(self.filter, item[1][0].lower()) and ff def GetFilterMessage(self, empty=False): + if self.rawfilter: if empty: message = '0 items' @@ -877,8 +952,8 @@ def SetupScrolling(self, *args, **kwargs): class SizeList(List): def __init__(self, columns, background, spacers=[0, 0], singleSelect=False, - showChange=False, borders=True, parent=None): - List.__init__(self, columns, background, spacers, singleSelect, showChange, borders, parent) + showChange=False, borders=True, parent=None, list_item_max=None): + List.__init__(self, columns, background, spacers, singleSelect, showChange, borders, parent, list_item_max=None) self.prevStates = {} self.library_manager = self.guiutility.library_manager @@ -1958,6 +2033,276 @@ def GetFilterMessage(self, empty=False): return header, message +class CreditMiningList(SizeList): + + def __init__(self, parent): + self.boosting_manager = BoostingManager.get_instance() + self.guiutility = GUIUtility.getInstance() + self.utility = self.guiutility.utility + + self.statefilter = None + self.newfilter = False + self.prevStates = {} + self.oldDS = {} + + self.initnumitems = False + self.tot_bytes_up = 0 + self.tot_bytes_dwn = 0 + self.channels = [] + + self.top_info_p = parent.FindWindowByName('top_info_p') or None + + + columns = [{'name': 'Speed up/down', 'width': '32em', 'autoRefresh': False}, + {'name': 'Bytes up/down', 'width': '32em', 'autoRefresh': False}, + {'name': 'Seeders/leechers', 'width': '27em', 'autoRefresh': False}, + {'name': 'Duplicate', 'showColumname': False, 'width': '2em'}, + {'name': 'Hash', 'width': '27em', 'fmt': lambda ih: ih.encode('hex')[:10]}, + {'name': 'Source', 'width': '40em', 'type': 'method', 'method': self.CreateSource}, + {'name': 'Investment status', 'width': '32em', 'autoRefresh': False}] + + columns = self.guiutility.SetColumnInfo(CreditMiningListItem, columns) + ColumnsManager.getInstance().setColumns(CreditMiningListItem, columns) + + SizeList.__init__(self, None, LIST_GREY, [0, 0], False, parent=parent) + + def GetManager(self): + if getattr(self, 'manager', None) == None: + self.manager = CreditMiningSearchManager(self) + return self.manager + + @warnWxThread + def CreateHeader(self, parent): + return None + + @warnWxThread + def CreateFooter(self, parent): + self.list.ShowMessage("No credit mining data available.") + footer = ListFooter(parent, radius=0) + footer.SetMinSize((-1, 0)) + return footer + + @warnWxThread + def CreateSource(self, parent, item): + torrent = self.boosting_manager.torrents.get(item.original_data.infohash, None) + text = torrent.get('source', '') + text = text[:30] + '..' if len(text) > 32 else text + return wx.StaticText(parent, -1, text) + + def OnExpand(self, item): + List.OnExpand(self, item) + return True + + @warnWxThread + def RefreshItems(self, dslist, magnetlist): + didStateChange, _, _ = SizeList.RefreshItems(self, dslist, magnetlist, rawdata=True) + + newFilter = self.newfilter + + new_keys = self.boosting_manager.torrents.keys() + old_keys = getattr(self, 'old_keys', []) + if len(new_keys) != len(old_keys): + self.GetManager().refresh_if_exists(new_keys, force=True) + self.old_keys = new_keys + + if didStateChange: + if self.statefilter != None: + self.list.SetData() # basically this means execute filter again + + boosting_dslist = [ds for ds in dslist if ds.get_download().get_def().get_infohash() in new_keys] + + # init source statistics + for _, src in self.boosting_manager.boosting_sources.items(): + src.storage_used = 0 + src.av_uprate = 0 + src.av_dwnrate = 0 + + # update torrent stats in boosting manager + for ds in boosting_dslist: + torrent_infohash = ds.get_download().get_def().get_infohash() + + if ds.get_seeding_statistics(): + self.boosting_manager.update_torrent_stats(torrent_infohash, ds.get_seeding_statistics()) + + for item in self.list.items.itervalues(): + ds = item.original_data.ds + torrent_infohash = item.original_data.infohash + source_str = self.boosting_manager.torrents[torrent_infohash]['source'] + + source = self.boosting_manager.get_source_object(source_str) + + # ds = DownloadState + if ds: + source.av_uprate += ds.get_current_speed('up') + source.av_dwnrate += ds.get_current_speed('down') + + if not ds in boosting_dslist: + continue + + # look for the current active stats to update visible list + if ds.get_seeding_statistics(): + seeding_stats_i = ds.get_seeding_statistics() + + bytes_up = bytes_down = 0 + + if self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']: + bytes_up = self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']['total_up'] + bytes_down = self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']['total_down'] + + bytes_up = max(seeding_stats_i['total_up'], bytes_up) + bytes_down = max(seeding_stats_i['total_down'], bytes_down) + + item.RefreshColumn(1, size_format(bytes_up) + ' / ' + size_format(bytes_down)) + if bytes_down: + item.RefreshColumn(6, '%f' %(float(bytes_up)/float(bytes_down))) + + #refresh seeder/leecher + it_seeder = self.boosting_manager.torrents[torrent_infohash]['num_seeders'] + it_leecher = self.boosting_manager.torrents[torrent_infohash]['num_leechers'] + item.RefreshColumn(2, '%d / %d' %(it_seeder, it_leecher)) + + item.SetSelectedColour(wx.Colour(255, 175, 175)) + item.SetDeselectedColour(wx.Colour(255, 200, 200)) + item.SetExpandedColour(wx.Colour(255, 150, 150)) + item.SetExpandedAndSelectedColour(wx.Colour(255, 125, 125)) + + speed_up = ds.get_current_speed('up') if ds else 0 + speed_down = ds.get_current_speed('down') if ds else 0 + + item.RefreshColumn(0, speed_format(speed_up) + ' / ' + speed_format(speed_down)) + + else: + item.SetSelectedColour(LIST_SELECTED) + item.SetDeselectedColour(LIST_DESELECTED) + item.SetExpandedColour(LIST_EXPANDED) + item.SetExpandedAndSelectedColour(LIST_DARKBLUE) + + item.RefreshColumn(0, '- / -') + + if torrent_infohash in self.boosting_manager.torrents: + is_dup = self.boosting_manager.torrents[torrent_infohash].get('is_duplicate', None) + item.RefreshColumn(3, ('*' if is_dup else '**') if is_dup != None else '') + + # compilation of all torrents seeding stats + seeding_stats = [] + + for _, torrent_dict in self.boosting_manager.torrents.items(): + seeding_stat_t = torrent_dict['last_seeding_stats'] + + if seeding_stat_t: + seeding_stats.append(seeding_stat_t) + + # seeding_stats = [ds.get_seeding_statistics() for ds in boosting_dslist if ds.get_seeding_statistics()] + self.tot_bytes_up = sum([stat['total_up'] for stat in seeding_stats]) + self.tot_bytes_dwn = sum([stat['total_down'] for stat in seeding_stats]) + + if self.top_info_p: + up_rate_txt = self.top_info_p.FindWindowByName('up_rate') + dwn_rate_txt = self.top_info_p.FindWindowByName('dwn_rate') + storage_used_txt = self.top_info_p.FindWindowByName('storage_used') + + try: + filter = self.rawfilter if self.rawfilter in self.boosting_manager.boosting_sources else unhexlify(self.rawfilter) + except: + filter = self.rawfilter + + if filter: + active_source = self.boosting_manager.get_source_object(filter) + + list = [tr['last_seeding_stats']['total_down'] for tr in self.boosting_manager.torrents.values() + if self.boosting_manager.get_source_object(tr['source']).source == active_source.source and tr['last_seeding_stats']] + + total_dl_source = sum(list) + up_rate_txt.SetLabel('Active upload rate : '+speed_format(active_source.av_uprate)) + dwn_rate_txt.SetLabel('Active download rate : '+speed_format(active_source.av_dwnrate)) + storage_used_txt.SetLabel('Storage Used : '+size_format(total_dl_source)) + + + header = self.parent.GetGrandParent().FindWindowByName('cm_header') + header.FindWindowByName('b_up').SetLabel('Total bytes up: ' + size_format(self.tot_bytes_up)) + header.FindWindowByName('b_down').SetLabel('Total bytes down: ' + size_format(self.tot_bytes_dwn)) + + if self.tot_bytes_dwn: + header.FindWindowByName('iv_sum').SetLabel(' Investment summary: %f' %(float(self.tot_bytes_up)/float(self.tot_bytes_dwn))) + + header.FindWindowByName('s_up').SetLabel('Current total speed up: ' + speed_format(sum([ds.get_current_speed('up') for ds in boosting_dslist]))) + header.FindWindowByName('s_down').SetLabel('Current total speed down: ' + speed_format(sum([ds.get_current_speed('down') for ds in boosting_dslist]))) + + if newFilter: + self.newfilter = False + + self.oldDS = dict([(infohash, item.original_data.ds) for infohash, item in self.list.items.iteritems()]) + + @warnWxThread + def SetData(self, data): + SizeList.SetData(self, data) + + if len(data) > 0: + data = [(file.infohash, ['- / -', '- / -', '%d / %d' % + (file.num_seeders, file.num_leechers), '', file.infohash, + self.boosting_manager.torrents.get(file.infohash,None).get('source', ''), '-1'], + file, CreditMiningListItem) for file in data] + else: + self.list.ShowMessage("No credit mining data available.") + self.SetNrResults(0) + + self.list.SetData(data) + + @warnWxThread + def RefreshData(self, key, data): + List.RefreshData(self, key, data) + + data = (data.infohash, ['-', '-', '%d / %d' % (data.num_seeders, data.num_leechers), '', data.infohash, + self.boosting_manager.torrents.get(data.infohash, None).get('source', ''),'-1'], data) + self.list.RefreshData(key, data) + + def SetNrResults(self, nr): + highlight = nr > self.nr_results and self.initnumitems + SizeList.SetNrResults(self, nr) + + actitem = self.guiutility.frame.actlist.GetItem(5) + num_items = getattr(actitem, 'num_items', None) + if num_items: + num_items.SetValue(str(nr)) + actitem.hSizer.Layout() + if highlight: + actitem.Highlight() + self.initnumitems = True + + def OnFilter(self,keyword): + pass + + def MatchFilter(self, item): + if not self.rawfilter: + return False + + source = item[1][5] + match = (hexlify(self.rawfilter) if len(hexlify(self.rawfilter)) == 40 else self.rawfilter) in source + + if self.boosting_manager.get_source_object(self.rawfilter): + match = match and self.boosting_manager.get_source_object(self.rawfilter).enabled + + return match + + def GotFilter(self, keyword=None): + self.rawfilter = keyword + if self.rawfilter == '' and not self.guiutility.getFamilyFilter(): + wx.CallAfter(self.list.SetFilter, None, None, keyword is None) + else: + wx.CallAfter(self.list.SetFilter, self.MatchFilter, self.GetFilterMessage, True) + + self.OnFilter(self.rawfilter) + + def GetFilterMessage(self, empty=False): + if empty: + return 'Empty','No credit mining torrent to show' + else: + return None,'end of list' + + def MatchFFilter(self, item): + return True + + class ChannelList(List): def __init__(self, parent): @@ -2172,18 +2517,24 @@ def __SetData(self): data_list = [(1, ['Home'], None, ActivityListItem), (2, ['Results'], None, ActivityListItem), (3, ['Channels'], None, ActivityListItem), - (4, ['Downloads'], None, ActivityListItem)] + (4, ['Downloads'], None, ActivityListItem), + (5, ['Credit Mining'], None, ActivityListItem)] if sys.platform != 'darwin': - data_list.append((5, ['Videoplayer'], None, ActivityListItem)) + data_list.append((6, ['Videoplayer'], None, ActivityListItem)) + + # data_list.append((7, ['CM List beta'], None, ActivityListItem)) self.list.SetData(data_list) self.ResizeListItems() self.DisableItem(2) + if not self.guiutility.frame.videoparentpanel and sys.platform != 'darwin': - self.DisableItem(5) + self.DisableItem(6) self.DisableCollapse() self.selectTab('home') + # self.list.GetItem(7).num_items.Show(False) + # Create expanded panels in advance channels_item = self.list.GetItem(3) self.expandedPanel_channels = ChannelsExpandedPanel(channels_item) @@ -2191,7 +2542,7 @@ def __SetData(self): self.expandedPanel_channels.Hide() if sys.platform != 'darwin': - videoplayer_item = self.list.GetItem(5) + videoplayer_item = self.list.GetItem(6) self.expandedPanel_videoplayer = VideoplayerExpandedPanel(videoplayer_item) videoplayer_item.AddEvents(self.expandedPanel_videoplayer) self.expandedPanel_videoplayer.Hide() @@ -2271,6 +2622,10 @@ def OnExpand(self, item): if self.guiutility.guiPage not in ['videoplayer']: self.guiutility.ShowPage('videoplayer') return self.expandedPanel_videoplayer + elif item.data[0] == 'Credit Mining': + self.guiutility.ShowPage('creditmining') + elif item.data[0] == 'CM List beta': + self.guiutility.ShowPage('cmbeta') return True def OnCollapse(self, item, panel, from_expand): @@ -2337,6 +2692,9 @@ def selectTab(self, tab): itemKey = 4 elif tab == 'videoplayer': itemKey = 5 + itemKey = 6 + # elif tab == 'cmbeta': + # itemKey = 7 if itemKey: wx.CallAfter(self.Select, itemKey, True) return @@ -2357,7 +2715,7 @@ def _DoPage(self, increment): if curPage < 0: curPage = len(pages) - 1 - pageNames = ['home', 'search_results', 'channels', 'my_files', 'videoplayer'] + pageNames = ['home', 'search_results', 'channels', 'my_files', 'creditmining', 'videoplayer']#, 'cmbeta'] for i in self.settings.keys(): pageNames.pop(i - 1) self.guiutility.ShowPage(pageNames[curPage]) diff --git a/Tribler/Main/vwxGUI/list_item.py b/Tribler/Main/vwxGUI/list_item.py index 5c2ab7528e1..74ac1fbf428 100644 --- a/Tribler/Main/vwxGUI/list_item.py +++ b/Tribler/Main/vwxGUI/list_item.py @@ -1124,6 +1124,13 @@ def GetIcons(self): def SetThumbnailIcon(self): pass +class CreditMiningListItem(ListItem): + + def AddComponents(self, leftSpacer, rightSpacer): + ListItem.AddComponents(self, 5, 5) + + def GetIcons(self): + return [] class ActivityListItem(ListItem): @@ -1132,7 +1139,7 @@ def __init__(self, *args, **kwargs): def AddComponents(self, leftSpacer, rightSpacer): ListItem.AddComponents(self, leftSpacer, rightSpacer) - if self.data[0] in ['Results', 'Channels', 'Downloads', 'Videoplayer']: + if self.data[0] in ['Results', 'Channels', 'Downloads', 'Credit Mining', 'Videoplayer', 'CM List beta']: self.num_items = TagText(self, -1, label='0', fill_colour=GRADIENT_DGREY, edge_colour=SEPARATOR_GREY) self.hSizer.Add(self.num_items, 0, wx.CENTER | wx.RIGHT, 5) self.hSizer.Layout() From 9a87521490e760ada929bd7f79ca4b123a5b9fe5 Mon Sep 17 00:00:00 2001 From: ardhipoetra Date: Fri, 18 Mar 2016 14:38:54 +0100 Subject: [PATCH 11/24] Extract peerlist to its own function - Manually fire trackers - Logging peer list - Fix seeding_stats reset bug in credit mining - Add more peers information --- .../Core/APIImplementation/LaunchManyCore.py | 9 +- .../Core/Libtorrent/LibtorrentDownloadImpl.py | 104 +++++--- Tribler/Core/Libtorrent/LibtorrentMgr.py | 18 -- Tribler/Policies/BoostingManager.py | 232 ++++++++++-------- Tribler/bootstraptribler.txt | 42 ++++ boosting.ini | 2 +- 6 files changed, 249 insertions(+), 158 deletions(-) create mode 100644 Tribler/bootstraptribler.txt diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index 78bdbef6ed1..2c657154ce7 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -319,7 +319,7 @@ def load_communities(): self.initComplete = True - def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidden=False, share_mode=False): + def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidden=False, share_mode=False,checkpoint_disabled=False): """ Called by any thread """ d = None with self.sesslock: @@ -343,7 +343,8 @@ def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidd # Store in list of Downloads, always. self.downloads[infohash] = d - setup_deferred = d.setup(dscfg, pstate, initialdlstatus, wrapperDelay=setupDelay, share_mode=share_mode) + setup_deferred = d.setup(dscfg, pstate, initialdlstatus, wrapperDelay=setupDelay, + share_mode=share_mode, checkpoint_disabled=checkpoint_disabled) setup_deferred.addCallback(self.on_download_wrapper_created) if d and not hidden and self.session.get_megacache(): @@ -594,7 +595,9 @@ def checkpoint(self, stop=False, checkpoint=True, gracetime=2.0): # Download, and additions are no problem (just won't be included # in list of states returned via callback. # - dllist = [dl for dl in self.downloads.values() if not dl.checkpoint_disabled] + dllist = self.downloads.values() + self._logger.debug("tlm: checkpointing %s stopping %s", len(dllist), stop) + network_checkpoint_callback_lambda = lambda: self.network_checkpoint_callback(dllist, stop, checkpoint, gracetime) self.threadpool.add_task(network_checkpoint_callback_lambda, 0.0) diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index 74811b38d08..8a00a6f99fc 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -11,10 +11,12 @@ from twisted.internet.defer import Deferred, CancelledError from Tribler.Core import NoDispersyRLock +from Tribler.Core.APIImplementation import maketorrent from Tribler.Core.DownloadConfig import DownloadStartupConfig, DownloadConfigInterface from Tribler.Core.DownloadState import DownloadState from Tribler.Core.Libtorrent import checkHandleAndSynchronize, waitForHandleAndSynchronize from Tribler.Core.TorrentDef import TorrentDefNoMetainfo, TorrentDef +from Tribler.Core.Utilities.torrent_utils import get_info_from_handle from Tribler.Core.Utilities import maketorrent from Tribler.Core.Utilities.torrent_utils import get_info_from_handle from Tribler.Core.osutils import fix_filebasename @@ -292,10 +294,20 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode atp["ti"] = torrentinfo has_resume_data = resume_data and isinstance(resume_data, dict) + + if has_resume_data is not None: + # we have resume data but somehow its not working (Credit mining case) + if not isinstance(resume_data, dict): + new_dict = pstate.get('engineresumedata', None) + if isinstance(new_dict, dict): + resume_data = new_dict + has_resume_data = resume_data and isinstance(resume_data, dict) + if has_resume_data: atp["resume_data"] = lt.bencode(resume_data) - self._logger.info("%s %s", self.tdef.get_name_as_unicode(), dict((k, v) - for k, v in resume_data.iteritems() if k not in ['pieces', 'piece_priority', 'peers']) if has_resume_data else None) + if not share_mode: + self._logger.info("%s %s", self.tdef.get_name_as_unicode(), dict((k, v) + for k, v in resume_data.iteritems() if k not in ['pieces', 'piece_priority', 'peers']) if has_resume_data else None) else: atp["url"] = self.tdef.get_url() or "magnet:?xt=urn:btih:" + hexlify(self.tdef.get_infohash()) atp["name"] = self.tdef.get_name_as_unicode() @@ -305,16 +317,16 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode if self.handle: self.set_selected_files() - # set_selected_files sets priorities to 1, so we must set - # share_mode again, but first we must unset it, otherwise - # set_share_mode doesn't do anything - if share_mode: - self.handle.set_share_mode(not share_mode) - self.handle.set_share_mode(share_mode) + # set_selected_files sets priorities to 1, so we must set + # share_mode again, but first we must unset it, otherwise + # set_share_mode doesn't do anything + if share_mode: + self.handle.set_share_mode(not share_mode) + self.handle.set_share_mode(share_mode) - # If we lost resume_data always resume download in order to force checking - if initialdlstatus != DLSTATUS_STOPPED or not resume_data: - self.handle.resume() + # If we lost resume_data always resume download in order to force checking + if initialdlstatus != DLSTATUS_STOPPED or not resume_data: + self.handle.resume() # If we only needed to perform checking, pause download after it is complete self.pause_after_next_hashcheck = initialdlstatus == DLSTATUS_STOPPED @@ -864,42 +876,62 @@ def network_get_vod_stats(self): d['npieces'] = ((self.length + 1023) / 1024) return d + @staticmethod + def create_peerlist_data(peer_info): + peer_dict = {} + + peer_dict['id'] = peer_info.pid + peer_dict['extended_version'] = peer_info.client + peer_dict['ip'] = peer_info.ip[0] + peer_dict['port'] = peer_info.ip[1] + # optimistic_unchoke = 0x800 seems unavailable in python bindings + peer_dict['optimistic'] = bool(peer_info.flags & 2048) + peer_dict['direction'] = 'L' if bool(peer_info.flags & peer_info.local_connection) else 'R' + peer_dict['uprate'] = peer_info.payload_up_speed + peer_dict['uinterested'] = bool(peer_info.flags & peer_info.remote_interested) + peer_dict['uchoked'] = bool(peer_info.flags & peer_info.remote_choked) + peer_dict['uhasqueries'] = peer_info.upload_queue_length > 0 + peer_dict['uflushed'] = peer_info.used_send_buffer > 0 + peer_dict['downrate'] = peer_info.payload_down_speed + peer_dict['dinterested'] = bool(peer_info.flags & peer_info.interesting) + peer_dict['dchoked'] = bool(peer_info.flags & peer_info.choked) + peer_dict['snubbed'] = bool(peer_info.flags & 4096) # snubbed = 0x1000 seems unavailable in python bindings + peer_dict['utotal'] = peer_info.total_upload + peer_dict['dtotal'] = peer_info.total_download + peer_dict['completed'] = peer_info.progress + peer_dict['have'] = peer_info.pieces + peer_dict['speed'] = peer_info.remote_dl_rate + peer_dict['country'] = peer_info.country + peer_dict['connection_type'] = peer_info.connection_type + + # add upload_only and/or seed + peer_dict['seed'] = bool(peer_info.flags & peer_info.seed) + peer_dict['upload_only'] = bool(peer_info.flags & peer_info.upload_only) + + # add read and write state (check unchoke/choke peers) + peer_dict['rstate_bw_limit'] = bool(peer_info.read_state & peer_info.bw_limit) + peer_dict['wstate_bw_limit'] = bool(peer_info.write_state & peer_info.bw_limit) + peer_dict['rstate_bw_network'] = bool(peer_info.read_state & peer_info.bw_network) + peer_dict['wstate_bw_network'] = bool(peer_info.write_state & peer_info.bw_network) + peer_dict['rstate'] = peer_info.read_state + peer_dict['wstate'] = peer_info.write_state + + + + return peer_dict + def network_create_spew_from_peerlist(self): plist = [] with self.dllock: peer_infos = self.handle.get_peer_info() for peer_info in peer_infos: - # Only consider fully connected peers. # Disabling for now, to avoid presenting the user with conflicting information # (partially connected peers are included in seeder/leecher stats). # if peer_info.flags & peer_info.connecting or peer_info.flags & peer_info.handshake: # continue + peer_dict = LibtorrentDownloadImpl.create_peerlist_data(peer_info) - peer_dict = {} - peer_dict['id'] = peer_info.pid - peer_dict['extended_version'] = peer_info.client - peer_dict['ip'] = peer_info.ip[0] - peer_dict['port'] = peer_info.ip[1] - # optimistic_unchoke = 0x800 seems unavailable in python bindings - peer_dict['optimistic'] = bool(peer_info.flags & 2048) - peer_dict['direction'] = 'L' if bool(peer_info.flags & peer_info.local_connection) else 'R' - peer_dict['uprate'] = peer_info.payload_up_speed - peer_dict['uinterested'] = bool(peer_info.flags & peer_info.remote_interested) - peer_dict['uchoked'] = bool(peer_info.flags & peer_info.remote_choked) - peer_dict['uhasqueries'] = peer_info.upload_queue_length > 0 - peer_dict['uflushed'] = peer_info.used_send_buffer > 0 - peer_dict['downrate'] = peer_info.payload_down_speed - peer_dict['dinterested'] = bool(peer_info.flags & peer_info.interesting) - peer_dict['dchoked'] = bool(peer_info.flags & peer_info.choked) - peer_dict['snubbed'] = bool(peer_info.flags & 4096) # snubbed = 0x1000 seems unavailable in python bindings - peer_dict['utotal'] = peer_info.total_upload - peer_dict['dtotal'] = peer_info.total_download - peer_dict['completed'] = peer_info.progress - peer_dict['have'] = peer_info.pieces - peer_dict['speed'] = peer_info.remote_dl_rate - peer_dict['country'] = peer_info.country - peer_dict['connection_type'] = peer_info.connection_type plist.append(peer_dict) return plist diff --git a/Tribler/Core/Libtorrent/LibtorrentMgr.py b/Tribler/Core/Libtorrent/LibtorrentMgr.py index 3540216a252..0199ac1d9b5 100644 --- a/Tribler/Core/Libtorrent/LibtorrentMgr.py +++ b/Tribler/Core/Libtorrent/LibtorrentMgr.py @@ -324,24 +324,6 @@ def process_alert(self, alert): else: self._logger.debug("LibtorrentMgr: alert for invalid torrent") - def reachability_check(self): - if self.ltsession and self.ltsession.status().has_incoming_connections: - self.trsession.lm.threadpool.add_task(self.trsession.lm.dialback_reachable_callback, 3) - else: - self.trsession.lm.threadpool.add_task(self.reachability_check, 10) - - def monitor_dht(self, chances_remaining=1): - # Sometimes the dht fails to start. To workaround this issue we monitor the #dht_nodes, and restart if needed. - if self.ltsession: - if self.get_dht_nodes() <= 25: - if self.get_dht_nodes() >= 5 and chances_remaining: - self._logger.info("LibtorrentMgr: giving the dht a chance (%d, %d)", self.ltsession.status().dht_nodes, chances_remaining) - self.trsession.lm.threadpool.add_task(lambda: self.monitor_dht(chances_remaining - 1), 5) - else: - self._logger.debug("could not find torrent %s", infohash) - else: - self._logger.debug("alert for invalid torrent") - def get_peers(self, infohash, callback, timeout=30): def on_metainfo_retrieved(metainfo, infohash=infohash, callback=callback): callback(infohash, metainfo.get('initial peers', [])) diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index d354e79da57..58091e52514 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -20,8 +20,10 @@ import libtorrent as lt from Tribler.Core.DownloadConfig import DownloadStartupConfig +from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl from Tribler.Core.TorrentChecker.session import MAX_TRACKER_MULTI_SCRAPE from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.Utilities import utilities from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_INSERT, NTFY_SCRAPE, NTFY_TORRENTS, NTFY_UPDATE from Tribler.Main.Utility.GuiDBTuples import Torrent from Tribler.Main.globals import DefaultDownloadStartupConfig @@ -33,7 +35,7 @@ from Tribler.dispersy.util import call_on_reactor_thread logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.DEBUG) formatter = logging.Formatter( "%(asctime)s.%(msecs).03dZ-%(levelname)s-%(message)s", datefmt="%Y%m%dT%H%M%S") @@ -97,10 +99,17 @@ def apply(self, torrents, max_active): if self.session.get_download(torrent["metainfo"].get_infohash()): torrents_stop.append(torrent) - # if not torrents_start and not torrents_stop: - # torrents_start = torrents.copy() + # if both results are empty for some reason (e.g, key_check too restrictive) + # or torrent started less than half available torrent (try to keep boosting alive) + # if it's already random, just let it be + if not isinstance(self, RandomPolicy) and ((not torrents_start and not torrents_stop) or + (len(torrents_start) < len(torrents)/2 and len(torrents_start) < max_active/2)): + logger.error("Start and stop torrent list are empty. Fallback to Random") + # fallback to random policy + rp = RandomPolicy(self.session) + torrents_start, torrents_stop = rp.apply(torrents, max_active) - return (torrents_start, torrents_stop) + return torrents_start, torrents_stop class RandomPolicy(BoostingPolicy): @@ -136,12 +145,17 @@ class BoostingManager(TaskManager): __single = None def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval=20, sw_interval=20, - max_per_source=100, max_active=2): + max_per_source=100, max_active=2, config_file=CONFIG_FILE): super(BoostingManager, self).__init__() + self._logger = logging.getLogger(self.__class__.__name__) BoostingManager.__single = self - self.gui_util = GUIUtility.getInstance() + self.gui_util = GUIUtility.getInstance(utility) + if not self.gui_util.registered: + self.gui_util.register() + + self.config_file = config_file self._saved_attributes = ["max_torrents_per_source", "max_torrents_active", "source_interval", @@ -152,14 +166,18 @@ def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval self.utility = utility self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), "credit_mining") - if not os.path.exists(self.credit_mining_path): - os.mkdir(self.credit_mining_path) + try: + if not os.path.exists(self.credit_mining_path): + os.mkdir(self.credit_mining_path) + except: + pass self.boosting_sources = {} self.torrents = {} self.policy = None self.share_mode_target = 3 + # TODO(ardhi) : hardcode some of the interval for now self.max_torrents_per_source = max_per_source self.max_torrents_active = max_active self.source_interval = src_interval @@ -175,8 +193,8 @@ def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval self.set_share_mode_params(share_mode_target=self.share_mode_target) - if os.path.exists(CONFIG_FILE): - logger.info("Config file %s", open(CONFIG_FILE).read()) + if os.path.exists(config_file): + logger.info("Config file %s", open(config_file).read()) else: logger.info("Config file missing") @@ -249,15 +267,17 @@ def save(self): logger.exception("Could not save state") def set_share_mode_params(self, share_mode_target=None, share_mode_bandwidth=None, share_mode_download=None, share_mode_seeders=None): - settings = self.session.lm.ltmgr.get_session().settings() + + # make set_settings call consistent + settings = {} if share_mode_target is not None: - settings.share_mode_target = share_mode_target + settings['share_mode_target'] = share_mode_target if share_mode_bandwidth is not None: - settings.share_mode_bandwidth = share_mode_bandwidth + settings['share_mode_bandwidth'] = share_mode_bandwidth if share_mode_download is not None: - settings.share_mode_download = share_mode_download + settings['share_mode_download'] = share_mode_download if share_mode_seeders is not None: - settings.share_mode_seeders = share_mode_seeders + settings['share_mode_seeders'] = share_mode_seeders self.session.lm.ltmgr.get_session().set_settings(settings) def add_source(self, source): @@ -278,6 +298,7 @@ def add_source(self, source): self.boosting_sources[source] = ChannelSource(*args) else: logger.error("Cannot add unknown source %s", source) + return logger.info("Added source %s", source) else: @@ -364,73 +385,79 @@ def on_torrent_insert(self, source, infohash, torrent): # source_str) def scrape_trackers(self): - for infohash, torrent in self.torrents.iteritems(): tf = torrent['metainfo'] - self.session.check_torrent_health(infohash) - - self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) - return None - - num_requests = 0 - trackers = defaultdict(list) - - for infohash, torrent in self.torrents.iteritems(): - if isinstance(torrent['metainfo'], TorrentDef) and torrent['enabled']: - tdef = torrent['metainfo'] - for tracker in tdef.get_trackers_as_single_tuple(): - trackers[tracker].append(infohash) - num_requests += 1 - - logger.info("Start tracker scraping for %s torrents", num_requests) - - results = defaultdict(lambda: [0, 0]) - for tracker, infohashes in trackers.iteritems(): - try: - reply = {} - #TODO(ardhi) : specific tracker? make it more general - if tracker.startswith("http://tracker.etree.org"): - for infohash in infohashes: - reply.update(scrape_tcp(tracker, (infohash,))) - elif tracker.startswith("udp://"): - for group in range(len(infohashes) // - MAX_TRACKER_MULTI_SCRAPE): - reply.update(scrape_udp(tracker, infohashes[ - group * MAX_TRACKER_MULTI_SCRAPE: - (group + 1) * MAX_TRACKER_MULTI_SCRAPE])) - reply.update(scrape_udp(tracker, infohashes[ - -(len(infohashes) % MAX_TRACKER_MULTI_SCRAPE):])) - else: - reply = scrape_tcp(tracker, infohashes) - - #RD : hack for readable infohash - logger.debug("Got reply from tracker %s : %s", tracker, - {hexlify(k): v for k, v in reply.items()}) - except Exception as e: - type, obj, _ = sys.exc_info() + # torrent handle + lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(lt.big_number(infohash)) - # pretty lazy hack for debugging purpose - ihash = infohashes[0] + # check health(seeder/leecher) + self.session.check_torrent_health(infohash) - source = self.torrents[ihash]['source'] - logger.error("%s Did not get reply from tracker %s. Reason : %s,%s", - hexlify(source) if len(hexlify(source)) == 40 else source, tracker, obj, str(type)) - else: - for infohash, info in reply.iteritems(): - if info['complete'] > results[infohash][0]: - results[infohash][0] = info['complete'] - results[infohash][1] = info['incomplete'] + if lt_torrent.is_valid() \ + and unhexlify(str(lt_torrent.status().info_hash)) in self.torrents: + status = lt_torrent.status() - for infohash, num_peers in results.iteritems(): - self.torrents[infohash]['num_seeders'] = num_peers[0] - self.torrents[infohash]['num_leechers'] = num_peers[1] - self.session.notifier.notify(NTFY_TORRENTS, NTFY_SCRAPE, infohash) + t = self.torrents[unhexlify(str(status.info_hash))] - logger.info("Finished tracker scraping for %s torrents", num_requests) + # some logging + def _logtorrentpeer(status, lt_torrent): + self._logger.info("%s numseedpeer %d/%d lastscrape: %s seed_rank : %s #conn : %d dl_rate : %s seed/peer : %d/%d", + str(status.info_hash), status.num_seeds, status.num_peers, status.last_scrape, status.seed_rank, + status.num_connections, status.download_rate, status.list_seeds, status.list_peers) + + peer_list = [] + for i in lt_torrent.get_peer_info(): + peer = LibtorrentDownloadImpl.create_peerlist_data(i) + peer_list.append(peer) + + # printing for debugging + out = "%s ip:%s uprate:%s dwnrate:%s progress:%s seed/ul:%s/%s>> " \ + %(str(status.info_hash), peer['ip'], peer['uprate'], peer['downrate'], + peer['completed'],peer['seed'], peer['upload_only']) + + if peer['uinterested']: + out += "INTERESTED in us |" + if peer['uchoked']: + out += "CHOKED on US |" + if peer['dinterested']: + out += "we are INTERESTED |" + if peer['dchoked']: + out += "we CHOKED him |" + + print out + + print "----------------------" + _logtorrentpeer(status, lt_torrent) + + peer_list = [] + for i in lt_torrent.get_peer_info(): + peer = LibtorrentDownloadImpl.create_peerlist_data(i) + peer_list.append(peer) + + # already downloaded + if 'download' in t: + trackers = t['download'].network_tracker_status() + + # find if any tracker is working + trackers_available = any([trackers[i][0] for i in trackers.keys() + if i.startswith('udp') or i.startswith('http')]) + + # we only rely on DHT + if not trackers_available: + #TODO(ardhi) : put some DHT scraper-like + + # use DHT data to translate number of seeder/leecher + t['num_seeders'], t['num_leechers'] = utilities.translate_peers_into_health(peer_list, status) + else: + # scrape again? + lt_torrent.scrape_tracker() self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) + # save the old part + return None + def set_archive(self, source, enable): if source in self.boosting_sources: self.boosting_sources[source].archive = enable @@ -448,9 +475,9 @@ def do_start(): tobj = torrent preload = tobj.get('preload', False) - logger.info("Starting %s preload %s", + logger.info("Starting %s preload %s has pstate %s" , hexlify(tobj["metainfo"].get_infohash()), - preload) + preload, True if tobj.get('pstate', None) else False) # not using Session.start_download because we need to specify pstate assert self.session.get_libtorrent() @@ -501,7 +528,7 @@ def _select_torrent(self): def load_config(self): config = ConfigParser.ConfigParser() - config.read(CONFIG_FILE) + config.read(self.config_file) validate_source = lambda s: unhexlify(s) if len(s) == 40 and not s.startswith("http") else s for k, v in config.items(__name__): if k in self._saved_attributes: @@ -534,7 +561,6 @@ def load_config(self): self.add_source(boosting_source) self.boosting_sources[boosting_source].enabled = False - def save_config(self): config = ConfigParser.ConfigParser() config.add_section(__name__) @@ -583,25 +609,16 @@ def log_statistics(self): for lt_torrent in lt_torrents: status = lt_torrent.status() + if unhexlify(str(status.info_hash)) in self.torrents: t = self.torrents[unhexlify(str(status.info_hash))] - # print({str(status.info_hash):{t['name']:t['num_seeders']}}) - # - # print("numpeer %d lastscrape: %s #upload : %d #conn : %d" %(status.num_peers, status.last_scrape, status.num_uploads, - # status.num_connections)) - - # pprint.pprint(lt_torrent.get_peer_info()) - - # tz = self.gui_util.torrentsearch_manager.getTorrentByInfohash(str(status.info_hash)) - # pprint.pprint(tz) - # pprint.pprint({str(status.info_hash):pprint.pformat(self.torrents[unhexlify(str(status.info_hash))])}) - # .update( - #self.torrents[unhexlify(str(status.info_hash))]['download'].network_tracker_status()) logger.debug("Status for %s : %s %s", status.info_hash, status.all_time_download, status.all_time_upload) non_zero_values = [] + + # assert error in libtorrent 1.0.9 for piece_priority in lt_torrent.piece_priorities(): if piece_priority != 0: non_zero_values.append(piece_priority) @@ -610,6 +627,12 @@ def log_statistics(self): status.info_hash, non_zero_values) self.session.lm.threadpool.add_task(self.log_statistics, self.logging_interval) + def update_torrent_stats(self, torrent_infohash_str, seeding_stats): + if 'time_seeding' in self.torrents[torrent_infohash_str]['last_seeding_stats']: + if seeding_stats['time_seeding'] >= self.torrents[torrent_infohash_str]['last_seeding_stats']['time_seeding']: + self.torrents[torrent_infohash_str]['last_seeding_stats'] = seeding_stats + else: + self.torrents[torrent_infohash_str]['last_seeding_stats'] = seeding_stats class BoostingSource(object): @@ -632,6 +655,8 @@ def __init__(self, session, tqueue, source, interval, max_torrents, callback): self.ready = False self.gui_util = GUIUtility.getInstance() + if not self.gui_util.registered: + self.gui_util.register() def kill_tasks(self): @@ -673,6 +698,7 @@ def _load(self, dispersy_cid): dispersy = self.session.get_dispersy_instance() @call_on_reactor_thread + def join_community(): try: self.community = dispersy.get_community(dispersy_cid, True) @@ -711,8 +737,8 @@ def get_channel_id(): try: join_community() self.ready = True - except: - logger.info("Channel %s was not ready, waits for next interval", hexlify(self.source)) + except Exception,ex: + logger.info("Channel %s was not ready, waits for next interval (%d chn)", hexlify(self.source), len(dispersy.get_communities())) self.session.lm.threadpool.add_task(lambda cid=self.source: self._load(cid), 10, task_name=self.source) @@ -731,10 +757,13 @@ def showTorrent(torrent): self.torrents[infohash]['creation_date'] = torrent.creation_date self.torrents[infohash]['length'] = torrent.tdef.get_length() self.torrents[infohash]['num_files'] = len(torrent.files) - self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] - self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] + self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] or 0 + self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] or 0 self.torrents[infohash]['enabled'] = self.enabled + # seeding stats from DownloadState + self.torrents[infohash]['last_seeding_stats'] = {} + del self.unavail_torrent[infohash] # logger.info("Torrent %s from %s ready to start", hexlify(infohash), hexlify(self.source)) @@ -750,8 +779,8 @@ def showTorrent(torrent): startWorker(doGui, self.gui_util.torrentsearch_manager.loadTorrent, wargs=(t,), wkwargs={'callback': showTorrent}) - if not self.session.lm.threadpool.is_pending_task_active(str(self.source)+"_checktor"): - self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=str(self.source)+"_checktor") + if not self.session.lm.threadpool.is_pending_task_active(hexlify(self.source)+"_checktor"): + self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=hexlify(self.source)+"_checktor") def _update(self): if len(self.torrents) < self.max_torrents: @@ -767,9 +796,12 @@ def _update(self): {self.channel_id: self.channelcast_db.getChannel(self.channel_id)})[2] # dict {key_infohash(binary):Torrent(object-GUIDBTuple)} - self.unavail_torrent = {t.infohash:t for t in listtor} + self.unavail_torrent.update({t.infohash:t for t in listtor if t.infohash not in self.torrents}) + + # it's highly probable the checktor function is running at this time (if it's already running) + if not self.session.lm.threadpool.is_pending_task_active(hexlify(self.source)+"_checktor"): + self.session.lm.threadpool.add_task(self._check_tor, 0, task_name=hexlify(self.source)+"_checktor") - self.session.lm.threadpool.add_task(self._check_tor, 0, task_name=str(self.source)+"_init_checktor") except: logger.info("Channel %s was not ready, waits for next interval", hexlify(self.source)) @@ -838,7 +870,7 @@ def _update(self): self.title = feed_status['title'] self.description = feed_status['description'] - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', 'enabled'] + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', 'enabled', 'last_seeding_stats'] self.total_torrents = len(feed_status['items']) @@ -888,7 +920,7 @@ def _update(self): if tdef: # Create a torrent dict. - torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled] + torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled, {}] self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) # manually generate an ID and put this into DB @@ -931,7 +963,7 @@ def _load(self, directory): def _update(self): if len(self.torrents) < self.max_torrents: - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', 'enabled'] + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', 'enabled', 'last_seeding_stats'] for torrent_filename in glob.glob(self.source + '/*.torrent'): if torrent_filename not in self.torrents: @@ -942,7 +974,7 @@ def _update(self): torrent_filename) continue # Create a torrent dict. - torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled] + torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled, {}] self.torrents[torrent_filename] = dict(zip(torrent_keys, torrent_values)) # Notify the BoostingManager. if self.callback: diff --git a/Tribler/bootstraptribler.txt b/Tribler/bootstraptribler.txt new file mode 100644 index 00000000000..cff68bfa681 --- /dev/null +++ b/Tribler/bootstraptribler.txt @@ -0,0 +1,42 @@ +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 +127.0.0.1 0 diff --git a/boosting.ini b/boosting.ini index db5e851479c..415c3581e60 100644 --- a/boosting.ini +++ b/boosting.ini @@ -9,5 +9,5 @@ logging_interval = 60 boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] boosting_enabled = ["http://bt.etree.org/rss/bt_etree_org.rdf"] boosting_disabled = [] -policy = seederratio +policy = random From 66007737efec6faeb9fa87347894dd271c980ccb Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 27 Jun 2016 16:11:38 +0200 Subject: [PATCH 12/24] Create function to translate peers into health - implement "feedback" (random) policy if we can't determine the actual policy - implement get_source_object to get source by flexible input (string/binary) - put DownloadState recent value into sources and update variable related with it. --- .../Core/TorrentChecker/torrent_checker.py | 3 +- Tribler/Core/Utilities/utilities.py | 53 +++++++++++++++++++ Tribler/Policies/BoostingManager.py | 7 +-- boosting.ini | 4 +- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/Tribler/Core/TorrentChecker/torrent_checker.py b/Tribler/Core/TorrentChecker/torrent_checker.py index b6cc20682ce..6a36e23ab77 100644 --- a/Tribler/Core/TorrentChecker/torrent_checker.py +++ b/Tribler/Core/TorrentChecker/torrent_checker.py @@ -357,7 +357,8 @@ def _create_session_for_request(self, infohash, tracker_url): # before creating a new session, check if the tracker is alive if not self._session.lm.tracker_manager.should_check_tracker(tracker_url): - self._logger.warn(u"skipping recently failed tracker %s", tracker_url) + self._logger.warn(u"skipping recently failed tracker %s by %d times", tracker_url, + self._session.lm.tracker_manager.get_tracker_info(tracker_url)['failures']) return session = create_tracker_session(tracker_url, self._on_result_from_session) diff --git a/Tribler/Core/Utilities/utilities.py b/Tribler/Core/Utilities/utilities.py index deeb5c58c4f..41a11746833 100644 --- a/Tribler/Core/Utilities/utilities.py +++ b/Tribler/Core/Utilities/utilities.py @@ -244,3 +244,56 @@ def fix_torrent(file_path): fixed_data = bencode(fixed_data) return fixed_data + +def translate_peers_into_health(peer_info_dicts, status=None): + # peer_info_dicts is a peer_info dictionary from LibTorrentDownloadImpl.create_peerlist_data + # status is libtorrent torrent status #TODO : unused for now + # purpose : where we want to measure a swarm's health but no tracker can be contacted + + num_seeders = 0 + num_leech = 0 + + num_all_peer = len(peer_info_dicts) + + upload_only = 0 + finished = 0 + unfinished_able_dl = 0 + interest_in_us = 0 + + # collecting some statistics + for p_info in peer_info_dicts: + upload_only_b = False + finished_b = False + interest_in_us_b = False + + if p_info['upload_only']: + upload_only+=1 + upload_only_b = True + if p_info['uinterested']: + interest_in_us+=1 + interest_in_us_b = True + + if p_info['completed'] == 1: + finished+=1 + finished_b = True + else: + unfinished_able_dl += 1 if upload_only_b else 0 + + ''' + seeders potentials: + 1. it's only want uploading right now (upload only) + 2. it's finished (we don't know whether it want to upload or not) + + + leecher potentials: + 1. it's interested in our piece + 2. it's unfinished but it's not 'upload only' (it can't leech for some reason) + 3. it's unfinished (less restrictive) + + make sure to change those description when change the algorithm + ''' + + num_seeders = max(upload_only, finished) + num_leech = max(interest_in_us, min(unfinished_able_dl, num_all_peer - finished)) + return (num_seeders, num_leech) + diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 58091e52514..c5b321ec188 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -283,7 +283,7 @@ def set_share_mode_params(self, share_mode_target=None, share_mode_bandwidth=Non def add_source(self, source): if source not in self.boosting_sources: args = (self.session, self.session.lm.threadpool, source, self.source_interval, self.max_torrents_per_source, self.on_torrent_insert) - # pylint: disable=star-args + # pylint: disable=star-args try: isdir = os.path.isdir(source) @@ -455,9 +455,6 @@ def _logtorrentpeer(status, lt_torrent): self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) - # save the old part - return None - def set_archive(self, source, enable): if source in self.boosting_sources: self.boosting_sources[source].archive = enable @@ -600,7 +597,7 @@ def save_config(self): elif isinstance(self.policy, SeederRatioPolicy): policy = "seederratio" config.set(__name__, "policy", policy) - with open(CONFIG_FILE, "w") as configf: + with open(CONFIG_FILE, "w ") as configf: config.write(configf) def log_statistics(self): diff --git a/boosting.ini b/boosting.ini index 415c3581e60..8064b71f82f 100644 --- a/boosting.ini +++ b/boosting.ini @@ -1,5 +1,5 @@ [Tribler.Policies.BoostingManager] -max_torrents_per_source = 100 +max_torrents_per_source = 10 max_torrents_active = 20 source_interval = 300 swarm_interval = 300 @@ -9,5 +9,5 @@ logging_interval = 60 boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] boosting_enabled = ["http://bt.etree.org/rss/bt_etree_org.rdf"] boosting_disabled = [] -policy = random +policy = seederratio From d7d9ccf36df6deebcea8e29da07f91fd7431fe8f Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 27 Jun 2016 16:14:44 +0200 Subject: [PATCH 13/24] Create holder in GUI if credit mining is not ready yet Also squashed: - Clean and add placeholder in GUI - Refactor code to only consider RSS and directory source to be able to remove --- Tribler/Main/Dialogs/BoostingDialogs.py | 23 +++++---------------- Tribler/Main/Utility/GuiDBTuples.py | 2 +- Tribler/Main/tribler_main.py | 11 +++++++--- Tribler/Main/vwxGUI/CreditMiningPanel.py | 10 ++++----- Tribler/Main/vwxGUI/GuiUtility.py | 26 +----------------------- Tribler/Main/vwxGUI/MainFrame.py | 6 ------ Tribler/Main/vwxGUI/home.py | 14 +++++++++++-- Tribler/Main/vwxGUI/list.py | 21 +++++++------------ Tribler/Main/vwxGUI/list_item.py | 2 +- 9 files changed, 40 insertions(+), 75 deletions(-) diff --git a/Tribler/Main/Dialogs/BoostingDialogs.py b/Tribler/Main/Dialogs/BoostingDialogs.py index 36b613ec6b6..3a91dbf10bc 100644 --- a/Tribler/Main/Dialogs/BoostingDialogs.py +++ b/Tribler/Main/Dialogs/BoostingDialogs.py @@ -4,7 +4,7 @@ from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY from Tribler.Main.vwxGUI.GuiUtility import GUIUtility -from Tribler.Policies.BoostingManager import BoostingManager +from Tribler.Policies.BoostingManager import BoostingManager, ChannelSource class AddBoostingSource(wx.Dialog): @@ -100,7 +100,7 @@ def __init__(self, parent): self.source = '' text = wx.StaticText(self, -1, 'Please select the boosting source you wish to remove:') - self.source_label = wx.StaticText(self, -1, 'Source:', style=wx.RB_GROUP) + self.source_label = wx.StaticText(self, -1, 'Source:') self.source_choice = wx.Choice(self, -1) ok_btn = wx.Button(self, -1, "OK") ok_btn.Bind(wx.EVT_BUTTON, self.OnOK) @@ -121,22 +121,9 @@ def __init__(self, parent): self.SetSizer(vSizer) channels = [] - for source in self.boosting_manager.boosting_sources.keys(): - if source.startswith('http://'): - self.sources.append(source) - elif len(source) == 20: - channels.append(source) - - def do_db(): - return self.guiutility.channelsearch_manager.getChannelsByCID(channels) - - def do_gui(delayedResult): - _, channels = delayedResult.get() - channels = sorted([(channel.name, channel.dispersy_cid) for channel in channels]) - self.sources = self.sources + [channel[1] for channel in channels] - self.source_choice.AppendItems([channel[0] for channel in channels]) - - startWorker(do_gui, do_db, retryOnBusy=True, priority=GUI_PRI_DISPERSY) + # retrieve all source except channel source + self.sources = [s.getSource() for s in self.boosting_manager.boosting_sources.values() + if not isinstance(s, ChannelSource)] self.source_choice.SetItems(self.sources) diff --git a/Tribler/Main/Utility/GuiDBTuples.py b/Tribler/Main/Utility/GuiDBTuples.py index 26b779f4d8c..aead355122b 100644 --- a/Tribler/Main/Utility/GuiDBTuples.py +++ b/Tribler/Main/Utility/GuiDBTuples.py @@ -127,7 +127,7 @@ def __init__(self, torrent_id, infohash, name, length, category, status, num_see @cacheProperty def torrent_id(self): - self._logger.error("Torrent: fetching getTorrentID from DB %s", self) + self._logger.debug("Torrent: fetching getTorrentID from DB %s", self) return self.torrent_db.getTorrentID(self.infohash) def update_torrent_id(self, torrent_id): diff --git a/Tribler/Main/tribler_main.py b/Tribler/Main/tribler_main.py index 764b2f27313..694f47f9994 100755 --- a/Tribler/Main/tribler_main.py +++ b/Tribler/Main/tribler_main.py @@ -28,6 +28,7 @@ import tempfile import traceback import urllib2 +import shutil from collections import defaultdict from random import randint from traceback import print_exc @@ -264,8 +265,6 @@ def __init__(self, params, installdir, autoload_discovery=True, # gracefully closes Tribler after 120 seconds. # wx.CallLater(120*1000, wx.GetApp().Exit) - self.boosting_manager = BoostingManager.get_instance() - self.ready = True except Exception as e: @@ -400,6 +399,8 @@ def PostInit2(self): # TODO(emilon): Use the LogObserver I already implemented # self.dispersy.callback.attach_exception_handler(self.frame.exceptionHandler) + startWorker(None, self.loadSessionCheckpoint, delay=5.0, workerType="ThreadPool") + @forceWxThread def sesscb_ntfy_myprefupdates(self, subject, changeType, objectID, *args): if self._frame_and_ready(): @@ -685,6 +686,9 @@ def sesscb_ntfy_torrentupdates(self, events): manager = self.frame.librarylist.GetManager() manager.torrentsUpdated(infohashes) + manager = self.frame.creditminingpanel.cmlist.GetManager() + manager.torrentsUpdated(infohashes) + from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent if self.frame.torrentdetailspanel.torrent and self.frame.torrentdetailspanel.torrent.infohash in infohashes: @@ -698,12 +702,13 @@ def sesscb_ntfy_torrentupdates(self, events): torrent = t.torrent if isinstance(t, CollectedTorrent) else t self.frame.librarydetailspanel.setTorrent(torrent) - def sesscb_ntfy_torrentfinished(self, subject, changeType, infohash, *args): + def sesscb_ntfy_torrentfinished(self, subject, changeType, objectID, *args): self.guiUtility.Notify( "Download Completed", "Torrent '%s' has finished downloading. Now seeding." % args[0], icon='seed') if self._frame_and_ready(): + infohash = objectID torrent = self.guiUtility.torrentsearch_manager.getTorrentByInfohash(infohash) # Check if we got the actual torrent as the bandwith investor # downloads aren't going to be there. diff --git a/Tribler/Main/vwxGUI/CreditMiningPanel.py b/Tribler/Main/vwxGUI/CreditMiningPanel.py index 502b852522a..6384ce2fb49 100644 --- a/Tribler/Main/vwxGUI/CreditMiningPanel.py +++ b/Tribler/Main/vwxGUI/CreditMiningPanel.py @@ -234,7 +234,7 @@ def __init__(self, parent): self.utility = self.guiutility.utility self.installdir = self.utility.getPath() - self.boosting_manager = BoostingManager.get_instance() + self.boosting_manager = BoostingManager.get_instance(self.utility.session) self.tdb = self.utility.session.open_dbhandler(NTFY_TORRENTS) @@ -250,6 +250,7 @@ def __init__(self, parent): self.main_sizer.Add(self.header, 0, wx.EXPAND) self.main_splitter = wx.SplitterWindow(self, style=wx.SP_BORDER) + self.main_splitter.SetMinimumPaneSize(300) self.sourcelist = CpanelCheckListCtrl(self.main_splitter, -1, agwStyle=wx.LC_REPORT | wx.LC_NO_HEADER | wx.LC_VRULES @@ -325,11 +326,10 @@ def AddComponents(self,parent): self.infoPanel.SetSizer(if_sizer) self.sourcelist.Hide() - self.loading_holder = wx.StaticText(self.main_splitter, -1, 'Loading..') + self.loading_holder = wx.StaticText(self.main_splitter, -1, 'Loading..') - parent.SplitVertically(self.loading_holder, self.infoPanel) - parent.SetMinimumPaneSize(100) - parent.SetSashGravity(0.25) + parent.SplitVertically(self.loading_holder, self.infoPanel,1) + parent.SetSashGravity(0.3) self.main_sizer.Add(parent, 1, wx.EXPAND) def OnItemSelected(self, event): diff --git a/Tribler/Main/vwxGUI/GuiUtility.py b/Tribler/Main/vwxGUI/GuiUtility.py index 8121667af5e..40a448cd957 100644 --- a/Tribler/Main/vwxGUI/GuiUtility.py +++ b/Tribler/Main/vwxGUI/GuiUtility.py @@ -251,34 +251,11 @@ def ShowPage(self, page, *args): self.frame.librarylist.Show(False) if page == 'creditmining': - # Show list - # self.frame.creditmininglist.Show(True) - # - # # Open infohash - # if args: - # self.frame.creditmininglist.GetManager().refresh_or_expand(args[0]) - # else: - # items = self.frame.creditmininglist.GetExpandedItems() - # if items: - # items[0][1].expanded = False - # self.frame.creditmininglist.Select(items[0][0]) - # - # # Open infohash - # if args: - # self.frame.creditmininglist.GetManager().refresh_or_expand(args[0]) - self.frame.creditminingpanel.Show(True) elif self.guiPage == 'creditmining': self.frame.creditminingpanel.Show(False) - if page == 'cmbeta': - # Show list - self.frame.creditminingpanel.Show(True) - elif self.guiPage == 'cmbeta': - # Hide list - self.frame.creditminingpanel.Show(False) - if page == 'home': self.frame.home.ResetSearchBox() self.frame.home.Show() @@ -682,8 +659,7 @@ def toggleFamilyFilter(self, newState=None, setCheck=False): if setCheck: self.frame.SRstatusbar.ff_checkbox.SetValue(newState) - # we don't use this panel in credit mining - # self.frame.home.aw_panel.refreshNow() + self.frame.home.aw_panel.refreshNow() if newState: self.utility.write_config('family_filter', 1) diff --git a/Tribler/Main/vwxGUI/MainFrame.py b/Tribler/Main/vwxGUI/MainFrame.py index 6dad4df652c..317de78f32c 100644 --- a/Tribler/Main/vwxGUI/MainFrame.py +++ b/Tribler/Main/vwxGUI/MainFrame.py @@ -190,8 +190,6 @@ def __init__(self, abc, parent, internalvideo): self.searchlist.Show(False) self.librarylist = LibraryList(self.splitter_top_window) self.librarylist.Show(False) - # self.creditmininglist = CreditMiningList(self) - # self.creditmininglist.Show(False) self.channellist = ChannelList(self.splitter_top_window) self.channellist.Show(False) self.selectedchannellist = SelectedChannelList(self.splitter_top_window) @@ -245,9 +243,6 @@ def OnShowSplitter(event): event.Skip() self.splitter.Bind(wx.EVT_SHOW, OnShowSplitter) - # self.creditmininglist = CreditMiningList(self) - # self.creditmininglist.Show(False) - self.stats = Stats(self) self.stats.Show(False) self.managechannel = ManageChannel(self) @@ -271,7 +266,6 @@ def OnShowSplitter(event): hSizer.Add(self.stats, 1, wx.EXPAND) hSizer.Add(self.networkgraph, 1, wx.EXPAND) hSizer.Add(self.splitter, 1, wx.EXPAND) - # hSizer.Add(self.creditmininglist, 1, wx.EXPAND) hSizer.Add(self.managechannel, 1, wx.EXPAND) diff --git a/Tribler/Main/vwxGUI/home.py b/Tribler/Main/vwxGUI/home.py index dc3b7320668..ab2bf3d9901 100644 --- a/Tribler/Main/vwxGUI/home.py +++ b/Tribler/Main/vwxGUI/home.py @@ -52,7 +52,7 @@ def __init__(self, parent): self.guiutility = GUIUtility.getInstance() self.gui_image_manager = GuiImageManager.getInstance() self.session = self.guiutility.utility.session - self.boosting_manager = BoostingManager.get_instance(self.session) + self.boosting_manager = None #dispersy_cid:Channel self.channels = {None:None} @@ -109,7 +109,7 @@ def __init__(self, parent): vSizer.Add(hSizer, 0, wx.ALIGN_CENTER) vSizer.AddStretchSpacer() - # channel panel is popular channel + # channel panel is for popular channel self.channel_panel = ScrolledPanel(self, 1) self.channel_panel.SetBackgroundColour(wx.WHITE) self.channel_panel.SetForegroundColour(parent.GetForegroundColour()) @@ -119,6 +119,9 @@ def __init__(self, parent): DetailHeader(self.channel_panel, "Select popular channels to mine"), 0, wx.EXPAND, 5) + self.loading_channel_txt = wx.StaticText(self.channel_panel, 1, 'Loading, please wait.') + v_chn_Sizer.Add(self.loading_channel_txt, 1, wx.TOP | wx.ALIGN_CENTER_HORIZONTAL, 10) + self.chn_sizer = wx.FlexGridSizer(0,self.COLUMN_SIZE,5,5) for i in range(0,self.COLUMN_SIZE): @@ -211,6 +214,9 @@ def CreateChannelItem(self, parent, channel, torrents, max_fav): from Tribler.Main.Utility.GuiDBTuples import Channel as ChannelObj assert isinstance(channel, ChannelObj), "Type channel should be ChannelObj %s" %channel + if not self.boosting_manager: + self.boosting_manager = BoostingManager.get_instance(self.session) + STRING_LENGTH = 35 vsizer = wx.BoxSizer(wx.VERTICAL) @@ -294,6 +300,7 @@ def do_gui(delayedResult): self.chn_sizer.Clear(True) self.chn_sizer.Layout() + self.loading_channel_txt.Show() for i in range(0,self.COLUMN_SIZE): if wx.MAJOR_VERSION > 2: if self.chn_sizer.IsColGrowable(i): @@ -316,6 +323,9 @@ def do_gui(delayedResult): self.chn_sizer.Add(self.CreateChannelItem(self.channel_panel,dict_channels.get(d), self.chn_torrents.get(d),max_favourite), 0, wx.ALL|wx.EXPAND) + + self.loading_channel_txt.Hide() + count += 1 if count >= MAX_CHANNEL_SHOW: break diff --git a/Tribler/Main/vwxGUI/list.py b/Tribler/Main/vwxGUI/list.py index ef0c6b65aa5..fb91a98aad4 100644 --- a/Tribler/Main/vwxGUI/list.py +++ b/Tribler/Main/vwxGUI/list.py @@ -2094,7 +2094,7 @@ def OnExpand(self, item): return True @warnWxThread - def RefreshItems(self, dslist, magnetlist): + def RefreshItems(self, dslist, magnetlist, rawdata=True): didStateChange, _, _ = SizeList.RefreshItems(self, dslist, magnetlist, rawdata=True) newFilter = self.newfilter @@ -2149,6 +2149,7 @@ def RefreshItems(self, dslist, magnetlist): bytes_up = self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']['total_up'] bytes_down = self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']['total_down'] + # we may have different value of total. Find the maximum gathered bytes_up = max(seeding_stats_i['total_up'], bytes_up) bytes_down = max(seeding_stats_i['total_down'], bytes_down) @@ -2192,7 +2193,6 @@ def RefreshItems(self, dslist, magnetlist): if seeding_stat_t: seeding_stats.append(seeding_stat_t) - # seeding_stats = [ds.get_seeding_statistics() for ds in boosting_dslist if ds.get_seeding_statistics()] self.tot_bytes_up = sum([stat['total_up'] for stat in seeding_stats]) self.tot_bytes_dwn = sum([stat['total_down'] for stat in seeding_stats]) @@ -2209,10 +2209,10 @@ def RefreshItems(self, dslist, magnetlist): if filter: active_source = self.boosting_manager.get_source_object(filter) - list = [tr['last_seeding_stats']['total_down'] for tr in self.boosting_manager.torrents.values() + seed_sp_list = [tr['last_seeding_stats']['total_down'] for tr in self.boosting_manager.torrents.values() if self.boosting_manager.get_source_object(tr['source']).source == active_source.source and tr['last_seeding_stats']] - total_dl_source = sum(list) + total_dl_source = sum(seed_sp_list) up_rate_txt.SetLabel('Active upload rate : '+speed_format(active_source.av_uprate)) dwn_rate_txt.SetLabel('Active download rate : '+speed_format(active_source.av_dwnrate)) storage_used_txt.SetLabel('Storage Used : '+size_format(total_dl_source)) @@ -2522,8 +2522,6 @@ def __SetData(self): if sys.platform != 'darwin': data_list.append((6, ['Videoplayer'], None, ActivityListItem)) - # data_list.append((7, ['CM List beta'], None, ActivityListItem)) - self.list.SetData(data_list) self.ResizeListItems() self.DisableItem(2) @@ -2533,8 +2531,6 @@ def __SetData(self): self.DisableCollapse() self.selectTab('home') - # self.list.GetItem(7).num_items.Show(False) - # Create expanded panels in advance channels_item = self.list.GetItem(3) self.expandedPanel_channels = ChannelsExpandedPanel(channels_item) @@ -2624,8 +2620,6 @@ def OnExpand(self, item): return self.expandedPanel_videoplayer elif item.data[0] == 'Credit Mining': self.guiutility.ShowPage('creditmining') - elif item.data[0] == 'CM List beta': - self.guiutility.ShowPage('cmbeta') return True def OnCollapse(self, item, panel, from_expand): @@ -2691,10 +2685,9 @@ def selectTab(self, tab): elif tab == 'my_files': itemKey = 4 elif tab == 'videoplayer': - itemKey = 5 itemKey = 6 - # elif tab == 'cmbeta': - # itemKey = 7 + elif tab == 'creditmining': + itemKey = 5 if itemKey: wx.CallAfter(self.Select, itemKey, True) return @@ -2715,7 +2708,7 @@ def _DoPage(self, increment): if curPage < 0: curPage = len(pages) - 1 - pageNames = ['home', 'search_results', 'channels', 'my_files', 'creditmining', 'videoplayer']#, 'cmbeta'] + pageNames = ['home', 'search_results', 'channels', 'my_files', 'creditmining', 'videoplayer'] for i in self.settings.keys(): pageNames.pop(i - 1) self.guiutility.ShowPage(pageNames[curPage]) diff --git a/Tribler/Main/vwxGUI/list_item.py b/Tribler/Main/vwxGUI/list_item.py index 74ac1fbf428..1fb64b97812 100644 --- a/Tribler/Main/vwxGUI/list_item.py +++ b/Tribler/Main/vwxGUI/list_item.py @@ -1139,7 +1139,7 @@ def __init__(self, *args, **kwargs): def AddComponents(self, leftSpacer, rightSpacer): ListItem.AddComponents(self, leftSpacer, rightSpacer) - if self.data[0] in ['Results', 'Channels', 'Downloads', 'Credit Mining', 'Videoplayer', 'CM List beta']: + if self.data[0] in ['Results', 'Channels', 'Downloads', 'Credit Mining', 'Videoplayer']: self.num_items = TagText(self, -1, label='0', fill_colour=GRADIENT_DGREY, edge_colour=SEPARATOR_GREY) self.hSizer.Add(self.num_items, 0, wx.CENTER | wx.RIGHT, 5) self.hSizer.Layout() From ae7576c5addd050455334dd13adf07991c5ce134 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 27 Jun 2016 16:14:52 +0200 Subject: [PATCH 14/24] Start boosting manager if ready only in other words : - Reduce creditmining load at Tribler startup - prioritize other things first - Start credit mining module with a certain condition Squashed commits: - Prevent libtorrent 1.0.9 bug in piece_priority - Add more documentation in BoostingManager - Add logging feature - Enable RSS swarms async download - Update torrent swarm info only when necessary - Use taskmanager whenever possible --- .../Core/Libtorrent/LibtorrentDownloadImpl.py | 8 +- Tribler/Policies/BoostingManager.py | 382 ++++++++++-------- 2 files changed, 227 insertions(+), 163 deletions(-) diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index 8a00a6f99fc..1ad44dad56e 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -909,10 +909,10 @@ def create_peerlist_data(peer_info): peer_dict['upload_only'] = bool(peer_info.flags & peer_info.upload_only) # add read and write state (check unchoke/choke peers) - peer_dict['rstate_bw_limit'] = bool(peer_info.read_state & peer_info.bw_limit) - peer_dict['wstate_bw_limit'] = bool(peer_info.write_state & peer_info.bw_limit) - peer_dict['rstate_bw_network'] = bool(peer_info.read_state & peer_info.bw_network) - peer_dict['wstate_bw_network'] = bool(peer_info.write_state & peer_info.bw_network) + # peer_dict['rstate_bw_limit'] = bool(peer_info.read_state & peer_info.bw_limit) + # peer_dict['wstate_bw_limit'] = bool(peer_info.write_state & peer_info.bw_limit) + # peer_dict['rstate_bw_network'] = bool(peer_info.read_state & peer_info.bw_network) + # peer_dict['wstate_bw_network'] = bool(peer_info.write_state & peer_info.bw_network) peer_dict['rstate'] = peer_info.read_state peer_dict['wstate'] = peer_info.write_state diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index c5b321ec188..d89490aa20f 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -19,13 +19,22 @@ import libtorrent as lt +from twisted.internet import reactor +from twisted.internet.task import LoopingCall +from twisted.web.client import Agent, readBody +from twisted.web.http_headers import Headers + from Tribler.Core.DownloadConfig import DownloadStartupConfig from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl +from Tribler.Core.TorrentChecker import torrent_checker from Tribler.Core.TorrentChecker.session import MAX_TRACKER_MULTI_SCRAPE -from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo from Tribler.Core.Utilities import utilities +from Tribler.Core.exceptions import TriblerException from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_INSERT, NTFY_SCRAPE, NTFY_TORRENTS, NTFY_UPDATE +from Tribler.Core.version import version_id from Tribler.Main.Utility.GuiDBTuples import Torrent +from Tribler.Main.Utility.GuiDBHandler import startWorker from Tribler.Main.globals import DefaultDownloadStartupConfig from Tribler.Main.vwxGUI.GuiUtility import GUIUtility from Tribler.Main.vwxGUI.SearchGridManager import ChannelManager @@ -166,37 +175,36 @@ def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval self.utility = utility self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), "credit_mining") - try: - if not os.path.exists(self.credit_mining_path): - os.mkdir(self.credit_mining_path) - except: - pass + if not os.path.exists(self.credit_mining_path): + os.mkdir(self.credit_mining_path) self.boosting_sources = {} self.torrents = {} self.policy = None - self.share_mode_target = 3 + self.share_mode_target = 2 - # TODO(ardhi) : hardcode some of the interval for now + # change some params here, if you want self.max_torrents_per_source = max_per_source self.max_torrents_active = max_active self.source_interval = src_interval self.swarm_interval = sw_interval self.initial_swarm_interval = 30 self.policy = policy(self.session) - self.tracker_interval = 300 + self.tracker_interval = 50 self.initial_tracker_interval = 25 - self.logging_interval = 60 - self.initial_logging_interval = 35 - - self.load_config() + self.logging_interval = 40 + self.initial_logging_interval = 20 self.set_share_mode_params(share_mode_target=self.share_mode_target) if os.path.exists(config_file): logger.info("Config file %s", open(config_file).read()) + self.load_config() else: - logger.info("Config file missing") + logger.warning("Initial config file missing") + + self.session.add_observer(self.OnTorrentNotify, NTFY_TORRENTS, [NTFY_UPDATE]) + # TODO(emilon): Refactor this to use taskmanager self.session.lm.threadpool.add_task(self._select_torrent, self.initial_swarm_interval) @@ -220,15 +228,20 @@ def shutdown(self): self.save_config() for torrent in self.torrents.itervalues(): - try: - self.stop_download(torrent) - except: - continue + self.stop_download(torrent) def get_source_object(self, sourcekey): return self.boosting_sources.get(sourcekey, None) def set_enable_mining(self, source, mining_bool=True, force_restart=False): + """ + Dynamically enable/disable mining source. + :param source: source, perhaps a url, byte-channelid, or directory + :param mining_bool: enable/disable + :param force_restart: do we really need to restart the mining? + """ + + # Flag : there are not any swarm stored for this source tor_not_exist = True for ihash, tor in self.torrents.iteritems(): @@ -236,10 +249,12 @@ def set_enable_mining(self, source, mining_bool=True, force_restart=False): tor_not_exist = False self.torrents[ihash]['enabled'] = mining_bool + # pause torrent download from disabled source if (not mining_bool): self.stop_download(tor) - # this only happen via new channel boosting interface + # this only happen via new channel boosting interface. (CreditMiningPanel) + # case : just start mining a particular source (e.g. PreviewChannel) if tor_not_exist and mining_bool and not (source in self.boosting_sources.keys()): self.add_source(source) self.set_archive(source, False) @@ -318,6 +333,10 @@ def remove_source(self, source_key): logger.info("Removing from possible swarms") def compare_torrents(self, t1, t2): + """ + comparing swarms. We don't want to download same swarm with different infohash + :return: whether those t1 and t2 similar enough + """ # pylint: disable=no-self-use, bad-builtin try: ff = lambda ft: ft[1] > 1024 * 1024 @@ -335,8 +354,14 @@ def compare_torrents(self, t1, t2): return False def on_torrent_insert(self, source, infohash, torrent): - # Remember where we got this torrent from - + """ + This function called when a source finally determined. Fetch some torrents from it, + then insert it to our data + :param source: + :param infohash: torrent infohash + :param torrent: torrent object (dictionary format) + :return: + """ try: isdir = os.path.isdir(source) except TypeError: @@ -348,6 +373,8 @@ def on_torrent_insert(self, source, infohash, torrent): source_str = source.encode('hex') else: source_str = 'unknown source' + + # Remember where we got this torrent from torrent['source'] = source_str boost_source = self.boosting_sources.get(source, None) @@ -366,6 +393,7 @@ def on_torrent_insert(self, source, infohash, torrent): else: self._logger.info("Not collected yet: %s %s ", infohash, torrent['name']) # TODO(emilon): Handle the case where the torrent hasn't been collected. (collected from the DHT) + # ardhi : so far, this case won't happen because torrent already defined in _update in BoostingSource # torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) pass @@ -381,10 +409,31 @@ def on_torrent_insert(self, source, infohash, torrent): self.stop_download(duplicate) self.torrents[infohash] = torrent - # logger.info("Got new torrent %s from %s", infohash.encode('hex'), - # source_str) + + def OnTorrentNotify(self, subject, change_type, infohash): + if infohash not in self.torrents: + return + + logger.debug("infohash %s updated", hexlify(infohash)) + + def do_gui(delayedResult): + torrent_obj = delayedResult.get() + infohash_str = torrent_obj.infohash_as_hex + + new_seed = torrent_obj.swarminfo[0] + new_leecher = torrent_obj.swarminfo[1] + + if new_seed - self.torrents[torrent_obj.infohash]['num_seeders'] \ + or new_leecher - self.torrents[torrent_obj.infohash]['num_leechers']: + self.torrents[torrent_obj.infohash]['num_seeders'] = new_seed + self.torrents[torrent_obj.infohash]['num_leechers'] = new_leecher + logger.info("infohash %s changed s:%d l:%d", infohash_str, torrent_obj.swarminfo[0],torrent_obj.swarminfo[1]) + + startWorker(do_gui, self.gui_util.torrentsearch_manager.getTorrentByInfohash, wargs=(infohash,)) + def scrape_trackers(self): + for infohash, torrent in self.torrents.iteritems(): tf = torrent['metainfo'] @@ -392,66 +441,38 @@ def scrape_trackers(self): lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(lt.big_number(infohash)) # check health(seeder/leecher) - self.session.check_torrent_health(infohash) - - if lt_torrent.is_valid() \ - and unhexlify(str(lt_torrent.status().info_hash)) in self.torrents: - status = lt_torrent.status() - - t = self.torrents[unhexlify(str(status.info_hash))] - - # some logging - def _logtorrentpeer(status, lt_torrent): - self._logger.info("%s numseedpeer %d/%d lastscrape: %s seed_rank : %s #conn : %d dl_rate : %s seed/peer : %d/%d", - str(status.info_hash), status.num_seeds, status.num_peers, status.last_scrape, status.seed_rank, - status.num_connections, status.download_rate, status.list_seeds, status.list_peers) - - peer_list = [] - for i in lt_torrent.get_peer_info(): - peer = LibtorrentDownloadImpl.create_peerlist_data(i) - peer_list.append(peer) - - # printing for debugging - out = "%s ip:%s uprate:%s dwnrate:%s progress:%s seed/ul:%s/%s>> " \ - %(str(status.info_hash), peer['ip'], peer['uprate'], peer['downrate'], - peer['completed'],peer['seed'], peer['upload_only']) - - if peer['uinterested']: - out += "INTERESTED in us |" - if peer['uchoked']: - out += "CHOKED on US |" - if peer['dinterested']: - out += "we are INTERESTED |" - if peer['dchoked']: - out += "we CHOKED him |" - - print out - - print "----------------------" - _logtorrentpeer(status, lt_torrent) - - peer_list = [] - for i in lt_torrent.get_peer_info(): - peer = LibtorrentDownloadImpl.create_peerlist_data(i) - peer_list.append(peer) - - # already downloaded - if 'download' in t: - trackers = t['download'].network_tracker_status() - - # find if any tracker is working - trackers_available = any([trackers[i][0] for i in trackers.keys() - if i.startswith('udp') or i.startswith('http')]) - - # we only rely on DHT - if not trackers_available: - #TODO(ardhi) : put some DHT scraper-like - - # use DHT data to translate number of seeder/leecher - t['num_seeders'], t['num_leechers'] = utilities.translate_peers_into_health(peer_list, status) - else: - # scrape again? - lt_torrent.scrape_tracker() + self.session.lm.torrent_checker.add_gui_request(infohash,True) + + # if lt_torrent.is_valid() \ + # and unhexlify(str(lt_torrent.status().info_hash)) in self.torrents: + # status = lt_torrent.status() + # + # t = self.torrents[unhexlify(str(status.info_hash))] + # + # peer_list = [] + # for i in lt_torrent.get_peer_info(): + # peer = LibtorrentDownloadImpl.create_peerlist_data(i) + # peer_list.append(peer) + # + # # already downloaded + # if 'download' in t: + # trackers = t['download'].network_tracker_status() + # + # # find if any tracker is working + # trackers_available = any([trackers[i][0] for i in trackers.keys() + # if i.startswith('udp') or i.startswith('http')]) + # + # # we only rely on DHT + # if not trackers_available: + # #TODO(ardhi) : put some DHT scraper-like + # # use DHT data to translate number of seeder/leecher + # # t['num_seeders'], t['num_leechers'] = utilities.translate_peers_into_health(peer_list, status) + # pass + # else: + # # scrape again? + # lt_torrent.scrape_tracker() + # + # t['num_seeders'], t['num_leechers'] = utilities.translate_peers_into_health(peer_list, status) self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) @@ -463,12 +484,16 @@ def set_archive(self, source, enable): logger.error("Could not set archive mode for unknown source %s", source) def start_download(self, torrent): + """ + Start downloading a particular torrent and add it to download list in Tribler + :param torrent: + """ def do_start(): dscfg = DownloadStartupConfig() dscfg.set_dest_dir(self.credit_mining_path) dscfg.set_safe_seeding(False) - #TODO (ardhi) debug variable so I can change this later + # just a debug variable tobj = torrent preload = tobj.get('preload', False) @@ -499,8 +524,13 @@ def do_stop(): self.session.lm.threadpool.add_task_in_thread(do_stop, 0) def _select_torrent(self): + """ + Function to select which torrent in the torrent list will be downloaded in the + next iteration. It depends on the source and applied policy + """ torrents = {} for infohash, torrent in self.torrents.iteritems(): + # we prioritize archive source if torrent.get('preload', False): if 'download' not in torrent: self.start_download(torrent) @@ -597,7 +627,7 @@ def save_config(self): elif isinstance(self.policy, SeederRatioPolicy): policy = "seederratio" config.set(__name__, "policy", policy) - with open(CONFIG_FILE, "w ") as configf: + with open(self.config_file, "w") as configf: config.write(configf) def log_statistics(self): @@ -610,18 +640,22 @@ def log_statistics(self): if unhexlify(str(status.info_hash)) in self.torrents: t = self.torrents[unhexlify(str(status.info_hash))] - logger.debug("Status for %s : %s %s", status.info_hash, + logger.debug("Status for %s : %s %s | ul_lim : %d, max_ul %d, maxcon %d", status.info_hash, status.all_time_download, status.all_time_upload) - non_zero_values = [] - - # assert error in libtorrent 1.0.9 - for piece_priority in lt_torrent.piece_priorities(): - if piece_priority != 0: - non_zero_values.append(piece_priority) - if non_zero_values: - logger.debug("Non zero priorities for %s : %s", - status.info_hash, non_zero_values) + + # piece_priorities will fail in libtorrent 1.0.9 + if lt.version == '1.0.9.0': + continue + else: + non_zero_values = [] + for piece_priority in lt_torrent.piece_priorities(): + if piece_priority != 0: + non_zero_values.append(piece_priority) + if non_zero_values: + logger.debug("Non zero priorities for %s : %s", + status.info_hash, non_zero_values) + self.session.lm.threadpool.add_task(self.log_statistics, self.logging_interval) def update_torrent_stats(self, torrent_infohash_str, seeding_stats): @@ -631,11 +665,13 @@ def update_torrent_stats(self, torrent_infohash_str, seeding_stats): else: self.torrents[torrent_infohash_str]['last_seeding_stats'] = seeding_stats + class BoostingSource(object): def __init__(self, session, tqueue, source, interval, max_torrents, callback): self.session = session self.session.lm.threadpool = tqueue + self.channelcast_db = session.lm.channelcast_db self.torrents = {} self.source = source @@ -655,10 +691,29 @@ def __init__(self, session, tqueue, source, interval, max_torrents, callback): if not self.gui_util.registered: self.gui_util.register() + self.boosting_manager = BoostingManager.get_instance() def kill_tasks(self): self.session.lm.threadpool.cancel_pending_task(self.source) + def _load_if_ready(self, source): + + nr_channels = self.channelcast_db.getNrChannels() + nr_connections = 0 + + for community in self.session.lm.dispersy.get_communities(): + from Tribler.community.search.community import SearchCommunity + if isinstance(community, SearchCommunity): + nr_connections = community.get_nr_connections() + + # condition example + if nr_channels > 100 and nr_connections > 5: + fun = self._load + else: + fun = self._load_if_ready + + self.session.lm.threadpool.add_task(lambda src=source: fun(src), 15, task_name=str(self.source)+"_load") + def _load(self, source): pass @@ -668,6 +723,7 @@ def _update(self): def getSource(self): return self.source + class ChannelSource(BoostingSource): def __init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback): @@ -676,14 +732,11 @@ def __init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callba self.channel_id = None self.channel = None - - self.channelcast_db = self.session.lm.channelcast_db - self.community = None self.database_updated = True self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) - self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load(cid), 0, task_name=self.source) + self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load_if_ready(cid), 0, task_name=str(self.source)+"_load") self.unavail_torrent = {} @@ -695,7 +748,6 @@ def _load(self, dispersy_cid): dispersy = self.session.get_dispersy_instance() @call_on_reactor_thread - def join_community(): try: self.community = dispersy.get_community(dispersy_cid, True) @@ -725,7 +777,9 @@ def get_channel_id(): self.channel = self.gui_util.channelsearch_manager.getChannel(self.channel_id) - self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") + if not self.boosting_manager.is_pending_task_active(str(self.source)+"_update"): + self.boosting_manager.register_task(str(self.source)+"_update",LoopingCall(self._update)).start(self.interval, now=True) + # self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") logger.info("Got channel id %s", self.channel_id) else: logger.warning("Could not get channel id, retrying in 10 s") @@ -736,11 +790,10 @@ def get_channel_id(): self.ready = True except Exception,ex: logger.info("Channel %s was not ready, waits for next interval (%d chn)", hexlify(self.source), len(dispersy.get_communities())) - self.session.lm.threadpool.add_task(lambda cid=self.source: self._load(cid), 10, task_name=self.source) - + self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load_if_ready(cid), 0, task_name=str(self.source)+"_load") def _check_tor(self): - from Tribler.Main.Utility.GuiDBHandler import startWorker + def doGui(delayedResult): # wait here requesttype = delayedResult.get(timeout=70) @@ -776,8 +829,8 @@ def showTorrent(torrent): startWorker(doGui, self.gui_util.torrentsearch_manager.loadTorrent, wargs=(t,), wkwargs={'callback': showTorrent}) - if not self.session.lm.threadpool.is_pending_task_active(hexlify(self.source)+"_checktor"): - self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=hexlify(self.source)+"_checktor") + # if not self.session.lm.threadpool.is_pending_task_active(hexlify(self.source)+"_checktor"): + # self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=hexlify(self.source)+"_checktor") def _update(self): if len(self.torrents) < self.max_torrents: @@ -796,13 +849,16 @@ def _update(self): self.unavail_torrent.update({t.infohash:t for t in listtor if t.infohash not in self.torrents}) # it's highly probable the checktor function is running at this time (if it's already running) - if not self.session.lm.threadpool.is_pending_task_active(hexlify(self.source)+"_checktor"): - self.session.lm.threadpool.add_task(self._check_tor, 0, task_name=hexlify(self.source)+"_checktor") + # if not running, start the checker + if not self.boosting_manager.is_pending_task_active(hexlify(self.source)+"_checktor"): + self.boosting_manager.register_task(hexlify(self.source)+"_checktor",LoopingCall(self._check_tor)).start(100, now=True) + + # self.session.lm.threadpool.add_task(self._check_tor, 0, task_name=hexlify(self.source)+"_checktor") except: logger.info("Channel %s was not ready, waits for next interval", hexlify(self.source)) - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") + # self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") def _on_database_updated(self, subject, change_type, infohash): if (subject, change_type, infohash) is None: @@ -830,7 +886,7 @@ def __init__(self, session, tqueue, rss_feed, interval, max_torrents, callback): self.feed_handle = None - self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load(feed), 0, task_name=str(self.source)+"_load") + self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load_if_ready(feed), 0, task_name=str(self.source)+"_load") self.title = "" self.description = "" @@ -853,7 +909,8 @@ def wait_for_feed(): self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=str(self.source)+"_wait_for_feed") else: # The feed is done updating. Now periodically start retrieving torrents. - self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") + self.boosting_manager.register_task(str(self.source)+"_update",LoopingCall(self._update), + 10,interval=self.interval) logger.info("Got RSS feed %s", feed_status['url']) wait_for_feed() @@ -871,67 +928,75 @@ def _update(self): self.total_torrents = len(feed_status['items']) + def __cb_body(body_bin, item): + tdef = None + try: + metainfo = lt.bdecode(body_bin) + tdef = TorrentDef.load_from_dict(metainfo) + tdef.save(torrent_filename) + except: + logger.error("Could not parse/save torrent, skipping %s", torrent_filename) + + if tdef: + # Create a torrent dict. + torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled, {}] + self.torrents[sha1(item['url']).digest()] = dict(zip(torrent_keys, torrent_values)) + + # manually generate an ID and put this into DB + self.session.lm.torrent_db.addOrGetTorrentID(sha1(item['url']).digest()) + self.session.lm.torrent_db.addExternalTorrent(tdef) + + # create Torrent object and store it + self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) + + # Notify the BoostingManager and provide the real infohash. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[sha1(item['url']).digest()]) + + def __success_cb(response, item): + defer = readBody(response) + defer.addCallback(__cb_body, item) + return defer + for item in feed_status['items']: # Not all RSS feeds provide us with the infohash, so we use a fake infohash based on the URL to identify the torrents. infohash = sha1(item['url']).digest() - if infohash not in self.torrents: # Store the torrents as rss-infohash_as_hex.torrent. - torrent_filename = os.path.join(BoostingManager.get_instance().credit_mining_path, 'rss-%s.torrent' % infohash.encode('hex')) + torrent_filename = os.path.join(self.boosting_manager.credit_mining_path, 'rss-%s.torrent' % infohash.encode('hex')) tdef = None if not os.path.exists(torrent_filename): - try: - # Download the torrent and create a TorrentDef. - if "torcache" in item['url']: - import urllib2 - import StringIO - import gzip - req = urllib2.Request(self.unescape(item['url'])) - req.add_header('Referer', 'http://torcache.net/') - req.add_header('Accept-encoding', 'deflate,gzip') - req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36') - - res = urllib2.urlopen(req) - cfile = StringIO.StringIO() - cfile.write(res.read()) - cfile.seek(0) - - f = gzip.GzipFile(fileobj=cfile, mode='rb') - else: - f = urllib.urlopen(self.unescape(item['url'])) - except: - logger.error("Could not get torrent, skipping %s", - item['url']) - continue - - try: - metainfo = lt.bdecode(f.read()) - tdef = TorrentDef.load_from_dict(metainfo) - tdef.save(torrent_filename) - - except: - logger.error("Could not parse/save torrent, skipping %s", torrent_filename) + + #create Agent to download torrent file + agent = Agent(reactor) + ses_agent = agent.request( + 'GET', #http://stackoverflow.com/a/845595 + urllib.quote(item['url'],safe="%/:=&?~#+!$,;'@()*[]"), + Headers({'User-Agent': ['Tribler ' + version_id]}), + None) + ses_agent.addCallback(__success_cb, item) else: + # torrent already exist in our system tdef = TorrentDef.load(torrent_filename) - if tdef: - # Create a torrent dict. - torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled, {}] - self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) + if tdef: + # Create a torrent dict. + torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled, {}] + self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) - # manually generate an ID and put this into DB - self.session.lm.torrent_db.addOrGetTorrentID(infohash) - self.session.lm.torrent_db.addExternalTorrent(tdef) + # manually generate an ID and put this into DB + self.session.lm.torrent_db.addOrGetTorrentID(infohash) + self.session.lm.torrent_db.addExternalTorrent(tdef) - # create Torrent object and store it - self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) + # create Torrent object and store it + self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) - # Notify the BoostingManager and provide the real infohash. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) + # Notify the BoostingManager and provide the real infohash. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") + # self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") def kill_tasks(self): BoostingSource.kill_tasks(self) @@ -944,8 +1009,7 @@ class DirectorySource(BoostingSource): def __init__(self, session, tqueue, directory, interval, max_torrents, callback): BoostingSource.__init__(self, session, tqueue, directory, interval, max_torrents, callback) - - self._load(directory) + self._load_if_ready(directory) def _load(self, directory): if os.path.isdir(directory): From 9b9c01a6fd9650ba7366d7523ab3803d49134a76 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 27 Jun 2016 16:17:06 +0200 Subject: [PATCH 15/24] Add dirdialog for convenience choosing source --- Tribler/Main/Dialogs/BoostingDialogs.py | 26 ++++++++---------------- Tribler/Main/vwxGUI/CreditMiningPanel.py | 9 +++++--- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/Tribler/Main/Dialogs/BoostingDialogs.py b/Tribler/Main/Dialogs/BoostingDialogs.py index 3a91dbf10bc..0b9d3367caf 100644 --- a/Tribler/Main/Dialogs/BoostingDialogs.py +++ b/Tribler/Main/Dialogs/BoostingDialogs.py @@ -28,6 +28,7 @@ def __init__(self, parent): self.rss_dir_radio = wx.RadioButton(self, -1, 'RSS dir:') self.rss_dir_edit = wx.TextCtrl(self, -1) self.rss_dir_edit.Bind(wx.EVT_TEXT, lambda evt: self.rss_dir_radio.SetValue(True)) + self.rss_dir_edit.Bind(wx.EVT_LEFT_DOWN, self.OnDir) self.archive_check = wx.CheckBox(self, -1, "Archive mode") ok_btn = wx.Button(self, -1, "OK") @@ -37,8 +38,6 @@ def __init__(self, parent): sourceGrid = wx.FlexGridSizer(2, 2, 0, 0) sourceGrid.AddGrowableCol(1) - # sourceGrid.Add(self.channel_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) - # sourceGrid.Add(self.channel_choice, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) sourceGrid.Add(self.rss_feed_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) sourceGrid.Add(self.rss_feed_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) sourceGrid.Add(self.rss_dir_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) @@ -55,23 +54,7 @@ def __init__(self, parent): vSizer.Add(btnSizer, 0, wx.EXPAND | wx.ALL, 5) self.SetSizer(vSizer) - # def do_db(): - # return self.guiutility.channelsearch_manager.getAllChannels() - # - # def do_gui(delayedResult): - # _, channels = delayedResult.get() - # self.channels = sorted([(channel.name, channel.dispersy_cid) for channel in channels]) - # self.channel_choice.SetItems([channel[0] for channel in self.channels]) - # - # startWorker(do_gui, do_db, retryOnBusy=True, priority=GUI_PRI_DISPERSY) - def OnOK(self, event): - # if self.channel_radio.GetValue(): - # selection = self.channel_choice.GetSelection() - # if selection < len(self.channels): - # self.source = self.channels[selection][1] - # el - if self.rss_feed_radio.GetValue(): self.source = self.rss_feed_edit.GetValue() else: @@ -88,6 +71,13 @@ def OnCancel(self, event): def GetValue(self): return self.source, self.archive_check.GetValue() + def OnDir(self,event): + self.rss_dir_dialog = wx.DirDialog(self, "Choose a directory:", + style=wx.DD_DEFAULT_STYLE | wx.DD_DIR_MUST_EXIST) + + if self.rss_dir_dialog.ShowModal() == wx.ID_OK: + self.rss_dir_edit.SetValue(self.rss_dir_dialog.GetPath()) + class RemoveBoostingSource(wx.Dialog): diff --git a/Tribler/Main/vwxGUI/CreditMiningPanel.py b/Tribler/Main/vwxGUI/CreditMiningPanel.py index 6384ce2fb49..53ff1bc9e8a 100644 --- a/Tribler/Main/vwxGUI/CreditMiningPanel.py +++ b/Tribler/Main/vwxGUI/CreditMiningPanel.py @@ -128,6 +128,8 @@ def do_update_gui(delayedResult): self.CreateSourceItem(s) self.getting_channels = False + self.RefreshData() + self.Layout() startWorker(do_update_gui, do_query_channels, retryOnBusy=True, priority=GUI_PRI_DISPERSY) @@ -437,7 +439,7 @@ def OnRemoveSource(event): dlg = RemoveBoostingSource(None) if dlg.ShowModal() == wx.ID_OK and dlg.GetValue(): self.boosting_manager.remove_source(dlg.GetValue()) - self.GetManager().refresh() + self.sourcelist.RefreshData() dlg.Destroy() addsource = LinkStaticText(header, 'Add', icon=None) @@ -489,5 +491,6 @@ def _PostInit(self): self.Bind(ULC.EVT_LIST_ITEM_SELECTED, self.OnItemSelected, self.sourcelist) - self.sourcelist.LoadMore() - self.sourcelist.RefreshData() + + self.guiutility.utility.session.lm.threadpool.add_task(self.sourcelist.LoadMore, 2, task_name=str(self)+"load_more") + self.Layout() \ No newline at end of file From c2e894b2c98c151967860025da364f5a030621b0 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 9 May 2016 14:26:02 +0200 Subject: [PATCH 16/24] Prevent GUI to do task when quitting Squashed Commits: - Fix resetting statistics data in list - Loading credit mining GUI if necessary - Enabling/Disabling feature credit mining in main - Refactor to move credit mining instance to LaunchManyCore - Remove singleton usage in GUI - Disable credit mining by default on GUI - Refactor GUI code to only provide RAW source --- Tribler/Main/Dialogs/BoostingDialogs.py | 155 +++++---- Tribler/Main/tribler_main.py | 20 +- Tribler/Main/vwxGUI/CreditMiningPanel.py | 423 +++++++++++++---------- Tribler/Main/vwxGUI/GuiUtility.py | 3 - Tribler/Main/vwxGUI/MainFrame.py | 2 - Tribler/Main/vwxGUI/home.py | 235 +++++++------ Tribler/Main/vwxGUI/list.py | 238 +++++++------ Tribler/Main/vwxGUI/settingsDialog.py | 17 +- 8 files changed, 602 insertions(+), 491 deletions(-) diff --git a/Tribler/Main/Dialogs/BoostingDialogs.py b/Tribler/Main/Dialogs/BoostingDialogs.py index 0b9d3367caf..1627861b5a0 100644 --- a/Tribler/Main/Dialogs/BoostingDialogs.py +++ b/Tribler/Main/Dialogs/BoostingDialogs.py @@ -1,91 +1,109 @@ -# Written by Egbert Bouman +""" +This module contains wx dialog gui for adding/removing boosting source + +Written by Egbert Bouman and Ardhi Putra Pratama H +""" import wx -from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY from Tribler.Main.vwxGUI.GuiUtility import GUIUtility -from Tribler.Policies.BoostingManager import BoostingManager, ChannelSource +from Tribler.Policies.BoostingManager import ChannelSource class AddBoostingSource(wx.Dialog): + """ + Class for adding the source for credit mining + """ def __init__(self, parent): - wx.Dialog.__init__(self, parent, -1, 'Add boosting source', size=(275, 275), name="AddBoostingSourceDialog") + wx.Dialog.__init__(self, parent, -1, 'Add boosting source', size=(475, 275), name="AddBoostingSourceDialog") - self.guiutility = GUIUtility.getInstance() self.channels = [] self.source = '' text = wx.StaticText(self, -1, 'Please enter a RSS feed URL or directory to start boosting swarms:') - # self.channel_radio = wx.RadioButton(self, -1, 'Subscribed Channel:', style=wx.RB_GROUP) - # self.channel_choice = wx.Choice(self, -1) - # self.channel_choice.Bind(wx.EVT_CHOICE, lambda evt: self.channel_radio.SetValue(True)) self.rss_feed_radio = wx.RadioButton(self, -1, 'RSS feed:') self.rss_feed_edit = wx.TextCtrl(self, -1) self.rss_feed_edit.Bind(wx.EVT_TEXT, lambda evt: self.rss_feed_radio.SetValue(True)) - self.rss_dir_radio = wx.RadioButton(self, -1, 'RSS dir:') + self.rss_dir_radio = wx.RadioButton(self, -1, 'Torrents local directory:') self.rss_dir_edit = wx.TextCtrl(self, -1) self.rss_dir_edit.Bind(wx.EVT_TEXT, lambda evt: self.rss_dir_radio.SetValue(True)) - self.rss_dir_edit.Bind(wx.EVT_LEFT_DOWN, self.OnDir) + self.rss_dir_edit.Bind(wx.EVT_LEFT_DOWN, self.on_open_dir) self.archive_check = wx.CheckBox(self, -1, "Archive mode") ok_btn = wx.Button(self, -1, "OK") - ok_btn.Bind(wx.EVT_BUTTON, self.OnOK) + ok_btn.Bind(wx.EVT_BUTTON, self.on_added_source) cancel_btn = wx.Button(self, -1, "Cancel") - cancel_btn.Bind(wx.EVT_BUTTON, self.OnCancel) - - sourceGrid = wx.FlexGridSizer(2, 2, 0, 0) - sourceGrid.AddGrowableCol(1) - sourceGrid.Add(self.rss_feed_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) - sourceGrid.Add(self.rss_feed_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) - sourceGrid.Add(self.rss_dir_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) - sourceGrid.Add(self.rss_dir_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) - btnSizer = wx.BoxSizer(wx.HORIZONTAL) - btnSizer.Add(ok_btn, 0, wx.RIGHT | wx.TOP | wx.BOTTOM, 5) - btnSizer.Add(cancel_btn, 0, wx.ALL, 5) - vSizer = wx.BoxSizer(wx.VERTICAL) - vSizer.Add(text, 0, wx.EXPAND | wx.ALL, 5) - vSizer.Add(sourceGrid, 0, wx.EXPAND | wx.ALL, 5) - vSizer.AddSpacer((-1, 5)) - vSizer.Add(self.archive_check, 0, wx.LEFT | wx.RIGHT, 10) - vSizer.AddStretchSpacer() - vSizer.Add(btnSizer, 0, wx.EXPAND | wx.ALL, 5) - self.SetSizer(vSizer) - - def OnOK(self, event): + cancel_btn.Bind(wx.EVT_BUTTON, self.on_cancel) + + source_grid = wx.FlexGridSizer(2, 2, 0, 0) + source_grid.AddGrowableCol(1) + source_grid.Add(self.rss_feed_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) + source_grid.Add(self.rss_feed_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + source_grid.Add(self.rss_dir_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) + source_grid.Add(self.rss_dir_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + btn_sizer = wx.BoxSizer(wx.HORIZONTAL) + btn_sizer.Add(ok_btn, 0, wx.RIGHT | wx.TOP | wx.BOTTOM, 5) + btn_sizer.Add(cancel_btn, 0, wx.ALL, 5) + v_sizer = wx.BoxSizer(wx.VERTICAL) + v_sizer.Add(text, 0, wx.EXPAND | wx.ALL, 5) + v_sizer.Add(source_grid, 0, wx.EXPAND | wx.ALL, 5) + v_sizer.AddSpacer((-1, 5)) + v_sizer.Add(self.archive_check, 0, wx.LEFT | wx.RIGHT, 10) + v_sizer.AddStretchSpacer() + v_sizer.Add(btn_sizer, 0, wx.EXPAND | wx.ALL, 5) + self.SetSizer(v_sizer) + + def on_added_source(self, _): + """ + this function called when user clicked 'OK' button for adding source + """ if self.rss_feed_radio.GetValue(): self.source = self.rss_feed_edit.GetValue() else: self.source = self.rss_dir_edit.GetValue() - self.guiutility.Notify( + GUIUtility.getInstance().Notify( "Successfully add source for credit mining %s" % self.source) self.EndModal(wx.ID_OK) - def OnCancel(self, event): + def on_cancel(self, _): + """ + this function called when user clicked 'Cancel' button when adding source + thus, cancelled + """ self.EndModal(wx.ID_CANCEL) - def GetValue(self): + def get_value(self): + """ + get the value of new source + """ return self.source, self.archive_check.GetValue() - def OnDir(self,event): - self.rss_dir_dialog = wx.DirDialog(self, "Choose a directory:", - style=wx.DD_DEFAULT_STYLE | wx.DD_DIR_MUST_EXIST) + def on_open_dir(self, event): + """ + opening local directory for choosing directory source + """ + rss_dir_dialog = wx.DirDialog(self, "Choose a directory:", + style=wx.DD_DEFAULT_STYLE | wx.DD_DIR_MUST_EXIST) - if self.rss_dir_dialog.ShowModal() == wx.ID_OK: - self.rss_dir_edit.SetValue(self.rss_dir_dialog.GetPath()) + if rss_dir_dialog.ShowModal() == wx.ID_OK: + self.rss_dir_edit.SetValue(rss_dir_dialog.GetPath()) class RemoveBoostingSource(wx.Dialog): - + """ + Class for adding the source for credit mining + """ def __init__(self, parent): - wx.Dialog.__init__(self, parent, -1, 'Remove boosting source', size=(475, 135), name="RemoveBoostingSourceDialog") + wx.Dialog.__init__(self, parent, -1, 'Remove boosting source', size=(475, 135), + name="RemoveBoostingSourceDialog") self.guiutility = GUIUtility.getInstance() - self.boosting_manager = BoostingManager.get_instance() + self.boosting_manager = self.guiutility.utility.session.lm.boosting_manager self.sources = [] self.source = '' @@ -93,38 +111,47 @@ def __init__(self, parent): self.source_label = wx.StaticText(self, -1, 'Source:') self.source_choice = wx.Choice(self, -1) ok_btn = wx.Button(self, -1, "OK") - ok_btn.Bind(wx.EVT_BUTTON, self.OnOK) + ok_btn.Bind(wx.EVT_BUTTON, self.on_remove_source) cancel_btn = wx.Button(self, -1, "Cancel") - cancel_btn.Bind(wx.EVT_BUTTON, self.OnCancel) - - sourceSizer = wx.BoxSizer(wx.HORIZONTAL) - sourceSizer.Add(self.source_label, 0, wx.ALIGN_CENTER_VERTICAL | wx.RIGHT | wx.TOP, 5) - sourceSizer.Add(self.source_choice, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) - btnSizer = wx.BoxSizer(wx.HORIZONTAL) - btnSizer.Add(ok_btn, 0, wx.RIGHT | wx.TOP | wx.BOTTOM, 5) - btnSizer.Add(cancel_btn, 0, wx.ALL, 5) - vSizer = wx.BoxSizer(wx.VERTICAL) - vSizer.Add(text, 0, wx.EXPAND | wx.ALL, 5) - vSizer.Add(sourceSizer, 0, wx.EXPAND | wx.ALL, 5) - vSizer.AddStretchSpacer() - vSizer.Add(btnSizer, 0, wx.EXPAND | wx.ALL, 5) - self.SetSizer(vSizer) - - channels = [] + cancel_btn.Bind(wx.EVT_BUTTON, self.on_cancel) + + sourcesizer = wx.BoxSizer(wx.HORIZONTAL) + sourcesizer.Add(self.source_label, 0, wx.ALIGN_CENTER_VERTICAL | wx.RIGHT | wx.TOP, 5) + sourcesizer.Add(self.source_choice, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + btnsizer = wx.BoxSizer(wx.HORIZONTAL) + btnsizer.Add(ok_btn, 0, wx.RIGHT | wx.TOP | wx.BOTTOM, 5) + btnsizer.Add(cancel_btn, 0, wx.ALL, 5) + vsizer = wx.BoxSizer(wx.VERTICAL) + vsizer.Add(text, 0, wx.EXPAND | wx.ALL, 5) + vsizer.Add(sourcesizer, 0, wx.EXPAND | wx.ALL, 5) + vsizer.AddStretchSpacer() + vsizer.Add(btnsizer, 0, wx.EXPAND | wx.ALL, 5) + self.SetSizer(vsizer) + # retrieve all source except channel source - self.sources = [s.getSource() for s in self.boosting_manager.boosting_sources.values() + self.sources = [s.get_source_text() for s in self.boosting_manager.boosting_sources.values() if not isinstance(s, ChannelSource)] self.source_choice.SetItems(self.sources) - def OnOK(self, event): + def on_remove_source(self, _): + """ + this function called when user clicked 'OK' button for removing source + """ selection = self.source_choice.GetSelection() if selection < len(self.sources): self.source = self.sources[selection] self.EndModal(wx.ID_OK) - def OnCancel(self, event): + def on_cancel(self, _): + """ + this function called when user clicked 'Cancel' button when adding source + thus, cancelled + """ self.EndModal(wx.ID_CANCEL) - def GetValue(self): + def get_value(self): + """ + get value when removing source + """ return self.source diff --git a/Tribler/Main/tribler_main.py b/Tribler/Main/tribler_main.py index 694f47f9994..9941b9c493c 100755 --- a/Tribler/Main/tribler_main.py +++ b/Tribler/Main/tribler_main.py @@ -9,7 +9,6 @@ # see LICENSE.txt for license information # from Tribler.Core.Modules.process_checker import ProcessChecker -from Tribler.Policies.BoostingManager import BoostingManager from Tribler.Main.Dialogs.NewVersionDialog import NewVersionDialog try: @@ -28,7 +27,6 @@ import tempfile import traceback import urllib2 -import shutil from collections import defaultdict from random import randint from traceback import print_exc @@ -399,8 +397,6 @@ def PostInit2(self): # TODO(emilon): Use the LogObserver I already implemented # self.dispersy.callback.attach_exception_handler(self.frame.exceptionHandler) - startWorker(None, self.loadSessionCheckpoint, delay=5.0, workerType="ThreadPool") - @forceWxThread def sesscb_ntfy_myprefupdates(self, subject, changeType, objectID, *args): if self._frame_and_ready(): @@ -686,8 +682,9 @@ def sesscb_ntfy_torrentupdates(self, events): manager = self.frame.librarylist.GetManager() manager.torrentsUpdated(infohashes) - manager = self.frame.creditminingpanel.cmlist.GetManager() - manager.torrentsUpdated(infohashes) + if self.utility.session.get_creditmining_enable(): + manager = self.frame.creditminingpanel.cmlist.GetManager() + manager.torrents_updated(infohashes) from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent @@ -710,7 +707,7 @@ def sesscb_ntfy_torrentfinished(self, subject, changeType, objectID, *args): if self._frame_and_ready(): infohash = objectID torrent = self.guiUtility.torrentsearch_manager.getTorrentByInfohash(infohash) - # Check if we got the actual torrent as the bandwith investor + # Check if we got the actual torrent as the bandwidth investor # downloads aren't going to be there. if torrent: self.guiUtility.library_manager.addDownloadState(torrent) @@ -823,15 +820,6 @@ def OnExit(self): self.webUI.stop() self.webUI.delInstance() - if self.boosting_manager: - - #remove credit mining data - #TODO(ardhi) : not persistent mode only - import shutil; shutil.rmtree(self.boosting_manager.credit_mining_path, ignore_errors=True) - - self.boosting_manager.shutdown() - self.boosting_manager.del_instance() - if self.frame: self.frame.Destroy() self.frame = None diff --git a/Tribler/Main/vwxGUI/CreditMiningPanel.py b/Tribler/Main/vwxGUI/CreditMiningPanel.py index 53ff1bc9e8a..b6b5e64afb8 100644 --- a/Tribler/Main/vwxGUI/CreditMiningPanel.py +++ b/Tribler/Main/vwxGUI/CreditMiningPanel.py @@ -1,160 +1,167 @@ -# Written by Ardhi Putra Pratama Hartono +""" +This module contains credit mining panel and list in wx +Written by Ardhi Putra Pratama H +""" -import os -import sys import logging +from binascii import hexlify + +# pylint complaining if wx imported before binascii import wx -from binascii import hexlify, unhexlify -from wx.lib.agw.ultimatelistctrl import ULC_VIRTUAL, EVT_LIST_ITEM_CHECKED -from Tribler import LIBRARYNAME +from wx.lib.agw import ultimatelistctrl as ULC +from wx.lib.agw.ultimatelistctrl import EVT_LIST_ITEM_CHECKED + from Tribler.Core.exceptions import NotYetImplementedException from Tribler.Core.simpledefs import NTFY_TORRENTS from Tribler.Main.Dialogs.BoostingDialogs import RemoveBoostingSource, AddBoostingSource - -from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent, Torrent, Channel -from Tribler.Main.Utility.GuiDBHandler import startWorker, cancelWorker, GUI_PRI_DISPERSY -from Tribler.Main.vwxGUI import forceWxThread, TRIBLER_RED, SEPARATOR_GREY, GRADIENT_LGREY, GRADIENT_DGREY, format_time +from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY +from Tribler.Main.Utility.GuiDBTuples import Channel +from Tribler.Main.vwxGUI import SEPARATOR_GREY, GRADIENT_LGREY, GRADIENT_DGREY, format_time from Tribler.Main.vwxGUI.GuiUtility import GUIUtility -from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager from Tribler.Main.vwxGUI.list import CreditMiningList -from Tribler.Main.vwxGUI.widgets import ActionButton, FancyPanel, TextCtrlAutoComplete, ProgressButton, LinkStaticText, \ - _set_font -from Tribler.Main.Dialogs.AddTorrent import AddTorrent -from Tribler.Main.Dialogs.RemoveTorrent import RemoveTorrent -from Tribler.Policies.BoostingManager import BoostingManager, RSSFeedSource, DirectorySource, ChannelSource, \ - BoostingSource - -try: - from agw import ultimatelistctrl as ULC -except ImportError: - from wx.lib.agw import ultimatelistctrl as ULC +from Tribler.Main.vwxGUI.widgets import FancyPanel, LinkStaticText, _set_font +from Tribler.Policies.BoostingSource import RSSFeedSource, DirectorySource, ChannelSource, BoostingSource +RETURNED_CHANNELS = 30 class CpanelCheckListCtrl(wx.ScrolledWindow, ULC.UltimateListCtrl): - def __init__(self, parent, id=wx.ID_ANY, pos=wx.DefaultPosition, size=wx.DefaultSize, style=0, \ - agwStyle=0): - ULC.UltimateListCtrl.__init__(self, parent, id, pos, size, style, agwStyle) - self.boosting_manager = BoostingManager.get_instance() + """ + The checklist of credit mining sources. Check to enable, uncheck to disable. + It is grouped by type : RSS, directory, and Channels + """ + def __init__(self, parent, wxid=wx.ID_ANY, style=0, agwStyle=0): + ULC.UltimateListCtrl.__init__(self, parent, wxid, wx.DefaultPosition, wx.DefaultSize, style, agwStyle) + self.guiutility = GUIUtility.getInstance() - self.utility = self.guiutility.utility + self.boosting_manager = self.guiutility.utility.session.lm.boosting_manager + self.channel_list = {} self._logger = logging.getLogger(self.__class__.__name__) - self.InsertColumn(0,'col') + self.InsertColumn(0, 'col') self.SetColumnWidth(0, -3) - self.label_rss_idx = 0 - self.InsertStringItem(self.label_rss_idx,"RSS") - it = self.GetItem(self.label_rss_idx) - it.Enable(False) - it.SetData("RSS") - self.SetItem(it) - - self.label_dir_idx = 1 - self.InsertStringItem(self.label_dir_idx,"Directory") - it = self.GetItem(self.label_dir_idx) - it.Enable(False) - it.SetData("Directory") - self.SetItem(it) - - self.label_channel_idx = 2 - self.InsertStringItem(self.label_channel_idx,"Channel") - it = self.GetItem(self.label_channel_idx) - it.Enable(False) - it.SetData("Channel") - self.SetItem(it) - - self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.OnItemActivated) + # index holder for labels. 0 for RSS, 1 for directory, 2 for channel + self.labels = [0, 1, 2] + + self.InsertStringItem(self.labels[0], "RSS") + item = self.GetItem(self.labels[0]) + item.Enable(False) + item.SetData("RSS") + self.SetItem(item) + + self.InsertStringItem(self.labels[1], "Directory") + item = self.GetItem(self.labels[1]) + item.Enable(False) + item.SetData("Directory") + self.SetItem(item) + + self.InsertStringItem(self.labels[2], "Channel") + item = self.GetItem(self.labels[2]) + item.Enable(False) + item.SetData("Channel") + self.SetItem(item) + self.Bind(EVT_LIST_ITEM_CHECKED, self.OnGetItemCheck) self.getting_channels = False - self._mainWin.Bind(wx.EVT_SCROLLWIN, self.OnScroll) - - def OnItemActivated(self, evt): - # double click - pass + self._mainWin.Bind(wx.EVT_SCROLLWIN, self.on_scroll) - def OnScroll(self, evt): + def on_scroll(self, evt): + """ + scroller watcher. Might be useful for unlimited load channels + """ vpos = self._mainWin.GetScrollPos(wx.VERTICAL) list_total = self.GetItemCount() list_pp = self.GetCountPerPage() - topitem_idx, bottomitem_idx = self._mainWin.GetVisibleLinesRange() + topitem_idx, _ = self._mainWin.GetVisibleLinesRange() - total_page = list_total/list_pp + total_page = list_total / list_pp - # print "vpos %d totlist %d topidx %d pp %d" "btmidx %s" %(vpos, total_page*list_pp, topitem_idx, list_pp, bottomitem_idx) - if (vpos >= list_total and total_page*list_pp < vpos and vpos > topitem_idx+list_pp)\ - or vpos == 0: - #not so accurate but this will do + # print "vpos %d totlist %d topidx %d pp %d" "btmidx %s" + # %(vpos, total_page*list_pp, topitem_idx, list_pp, bottomitem_idx) + if (vpos >= list_total and total_page * list_pp < vpos and + vpos > topitem_idx + list_pp) or vpos == 0: + # not so accurate but this will do if self.getting_channels: evt.Skip() return - self.LoadMore() + self.load_more() evt.Skip() - def LoadMore(self): + def load_more(self): + """ + load more channels to the list + """ self._logger.info("getting new channels..") self.getting_channels = True def do_query_channels(): - RETURNED_CHANNELS = 30 - + """ + querying channels in the background. Only return as much as RETURNED_CHANNELS + """ _, channels = self.guiutility.channelsearch_manager.getPopularChannels(20) dict_channels = {channel.dispersy_cid: channel for channel in channels} new_channels_ids = list(set(dict_channels.keys()) - set(self.channel_list.keys())) - return_list = [dict_channels.get(new_channels_ids[i]) for i in range(0, - len(new_channels_ids) if len(new_channels_ids) < RETURNED_CHANNELS else RETURNED_CHANNELS)] + return_list = [dict_channels.get(new_channels_ids[i]) + for i in xrange(0, min(len(new_channels_ids), RETURNED_CHANNELS))] if return_list: - return [l for l in sorted(return_list, key=lambda x: x.nr_favorites, reverse=True)]# if "tribler" in l.name.lower() or "linux" in l.name.lower()] - else: - None + return [l for l in sorted(return_list, key=lambda x: x.nr_favorites, + reverse=True)] - def do_update_gui(delayedResult): - channels = delayedResult.get() + def do_update_gui(delayed_result): + """ + add fetched channels to GUI + """ + channels = delayed_result.get() if channels: - for s in channels: + for channel in channels: # s is channel object - self.channel_list[s.dispersy_cid] = s - self.CreateSourceItem(s) + self.channel_list[channel.dispersy_cid] = channel + self.create_source_item(channel) self.getting_channels = False - self.RefreshData() + self.refresh_sourcelist_data() self.Layout() startWorker(do_update_gui, do_query_channels, retryOnBusy=True, priority=GUI_PRI_DISPERSY) - def CreateSourceItem(self, source): + def create_source_item(self, source): + """ + put the source object in the sourcelist available for enable/disable + """ item_count = self.GetItemCount() if isinstance(source, RSSFeedSource): - self.InsertStringItem(self.label_dir_idx, source.getSource(), 1) - item = self.GetItem(self.label_dir_idx) + # update label for directory as we pushed it down + self.InsertStringItem(self.labels[1], source.get_source_text(), 1) + item = self.GetItem(self.labels[1]) item.Check(source.enabled) item.SetData(source) self.SetItem(item) - self.label_dir_idx += 1 - self.label_channel_idx += 1 + self.labels[1] += 1 + self.labels[2] += 1 elif isinstance(source, DirectorySource): - self.InsertStringItem(self.label_channel_idx, source.getSource(), 1) - item = self.GetItem(self.label_channel_idx) + self.InsertStringItem(self.labels[2], source.get_source_text(), 1) + item = self.GetItem(self.labels[2]) item.Check(source.enabled) item.SetData(source) self.SetItem(item) - self.label_channel_idx += 1 + self.labels[2] += 1 elif isinstance(source, ChannelSource): - self.InsertStringItem(self.label_channel_idx+1, source.getSource() or "Loading..", 1) - item = self.GetItem(self.label_channel_idx+1) + self.InsertStringItem(self.labels[2] + 1, source.get_source_text() or "Loading..", 1) + item = self.GetItem(self.labels[2] + 1) item.Check(source.enabled) item.SetData(source) self.SetItem(item) @@ -172,9 +179,15 @@ def OnGetItemCheck(self, evt): data = item.GetData() flag = item.IsChecked() + # if it was channel that not stored in cm variables + if not isinstance(data, BoostingSource) and flag: + source = data.dispersy_cid + self.boosting_manager.add_source(source) + self.boosting_manager.set_archive(source, False) + self.boosting_manager.set_enable_mining( data.dispersy_cid if not isinstance(data, BoostingSource) - else data.source, flag, True) + else data.source, flag, True) if isinstance(data, Channel): # channel -> channel source @@ -183,33 +196,33 @@ def OnGetItemCheck(self, evt): item.SetData(channel_src) self.SetItem(item) + def refresh_sourcelist_data(self, rerun=True): + """ + delete all the source in the list and adding a new one + """ - def RefreshData(self, rerun=True): - for i in range(0, self.GetItemCount()): + # don't refresh if we are quitting + if GUIUtility.getInstance().utility.abcquitting: + return + + for i in xrange(0, self.GetItemCount()): item = self.GetItem(i) data = item.GetData() - if isinstance(data, BoostingSource): - source = data.getSource() - if isinstance(data, RSSFeedSource): - sobj = self.boosting_manager.boosting_sources[source] - elif isinstance(data, DirectorySource): - sobj = self.boosting_manager.boosting_sources[source] - elif isinstance(data, ChannelSource): - source = data.source - sobj = self.boosting_manager.boosting_sources[source] - if item.GetText() == "Loading..": - item.SetText(data.getSource() or "Loading..") - self.SetItem(item) - - elif isinstance(data, Channel): - pass - - if rerun: - self.utility.session.lm.threadpool.add_task(self.RefreshData, 30, - task_name=str(self)+"_refresh_data_ULC") - - def FixChannelPos(self, source): + if isinstance(data, ChannelSource): + if item.GetText() == "Loading..": + item.SetText(data.get_source_text() or "Loading..") + self.SetItem(item) + + if rerun and not self.guiutility.utility.session.lm.threadpool.is_pending_task_active( + str(self) + "_refresh_data_ULC"): + self.guiutility.utility.session.lm.threadpool.add_task(self.refresh_sourcelist_data, 30, + task_name=str(self) + "_refresh_data_ULC") + + def fix_channel_position(self, source): + """ + This function called when new checked channel want to pushed above + """ chn_source = self.boosting_manager.boosting_sources[source] chn = self.channel_list[source] @@ -217,8 +230,8 @@ def FixChannelPos(self, source): idx = self.FindItemData(-1, chn) self.DeleteItem(idx) - self.InsertStringItem(self.label_channel_idx+1,chn_source.getSource() or "Loading..", 1) - item = self.GetItem(self.label_channel_idx+1) + self.InsertStringItem(self.labels[2] + 1, chn_source.get_source_text() or "Loading..", 1) + item = self.GetItem(self.labels[2] + 1) item.Check(chn_source.enabled) item.SetData(chn_source) self.SetItem(item) @@ -227,6 +240,9 @@ def FixChannelPos(self, source): class CreditMiningPanel(FancyPanel): + """ + A class representing panel control for credit mining + """ def __init__(self, parent): self._logger = logging.getLogger(self.__class__.__name__) @@ -236,18 +252,21 @@ def __init__(self, parent): self.utility = self.guiutility.utility self.installdir = self.utility.getPath() - self.boosting_manager = BoostingManager.get_instance(self.utility.session) - - self.tdb = self.utility.session.open_dbhandler(NTFY_TORRENTS) - FancyPanel.__init__(self, parent, border=wx.BOTTOM) self.SetBorderColour(SEPARATOR_GREY) self.SetBackgroundColour(GRADIENT_LGREY, GRADIENT_DGREY) + if not self.utility.session.get_creditmining_enable(): + wx.StaticText(self, -1, 'Credit mining inactive') + return + + self.tdb = self.utility.session.open_dbhandler(NTFY_TORRENTS) + self.boosting_manager = self.utility.session.lm.boosting_manager + self.main_sizer = wx.BoxSizer(wx.VERTICAL) - self.header = self.CreateHeader(self) + self.header = self.create_header_info(self) if self.header: self.main_sizer.Add(self.header, 0, wx.EXPAND) @@ -255,22 +274,23 @@ def __init__(self, parent): self.main_splitter.SetMinimumPaneSize(300) self.sourcelist = CpanelCheckListCtrl(self.main_splitter, -1, - agwStyle=wx.LC_REPORT | wx.LC_NO_HEADER | wx.LC_VRULES - | wx.LC_HRULES | wx.LC_SINGLE_SEL | ULC.ULC_HAS_VARIABLE_ROW_HEIGHT) + agwStyle=wx.LC_REPORT | wx.LC_NO_HEADER | wx.LC_VRULES | wx.LC_HRULES + | wx.LC_SINGLE_SEL | ULC.ULC_HAS_VARIABLE_ROW_HEIGHT) - self.AddComponents(self.main_splitter) + self.add_components(self.main_splitter) self.SetSizer(self.main_sizer) + self.guiutility.utility.session.lm.threadpool.add_task(self._post_init, 2, + task_name=str(self) + "_post_init") - self.guiutility.utility.session.lm.threadpool.add_task(self._PostInit, 2, - task_name=str(self)+"_post_init") - - - def AddComponents(self,parent): - self.infoPanel = FancyPanel(parent, style=wx.BORDER_SUNKEN) + def add_components(self, parent): + """ + adding GUI components to the control panel + """ + self.info_panel = FancyPanel(parent, style=wx.BORDER_SUNKEN) if_sizer = wx.BoxSizer(wx.VERTICAL) - self.top_info_p = FancyPanel(self.infoPanel, border=wx.ALL, style=wx.BORDER_SUNKEN, name="top_info_p") + self.top_info_p = FancyPanel(self.info_panel, border=wx.ALL, style=wx.BORDER_SUNKEN, name="top_info_p") tinfo_sizer = wx.BoxSizer(wx.VERTICAL) self.tnfo_subpanel_top = FancyPanel(self.top_info_p, border=wx.ALL) @@ -300,14 +320,14 @@ def AddComponents(self,parent): self.debug_info = wx.StaticText(self.tnfo_subpanel_top, -1, 'Debug Info : -') stat_sizer.Add(self.debug_info) - tinfo_spanel_sizer.Add(stat_sizer,-1) + tinfo_spanel_sizer.Add(stat_sizer, -1) tinfo_spanel_sizer.Add(wx.StaticText(self.tnfo_subpanel_top, -1, 'Credit Mining Status: ')) self.status_cm = wx.StaticText(self.tnfo_subpanel_top, -1, '-') tinfo_spanel_sizer.Add(self.status_cm) self.tnfo_subpanel_top.SetSizer(tinfo_spanel_sizer) tinfo_sizer.Add(self.tnfo_subpanel_top, 1, wx.EXPAND) - tinfo_sizer.Add(wx.StaticLine(self.top_info_p), 0, wx.ALL|wx.EXPAND, 5) + tinfo_sizer.Add(wx.StaticLine(self.top_info_p), 0, wx.ALL | wx.EXPAND, 5) self.up_rate = wx.StaticText(self.top_info_p, -1, 'Upload rate : -', name="up_rate") tinfo_sizer.Add(self.up_rate) @@ -320,32 +340,39 @@ def AddComponents(self,parent): if_sizer.Add(self.top_info_p, 1, wx.EXPAND) - self.cmlist = CreditMiningList(self.infoPanel) + self.cmlist = CreditMiningList(self.info_panel) self.cmlist.do_or_schedule_refresh(True) self.cmlist.library_manager.add_download_state_callback(self.cmlist.RefreshItems) if_sizer.Add(self.cmlist, 1, wx.EXPAND) - self.infoPanel.SetSizer(if_sizer) + self.info_panel.SetSizer(if_sizer) self.sourcelist.Hide() self.loading_holder = wx.StaticText(self.main_splitter, -1, 'Loading..') - parent.SplitVertically(self.loading_holder, self.infoPanel,1) + parent.SplitVertically(self.loading_holder, self.info_panel, 1) parent.SetSashGravity(0.3) self.main_sizer.Add(parent, 1, wx.EXPAND) - def OnItemSelected(self, event): + def on_sourceitem_selected(self, event): + """ + This function is called when a user select 'source' in the list. + The credit mining list will only show this particular source + """ idx = event.m_itemIndex data = self.sourcelist.GetItem(idx).GetData() if isinstance(data, ChannelSource): self.cmlist.GotFilter(data.source) else: - self.cmlist.GotFilter(data.getSource() if isinstance(data, BoostingSource) else '') + self.cmlist.GotFilter(data.get_source_text() if isinstance(data, BoostingSource) else '') - self.ShowInfo(data) + self.show_source_info(data) - def ShowInfo(self, data): + def show_source_info(self, data): + """ + shows information about selected source (not necessarily activated/enabled) in the panel + """ if isinstance(data, ChannelSource): self.last_updt.Show() @@ -354,15 +381,14 @@ def ShowInfo(self, data): self.rss_desc.Hide() self.source_label.SetLabel("Source : Channel (stored)") - self.source_name.SetLabel("Name : "+data.getSource()) - self.torrent_num.SetLabel("# Torrents : "+str(data.channel.nr_torrents)) - self.last_updt.SetLabel("Latest update : "+format_time(data.channel.modified)) - self.votes_num.SetLabel('Favorite votes : '+str(data.channel.nr_favorites)) + self.source_name.SetLabel("Name : " + data.get_source_text()) + self.torrent_num.SetLabel("# Torrents : " + str(data.channel.nr_torrents)) + self.last_updt.SetLabel("Latest update : " + format_time(data.channel.modified)) + self.votes_num.SetLabel('Favorite votes : ' + str(data.channel.nr_favorites)) self.status_cm.SetLabel("Active" if data.enabled else "Inactive") - debug_str = hexlify(data.source) - self.debug_info.SetLabel("Debug Info : \n"+debug_str) + self.debug_info.SetLabel("Debug Info : \n" + debug_str) elif isinstance(data, Channel): self.last_updt.Show() @@ -371,14 +397,14 @@ def ShowInfo(self, data): self.rss_desc.Hide() self.source_label.SetLabel("Source : Channel") - self.source_name.SetLabel("Name : "+data.name) - self.torrent_num.SetLabel("# Torrents : "+str(data.nr_torrents)) - self.last_updt.SetLabel("Latest update : "+format_time(data.modified)) - self.votes_num.SetLabel('Favorite votes : '+str(data.nr_favorites)) + self.source_name.SetLabel("Name : " + data.name) + self.torrent_num.SetLabel("# Torrents : " + str(data.nr_torrents)) + self.last_updt.SetLabel("Latest update : " + format_time(data.modified)) + self.votes_num.SetLabel('Favorite votes : ' + str(data.nr_favorites)) self.status_cm.SetLabel("Inactive") debug_str = hexlify(data.dispersy_cid) - self.debug_info.SetLabel("Debug Info : \n"+debug_str) + self.debug_info.SetLabel("Debug Info : \n" + debug_str) elif isinstance(data, RSSFeedSource): self.last_updt.Hide() @@ -387,14 +413,14 @@ def ShowInfo(self, data): self.rss_desc.Show() self.source_label.SetLabel("Source : RSS Web Feed") - self.source_name.SetLabel("Source URL : "+data.getSource()) - self.torrent_num.SetLabel("# Torrents : "+str(data.total_torrents)) - self.rss_title.SetLabel("Title : "+data.title) - self.rss_desc.SetLabel("Description : "+data.description) + self.source_name.SetLabel("Source URL : " + data.get_source_text()) + self.torrent_num.SetLabel("# Torrents : %s" % len(data.torrents)) + self.rss_title.SetLabel("Title : " + data.title) + self.rss_desc.SetLabel("Description : " + data.description) self.status_cm.SetLabel("Active" if data.enabled else "Inactive") debug_str = "-" - self.debug_info.SetLabel("Debug Info : \n"+debug_str) + self.debug_info.SetLabel("Debug Info : \n" + debug_str) elif isinstance(data, DirectorySource): self.last_updt.Hide() @@ -403,65 +429,73 @@ def ShowInfo(self, data): self.rss_desc.Hide() self.source_label.SetLabel("Source : Directory") - self.source_name.SetLabel("Name : "+data.getSource()) - self.torrent_num.SetLabel("# Torrents : "+str(12345)) + self.source_name.SetLabel("Name : " + data.get_source_text()) + self.torrent_num.SetLabel("# Torrents : %d" % len(data.torrents)) self.status_cm.SetLabel("Active" if data.enabled else "Inactive") debug_str = "-" - self.debug_info.SetLabel("Debug Info : \n"+debug_str) + self.debug_info.SetLabel("Debug Info : \n" + debug_str) else: self._logger.debug("Not implemented yet") - pass # show/hide items self.tnfo_subpanel_top.Layout() - - def CreateHeader(self, parent): + def create_header_info(self, parent): + """ + function to create wx header/info panel above the credit mining list + """ if self.guiutility.frame.top_bg: header = FancyPanel(parent, border=wx.BOTTOM, name="cm_header") text = wx.StaticText(header, -1, 'Investment overview') - def OnAddSource(event): + def on_add_source(_): + """ + callback when a user wants to add new source + """ dlg = AddBoostingSource(None) if dlg.ShowModal() == wx.ID_OK: - source, archive = dlg.GetValue() + source, archive = dlg.get_value() if source: self.boosting_manager.add_source(source) self.boosting_manager.set_archive(source, archive) - self.sourcelist.CreateSourceItem(self.boosting_manager.boosting_sources[source]) + self.sourcelist.create_source_item(self.boosting_manager.boosting_sources[source]) dlg.Destroy() - def OnRemoveSource(event): + def on_remove_source(_): + """ + callback when a user wants to remove source + """ dlg = RemoveBoostingSource(None) - if dlg.ShowModal() == wx.ID_OK and dlg.GetValue(): - self.boosting_manager.remove_source(dlg.GetValue()) - self.sourcelist.RefreshData() + if dlg.ShowModal() == wx.ID_OK and dlg.get_value(): + self.boosting_manager.remove_source(dlg.get_value()) + self.sourcelist.refresh_sourcelist_data() dlg.Destroy() addsource = LinkStaticText(header, 'Add', icon=None) - addsource.Bind(wx.EVT_LEFT_UP, OnAddSource) + addsource.Bind(wx.EVT_LEFT_UP, on_add_source) removesource = LinkStaticText(header, 'Remove', icon=None) - removesource.Bind(wx.EVT_LEFT_UP, OnRemoveSource) - self.b_up = wx.StaticText(header, -1, 'Total bytes up: -',name="b_up") - self.b_down = wx.StaticText(header, -1, 'Total bytes down: -',name="b_down") - self.s_up = wx.StaticText(header, -1, 'Total speed up: -',name="s_up") - self.s_down = wx.StaticText(header, -1, 'Total speed down: -',name="s_down") - self.iv_sum = wx.StaticText(header, -1, 'Investment summary: -',name="iv_sum") + removesource.Bind(wx.EVT_LEFT_UP, on_remove_source) + + self.b_up = wx.StaticText(header, -1, 'Total bytes up: -', name="b_up") + self.b_down = wx.StaticText(header, -1, 'Total bytes down: -', name="b_down") + self.s_up = wx.StaticText(header, -1, 'Total speed up: -', name="s_up") + self.s_down = wx.StaticText(header, -1, 'Total speed down: -', name="s_down") + self.iv_sum = wx.StaticText(header, -1, 'Investment summary: -', name="iv_sum") _set_font(text, size_increment=2, fontweight=wx.FONTWEIGHT_BOLD) sizer = wx.BoxSizer(wx.VERTICAL) sizer.AddStretchSpacer() - titleSizer = wx.BoxSizer(wx.HORIZONTAL) - titleSizer.Add(text, 0, wx.ALIGN_BOTTOM | wx.RIGHT, 5) - titleSizer.Add(wx.StaticText(header, -1, '('), 0, wx.ALIGN_BOTTOM) - titleSizer.Add(addsource, 0, wx.ALIGN_BOTTOM) - titleSizer.Add(wx.StaticText(header, -1, '/'), 0, wx.ALIGN_BOTTOM) - titleSizer.Add(removesource, 0, wx.ALIGN_BOTTOM) - titleSizer.Add(wx.StaticText(header, -1, ' boosting source)'), 0, wx.ALIGN_BOTTOM) - sizer.Add(titleSizer, 0, wx.LEFT | wx.BOTTOM, 5) + titlesizer = wx.BoxSizer(wx.HORIZONTAL) + titlesizer.Add(text, 0, wx.ALIGN_BOTTOM | wx.RIGHT, 5) + titlesizer.Add(wx.StaticText(header, -1, '('), 0, wx.ALIGN_BOTTOM) + titlesizer.Add(addsource, 0, wx.ALIGN_BOTTOM) + titlesizer.Add(wx.StaticText(header, -1, '/'), 0, wx.ALIGN_BOTTOM) + titlesizer.Add(removesource, 0, wx.ALIGN_BOTTOM) + titlesizer.Add(wx.StaticText(header, -1, ' boosting source)'), 0, wx.ALIGN_BOTTOM) + sizer.Add(titlesizer, 0, wx.LEFT | wx.BOTTOM, 5) sizer.Add(self.b_up, 0, wx.LEFT, 5) sizer.Add(self.b_down, 0, wx.LEFT, 5) sizer.Add(self.s_up, 0, wx.LEFT, 5) @@ -475,22 +509,27 @@ def OnRemoveSource(event): return header - def _PostInit(self): + def _post_init(self): + if GUIUtility.getInstance().utility.abcquitting: + return - for i in self.boosting_manager.boosting_sources: - if not self.boosting_manager.boosting_sources[i].ready: - self.guiutility.utility.session.lm.threadpool.add_task(self._PostInit, 2, task_name=str(self)+"_post_init") - return + some_ready = any([i.ready for i in self.boosting_manager.boosting_sources.values()]) - for source, source_obj in self.boosting_manager.boosting_sources.items(): - self.sourcelist.CreateSourceItem(source_obj) + # if none are ready, keep waiting or If no source available + if not some_ready and len(self.boosting_manager.boosting_sources.values()): + self.guiutility.utility.session.lm.threadpool.add_task(self._post_init, 2, + task_name=str(self) + "_post_init") + return + + for _, source_obj in self.boosting_manager.boosting_sources.items(): + self.sourcelist.create_source_item(source_obj) self.sourcelist.Show() self.main_splitter.ReplaceWindow(self.loading_holder, self.sourcelist) self.loading_holder.Close() - self.Bind(ULC.EVT_LIST_ITEM_SELECTED, self.OnItemSelected, self.sourcelist) - + self.Bind(ULC.EVT_LIST_ITEM_SELECTED, self.on_sourceitem_selected, self.sourcelist) - self.guiutility.utility.session.lm.threadpool.add_task(self.sourcelist.LoadMore, 2, task_name=str(self)+"load_more") - self.Layout() \ No newline at end of file + self.guiutility.utility.session.lm.threadpool.add_task(self.sourcelist.load_more, 2, + task_name=str(self) + "load_more") + self.Layout() diff --git a/Tribler/Main/vwxGUI/GuiUtility.py b/Tribler/Main/vwxGUI/GuiUtility.py index 40a448cd957..b59b0f002e8 100644 --- a/Tribler/Main/vwxGUI/GuiUtility.py +++ b/Tribler/Main/vwxGUI/GuiUtility.py @@ -296,9 +296,6 @@ def ShowPage(self, page, *args): self.frame.selectedchannellist.Focus() elif page == 'my_files': self.frame.librarylist.Focus() - elif page == 'creditmining': - pass - # self.frame.creditmininglist.Focus() @forceWxThread def on_show_startup_splash(self, subject, changetype, objectID, *args): diff --git a/Tribler/Main/vwxGUI/MainFrame.py b/Tribler/Main/vwxGUI/MainFrame.py index 317de78f32c..5ab07a92080 100644 --- a/Tribler/Main/vwxGUI/MainFrame.py +++ b/Tribler/Main/vwxGUI/MainFrame.py @@ -42,8 +42,6 @@ from Tribler.Main.Dialogs.ConfirmationDialog import ConfirmationDialog from Tribler.Main.Dialogs.FeedbackWindow import FeedbackWindow from Tribler.Main.Dialogs.systray import ABCTaskBarIcon -from Tribler.Main.Utility.GuiDBHandler import startWorker -from Tribler.Main.globals import DefaultDownloadStartupConfig from Tribler.Main.vwxGUI import DEFAULT_BACKGROUND, SEPARATOR_GREY from Tribler.Main.vwxGUI.CreditMiningPanel import CreditMiningPanel from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread diff --git a/Tribler/Main/vwxGUI/home.py b/Tribler/Main/vwxGUI/home.py index ab2bf3d9901..dbedc9498b7 100644 --- a/Tribler/Main/vwxGUI/home.py +++ b/Tribler/Main/vwxGUI/home.py @@ -1,61 +1,63 @@ # Written by Niels Zeilemaker -import wx -import sys -import os import datetime -from binascii import hexlify -from wx.lib.scrolledpanel import ScrolledPanel - -import math - -from Tribler.Policies.BoostingManager import BoostingManager -from Tribler.community.tunnel.hidden_community import HiddenTunnelCommunity -from Tribler.community.tunnel.routing import Hop -from Tribler.community.multichain.community import MultiChainCommunity - -import random import logging +import os +import random +import sys import binascii + from time import strftime, time from traceback import print_exc +# pylint complaining if wx imported before those three +import wx + from Tribler.Category.Category import Category +from Tribler.Core.CacheDB.sqlitecachedb import bin2str +from Tribler.Core.Session import Session +from Tribler.Core.Video.VideoUtility import considered_xxx from Tribler.Core.simpledefs import (NTFY_TORRENTS, NTFY_CHANNELCAST, NTFY_INSERT, NTFY_TUNNEL, NTFY_CREATED, NTFY_EXTENDED, NTFY_BROKEN, NTFY_SELECT, NTFY_JOINED, NTFY_EXTENDED_FOR, NTFY_IP_REMOVED, NTFY_RP_REMOVED, NTFY_IP_RECREATE, NTFY_DHT_LOOKUP, NTFY_KEY_REQUEST, NTFY_KEY_RESPOND, NTFY_KEY_RESPONSE, NTFY_CREATE_E2E, NTFY_ONCREATED_E2E, NTFY_IP_CREATED, NTFY_RP_CREATED, NTFY_REMOVE) -from Tribler.Core.Session import Session - -from Tribler.Main.vwxGUI import SEPARATOR_GREY, DEFAULT_BACKGROUND, LIST_BLUE, THUMBNAIL_FILETYPES, warnWxThread -from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY -from Tribler.Main.vwxGUI.list_header import DetailHeader +from Tribler.Main.Utility.utility import size_format +from Tribler.Main.vwxGUI import SEPARATOR_GREY, DEFAULT_BACKGROUND, LIST_BLUE, THUMBNAIL_FILETYPES +from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager +from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread from Tribler.Main.vwxGUI.list_body import ListBody -from Tribler.Main.vwxGUI.list_item import ThumbnailListItemNoTorrent from Tribler.Main.vwxGUI.list_footer import ListFooter +from Tribler.Main.vwxGUI.list_header import DetailHeader +from Tribler.Main.vwxGUI.list_item import ThumbnailListItemNoTorrent from Tribler.Main.vwxGUI.widgets import (SelectableListCtrl, TextCtrlAutoComplete, BetterText as StaticText, LinkStaticText, ActionButton, HorizontalGauge, TagText) -from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager -from Tribler.Core.CacheDB.sqlitecachedb import bin2str -from Tribler.Core.Video.VideoUtility import considered_xxx +from Tribler.Policies.credit_mining_util import string_to_source +from Tribler.community.multichain.community import MultiChainCommunity +from Tribler.community.tunnel.hidden_community import HiddenTunnelCommunity +from Tribler.community.tunnel.routing import Hop -from Tribler.Main.Utility.utility import size_format +# width size of channel grid +COLUMN_SIZE = 3 +# how long the string before it cut +CHANNEL_STRING_LENGTH = 35 +# number of popular torrent fetched to know the 'content' of channels +TORRENT_FETCHED = 5 +# max number of channel shown in the panel +MAX_CHANNEL_SHOW = 9 class Home(wx.Panel): - COLUMN_SIZE = 3 - def __init__(self, parent): wx.Panel.__init__(self, parent) self.guiutility = GUIUtility.getInstance() self.gui_image_manager = GuiImageManager.getInstance() self.session = self.guiutility.utility.session - self.boosting_manager = None + self.boosting_manager = self.session.lm.boosting_manager #dispersy_cid:Channel - self.channels = {None:None} + self.channels = {} #dispersy_cid:Popular Torrents self.chn_torrents = {} @@ -110,41 +112,41 @@ def __init__(self, parent): vSizer.AddStretchSpacer() # channel panel is for popular channel - self.channel_panel = ScrolledPanel(self, 1) + self.channel_panel = wx.lib.scrolledpanel.ScrolledPanel(self, 1) self.channel_panel.SetBackgroundColour(wx.WHITE) self.channel_panel.SetForegroundColour(parent.GetForegroundColour()) - v_chn_Sizer = wx.BoxSizer(wx.VERTICAL) - v_chn_Sizer.Add( + v_chn_sizer = wx.BoxSizer(wx.VERTICAL) + v_chn_sizer.Add( DetailHeader(self.channel_panel, "Select popular channels to mine"), 0, wx.EXPAND, 5) - self.loading_channel_txt = wx.StaticText(self.channel_panel, 1, 'Loading, please wait.') - v_chn_Sizer.Add(self.loading_channel_txt, 1, wx.TOP | wx.ALIGN_CENTER_HORIZONTAL, 10) + self.loading_channel_txt = wx.StaticText(self.channel_panel, 1, + 'Loading, please wait.' + if self.boosting_manager else "Credit Mining inactive") + + v_chn_sizer.Add(self.loading_channel_txt, 1, wx.TOP | wx.ALIGN_CENTER_HORIZONTAL, 10) - self.chn_sizer = wx.FlexGridSizer(0,self.COLUMN_SIZE,5,5) + self.chn_sizer = wx.FlexGridSizer(0, COLUMN_SIZE, 5, 5) - for i in range(0,self.COLUMN_SIZE): + for i in xrange(0, COLUMN_SIZE): if wx.MAJOR_VERSION > 2: if self.chn_sizer.IsColGrowable(i): - self.chn_sizer.AddGrowableCol(i,1) + self.chn_sizer.AddGrowableCol(i, 1) else: - self.chn_sizer.AddGrowableCol(i,1) + self.chn_sizer.AddGrowableCol(i, 1) - v_chn_Sizer.Add(self.chn_sizer, 0, wx.EXPAND, 5) + v_chn_sizer.Add(self.chn_sizer, 0, wx.EXPAND, 5) - self.channel_panel.SetSizer(v_chn_Sizer) + self.channel_panel.SetSizer(v_chn_sizer) self.channel_panel.SetupScrolling() vSizer.Add(self.channel_panel, 5, wx.EXPAND) - # # TODO (ardhi) : enable this once the layout finishes # video thumbnail panel self.aw_panel = ArtworkPanel(self) self.aw_panel.SetMinSize((-1, 275)) - # TODO(lipu): enable this when metadata PR is merged - #self.aw_panel.Show(self.guiutility.ReadGuiSetting('show_artwork', True)) - self.aw_panel.Show(True) + self.aw_panel.Show(self.guiutility.ReadGuiSetting('show_artwork', False)) vSizer.Add(self.aw_panel, 0, wx.EXPAND) self.SetSizer(vSizer) @@ -153,9 +155,10 @@ def __init__(self, parent): self.SearchFocus() self.channel_list_ready = False - self.session.lm.threadpool.add_task(self.RefreshChannels, 10, - task_name=str(self.__class__)+"_refreshchannel") + if self.boosting_manager: + self.session.lm.threadpool.add_task(self.refresh_channels_home, 10, + task_name=str(self.__class__)+"_refreshchannel") def OnRightClick(self, event): menu = wx.Menu() @@ -173,12 +176,12 @@ def toggleArtwork(event): self.guiutility.WriteGuiSetting("show_artwork", show) self.Layout() - def toggleChannels(event): + def togglechannels(_): show = not self.channel_panel.IsShown() self.channel_panel.Show(show) self.Layout() - menu.Bind(wx.EVT_MENU, toggleChannels, id=itemid_popchn) + menu.Bind(wx.EVT_MENU, togglechannels, id=itemid_popchn) menu.Bind(wx.EVT_MENU, toggleArtwork, id=itemid_rcvid) if menu: @@ -202,48 +205,38 @@ def SearchFocus(self): self.searchBox.SetFocus() self.searchBox.SelectAll() - def CreateChannelItem(self, parent, channel, torrents, max_fav): + def create_channel_item(self, parent, channel, torrents, max_fav): """ - Function to create channel (and it's torrents) checkbox on home panel - :param parent: where we put this element - :param channel: channel object - :param torrents: torrents of that particular channel - :param max_fav: max possible votes for ALL channel (to count relative popularity) + Function to create channel (and its torrents) checkbox on home panel """ from Tribler.Main.Utility.GuiDBTuples import Channel as ChannelObj - assert isinstance(channel, ChannelObj), "Type channel should be ChannelObj %s" %channel - - if not self.boosting_manager: - self.boosting_manager = BoostingManager.get_instance(self.session) - - STRING_LENGTH = 35 + assert isinstance(channel, ChannelObj), "Type channel should be ChannelObj %s" % channel vsizer = wx.BoxSizer(wx.VERTICAL) hsizer = wx.BoxSizer(wx.HORIZONTAL) chn_pn = wx.Panel(parent, -1, style=wx.SUNKEN_BORDER) - cb_chn = wx.CheckBox(chn_pn, 1, '',name=hexlify(channel.dispersy_cid)) + cb_chn = wx.CheckBox(chn_pn, 1, '', name=binascii.hexlify(channel.dispersy_cid)) obj = self.boosting_manager.get_source_object(channel.dispersy_cid) cb_chn.SetValue(False if not obj else obj.enabled) - normalministar = self.gui_image_manager.getImage(u"ministar.png") - ministar = self.gui_image_manager.getImage(u"ministarEnabled.png") - control = HorizontalGauge(chn_pn, normalministar, ministar, 5) + control = HorizontalGauge(chn_pn, self.gui_image_manager.getImage(u"ministar.png"), + self.gui_image_manager.getImage(u"ministarEnabled.png"), 5) # count popularity pop = channel.nr_favorites - if pop <= 0: + if pop <= 0 or max_fav == 0: control.SetPercentage(0) else: control.SetPercentage(pop/float(max_fav)) control.SetToolTipString('%s users marked this channel as one of their favorites.' % pop) hsizer.Add(cb_chn, 0, wx.ALIGN_LEFT) - hsizer.Add(TagText(chn_pn, -1, label='channel', fill_colour=wx.Colour(210, 252, 120)),0, - wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL) + hsizer.Add(TagText(chn_pn, -1, label='channel', fill_colour=wx.Colour(210, 252, 120)), 0, + wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL) hsizer.AddSpacer(5) hsizer.Add(wx.StaticText(chn_pn, -1, channel.name.encode('utf-8')), 0, wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL) hsizer.AddSpacer(30) @@ -252,77 +245,80 @@ def CreateChannelItem(self, parent, channel, torrents, max_fav): vsizer.Add(hsizer, 0, wx.EXPAND) - for t in torrents: - t = wx.StaticText(chn_pn, 1, t.name[:STRING_LENGTH] + (t.name[STRING_LENGTH:] and '...')) - vsizer.Add(t, 0, wx.EXPAND | wx.LEFT, 25) + for trnts in torrents: + trnts = wx.StaticText(chn_pn, 1, trnts.name[:CHANNEL_STRING_LENGTH] + (trnts.name[CHANNEL_STRING_LENGTH:] + and '...')) + vsizer.Add(trnts, 0, wx.EXPAND | wx.LEFT, 25) chn_pn.SetSizer(vsizer) - self.Bind(wx.EVT_CHECKBOX, self.OnCheckBox, cb_chn) + self.Bind(wx.EVT_CHECKBOX, self.on_check_channels_cm, cb_chn) return chn_pn - def RefreshChannels(self): + def refresh_channels_home(self): """ This function will be called to get popular channel list in Home - """ - # number of popular torrent fetched to know the 'content' of channels - TORRENT_FETCHED = 5 - - # max number of channel shown in the panel - MAX_CHANNEL_SHOW = 9 - def do_query(): - _, channels = self.guiutility.channelsearch_manager.getPopularChannels(2*MAX_CHANNEL_SHOW) + """ + querying channels to show at home page. Blocking + :return: dict_channels, dict_torrents, new_channels_ids + """ + _, channels = self.guiutility.channelsearch_manager.getPopularChannels(2 * MAX_CHANNEL_SHOW) - dict_channels = {channel.dispersy_cid:channel for channel in channels} + dict_channels = {channel.dispersy_cid: channel for channel in channels} dict_torrents = {} new_channels_ids = list(set(dict_channels.keys()) - set(self.channels.keys() if not self.channel_list_ready else [])) - for c in new_channels_ids: - channel = dict_channels.get(c) - torrents = self.guiutility.channelsearch_manager.getRecentReceivedTorrentsFromChannel\ - (channel, limit=TORRENT_FETCHED)[2] - dict_torrents[c] = torrents - return (dict_channels, dict_torrents, new_channels_ids) - - def do_gui(delayedResult): - (dict_channels,dict_torrents, new_channels_ids) = delayedResult.get() + for chan_id in new_channels_ids: + channel = dict_channels.get(chan_id) + torrents = self.guiutility.channelsearch_manager.getRecentReceivedTorrentsFromChannel( + channel, limit=TORRENT_FETCHED)[2] + dict_torrents[chan_id] = torrents + return dict_channels, dict_torrents, new_channels_ids + + def do_gui(delayed_result): + """ + put those new channels in the GUI + """ + (dict_channels, dict_torrents, new_channels_ids) = delayed_result.get() count = 0 if self.channel_list_ready: # reset it. Not reseting torrent_dict because it dynamically added anyway self.channels = {} - for c in new_channels_ids: - self.channels[c] = dict_channels.get(c) + for chn_id in new_channels_ids: + channel = dict_channels.get(chn_id) + self.channels[chn_id] = channel self.chn_torrents.update(dict_torrents) self.chn_sizer.Clear(True) self.chn_sizer.Layout() self.loading_channel_txt.Show() - for i in range(0,self.COLUMN_SIZE): + for i in xrange(0, COLUMN_SIZE): if wx.MAJOR_VERSION > 2: if self.chn_sizer.IsColGrowable(i): - self.chn_sizer.AddGrowableCol(i,1) + self.chn_sizer.AddGrowableCol(i, 1) else: - self.chn_sizer.AddGrowableCol(i,1) + self.chn_sizer.AddGrowableCol(i, 1) sortedchannels = sorted(self.channels.values(), - key=lambda x: x.nr_favorites if x else 0, reverse=True) + key=lambda z: z.nr_favorites if z else 0, reverse=True) max_favourite = sortedchannels[0].nr_favorites if sortedchannels else 0 - for c in [x for x in sortedchannels]: - d = c.dispersy_cid + for chn_id in [x for x in sortedchannels]: + d = chn_id.dispersy_cid # if we can't find channel details, ignore it, or # if no torrent available for that channel if not dict_channels.get(d) or not len(self.chn_torrents.get(d)): continue - self.chn_sizer.Add(self.CreateChannelItem(self.channel_panel,dict_channels.get(d), - self.chn_torrents.get(d),max_favourite), - 0, wx.ALL|wx.EXPAND) + if self.session.get_creditmining_enable(): + self.chn_sizer.Add( + self.create_channel_item(self.channel_panel, dict_channels.get(d), self.chn_torrents.get(d), + max_favourite), 0, wx.ALL | wx.EXPAND) self.loading_channel_txt.Hide() @@ -334,32 +330,43 @@ def do_gui(delayedResult): self.chn_sizer.Layout() self.channel_panel.SetupScrolling() + # quit refreshing if Tribler quitting + if GUIUtility.getInstance().utility.abcquitting: + return + if self.guiutility.frame.ready and isinstance(self.guiutility.GetSelectedPage(), Home): startWorker(do_gui, do_query, retryOnBusy=True, priority=GUI_PRI_DISPERSY) repeat = len(self.channels) < MAX_CHANNEL_SHOW self.channel_list_ready = not repeat - if repeat: - self.session.lm.threadpool.add_task_in_thread(self.RefreshChannels, 10, - task_name=str(self.__class__)+"_refreshchannel") - else : - # try to update the popular channel once in a while - self.session.lm.threadpool.add_task_in_thread(self.RefreshChannels, 10, - task_name=str(self.__class__)+"_refreshchannel") - def OnCheckBox(self, evt): - cb = evt.GetEventObject() - self.boosting_manager.set_enable_mining(binascii.unhexlify(cb.GetName()), evt.IsChecked()) + # try to update the popular channel once in a while + self.session.lm.threadpool.add_task_in_thread(self.refresh_channels_home, 10, + task_name=str(self.__class__)+"_refreshchannel") + + def on_check_channels_cm(self, evt): + """ + this callback called if a channel in home was checked/unchecked + """ + cbox = evt.GetEventObject() + source_str = cbox.GetName() + + # if we don't have the channel in boosting source, and its checked for the first time + if not self.boosting_manager.get_source_object(string_to_source(source_str)) and evt.IsChecked(): + source = binascii.unhexlify(source_str) + self.boosting_manager.add_source(source) + self.boosting_manager.set_archive(source, False) + self.boosting_manager.set_enable_mining(binascii.unhexlify(source_str), evt.IsChecked()) if evt.IsChecked(): - chn_src = self.boosting_manager.boosting_sources[binascii.unhexlify(cb.GetName())] + chn_src = self.boosting_manager.boosting_sources[binascii.unhexlify(cbox.GetName())] sourcelist = self.guiutility.frame.creditminingpanel.sourcelist - if binascii.unhexlify(cb.GetName()) in sourcelist.channel_list: - sourcelist.FixChannelPos(binascii.unhexlify(cb.GetName())) + if binascii.unhexlify(cbox.GetName()) in sourcelist.channel_list: + sourcelist.fix_channel_position(binascii.unhexlify(cbox.GetName())) else: - sourcelist.CreateSourceItem(chn_src) + sourcelist.create_source_item(chn_src) class Stats(wx.Panel): diff --git a/Tribler/Main/vwxGUI/list.py b/Tribler/Main/vwxGUI/list.py index fb91a98aad4..de840da03be 100644 --- a/Tribler/Main/vwxGUI/list.py +++ b/Tribler/Main/vwxGUI/list.py @@ -1,18 +1,15 @@ -# Written by Niels Zeilemaker +# Written by Niels Zeilemaker and Ardhi Putra Pratama H import copy import logging -import os import re import sys + from binascii import hexlify, unhexlify from colorsys import hsv_to_rgb, rgb_to_hsv from math import log from time import time import wx -from wx._core import LayoutConstraints -from wx.lib.mixins.listctrl import CheckListCtrlMixin -from wx.lib.scrolledpanel import ScrolledPanel from wx.lib.wordwrap import wordwrap from Tribler.Category.Category import Category @@ -37,7 +34,7 @@ CreditMiningListItem) from Tribler.Main.vwxGUI.widgets import (BetterText, FancyPanel, HorizontalGauge, LinkStaticText, SwarmHealth, TagText, TorrentStatus, TransparentStaticBitmap, TransparentText, _set_font) -from Tribler.Policies.BoostingManager import BoostingManager, BoostingSource +from Tribler.Policies.credit_mining_util import string_to_source DEBUG_RELEVANCE = False MAX_REFRESH_PARTIAL = 5 @@ -309,70 +306,77 @@ def downloadStarted(self, _): class CreditMiningSearchManager(BaseManager): - - def __init__(self, list): - BaseManager.__init__(self, list) - self.boosting_manager = BoostingManager.get_instance() + """ + search manager for credit mining purpose + """ + def __init__(self, llist): + BaseManager.__init__(self, llist) + self.boosting_manager = self.guiutility.utility.session.lm.boosting_manager self.library_manager = self.guiutility.library_manager def refresh(self): - startWorker(self._on_data, self.getHitsInCategory, uId=u"CreditMiningSearchManager_refresh", retryOnBusy=True, priority=GUI_PRI_DISPERSY) + startWorker(self._on_data, self.get_torrents_list_boosting, uId=u"CreditMiningSearchManager_refresh", + retryOnBusy=True, priority=GUI_PRI_DISPERSY) - def getTorrentFromInfohash(self, infohash): + def get_torrent_from_infohash(self, infohash): + """ + get LibraryTorrent object from torrent infohash (byte) + """ torrent = self.boosting_manager.torrents.get(infohash, None) if torrent: - t = LibraryTorrent('', infohash, name=torrent['name'], length=torrent['length'], category='', status='', - num_seeders=torrent['num_seeders'], num_leechers=torrent['num_leechers']) - t.torrent_db = self.library_manager.torrent_db - t.channelcast_db = self.library_manager.channelcast_db + ltorrent = LibraryTorrent('', infohash, name=torrent['name'], length=torrent['length'], category='', + status='', num_seeders=torrent['num_seeders'], + num_leechers=torrent['num_leechers']) + ltorrent.torrent_db = self.library_manager.torrent_db + ltorrent.channelcast_db = self.library_manager.channelcast_db #touch channel instance - t.channel - self.library_manager.addDownloadState(t) - return t + ltorrent.channel #pylint: disable=pointless-statement + self.library_manager.addDownloadState(ltorrent) + return ltorrent - def getHitsInCategory(self): - hits = [self.getTorrentFromInfohash(infohash) for infohash in self.boosting_manager.torrents.keys()] + def get_torrents_list_boosting(self): + """ + Get a list of LibraryTorrent(s) in Boosting Manager + """ + hits = [self.get_torrent_from_infohash(infohash) for infohash in self.boosting_manager.torrents.keys()] return [len(hits), hits] def refresh_partial(self, ids): for infohash in ids: - startWorker(self.list.RefreshDelayedData, self.getTorrentFromInfohash, cargs=(infohash,), wargs=(infohash,), retryOnBusy=True, priority=GUI_PRI_DISPERSY) + startWorker(self.list.RefreshDelayedData, self.get_torrent_from_infohash, cargs=(infohash,), + wargs=(infohash, ), retryOnBusy=True, priority=GUI_PRI_DISPERSY) - def refresh_if_exists(self, infohashes, force=False): - if any([self.boosting_manager.torrents.has_key(infohash) for infohash in infohashes]): - print >> sys.stderr, long(time()), "Scheduling a refresh, missing some infohashes in the Credit Mining overview" + def refresh_if_exists(self, infohashes): + """ + refresh list if there's a swarm available to mine + """ + if any([infohash in self.boosting_manager.torrents for infohash in infohashes]): + self._logger.info("Scheduling a refresh, missing some infohashes in the Credit Mining overview") self.refresh() else: - print >> sys.stderr, long(time()), "Not scheduling a refresh" - - def refresh_or_expand(self, infohash): - if not self.list.InList(infohash): - def select(delayedResult): - delayedResult.get() - self.refresh_or_expand(infohash) - - startWorker(select, self.refresh_partial, wargs=([infohash],), priority=GUI_PRI_DISPERSY) - else: - self.list.Select(infohash) + self._logger.debug("Not scheduling a refresh") @forceWxThread - def _on_data(self, delayedResult): - total_items, data = delayedResult.get() + def _on_data(self, delayed_result): + _, data = delayed_result.get() self.list.SetData(data) self.list.Layout() - def torrentUpdated(self, infohash): + def torrent_updated(self, infohash): + """ + function to handle single updated torrent information + """ if self.list.InList(infohash): self.do_or_schedule_partial([infohash]) - def torrentsUpdated(self, infohashes): + def torrents_updated(self, infohashes): + """ + function when many torrents updated + """ infohashes = [infohash for infohash in infohashes if self.list.InList(infohash)] self.do_or_schedule_partial(infohashes) - def downloadStarted(self, infohash): - self.refresh() - class ChannelSearchManager(BaseManager): @@ -624,8 +628,8 @@ def CreateHeader(self, parent): def CreateList(self, parent=None, listRateLimit=1): if not parent: parent = self - return ListBody(parent, self, self.columns, self.spacers[0], self.spacers[1], self.singleSelect, self.showChange, listRateLimit=listRateLimit, - list_item_max=self.list_item_max) + return ListBody(parent, self, self.columns, self.spacers[0], self.spacers[1], self.singleSelect, + self.showChange, listRateLimit=listRateLimit, list_item_max=self.list_item_max) def CreateFooter(self, parent): return ListFooter(parent) @@ -2034,31 +2038,34 @@ def GetFilterMessage(self, empty=False): class CreditMiningList(SizeList): - + """ + List of all swarms that are available for mining. + """ def __init__(self, parent): - self.boosting_manager = BoostingManager.get_instance() self.guiutility = GUIUtility.getInstance() + self.boosting_manager = self.guiutility.utility.session.lm.boosting_manager self.utility = self.guiutility.utility self.statefilter = None self.newfilter = False self.prevStates = {} - self.oldDS = {} + self.old_dlstate = {} + self.old_keys = [] self.initnumitems = False self.tot_bytes_up = 0 self.tot_bytes_dwn = 0 self.channels = [] + self.manager = None self.top_info_p = parent.FindWindowByName('top_info_p') or None - columns = [{'name': 'Speed up/down', 'width': '32em', 'autoRefresh': False}, {'name': 'Bytes up/down', 'width': '32em', 'autoRefresh': False}, {'name': 'Seeders/leechers', 'width': '27em', 'autoRefresh': False}, {'name': 'Duplicate', 'showColumname': False, 'width': '2em'}, {'name': 'Hash', 'width': '27em', 'fmt': lambda ih: ih.encode('hex')[:10]}, - {'name': 'Source', 'width': '40em', 'type': 'method', 'method': self.CreateSource}, + {'name': 'Source', 'width': '40em', 'type': 'method', 'method': self.create_source_txt}, {'name': 'Investment status', 'width': '32em', 'autoRefresh': False}] columns = self.guiutility.SetColumnInfo(CreditMiningListItem, columns) @@ -2067,7 +2074,7 @@ def __init__(self, parent): SizeList.__init__(self, None, LIST_GREY, [0, 0], False, parent=parent) def GetManager(self): - if getattr(self, 'manager', None) == None: + if getattr(self, 'manager', None) is None: self.manager = CreditMiningSearchManager(self) return self.manager @@ -2083,7 +2090,10 @@ def CreateFooter(self, parent): return footer @warnWxThread - def CreateSource(self, parent, item): + def create_source_txt(self, parent, item): + """ + create a source string, cut it if the length is longer than 30 + """ torrent = self.boosting_manager.torrents.get(item.original_data.infohash, None) text = torrent.get('source', '') text = text[:30] + '..' if len(text) > 32 else text @@ -2095,21 +2105,22 @@ def OnExpand(self, item): @warnWxThread def RefreshItems(self, dslist, magnetlist, rawdata=True): - didStateChange, _, _ = SizeList.RefreshItems(self, dslist, magnetlist, rawdata=True) + didstatechange, _, _ = SizeList.RefreshItems(self, dslist, magnetlist, rawdata=True) - newFilter = self.newfilter + newfilter = self.newfilter new_keys = self.boosting_manager.torrents.keys() old_keys = getattr(self, 'old_keys', []) if len(new_keys) != len(old_keys): - self.GetManager().refresh_if_exists(new_keys, force=True) + self.GetManager().refresh_if_exists(new_keys) self.old_keys = new_keys - if didStateChange: - if self.statefilter != None: - self.list.SetData() # basically this means execute filter again + if didstatechange: + if self.statefilter: + self.list.SetData() # basically this means execute filter again - boosting_dslist = [ds for ds in dslist if ds.get_download().get_def().get_infohash() in new_keys] + boosting_dslist = [dl_state for dl_state in dslist if dl_state.get_download().get_def().get_infohash() + in new_keys] # init source statistics for _, src in self.boosting_manager.boosting_sources.items(): @@ -2118,36 +2129,37 @@ def RefreshItems(self, dslist, magnetlist, rawdata=True): src.av_dwnrate = 0 # update torrent stats in boosting manager - for ds in boosting_dslist: - torrent_infohash = ds.get_download().get_def().get_infohash() + for dl_state in boosting_dslist: + torrent_infohash = dl_state.get_download().get_def().get_infohash() - if ds.get_seeding_statistics(): - self.boosting_manager.update_torrent_stats(torrent_infohash, ds.get_seeding_statistics()) + if dl_state.get_seeding_statistics(): + self.boosting_manager.update_torrent_stats(torrent_infohash, dl_state.get_seeding_statistics()) for item in self.list.items.itervalues(): - ds = item.original_data.ds + dl_state = item.original_data.ds torrent_infohash = item.original_data.infohash source_str = self.boosting_manager.torrents[torrent_infohash]['source'] - source = self.boosting_manager.get_source_object(source_str) + source = self.boosting_manager.get_source_object(string_to_source(source_str)) # ds = DownloadState - if ds: - source.av_uprate += ds.get_current_speed('up') - source.av_dwnrate += ds.get_current_speed('down') + if dl_state: + source.av_uprate += dl_state.get_current_speed('up') + source.av_dwnrate += dl_state.get_current_speed('down') - if not ds in boosting_dslist: + if dl_state not in boosting_dslist: continue # look for the current active stats to update visible list - if ds.get_seeding_statistics(): - seeding_stats_i = ds.get_seeding_statistics() + if dl_state.get_seeding_statistics(): + seeding_stats_i = dl_state.get_seeding_statistics() bytes_up = bytes_down = 0 if self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']: bytes_up = self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']['total_up'] - bytes_down = self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']['total_down'] + bytes_down = self.boosting_manager.torrents[torrent_infohash][ + 'last_seeding_stats']['total_down'] # we may have different value of total. Find the maximum gathered bytes_up = max(seeding_stats_i['total_up'], bytes_up) @@ -2167,8 +2179,8 @@ def RefreshItems(self, dslist, magnetlist, rawdata=True): item.SetExpandedColour(wx.Colour(255, 150, 150)) item.SetExpandedAndSelectedColour(wx.Colour(255, 125, 125)) - speed_up = ds.get_current_speed('up') if ds else 0 - speed_down = ds.get_current_speed('down') if ds else 0 + speed_up = dl_state.get_current_speed('up') if dl_state else 0 + speed_down = dl_state.get_current_speed('down') if dl_state else 0 item.RefreshColumn(0, speed_format(speed_up) + ' / ' + speed_format(speed_down)) @@ -2202,58 +2214,83 @@ def RefreshItems(self, dslist, magnetlist, rawdata=True): storage_used_txt = self.top_info_p.FindWindowByName('storage_used') try: - filter = self.rawfilter if self.rawfilter in self.boosting_manager.boosting_sources else unhexlify(self.rawfilter) - except: - filter = self.rawfilter + filter_src = self.rawfilter if self.rawfilter in self.boosting_manager.boosting_sources \ + else unhexlify(self.rawfilter) + except TypeError: + # can't unhex the string, go with RAW as default + filter_src = self.rawfilter - if filter: - active_source = self.boosting_manager.get_source_object(filter) + # filter_src is RAW format + if filter_src: + active_source = self.boosting_manager.get_source_object(filter_src) seed_sp_list = [tr['last_seeding_stats']['total_down'] for tr in self.boosting_manager.torrents.values() - if self.boosting_manager.get_source_object(tr['source']).source == active_source.source and tr['last_seeding_stats']] + if self.boosting_manager.get_source_object(string_to_source(tr['source'])).source + == active_source.source and tr['last_seeding_stats']] total_dl_source = sum(seed_sp_list) up_rate_txt.SetLabel('Active upload rate : '+speed_format(active_source.av_uprate)) dwn_rate_txt.SetLabel('Active download rate : '+speed_format(active_source.av_dwnrate)) storage_used_txt.SetLabel('Storage Used : '+size_format(total_dl_source)) - header = self.parent.GetGrandParent().FindWindowByName('cm_header') header.FindWindowByName('b_up').SetLabel('Total bytes up: ' + size_format(self.tot_bytes_up)) header.FindWindowByName('b_down').SetLabel('Total bytes down: ' + size_format(self.tot_bytes_dwn)) if self.tot_bytes_dwn: - header.FindWindowByName('iv_sum').SetLabel(' Investment summary: %f' %(float(self.tot_bytes_up)/float(self.tot_bytes_dwn))) + header.FindWindowByName('iv_sum').SetLabel(' Investment summary: %f' + %(float(self.tot_bytes_up)/float(self.tot_bytes_dwn))) - header.FindWindowByName('s_up').SetLabel('Current total speed up: ' + speed_format(sum([ds.get_current_speed('up') for ds in boosting_dslist]))) - header.FindWindowByName('s_down').SetLabel('Current total speed down: ' + speed_format(sum([ds.get_current_speed('down') for ds in boosting_dslist]))) + header.FindWindowByName('s_up').SetLabel('Current total speed up: ' + speed_format( + sum([dl_state.get_current_speed('up') for dl_state in boosting_dslist]))) + header.FindWindowByName('s_down').SetLabel('Current total speed down: ' + speed_format( + sum([dl_state.get_current_speed('down') for dl_state in boosting_dslist]))) - if newFilter: + if newfilter: self.newfilter = False - self.oldDS = dict([(infohash, item.original_data.ds) for infohash, item in self.list.items.iteritems()]) + self.old_dlstate = dict([(infohash, item.original_data.ds) for infohash, item in self.list.items.iteritems()]) @warnWxThread def SetData(self, data): SizeList.SetData(self, data) + data_new = [] + if len(data) > 0: - data = [(file.infohash, ['- / -', '- / -', '%d / %d' % - (file.num_seeders, file.num_leechers), '', file.infohash, - self.boosting_manager.torrents.get(file.infohash,None).get('source', ''), '-1'], - file, CreditMiningListItem) for file in data] + for ffile in data: + torrent_cm_dict = self.boosting_manager.torrents.get(ffile.infohash, None) + current_speed = "- / -" + if torrent_cm_dict and torrent_cm_dict["last_seeding_stats"]: + bytes_up_dwn = "%s / %s" %(size_format(torrent_cm_dict['last_seeding_stats']['total_up']), + size_format(torrent_cm_dict['last_seeding_stats']['total_down'])) + if torrent_cm_dict['last_seeding_stats']['total_down']: + ratio = '%f' % (float(torrent_cm_dict['last_seeding_stats']['total_up']) / float( + torrent_cm_dict['last_seeding_stats']['total_down'])) + else: + ratio = '%f' % 0 + else: + bytes_up_dwn = "- / -" + ratio = "-1" + + seeder_leecher = '%d / %d' %(ffile.num_seeders, ffile.num_leechers) + + init_data = [current_speed, bytes_up_dwn, seeder_leecher, '', ffile.infohash, + torrent_cm_dict.get('source', ''), ratio] + + data_new.append((ffile.infohash, init_data, ffile, CreditMiningListItem)) else: self.list.ShowMessage("No credit mining data available.") self.SetNrResults(0) - self.list.SetData(data) + self.list.SetData(data_new) @warnWxThread def RefreshData(self, key, data): List.RefreshData(self, key, data) data = (data.infohash, ['-', '-', '%d / %d' % (data.num_seeders, data.num_leechers), '', data.infohash, - self.boosting_manager.torrents.get(data.infohash, None).get('source', ''),'-1'], data) + self.boosting_manager.torrents.get(data.infohash, None).get('source', ''), '-1'], data) self.list.RefreshData(key, data) def SetNrResults(self, nr): @@ -2269,7 +2306,7 @@ def SetNrResults(self, nr): actitem.Highlight() self.initnumitems = True - def OnFilter(self,keyword): + def OnFilter(self, keyword): pass def MatchFilter(self, item): @@ -2295,9 +2332,9 @@ def GotFilter(self, keyword=None): def GetFilterMessage(self, empty=False): if empty: - return 'Empty','No credit mining torrent to show' + return 'Empty', 'No credit mining torrent to show' else: - return None,'end of list' + return None, 'end of list' def MatchFFilter(self, item): return True @@ -2517,8 +2554,11 @@ def __SetData(self): data_list = [(1, ['Home'], None, ActivityListItem), (2, ['Results'], None, ActivityListItem), (3, ['Channels'], None, ActivityListItem), - (4, ['Downloads'], None, ActivityListItem), - (5, ['Credit Mining'], None, ActivityListItem)] + (4, ['Downloads'], None, ActivityListItem)] + + if self.utility.session.get_creditmining_enable(): + data_list.append((5, ['Credit Mining'], None, ActivityListItem)) + if sys.platform != 'darwin': data_list.append((6, ['Videoplayer'], None, ActivityListItem)) @@ -2684,10 +2724,10 @@ def selectTab(self, tab): itemKey = 3 elif tab == 'my_files': itemKey = 4 - elif tab == 'videoplayer': - itemKey = 6 elif tab == 'creditmining': itemKey = 5 + elif tab == 'videoplayer': + itemKey = 6 if itemKey: wx.CallAfter(self.Select, itemKey, True) return diff --git a/Tribler/Main/vwxGUI/settingsDialog.py b/Tribler/Main/vwxGUI/settingsDialog.py index 3870e499fbc..5cf34a6be8e 100644 --- a/Tribler/Main/vwxGUI/settingsDialog.py +++ b/Tribler/Main/vwxGUI/settingsDialog.py @@ -65,7 +65,7 @@ def add_label(parent, sizer, label): class SettingsDialog(wx.Dialog): def __init__(self): - super(SettingsDialog, self).__init__(None, size=(600, 600), + super(SettingsDialog, self).__init__(None, size=(600, 700), title="Settings", name="settingsDialog", style=wx.DEFAULT_DIALOG_STYLE) self.SetExtraStyle(self.GetExtraStyle() | wx.WS_EX_VALIDATE_RECURSIVELY) self._logger = logging.getLogger(self.__class__.__name__) @@ -318,6 +318,12 @@ def saveAll(self, event, skip_restart_dialog=False): scfg.set_enable_multichain(use_multichain) restart = True + # Credit Mining + use_boosting = self._use_boosting.IsChecked() + if use_boosting != self.utility.session.get_creditmining_enable(): + scfg.set_creditmining_enable(use_boosting) + restart = True + scfg.save(cfgfilename) self.utility.flush_config() @@ -737,6 +743,13 @@ def __create_s5(self, tree_root, sizer): exp_panel, label="Tribler connects to Emercoin over its JSON-RPC API.\nThis requires you to enable it by editing the emercoin.conf file and setting\nserver=1, rpcport, rpcuser, rpcpassword, and rpcconnect.") exp_vsizer.Add(exp_s2_faq_text, 0, wx.EXPAND | wx.TOP, 10) + exp_s3_sizer = create_subsection(exp_panel, exp_vsizer, "Credit Mining", 1, 3) + boosting_text = wx.StaticText(exp_panel, -1, 'Credit Mining is a mechanism to boost your ratio by ' + '\nautomatically download and upload data.') + exp_s3_sizer.Add(boosting_text, 0, wx.EXPAND | wx.TOP, 5) + self._use_boosting = wx.CheckBox(exp_panel, label="Enable credit mining") + exp_s3_sizer.Add(self._use_boosting, 0, wx.EXPAND) + # load values self._use_webui.SetValue(self.utility.read_config('use_webui')) self._webui_port.SetValue(str(self.utility.read_config('webui_port'))) @@ -747,6 +760,8 @@ def __create_s5(self, tree_root, sizer): self._emc_username.SetValue(self.utility.read_config('emc_username')) self._emc_password.SetValue(self.utility.read_config('emc_password')) + self._use_boosting.SetValue(self.utility.session.get_creditmining_enable()) + return exp_panel, item_id def __create_s6(self, tree_root, sizer): From a33b4c763837011e4ed41a889577893df7ed0dbf Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Tue, 28 Jun 2016 00:14:55 +0200 Subject: [PATCH 17/24] Clean onexit in BoostingManager --- Tribler/Core/Libtorrent/LibtorrentMgr.py | 7 +- Tribler/Policies/BoostingManager.py | 1039 +++++++--------------- 2 files changed, 333 insertions(+), 713 deletions(-) diff --git a/Tribler/Core/Libtorrent/LibtorrentMgr.py b/Tribler/Core/Libtorrent/LibtorrentMgr.py index 0199ac1d9b5..a0d162d29b6 100644 --- a/Tribler/Core/Libtorrent/LibtorrentMgr.py +++ b/Tribler/Core/Libtorrent/LibtorrentMgr.py @@ -322,12 +322,7 @@ def process_alert(self, alert): else: self._logger.debug("LibtorrentMgr: could not find torrent %s", infohash) else: - self._logger.debug("LibtorrentMgr: alert for invalid torrent") - - def get_peers(self, infohash, callback, timeout=30): - def on_metainfo_retrieved(metainfo, infohash=infohash, callback=callback): - callback(infohash, metainfo.get('initial peers', [])) - self.get_metainfo(infohash, on_metainfo_retrieved, timeout, notify=False) + self._logger.debug("Alert for invalid torrent") def get_metainfo(self, infohash_or_magnet, callback, timeout=30, timeout_callback=None, notify=True): if not self.is_dht_ready() and timeout > 5: diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index d89490aa20f..925340f415c 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -1,21 +1,12 @@ # -*- coding: utf-8 -*- -# Written by Egbert Bouman, Mihai Capotă, Elric Milon -# pylint: disable=too-few-public-methods, too-many-instance-attributes -# pylint: disable=too-many-arguments, too-many-branches +# Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H """Manage boosting of swarms""" - import ConfigParser -import HTMLParser -import glob import json import logging import os import random -import time -import urllib from binascii import hexlify, unhexlify -from collections import defaultdict -from hashlib import sha1 import libtorrent as lt @@ -26,75 +17,70 @@ from Tribler.Core.DownloadConfig import DownloadStartupConfig from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl -from Tribler.Core.TorrentChecker import torrent_checker -from Tribler.Core.TorrentChecker.session import MAX_TRACKER_MULTI_SCRAPE -from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo from Tribler.Core.Utilities import utilities -from Tribler.Core.exceptions import TriblerException -from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_INSERT, NTFY_SCRAPE, NTFY_TORRENTS, NTFY_UPDATE -from Tribler.Core.version import version_id -from Tribler.Main.Utility.GuiDBTuples import Torrent -from Tribler.Main.Utility.GuiDBHandler import startWorker +from Tribler.Core.Utilities.install_dir import determine_install_dir +from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_TORRENTS, NTFY_UPDATE, NTFY_CHANNELCAST from Tribler.Main.globals import DefaultDownloadStartupConfig -from Tribler.Main.vwxGUI.GuiUtility import GUIUtility -from Tribler.Main.vwxGUI.SearchGridManager import ChannelManager -from Tribler.Utilities.scraper import scrape_tcp, scrape_udp -from Tribler.community.allchannel.community import AllChannelCommunity -from Tribler.community.channel.community import ChannelCommunity -from Tribler.dispersy.util import call_on_reactor_thread - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -formatter = logging.Formatter( - "%(asctime)s.%(msecs).03dZ-%(levelname)s-%(message)s", - datefmt="%Y%m%dT%H%M%S") -formatter.converter = time.gmtime -handler = logging.FileHandler("boosting.log", mode="w") -handler.setFormatter(formatter) -logger.addHandler(handler) - -# logging.getLogger(TimedTaskQueue.__name__+"BoostingManager").setLevel( -# logging.DEBUG) - -number_types = (int, long, float) +from Tribler.Policies.BoostingSource import ChannelSource +from Tribler.Policies.BoostingSource import DirectorySource +from Tribler.Policies.BoostingSource import RSSFeedSource +from Tribler.dispersy.taskmanager import TaskManager -# CONFIG_FILE = "boosting.ini" +NUMBER_TYPES = (int, long, float) -from Tribler.Core.Utilities.install_dir import determine_install_dir +# CONFIG_FILE = "boosting.ini" TRIBLER_ROOT = determine_install_dir() CONFIG_FILE = os.path.join(TRIBLER_ROOT, "boosting.ini") -def lev(a, b): - "Calculates the Levenshtein distance between a and b." - n, m = len(a), len(b) - if n > m: - # Make sure n <= m, to use O(min(n,m)) space - a, b = b, a - n, m = m, n - - current = range(n + 1) - for i in range(1, m + 1): - previous, current = current, [i] + [0] * n - for j in range(1, n + 1): + +def levenshtein_dist(t1_fname, t2_fname): + """ + Calculates the Levenshtein distance between a and b. + """ + len_t1_fname, len_t2_fname = len(t1_fname), len(t2_fname) + if len_t1_fname > len_t2_fname: + # Make sure len_t1_fname <= len_t2_fname, to use O(min(len_t1_fname,len_t2_fname)) space + t1_fname, t2_fname = t2_fname, t1_fname + len_t1_fname, len_t2_fname = len_t2_fname, len_t1_fname + + current = range(len_t1_fname + 1) + for i in range(1, len_t2_fname + 1): + previous, current = current, [i] + [0] * len_t1_fname + for j in range(1, len_t1_fname + 1): add, delete = previous[j] + 1, current[j - 1] + 1 change = previous[j - 1] - if a[j - 1] != b[i - 1]: - change = change + 1 + if t1_fname[j - 1] != t2_fname[i - 1]: + change += 1 current[j] = min(add, delete, change) - return current[n] + return current[len_t1_fname] +def source_to_string(source_obj): + return hexlify(source_obj) if len(source_obj) == 20 and not (source_obj.startswith('http://') + or source_obj.startswith('https://')) else source_obj + +def string_to_source(source_str): + return source_str.decode('hex') \ + if len(source_str) == 40 and not (os.path.isdir(source_str) or source_str.startswith('http://')) else source_str class BoostingPolicy(object): + """ + Base class for determining what swarm selection policy will be applied + """ def __init__(self, session): self.session = session self.key = lambda x: None # function that checks if key can be applied to torrent - self.key_check = lambda x: None + self.key_check = lambda x: False self.reverse = None - def apply(self, torrents, max_active): + self._logger = logging.getLogger(self.__class__.__name__) + + def apply(self, torrents, max_active, force=False): + """ + apply the policy to the torrents stored + """ sorted_torrents = sorted([torrent for torrent in torrents.itervalues() if self.key_check(torrent)], key=self.key, reverse=self.reverse) @@ -108,21 +94,26 @@ def apply(self, torrents, max_active): if self.session.get_download(torrent["metainfo"].get_infohash()): torrents_stop.append(torrent) + if force: + return torrents_start, torrents_stop + # if both results are empty for some reason (e.g, key_check too restrictive) # or torrent started less than half available torrent (try to keep boosting alive) # if it's already random, just let it be if not isinstance(self, RandomPolicy) and ((not torrents_start and not torrents_stop) or - (len(torrents_start) < len(torrents)/2 and len(torrents_start) < max_active/2)): - logger.error("Start and stop torrent list are empty. Fallback to Random") + (len(torrents_start) < len(torrents) / 2 and len( + torrents_start) < max_active / 2)): + self._logger.error("Start and stop torrent list are empty. Fallback to Random") # fallback to random policy - rp = RandomPolicy(self.session) - torrents_start, torrents_stop = rp.apply(torrents, max_active) + torrents_start, torrents_stop = RandomPolicy(self.session).apply(torrents, max_active) return torrents_start, torrents_stop class RandomPolicy(BoostingPolicy): - + """ + A credit mining policy that chooses swarm randomly + """ def __init__(self, session): BoostingPolicy.__init__(self, session) self.key = lambda v: random.random() @@ -131,7 +122,11 @@ def __init__(self, session): class CreationDatePolicy(BoostingPolicy): + """ + A credit mining policy that chooses swarm by its creation date + The idea is, older swarm need to be boosted. + """ def __init__(self, session): BoostingPolicy.__init__(self, session) self.key = lambda v: v['creation_date'] @@ -140,31 +135,57 @@ def __init__(self, session): class SeederRatioPolicy(BoostingPolicy): - + """ + Default policy. Find the most underseeded swarm to boost. + """ def __init__(self, session): BoostingPolicy.__init__(self, session) self.key = lambda v: v['num_seeders'] / float(v['num_seeders'] + v['num_leechers']) - self.key_check = lambda v: isinstance(v['num_seeders'], number_types) and isinstance(v['num_leechers'], number_types) and v['num_seeders'] + v['num_leechers'] > 0 + self.key_check = lambda v: isinstance(v['num_seeders'], NUMBER_TYPES) and isinstance( + v['num_leechers'], NUMBER_TYPES) and v['num_seeders'] + v['num_leechers'] > 0 self.reverse = False +class BoostingSettings(object): + """ + Class contains settings on boosting manager + """ + def __init__(self, session, policy=SeederRatioPolicy): + self.session = session + + self.config_file = CONFIG_FILE + self.max_torrents_active = 30 + self.max_torrents_per_source = 100 + self.source_interval = 20 + self.swarm_interval = 20 + self.initial_swarm_interval = 30 + self.policy = policy(session) + self.tracker_interval = 50 + self.initial_tracker_interval = 25 + self.logging_interval = 40 + self.initial_logging_interval = 20 + + self.min_connection_start = 5 + self.min_channels_start = 100 + + self.share_mode_target = 2 + self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), + "credit_mining") + + class BoostingManager(TaskManager): + """ + Class to manage all the credit mining activities + """ __single = None - def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval=20, sw_interval=20, - max_per_source=100, max_active=2, config_file=CONFIG_FILE): + def __init__(self, session, settings=None): super(BoostingManager, self).__init__() - self._logger = logging.getLogger(self.__class__.__name__) BoostingManager.__single = self - self.gui_util = GUIUtility.getInstance(utility) - if not self.gui_util.registered: - self.gui_util.register() - - self.config_file = config_file self._saved_attributes = ["max_torrents_per_source", "max_torrents_active", "source_interval", @@ -172,63 +193,76 @@ def __init__(self, session, utility=None, policy=SeederRatioPolicy, src_interval "tracker_interval", "logging_interval"] self.session = session - self.utility = utility - self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), "credit_mining") + assert self.session.get_libtorrent() - if not os.path.exists(self.credit_mining_path): - os.mkdir(self.credit_mining_path) + self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) + self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) + + # use provided settings or a default one + self.settings = settings or BoostingSettings(session) + + if not os.path.exists(self.settings.credit_mining_path): + os.makedirs(self.settings.credit_mining_path) self.boosting_sources = {} self.torrents = {} - self.policy = None - self.share_mode_target = 2 - - # change some params here, if you want - self.max_torrents_per_source = max_per_source - self.max_torrents_active = max_active - self.source_interval = src_interval - self.swarm_interval = sw_interval - self.initial_swarm_interval = 30 - self.policy = policy(self.session) - self.tracker_interval = 50 - self.initial_tracker_interval = 25 - self.logging_interval = 40 - self.initial_logging_interval = 20 - self.set_share_mode_params(share_mode_target=self.share_mode_target) + local_settings = {} + local_settings['share_mode_target'] = self.settings.share_mode_target + self.session.lm.ltmgr.get_session().set_settings(local_settings) - if os.path.exists(config_file): - logger.info("Config file %s", open(config_file).read()) + if os.path.exists(self.settings.config_file): + self._logger.info("Config file %s", open(self.settings.config_file).read()) self.load_config() else: - logger.warning("Initial config file missing") - - self.session.add_observer(self.OnTorrentNotify, NTFY_TORRENTS, [NTFY_UPDATE]) + self._logger.warning("Initial config file missing") + self.session.add_observer(self.on_torrent_notify, NTFY_TORRENTS, [NTFY_UPDATE]) # TODO(emilon): Refactor this to use taskmanager - self.session.lm.threadpool.add_task(self._select_torrent, self.initial_swarm_interval) + self.session.lm.threadpool.add_task(self._select_torrent, self.settings.initial_swarm_interval, + "CreditMining_select_init") self.session.lm.threadpool.add_task(self.scrape_trackers, - self.initial_tracker_interval) + self.settings.initial_tracker_interval, "CreditMining_scrape") self.session.lm.threadpool.add_task(self.log_statistics, - self.initial_logging_interval) + self.settings.initial_logging_interval, "CreditMining_log_init") + @staticmethod def get_instance(*args, **kw): + """ + get single instance of Boostingmanager + """ if BoostingManager.__single is None: BoostingManager(*args, **kw) return BoostingManager.__single - get_instance = staticmethod(get_instance) + @staticmethod def del_instance(): + """ + resetting, then deleting single instance + """ BoostingManager.__single = None - del_instance = staticmethod(del_instance) def shutdown(self): - # save configuration before stopping stuffs + """ + save configuration before stopping stuffs + """ self.save_config() + self._logger.info("Shutting down boostingmanager") - for torrent in self.torrents.itervalues(): - self.stop_download(torrent) + for sourcekey in self.boosting_sources.keys(): + self.remove_source(sourcekey) + + if self.session.lm.threadpool.is_pending_task_active("CreditMining_select_init"): + self.session.lm.threadpool.cancel_pending_task("CreditMining_select_init") + if self.session.lm.threadpool.is_pending_task_active("CreditMining_scrape"): + self.session.lm.threadpool.cancel_pending_task("CreditMining_scrape") + if self.session.lm.threadpool.is_pending_task_active("CreditMining_log_init"): + self.session.lm.threadpool.cancel_pending_task("CreditMining_log_init") + + self.cancel_all_pending_tasks() + # for torrent in self.torrents.itervalues(): + # self.stop_download(torrent) def get_source_object(self, sourcekey): return self.boosting_sources.get(sourcekey, None) @@ -236,69 +270,28 @@ def get_source_object(self, sourcekey): def set_enable_mining(self, source, mining_bool=True, force_restart=False): """ Dynamically enable/disable mining source. - :param source: source, perhaps a url, byte-channelid, or directory - :param mining_bool: enable/disable - :param force_restart: do we really need to restart the mining? """ - - # Flag : there are not any swarm stored for this source - tor_not_exist = True - for ihash, tor in self.torrents.iteritems(): - if tor['source'] == source: - tor_not_exist = False + if tor['source'] == source_to_string(source): self.torrents[ihash]['enabled'] = mining_bool # pause torrent download from disabled source - if (not mining_bool): + if not mining_bool: self.stop_download(tor) - # this only happen via new channel boosting interface. (CreditMiningPanel) - # case : just start mining a particular source (e.g. PreviewChannel) - if tor_not_exist and mining_bool and not (source in self.boosting_sources.keys()): - self.add_source(source) - self.set_archive(source, False) - self.set_enable_mining(source, mining_bool) - - string_to_source = lambda s: s.decode('hex') if len(s) == 40 and not (os.path.isdir(s) or s.startswith('http://')) else s self.boosting_sources[string_to_source(source)].enabled = mining_bool - logger.info("Set mining source %s %s", source, mining_bool) + self._logger.info("Set mining source %s %s", source, mining_bool) if force_restart: self._select_torrent() - def save(self): - if self.utility: - try: - source_to_string = lambda s: s.encode('hex') if len(s) == 20 and not (os.path.isdir(s) or s.startswith('http://')) else s - self.utility.write_config( - 'boosting_sources', - json.dumps([source_to_string(source) for - source in self.boosting_sources.keys()]), - flush=True) - logger.info("Saved sources %s", self.boosting_sources.keys()) - except: - logger.exception("Could not save state") - - def set_share_mode_params(self, share_mode_target=None, share_mode_bandwidth=None, share_mode_download=None, share_mode_seeders=None): - - # make set_settings call consistent - settings = {} - if share_mode_target is not None: - settings['share_mode_target'] = share_mode_target - if share_mode_bandwidth is not None: - settings['share_mode_bandwidth'] = share_mode_bandwidth - if share_mode_download is not None: - settings['share_mode_download'] = share_mode_download - if share_mode_seeders is not None: - settings['share_mode_seeders'] = share_mode_seeders - self.session.lm.ltmgr.get_session().set_settings(settings) - def add_source(self, source): + """ + add new source into the boosting manager + """ if source not in self.boosting_sources: - args = (self.session, self.session.lm.threadpool, source, self.source_interval, self.max_torrents_per_source, self.on_torrent_insert) - # pylint: disable=star-args + args = (self.session, source, self.settings, self.on_torrent_insert) try: isdir = os.path.isdir(source) @@ -312,93 +305,67 @@ def add_source(self, source): elif len(source) == 20: self.boosting_sources[source] = ChannelSource(*args) else: - logger.error("Cannot add unknown source %s", source) + self._logger.error("Cannot add unknown source %s", source) return - logger.info("Added source %s", source) + self._logger.info("Added source %s", source) else: - logger.info("Already have source %s", source) + self._logger.info("Already have source %s", source) def remove_source(self, source_key): + """ + remove source by stop the downloading and remove its metainfo for all its swarms + """ if source_key in self.boosting_sources: source = self.boosting_sources.pop(source_key) source.kill_tasks() - logger.info("Removed source %s", source_key) + self._logger.info("Removed source %s", source_key) - rm_torrents = [torrent for _, torrent in self.torrents.items() if torrent['source'] == source_key] - map(self.stop_download,rm_torrents) - logger.info("Torrents download stopped") + rm_torrents = [torrent for _, torrent in self.torrents.items() + if torrent['source'] == source_to_string(source_key)] - map(lambda x:self.torrents.pop(x["metainfo"].get_infohash(), None), rm_torrents) - logger.info("Removing from possible swarms") + for torrent in rm_torrents: + self.stop_download(torrent) + self.torrents.pop(torrent["metainfo"].get_infohash(), None) + + self._logger.info("Torrents download stopped and removed") - def compare_torrents(self, t1, t2): + def on_torrent_insert(self, source, infohash, torrent): """ - comparing swarms. We don't want to download same swarm with different infohash - :return: whether those t1 and t2 similar enough + This function called when a source is finally determined. Fetch some torrents from it, + then insert it to our data """ - # pylint: disable=no-self-use, bad-builtin - try: - ff = lambda ft: ft[1] > 1024 * 1024 - files1 = filter(ff, t1['metainfo'].get_files_with_length()) - files2 = filter(ff, t2['metainfo'].get_files_with_length()) + def compare_torrents(torrent_1, torrent_2): + """ + comparing swarms. We don't want to download same swarm with different infohash + :return: whether those t1 and t2 similar enough + """ + files1 = [files for files in torrent_1['metainfo'].get_files_with_length() if files[1] > 1024 * 1024] + files2 = [files for files in torrent_2['metainfo'].get_files_with_length() if files[1] > 1024 * 1024] if len(files1) == len(files2): for ft1 in files1: for ft2 in files2: - if ft1[1] != ft2[1] or lev(ft1[0], ft2[0]) > 5: + if ft1[1] != ft2[1] or levenshtein_dist(ft1[0], ft2[0]) > 5: return False return True return False - except: - return False - - def on_torrent_insert(self, source, infohash, torrent): - """ - This function called when a source finally determined. Fetch some torrents from it, - then insert it to our data - :param source: - :param infohash: torrent infohash - :param torrent: torrent object (dictionary format) - :return: - """ - try: - isdir = os.path.isdir(source) - except TypeError: - isdir = False - - if isdir or source.startswith('http://') or source.startswith('https://'): - source_str = source - elif len(source) == 20: - source_str = source.encode('hex') - else: - source_str = 'unknown source' # Remember where we got this torrent from - torrent['source'] = source_str + self._logger.debug("remember torrent %s from %s", torrent, source_to_string(source)) + + torrent['source'] = source_to_string(source) boost_source = self.boosting_sources.get(source, None) if not boost_source: - self._logger.info("Dropping torrent insert from removed source: %s" % repr(torrent)) + self._logger.info("Dropping torrent insert from removed source: %s", repr(torrent)) return elif boost_source.archive: torrent['preload'] = True torrent['prio'] = 100 - # Preload the TorrentDef. - if not isinstance(torrent.get('metainfo', None), TorrentDef): - torrent_data = self.session.lm.torrent_store.get(infohash) - if torrent_data: - torrent['metainfo'] = TorrentDef.load_from_memory(torrent_data) - else: - self._logger.info("Not collected yet: %s %s ", infohash, torrent['name']) - # TODO(emilon): Handle the case where the torrent hasn't been collected. (collected from the DHT) - # ardhi : so far, this case won't happen because torrent already defined in _update in BoostingSource - # torrent['metainfo'] = TorrentDefNoMetainfo(infohash, torrent['name']) - pass - # If duplicates exist, set is_duplicate to True, except for the one with the most seeders. - duplicates = [other for other in self.torrents.values() if self.compare_torrents(torrent, other)] + duplicates = [other for other in self.torrents.values() if compare_torrents(torrent, other)] if duplicates: duplicates += [torrent] healthiest_torrent = max([(torrent['num_seeders'], torrent) for torrent in duplicates])[1] @@ -410,116 +377,113 @@ def on_torrent_insert(self, source, infohash, torrent): self.torrents[infohash] = torrent - def OnTorrentNotify(self, subject, change_type, infohash): + def on_torrent_notify(self, subject, change_type, infohash): + """ + Notify us when we have new seeder/leecher value in torrent from tracker + """ if infohash not in self.torrents: return - logger.debug("infohash %s updated", hexlify(infohash)) - - def do_gui(delayedResult): - torrent_obj = delayedResult.get() - infohash_str = torrent_obj.infohash_as_hex + self._logger.debug("infohash %s %s %s updated", subject, change_type, hexlify(infohash)) - new_seed = torrent_obj.swarminfo[0] - new_leecher = torrent_obj.swarminfo[1] + tdict = self.torrent_db.getTorrent(infohash, keys=['C.torrent_id', 'infohash', 'name', + 'length', 'category', 'status', 'num_seeders', + 'num_leechers']) - if new_seed - self.torrents[torrent_obj.infohash]['num_seeders'] \ - or new_leecher - self.torrents[torrent_obj.infohash]['num_leechers']: - self.torrents[torrent_obj.infohash]['num_seeders'] = new_seed - self.torrents[torrent_obj.infohash]['num_leechers'] = new_leecher - logger.info("infohash %s changed s:%d l:%d", infohash_str, torrent_obj.swarminfo[0],torrent_obj.swarminfo[1]) + if tdict: + infohash_str = hexlify(tdict['infohash']) - startWorker(do_gui, self.gui_util.torrentsearch_manager.getTorrentByInfohash, wargs=(infohash,)) + new_seed = tdict['num_seeders'] + new_leecher = tdict['num_leechers'] + if new_seed - self.torrents[tdict['infohash']]['num_seeders'] \ + or new_leecher - self.torrents[tdict['infohash']]['num_leechers']: + self.torrents[tdict['infohash']]['num_seeders'] = new_seed + self.torrents[tdict['infohash']]['num_leechers'] = new_leecher + self._logger.info("infohash %s : seeder/leecher changed seed:%d leech:%d", + infohash_str, new_seed, new_leecher) def scrape_trackers(self): + """ + Manually scrape tracker by requesting to tracker manager + """ - for infohash, torrent in self.torrents.iteritems(): - tf = torrent['metainfo'] - + for infohash, _ in self.torrents.iteritems(): # torrent handle lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(lt.big_number(infohash)) + peer_list = [] + for i in lt_torrent.get_peer_info(): + peer = LibtorrentDownloadImpl.create_peerlist_data(i) + peer_list.append(peer) + + num_seed, num_leech = utilities.translate_peers_into_health(peer_list) + self._logger.debug("Seeder/leecher data translated from peers : seeder %s, leecher %s", num_seed, num_leech) + # check health(seeder/leecher) - self.session.lm.torrent_checker.add_gui_request(infohash,True) - - # if lt_torrent.is_valid() \ - # and unhexlify(str(lt_torrent.status().info_hash)) in self.torrents: - # status = lt_torrent.status() - # - # t = self.torrents[unhexlify(str(status.info_hash))] - # - # peer_list = [] - # for i in lt_torrent.get_peer_info(): - # peer = LibtorrentDownloadImpl.create_peerlist_data(i) - # peer_list.append(peer) - # - # # already downloaded - # if 'download' in t: - # trackers = t['download'].network_tracker_status() - # - # # find if any tracker is working - # trackers_available = any([trackers[i][0] for i in trackers.keys() - # if i.startswith('udp') or i.startswith('http')]) - # - # # we only rely on DHT - # if not trackers_available: - # #TODO(ardhi) : put some DHT scraper-like - # # use DHT data to translate number of seeder/leecher - # # t['num_seeders'], t['num_leechers'] = utilities.translate_peers_into_health(peer_list, status) - # pass - # else: - # # scrape again? - # lt_torrent.scrape_tracker() - # - # t['num_seeders'], t['num_leechers'] = utilities.translate_peers_into_health(peer_list, status) - - self.session.lm.threadpool.add_task(self.scrape_trackers, self.tracker_interval) + self.session.lm.torrent_checker.add_gui_request(infohash, True) + + self.session.lm.threadpool.add_task(self.scrape_trackers, self.settings.tracker_interval, "CreditMining_scrape") def set_archive(self, source, enable): + """ + setting archive of a particular source. Affect all the torrents in this source + """ if source in self.boosting_sources: self.boosting_sources[source].archive = enable - logger.info("Set archive mode for %s to %s", source, enable) + self._logger.info("Set archive mode for %s to %s", source, enable) else: - logger.error("Could not set archive mode for unknown source %s", source) + self._logger.error("Could not set archive mode for unknown source %s", source) def start_download(self, torrent): """ Start downloading a particular torrent and add it to download list in Tribler - :param torrent: """ def do_start(): + """ + add the actual torrent to the manager to download it later + :return: + """ dscfg = DownloadStartupConfig() - dscfg.set_dest_dir(self.credit_mining_path) + dscfg.set_dest_dir(self.settings.credit_mining_path) dscfg.set_safe_seeding(False) - # just a debug variable - tobj = torrent - - preload = tobj.get('preload', False) - logger.info("Starting %s preload %s has pstate %s" , - hexlify(tobj["metainfo"].get_infohash()), - preload, True if tobj.get('pstate', None) else False) + preload = torrent.get('preload', False) # not using Session.start_download because we need to specify pstate - assert self.session.get_libtorrent() + if self.session.lm.download_exists(torrent["metainfo"].get_infohash()): + self._logger.error("Already downloading %s. Cancel start_download", + hexlify(torrent["metainfo"].get_infohash())) + return + + self._logger.info("Starting %s preload %s has pstate %s", + hexlify(torrent["metainfo"].get_infohash()), preload, + True if torrent.get('pstate', None) else False) - tobj['download'] = self.session.lm.add(tobj['metainfo'], dscfg, pstate=tobj.get('pstate', None), + torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=torrent.get('pstate', None), hidden=True, share_mode=not preload, checkpoint_disabled=True) - tobj['download'].set_priority(tobj.get('prio', 1)) + torrent['download'].set_priority(torrent.get('prio', 1)) self.session.lm.threadpool.add_task_in_thread(do_start, 0) def stop_download(self, torrent): + """ + Stopping torrent that currently downloading + """ + def do_stop(): + """ + The actual function to stop torrent downloading + :return: + """ ihash = lt.big_number(torrent["metainfo"].get_infohash()) - logger.info("Stopping %s", str(ihash)) + self._logger.info("Stopping %s", str(ihash)) download = torrent.pop('download', False) lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(ihash) if download and lt_torrent.is_valid(): - logger.debug("Writing resume data") + self._logger.debug("Writing resume data") torrent['pstate'] = {'engineresumedata': download.write_resume_data()} - self.session.remove_download(download) + self.session.remove_download(download, hidden=True) self.session.lm.threadpool.add_task_in_thread(do_stop, 0) @@ -540,61 +504,98 @@ def _select_torrent(self): if torrent.get('enabled', True): torrents[infohash] = torrent - if self.policy is not None and torrents: + if self.settings.policy is not None and torrents: # Determine which torrent to start and which to stop. - torrents_start, torrents_stop = self.policy.apply( - torrents, self.max_torrents_active) + torrents_start, torrents_stop = self.settings.policy.apply( + torrents, self.settings.max_torrents_active) for torrent in torrents_stop: self.stop_download(torrent) for torrent in torrents_start: self.start_download(torrent) - logger.info("Selecting from %s torrents %s start download", len(torrents), len(torrents_start)) + self._logger.info("Selecting from %s torrents %s start download", len(torrents), len(torrents_start)) - self.session.lm.threadpool.add_task(self._select_torrent, self.swarm_interval) + self.session.lm.threadpool.add_task(self._select_torrent, self.settings.swarm_interval) def load_config(self): + """ + load config in file configuration and apply it to manager + """ config = ConfigParser.ConfigParser() - config.read(self.config_file) - validate_source = lambda s: unhexlify(s) if len(s) == 40 and not s.startswith("http") else s - for k, v in config.items(__name__): - if k in self._saved_attributes: - object.__setattr__(self, k, int(v)) - elif k == "policy": - if v == "random": - self.policy = RandomPolicy(self.session) - elif v == "creation": - self.policy = CreationDatePolicy(self.session) - elif v == "seederratio": - self.policy = SeederRatioPolicy(self.session) - elif k == "boosting_sources": - for boosting_source in json.loads(v): - boosting_source = validate_source(boosting_source) + config.read(self.settings.config_file) + validate_source = lambda source_str: unhexlify(source_str) if len(source_str) == 40 and \ + not source_str.startswith("http") else source_str + + def _add_sources(value): + """ + adding sources in configuration file + """ + for boosting_source in json.loads(value): + boosting_source = validate_source(boosting_source) + self.add_source(boosting_source) + + def _archive_sources(value): + """ + setting archive to sources + """ + for archive_source in json.loads(value): + archive_source = validate_source(archive_source) + self.set_archive(archive_source, True) + + def _set_enable_boosting(value, enabled): + """ + set disable/enable source + """ + for boosting_source in json.loads(value): + boosting_source = validate_source(boosting_source) + if not self.boosting_sources[boosting_source]: self.add_source(boosting_source) - elif k == "archive_sources": - for archive_source in json.loads(v): - archive_source = validate_source(archive_source) - self.set_archive(archive_source, True) - elif k == "boosting_enabled": - for boosting_source in json.loads(v): - boosting_source = validate_source(boosting_source) - if not self.boosting_sources[boosting_source]: - self.add_source(boosting_source) - self.boosting_sources[boosting_source].enabled = True - elif k == "boosting_disabled": - for boosting_source in json.loads(v): - boosting_source = validate_source(boosting_source) - if not self.boosting_sources[boosting_source]: - self.add_source(boosting_source) - self.boosting_sources[boosting_source].enabled = False + self.boosting_sources[boosting_source].enabled = enabled + + switch = { + "boosting_sources": { + "cmd": _add_sources, + "args": (None,) + }, + "archive_sources": { + "cmd": _archive_sources, + "args": (None,) + }, + "boosting_enabled": { + "cmd": _set_enable_boosting, + "args": (None, True) + }, + "boosting_disabled": { + "cmd": _set_enable_boosting, + "args": (None, False) + }, + } + + switch_policy = { + "random" : RandomPolicy, + "creation" : CreationDatePolicy, + "seederratio" : SeederRatioPolicy + } + + for k, val in config.items(__name__): + try: + if k in self._saved_attributes: + object.__setattr__(self.settings, k, int(val)) + elif k == "policy": + self.settings.policy = switch_policy[val](self.session) + else: + switch[k]["cmd"](*((switch[k]['args'][0] or val,) + switch[k]['args'][1:])) + except KeyError: + self._logger.error("Key %s can't be applied", k) def save_config(self): + """ + save the environment parameters in config file + """ config = ConfigParser.ConfigParser() config.add_section(__name__) for k in self._saved_attributes: - config.set(__name__, k, BoostingManager.__getattribute__(self, k)) - - source_to_string = lambda s: hexlify(s) if len(s) == 20 and not (s.startswith('http://') or s.startswith('https://')) else s + config.set(__name__, k, BoostingSettings.__getattribute__(self.settings, k)) archive_sources = [] lboosting_sources = [] @@ -620,14 +621,15 @@ def save_config(self): if archive_sources: config.set(__name__, "archive_sources", json.dumps(archive_sources)) - if isinstance(self.policy, RandomPolicy): + policy = "None" + if isinstance(self.settings.policy, RandomPolicy): policy = "random" - elif isinstance(self.policy, CreationDatePolicy): + elif isinstance(self.settings.policy, CreationDatePolicy): policy = "creation" - elif isinstance(self.policy, SeederRatioPolicy): + elif isinstance(self.settings.policy, SeederRatioPolicy): policy = "seederratio" config.set(__name__, "policy", policy) - with open(self.config_file, "w") as configf: + with open(self.settings.config_file, "w") as configf: config.write(configf) def log_statistics(self): @@ -638,11 +640,9 @@ def log_statistics(self): status = lt_torrent.status() if unhexlify(str(status.info_hash)) in self.torrents: - t = self.torrents[unhexlify(str(status.info_hash))] - - logger.debug("Status for %s : %s %s | ul_lim : %d, max_ul %d, maxcon %d", status.info_hash, - status.all_time_download, - status.all_time_upload) + self._logger.debug("Status for %s : %s %s | ul_lim : %d, max_ul %d, maxcon %d", status.info_hash, + status.all_time_download, + status.all_time_upload) # piece_priorities will fail in libtorrent 1.0.9 if lt.version == '1.0.9.0': @@ -653,392 +653,17 @@ def log_statistics(self): if piece_priority != 0: non_zero_values.append(piece_priority) if non_zero_values: - logger.debug("Non zero priorities for %s : %s", - status.info_hash, non_zero_values) + self._logger.debug("Non zero priorities for %s : %s", status.info_hash, non_zero_values) - self.session.lm.threadpool.add_task(self.log_statistics, self.logging_interval) + self.session.lm.threadpool.add_task(self.log_statistics, self.settings.logging_interval) def update_torrent_stats(self, torrent_infohash_str, seeding_stats): + """ + function to update swarm statistics + """ if 'time_seeding' in self.torrents[torrent_infohash_str]['last_seeding_stats']: - if seeding_stats['time_seeding'] >= self.torrents[torrent_infohash_str]['last_seeding_stats']['time_seeding']: + if seeding_stats['time_seeding'] >= self.torrents[torrent_infohash_str][ + 'last_seeding_stats']['time_seeding']: self.torrents[torrent_infohash_str]['last_seeding_stats'] = seeding_stats else: self.torrents[torrent_infohash_str]['last_seeding_stats'] = seeding_stats - - -class BoostingSource(object): - - def __init__(self, session, tqueue, source, interval, max_torrents, callback): - self.session = session - self.session.lm.threadpool = tqueue - self.channelcast_db = session.lm.channelcast_db - - self.torrents = {} - self.source = source - self.interval = interval - self.max_torrents = max_torrents - self.callback = callback - self.archive = False - - self.enabled = True - - self.av_uprate = 0 - self.av_dwnrate = 0 - self.storage_used = 0 - self.ready = False - - self.gui_util = GUIUtility.getInstance() - if not self.gui_util.registered: - self.gui_util.register() - - self.boosting_manager = BoostingManager.get_instance() - - def kill_tasks(self): - self.session.lm.threadpool.cancel_pending_task(self.source) - - def _load_if_ready(self, source): - - nr_channels = self.channelcast_db.getNrChannels() - nr_connections = 0 - - for community in self.session.lm.dispersy.get_communities(): - from Tribler.community.search.community import SearchCommunity - if isinstance(community, SearchCommunity): - nr_connections = community.get_nr_connections() - - # condition example - if nr_channels > 100 and nr_connections > 5: - fun = self._load - else: - fun = self._load_if_ready - - self.session.lm.threadpool.add_task(lambda src=source: fun(src), 15, task_name=str(self.source)+"_load") - - def _load(self, source): - pass - - def _update(self): - pass - - def getSource(self): - return self.source - - -class ChannelSource(BoostingSource): - - def __init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback): - BoostingSource.__init__(self, session, tqueue, dispersy_cid, interval, max_torrents, callback) - - self.channel_id = None - - self.channel = None - self.community = None - self.database_updated = True - - self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) - self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load_if_ready(cid), 0, task_name=str(self.source)+"_load") - - self.unavail_torrent = {} - - def kill_tasks(self): - BoostingSource.kill_tasks(self) - self.session.remove_observer(self._on_database_updated) - - def _load(self, dispersy_cid): - dispersy = self.session.get_dispersy_instance() - - @call_on_reactor_thread - def join_community(): - try: - self.community = dispersy.get_community(dispersy_cid, True) - self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=str(self.source)+"_get_channel_id") - - except KeyError: - - allchannelcommunity = None - for community in dispersy.get_communities(): - if isinstance(community, AllChannelCommunity): - allchannelcommunity = community - break - - if allchannelcommunity: - # pylint: disable=protected-access - self.community = ChannelCommunity.init_community(dispersy, dispersy.get_member(mid=dispersy_cid), allchannelcommunity._my_member, True) - logger.info("Joined channel community %s", - dispersy_cid.encode("HEX")) - self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=str(self.source)+"_get_channel_id") - else: - logger.error("Could not find AllChannelCommunity") - - def get_channel_id(): - # pylint: disable=protected-access - if self.community and self.community._channel_id and self.gui_util.registered: - self.channel_id = self.community._channel_id - - self.channel = self.gui_util.channelsearch_manager.getChannel(self.channel_id) - - if not self.boosting_manager.is_pending_task_active(str(self.source)+"_update"): - self.boosting_manager.register_task(str(self.source)+"_update",LoopingCall(self._update)).start(self.interval, now=True) - # self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") - logger.info("Got channel id %s", self.channel_id) - else: - logger.warning("Could not get channel id, retrying in 10 s") - self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=str(self.source)+"_get_channel_id") - - try: - join_community() - self.ready = True - except Exception,ex: - logger.info("Channel %s was not ready, waits for next interval (%d chn)", hexlify(self.source), len(dispersy.get_communities())) - self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load_if_ready(cid), 0, task_name=str(self.source)+"_load") - - def _check_tor(self): - - def doGui(delayedResult): - # wait here - requesttype = delayedResult.get(timeout=70) - - def showTorrent(torrent): - if (torrent.files): - infohash = torrent.infohash - self.torrents[infohash] = {} - self.torrents[infohash]['name'] = torrent.name - self.torrents[infohash]['metainfo'] = torrent.tdef - self.torrents[infohash]['creation_date'] = torrent.creation_date - self.torrents[infohash]['length'] = torrent.tdef.get_length() - self.torrents[infohash]['num_files'] = len(torrent.files) - self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] or 0 - self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] or 0 - self.torrents[infohash]['enabled'] = self.enabled - - # seeding stats from DownloadState - self.torrents[infohash]['last_seeding_stats'] = {} - - del self.unavail_torrent[infohash] - - # logger.info("Torrent %s from %s ready to start", hexlify(infohash), hexlify(self.source)) - - if self.callback: - self.callback(self.source, infohash, self.torrents[infohash]) - self.database_updated = False - - logger.info("Unavailable #torrents : %d from %s", len(self.unavail_torrent), hexlify(self.source)) - - if len(self.unavail_torrent) and self.enabled: - for k,t in self.unavail_torrent.items(): - startWorker(doGui, self.gui_util.torrentsearch_manager.loadTorrent, - wargs=(t,), wkwargs={'callback': showTorrent}) - - # if not self.session.lm.threadpool.is_pending_task_active(hexlify(self.source)+"_checktor"): - # self.session.lm.threadpool.add_task(self._check_tor, 100, task_name=hexlify(self.source)+"_checktor") - - def _update(self): - if len(self.torrents) < self.max_torrents: - - if self.database_updated: - - CHANTOR_DB = ['ChannelTorrents.channel_id', 'Torrent.torrent_id', 'infohash', '""', 'length', 'category', 'status', 'num_seeders', 'num_leechers', 'ChannelTorrents.id', 'ChannelTorrents.dispersy_id', 'ChannelTorrents.name', 'Torrent.name', 'ChannelTorrents.description', 'ChannelTorrents.time_stamp', 'ChannelTorrents.inserted'] - - try: - torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, CHANTOR_DB, self.max_torrents) - - listtor = self.gui_util.channelsearch_manager._createTorrents(torrent_values, True, - {self.channel_id: self.channelcast_db.getChannel(self.channel_id)})[2] - - # dict {key_infohash(binary):Torrent(object-GUIDBTuple)} - self.unavail_torrent.update({t.infohash:t for t in listtor if t.infohash not in self.torrents}) - - # it's highly probable the checktor function is running at this time (if it's already running) - # if not running, start the checker - if not self.boosting_manager.is_pending_task_active(hexlify(self.source)+"_checktor"): - self.boosting_manager.register_task(hexlify(self.source)+"_checktor",LoopingCall(self._check_tor)).start(100, now=True) - - # self.session.lm.threadpool.add_task(self._check_tor, 0, task_name=hexlify(self.source)+"_checktor") - - except: - logger.info("Channel %s was not ready, waits for next interval", hexlify(self.source)) - - # self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") - - def _on_database_updated(self, subject, change_type, infohash): - if (subject, change_type, infohash) is None: - # Unused arguments - pass - self.database_updated = True - - def getSource(self): - return self.channel.name if self.channel else None - - -class RSSFeedSource(BoostingSource): - # supported list (tested) : - # http://bt.etree.org/rss/bt_etree_org.rdf - # http://www.mininova.org/rss.xml - # https://kat.cr (via torcache) - - # not supported (till now) - # https://eztv.ag/ezrss.xml (https link) - - def __init__(self, session, tqueue, rss_feed, interval, max_torrents, callback): - BoostingSource.__init__(self, session, tqueue, rss_feed, interval, max_torrents, callback) - - self.unescape = HTMLParser.HTMLParser().unescape - - self.feed_handle = None - - self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load_if_ready(feed), 0, task_name=str(self.source)+"_load") - - self.title = "" - self.description = "" - self.total_torrents = 0 - - def _load(self, rss_feed): - self.feed_handle = self.session.lm.ltmgr.get_session().add_feed({'url': rss_feed, 'auto_download': False, 'auto_map_handles': False}) - - def wait_for_feed(): - # Wait until the RSS feed is longer updating. - feed_status = self.feed_handle.get_feed_status() - if feed_status['updating']: - self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=str(self.source)+"_wait_for_feed") - elif len(feed_status['error']) > 0: - logger.error("Got error for RSS feed %s : %s", - feed_status['url'], feed_status['error']) - if "503" in feed_status["error"]: - time.sleep(5 * random.random()) - self.feed_handle.update_feed() - self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=str(self.source)+"_wait_for_feed") - else: - # The feed is done updating. Now periodically start retrieving torrents. - self.boosting_manager.register_task(str(self.source)+"_update",LoopingCall(self._update), - 10,interval=self.interval) - logger.info("Got RSS feed %s", feed_status['url']) - - wait_for_feed() - self.ready = True - - def _update(self): - if len(self.torrents) < self.max_torrents: - - feed_status = self.feed_handle.get_feed_status() - - self.title = feed_status['title'] - self.description = feed_status['description'] - - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', 'enabled', 'last_seeding_stats'] - - self.total_torrents = len(feed_status['items']) - - def __cb_body(body_bin, item): - tdef = None - try: - metainfo = lt.bdecode(body_bin) - tdef = TorrentDef.load_from_dict(metainfo) - tdef.save(torrent_filename) - except: - logger.error("Could not parse/save torrent, skipping %s", torrent_filename) - - if tdef: - # Create a torrent dict. - torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled, {}] - self.torrents[sha1(item['url']).digest()] = dict(zip(torrent_keys, torrent_values)) - - # manually generate an ID and put this into DB - self.session.lm.torrent_db.addOrGetTorrentID(sha1(item['url']).digest()) - self.session.lm.torrent_db.addExternalTorrent(tdef) - - # create Torrent object and store it - self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) - - # Notify the BoostingManager and provide the real infohash. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[sha1(item['url']).digest()]) - - def __success_cb(response, item): - defer = readBody(response) - defer.addCallback(__cb_body, item) - return defer - - for item in feed_status['items']: - # Not all RSS feeds provide us with the infohash, so we use a fake infohash based on the URL to identify the torrents. - infohash = sha1(item['url']).digest() - if infohash not in self.torrents: - # Store the torrents as rss-infohash_as_hex.torrent. - torrent_filename = os.path.join(self.boosting_manager.credit_mining_path, 'rss-%s.torrent' % infohash.encode('hex')) - tdef = None - if not os.path.exists(torrent_filename): - - #create Agent to download torrent file - agent = Agent(reactor) - ses_agent = agent.request( - 'GET', #http://stackoverflow.com/a/845595 - urllib.quote(item['url'],safe="%/:=&?~#+!$,;'@()*[]"), - Headers({'User-Agent': ['Tribler ' + version_id]}), - None) - ses_agent.addCallback(__success_cb, item) - - else: - # torrent already exist in our system - tdef = TorrentDef.load(torrent_filename) - - if tdef: - # Create a torrent dict. - torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled, {}] - self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) - - # manually generate an ID and put this into DB - self.session.lm.torrent_db.addOrGetTorrentID(infohash) - self.session.lm.torrent_db.addExternalTorrent(tdef) - - # create Torrent object and store it - self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) - - # Notify the BoostingManager and provide the real infohash. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) - - # self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") - - def kill_tasks(self): - BoostingSource.kill_tasks(self) - - #stop updating - self.session.lm.threadpool.cancel_pending_task(str(self.source)+"_update") - - -class DirectorySource(BoostingSource): - - def __init__(self, session, tqueue, directory, interval, max_torrents, callback): - BoostingSource.__init__(self, session, tqueue, directory, interval, max_torrents, callback) - self._load_if_ready(directory) - - def _load(self, directory): - if os.path.isdir(directory): - # Wait for __init__ to finish so the source is registered with the - # BoostinManager, otherwise adding torrents won't work - self.session.lm.threadpool.add_task(self._update, 1, task_name=str(self.source)+"_update") - logger.info("Got directory %s", directory) - self.ready = True - else: - logger.error("Could not find directory %s", directory) - - def _update(self): - if len(self.torrents) < self.max_torrents: - - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', 'enabled', 'last_seeding_stats'] - - for torrent_filename in glob.glob(self.source + '/*.torrent'): - if torrent_filename not in self.torrents: - try: - tdef = TorrentDef.load(torrent_filename) - except: - logger.error("Could not load torrent, skipping %s", - torrent_filename) - continue - # Create a torrent dict. - torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), len(tdef.get_files()), -1, -1, self.enabled, {}] - self.torrents[torrent_filename] = dict(zip(torrent_keys, torrent_values)) - # Notify the BoostingManager. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[torrent_filename]) - - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source)+"_update") From 6cc9d7d0971cee33e5123947242988351bb37946 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Tue, 3 May 2016 16:39:13 +0200 Subject: [PATCH 18/24] Add and clean documentation in some functions needed This commit fixes: - some english grammar error in documentation - move etree restriction in torrent_checker - fixes parameter needed in credit mining - Add etree as exception in torrent_checker, add param to force scrape --- .../Core/APIImplementation/LaunchManyCore.py | 3 +- .../Core/Libtorrent/LibtorrentDownloadImpl.py | 50 ++++++++----------- Tribler/Core/TorrentChecker/session.py | 6 ++- .../Core/TorrentChecker/torrent_checker.py | 6 +-- Tribler/Core/Utilities/utilities.py | 43 +++++++--------- Tribler/Utilities/scraper.py | 50 ------------------- 6 files changed, 49 insertions(+), 109 deletions(-) delete mode 100644 Tribler/Utilities/scraper.py diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index 2c657154ce7..69ab30a41e5 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -319,7 +319,8 @@ def load_communities(): self.initComplete = True - def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidden=False, share_mode=False,checkpoint_disabled=False): + def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidden=False, + share_mode=False, checkpoint_disabled=False): """ Called by any thread """ d = None with self.sesslock: diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index 1ad44dad56e..f2cbedaf250 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -152,7 +152,8 @@ def __init__(self, session, tdef): self.deferreds_resume = [] def __str__(self): - return "LibtorrentDownloadImpl " % (self.correctedinfoname, self.get_hops(), self.checkpoint_disabled) + return "LibtorrentDownloadImpl " % \ + (self.correctedinfoname, self.get_hops(), self.checkpoint_disabled) def __repr__(self): return self.__str__() @@ -160,13 +161,13 @@ def __repr__(self): def get_def(self): return self.tdef - def set_checkpoint_disabled(self, val=True): - self.checkpoint_disabled = val + def set_checkpoint_disabled(self, disabled=True): + self.checkpoint_disabled = disabled def get_checkpoint_disabled(self): return self.checkpoint_disabled - def setup(self, dcfg=None, pstate=None, initialdlstatus=None, lm_network_engine_wrapper_created_callback=None, + def setup(self, dcfg=None, pstate=None, initialdlstatus=None, wrapperDelay=0, share_mode=False, checkpoint_disabled=False): """ Create a Download object. Used internally by Session. @@ -202,7 +203,8 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None, lm_network_engine_ def schedule_create_engine(): self.cew_scheduled = True - create_engine_wrapper_deferred = self.network_create_engine_wrapper(self.pstate_for_restart, initialdlstatus, share_mode=share_mode) + create_engine_wrapper_deferred = self.network_create_engine_wrapper( + self.pstate_for_restart, initialdlstatus, share_mode=share_mode) create_engine_wrapper_deferred.chainDeferred(deferred) @@ -254,7 +256,7 @@ def do_check(): do_check() return can_create_deferred - def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode=False,checkpoint_disabled=False): + def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode=False, checkpoint_disabled=False): with self.dllock: self._logger.debug("LibtorrentDownloadImpl: network_create_engine_wrapper()") @@ -295,6 +297,7 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode atp["ti"] = torrentinfo has_resume_data = resume_data and isinstance(resume_data, dict) + # TODO(ardhi) : properly store pstate in BoostingManager if has_resume_data is not None: # we have resume data but somehow its not working (Credit mining case) if not isinstance(resume_data, dict): @@ -305,9 +308,6 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode if has_resume_data: atp["resume_data"] = lt.bencode(resume_data) - if not share_mode: - self._logger.info("%s %s", self.tdef.get_name_as_unicode(), dict((k, v) - for k, v in resume_data.iteritems() if k not in ['pieces', 'piece_priority', 'peers']) if has_resume_data else None) else: atp["url"] = self.tdef.get_url() or "magnet:?xt=urn:btih:" + hexlify(self.tdef.get_infohash()) atp["name"] = self.tdef.get_name_as_unicode() @@ -315,14 +315,8 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode self.handle = self.ltmgr.add_torrent(self, atp) if self.handle: - self.set_selected_files() - - # set_selected_files sets priorities to 1, so we must set - # share_mode again, but first we must unset it, otherwise - # set_share_mode doesn't do anything - if share_mode: - self.handle.set_share_mode(not share_mode) - self.handle.set_share_mode(share_mode) + # if in share mode, don't change priority of the file + self.set_selected_files(share_mode=share_mode) # If we lost resume_data always resume download in order to force checking if initialdlstatus != DLSTATUS_STOPPED or not resume_data: @@ -664,7 +658,7 @@ def set_corrected_infoname(self): self.correctedinfoname = self.get_corrected_filename() @checkHandleAndSynchronize() - def set_selected_files(self, selected_files=None): + def set_selected_files(self, selected_files=None, share_mode=False): if not isinstance(self.tdef, TorrentDefNoMetainfo): if selected_files is None: @@ -691,7 +685,7 @@ def set_selected_files(self, selected_files=None): torrent_storage = get_info_from_handle(self.handle).files() - # TODO(ardhi) : as from 1.0, files returning file_storage (lazy-iterable) + # as from libtorrent 1.0, files returning file_storage (lazy-iterable) if hasattr(lt, 'file_storage') and isinstance(torrent_storage, lt.file_storage): cur_path = torrent_storage.at(index).path.decode('utf-8') else: @@ -716,7 +710,8 @@ def set_selected_files(self, selected_files=None): except TypeError: self.handle.rename_file(index, new_path.encode("utf-8")) - self.handle.prioritize_files(filepriorities) + if not share_mode: + self.handle.prioritize_files(filepriorities) self.unwanteddir_abs = unwanteddir_abs @@ -878,6 +873,10 @@ def network_get_vod_stats(self): @staticmethod def create_peerlist_data(peer_info): + """ + A function to convert peer_info libtorrent object into dictionary + This data is used to identify peers with combination of several flags + """ peer_dict = {} peer_dict['id'] = peer_info.pid @@ -909,15 +908,10 @@ def create_peerlist_data(peer_info): peer_dict['upload_only'] = bool(peer_info.flags & peer_info.upload_only) # add read and write state (check unchoke/choke peers) - # peer_dict['rstate_bw_limit'] = bool(peer_info.read_state & peer_info.bw_limit) - # peer_dict['wstate_bw_limit'] = bool(peer_info.write_state & peer_info.bw_limit) - # peer_dict['rstate_bw_network'] = bool(peer_info.read_state & peer_info.bw_network) - # peer_dict['wstate_bw_network'] = bool(peer_info.write_state & peer_info.bw_network) + # read and write state is char with value 0, 1, 2, 4. May be empty peer_dict['rstate'] = peer_info.read_state peer_dict['wstate'] = peer_info.write_state - - return peer_dict def network_create_spew_from_peerlist(self): @@ -1193,9 +1187,9 @@ def dlconfig_changed_callback(self, section, name, new_value, old_value): return False return True + @checkHandleAndSynchronize def get_share_mode(self): - if self.handle: - return self.handle.status().share_mode + return self.handle.status().share_mode @checkHandleAndSynchronize def set_share_mode(self, share_mode): diff --git a/Tribler/Core/TorrentChecker/session.py b/Tribler/Core/TorrentChecker/session.py index f884d0f4c0d..21322d2233a 100644 --- a/Tribler/Core/TorrentChecker/session.py +++ b/Tribler/Core/TorrentChecker/session.py @@ -95,7 +95,11 @@ def can_add_request(self): Checks if we still can add requests to this session. :return: True or False. """ - return not self._is_initiated and len(self._infohash_list) < MAX_TRACKER_MULTI_SCRAPE + + #TODO(ardhi) : quickfix for etree.org can't handle multiple infohash in single call + etree_condition = "etree" not in self.tracker_url + + return not self._is_initiated and len(self._infohash_list) < MAX_TRACKER_MULTI_SCRAPE and etree_condition def has_request(self, infohash): return infohash in self._infohash_list diff --git a/Tribler/Core/TorrentChecker/torrent_checker.py b/Tribler/Core/TorrentChecker/torrent_checker.py index 6a36e23ab77..1a00bc5d2d7 100644 --- a/Tribler/Core/TorrentChecker/torrent_checker.py +++ b/Tribler/Core/TorrentChecker/torrent_checker.py @@ -140,10 +140,11 @@ def _task_select_torrents(self): self._logger.debug(u"Selected %d new torrents to check on tracker: %s", scheduled_torrents, tracker_url) @call_on_reactor_thread - def add_gui_request(self, infohash): + def add_gui_request(self, infohash, scrape_now=False): """ Public API for adding a GUI request. :param infohash: Torrent infohash. + :param scrape_now: Flag whether we want to force scraping immediately """ result = self._torrent_db.getTorrent(infohash, (u'torrent_id', u'last_tracker_check'), False) if result is None: @@ -153,7 +154,7 @@ def add_gui_request(self, infohash): torrent_id = result[u'torrent_id'] last_check = result[u'last_tracker_check'] time_diff = time.time() - last_check - if time_diff < self._torrent_check_interval: + if time_diff < self._torrent_check_interval and not scrape_now: self._logger.debug(u"time interval too short, skip GUI request. infohash: %s", hexlify(infohash)) return @@ -373,7 +374,6 @@ def _create_session_for_request(self, infohash, tracker_url): self._logger.debug(u"Session created for infohash %s", hexlify(infohash)) - def _update_pending_response(self, infohash): if infohash in self._pending_response_dict: self._pending_response_dict[infohash][u'remaining_responses'] += 1 diff --git a/Tribler/Core/Utilities/utilities.py b/Tribler/Core/Utilities/utilities.py index 41a11746833..c23894d1dad 100644 --- a/Tribler/Core/Utilities/utilities.py +++ b/Tribler/Core/Utilities/utilities.py @@ -245,11 +245,11 @@ def fix_torrent(file_path): return fixed_data -def translate_peers_into_health(peer_info_dicts, status=None): - # peer_info_dicts is a peer_info dictionary from LibTorrentDownloadImpl.create_peerlist_data - # status is libtorrent torrent status #TODO : unused for now - # purpose : where we want to measure a swarm's health but no tracker can be contacted - +def translate_peers_into_health(peer_info_dicts): + """ + peer_info_dicts is a peer_info dictionary from LibTorrentDownloadImpl.create_peerlist_data + purpose : where we want to measure a swarm's health but no tracker can be contacted + """ num_seeders = 0 num_leech = 0 @@ -263,37 +263,28 @@ def translate_peers_into_health(peer_info_dicts, status=None): # collecting some statistics for p_info in peer_info_dicts: upload_only_b = False - finished_b = False - interest_in_us_b = False if p_info['upload_only']: - upload_only+=1 + upload_only += 1 upload_only_b = True if p_info['uinterested']: - interest_in_us+=1 - interest_in_us_b = True - + interest_in_us += 1 if p_info['completed'] == 1: - finished+=1 - finished_b = True + finished += 1 else: unfinished_able_dl += 1 if upload_only_b else 0 - ''' - seeders potentials: - 1. it's only want uploading right now (upload only) - 2. it's finished (we don't know whether it want to upload or not) - - - leecher potentials: - 1. it's interested in our piece - 2. it's unfinished but it's not 'upload only' (it can't leech for some reason) - 3. it's unfinished (less restrictive) + # seeders potentials: + # 1. it's only want uploading right now (upload only) + # 2. it's finished (we don't know whether it want to upload or not) + # leecher potentials: + # 1. it's interested in our piece + # 2. it's unfinished but it's not 'upload only' (it can't leech for some reason) + # 3. it's unfinished (less restrictive) - make sure to change those description when change the algorithm - ''' + # make sure to change those description when changing the algorithm num_seeders = max(upload_only, finished) num_leech = max(interest_in_us, min(unfinished_able_dl, num_all_peer - finished)) - return (num_seeders, num_leech) + return num_seeders, num_leech diff --git a/Tribler/Utilities/scraper.py b/Tribler/Utilities/scraper.py deleted file mode 100644 index bad6dfafde6..00000000000 --- a/Tribler/Utilities/scraper.py +++ /dev/null @@ -1,50 +0,0 @@ -import urllib -import socket, random, struct - -from libtorrent import bdecode - -def scrape_udp(tracker, infohashes): - tracker = tracker.lower() - host, port = tracker[6:].split(':') - addr = (host, int(port.split('/')[0])) - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(6) - - # Connection request - conn_req = struct.pack("!QII", 0x41727101980, 0, random.getrandbits(32)) - sock.sendto(conn_req, addr) - - # Process connection response - conn_resp, _ = sock.recvfrom(1024) - action, _, connection_id = struct.unpack_from("!IIQ", conn_resp) - if action != 0: - return - - # Scrape request - transaction_id = random.getrandbits(32) - scrp_req = struct.pack("!QII", connection_id, 2, transaction_id) + ''.join([struct.pack("!20s", i) for i in infohashes]) - sock.sendto(scrp_req, addr) - - # Process scrape response - scrp_resp, _ = sock.recvfrom(1024) - action, scrp_transaction_id = struct.unpack_from("!II", scrp_resp) - if action != 2 or scrp_transaction_id != transaction_id: - return - - result = {} - for index, infohash in enumerate(infohashes): - t = struct.unpack_from("!III", scrp_resp, 8 + (index * 12)) - result[infohash] = dict(zip(['complete', 'downloaded', 'incomplete'], t)) - return result - -def scrape_tcp(tracker, infohashes): - infohashes_quoted = [urllib.quote(infohash) for infohash in infohashes] - tracker = tracker.replace("announce", "scrape") - tracker = tracker + ('?' if tracker.find('?') == -1 else '&') + 'info_hash=' + '&info_hash='.join(infohashes_quoted) - response = urllib.urlopen(tracker).read() - response = bdecode(response) - - result = {} - for h, i in response['files'].iteritems(): - result[h] = {"complete" : i["complete"], "downloaded" : i["downloaded"], "incomplete" : i["incomplete"]} - return result From 2310579799642f5c27974038bf7ddd44f2e3bc72 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Fri, 27 May 2016 17:57:51 +0200 Subject: [PATCH 19/24] Add credit mining preferences by SessionConfig Squashed commits/fixes : - permit cm pref to be changed in runtime - delete boosting.ini file - Remove unused-obsolete function and fixes function name --- .../Core/APIImplementation/LaunchManyCore.py | 5 +- Tribler/Core/SessionConfig.py | 158 +++++ Tribler/Core/defaults.py | 21 +- Tribler/Policies/BoostingManager.py | 221 ++---- Tribler/Policies/BoostingSource.py | 633 ++++++++++++++++++ Tribler/Policies/defs.py | 23 + boosting.ini | 13 - boosting.ini.1 | 11 - boosting.ini.2 | 11 - boosting.ini.3 | 11 - boosting.ini.one | 11 - 11 files changed, 901 insertions(+), 217 deletions(-) create mode 100644 Tribler/Policies/BoostingSource.py create mode 100644 Tribler/Policies/defs.py delete mode 100644 boosting.ini delete mode 100644 boosting.ini.1 delete mode 100644 boosting.ini.2 delete mode 100644 boosting.ini.3 delete mode 100644 boosting.ini.one diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index 69ab30a41e5..540f37288a0 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -802,7 +802,10 @@ def sessconfig_changed_callback(self, section, name, new_value, old_value): 'anon_listen_port']) or \ (section == 'torrent_collecting' and name in ['stop_collecting_threshold']) or \ (section == 'watch_folder') or \ - (section == 'tunnel_community' and name in ['socks5_listen_port']): + (section == 'tunnel_community' and name in ['socks5_listen_port']) or \ + (section == 'credit_mining' and name in ['max_torrents_per_source', 'max_torrents_active', + 'source_interval', 'swarm_interval', 'boosting_sources', + 'boosting_enabled', 'boosting_disabled', 'archive_sources']): return True else: return False diff --git a/Tribler/Core/SessionConfig.py b/Tribler/Core/SessionConfig.py index a3e0f9d9fca..bbad991eb59 100644 --- a/Tribler/Core/SessionConfig.py +++ b/Tribler/Core/SessionConfig.py @@ -23,6 +23,9 @@ from Tribler.Core.defaults import sessdefaults from Tribler.Core.osutils import get_appstate_dir, is_android from Tribler.Core.simpledefs import STATEDIR_SESSCONFIG +from Tribler.Policies.BoostingPolicy import CreationDatePolicy, BoostingPolicy +from Tribler.Policies.BoostingPolicy import RandomPolicy +from Tribler.Policies.BoostingPolicy import SeederRatioPolicy class SessionConfigInterface(object): @@ -842,6 +845,161 @@ def get_http_api_port(self): """ return self._obtain_port(u'http_api', u'port') + # + # Credit Mining + # + + def set_creditmining_enable(self, value): + """ + Sets to enable credit mining + """ + self.sessconfig.set(u'credit_mining', u'enabled', value) + + def get_creditmining_enable(self): + """ + Gets if credit mining is enabled + :return: (bool) True or False + """ + return self.sessconfig.get(u'credit_mining', u'enabled') + + def set_cm_max_torrents_active(self, max_torrents_active): + """ + Set credit mining max active torrents in a single session + """ + return self.sessconfig.set(u'credit_mining', u'max_torrents_active', max_torrents_active) + + def get_cm_max_torrents_active(self): + """ + get max number of torrents active in a single session + """ + return self.sessconfig.get(u'credit_mining', u'max_torrents_active') + + def set_cm_max_torrents_per_source(self, max_torrents_per_source): + """ + set a number of torrent that can be stored in a single source + """ + return self.sessconfig.set(u'credit_mining', u'max_torrents_per_source', max_torrents_per_source) + + def get_cm_max_torrents_per_source(self): + """ + get max number of torrent that can be stored in a single source + """ + return self.sessconfig.get(u'credit_mining', u'max_torrents_per_source') + + def set_cm_source_interval(self, source_interval): + """ + set interval of looking up new torrent in a swarm + """ + return self.sessconfig.set(u'credit_mining', u'source_interval', source_interval) + + def get_cm_source_interval(self): + """ + get interval of looking up new torrent in a swarm + """ + return self.sessconfig.get(u'credit_mining', u'source_interval') + + def set_cm_swarm_interval(self, swarm_interval): + """ + set the interval of choosing activity which swarm will be downloaded + """ + return self.sessconfig.set(u'credit_mining', u'swarm_interval', swarm_interval) + + def get_cm_swarm_interval(self): + """ + getting the interval of choosing activity which swarm will be downloaded + """ + return self.sessconfig.get(u'credit_mining', u'swarm_interval') + + def set_cm_tracker_interval(self, tracker_interval): + """ + set the manual (force) scraping interval. + """ + return self.sessconfig.set(u'credit_mining', u'tracker_interval', tracker_interval) + + def get_cm_tracker_interval(self): + """ + get the manual (force) scraping interval. + """ + return self.sessconfig.get(u'credit_mining', u'tracker_interval') + + def set_cm_logging_interval(self, logging_interval): + """ + set the credit mining logging interval (INFO,DEBUG) + """ + return self.sessconfig.set(u'credit_mining', u'logging_interval', logging_interval) + + def get_cm_logging_interval(self): + """ + get the credit mining logging interval (INFO,DEBUG) + """ + return self.sessconfig.get(u'credit_mining', u'logging_interval') + + def set_cm_share_mode_target(self, share_mode_target): + """ + set the share mode target in credit mining. Value can be referenced at : + http://www.libtorrent.org/reference-Settings.html#share_mode_target + """ + return self.sessconfig.set(u'credit_mining', u'share_mode_target', share_mode_target) + + def get_cm_share_mode_target(self): + """ + get the current share mode target that applies in all the swarm + """ + return self.sessconfig.get(u'credit_mining', u'share_mode_target') + + def set_cm_policy(self, policy_str): + """ + set the credit mining policy. Input can be policy name or class + """ + switch_policy = { + RandomPolicy: "random", + CreationDatePolicy: "creation", + SeederRatioPolicy: "seederratio" + } + + if isinstance(policy_str, BoostingPolicy): + policy_str = switch_policy[type(policy_str)] + + return self.sessconfig.set(u'credit_mining', u'policy', policy_str) + + def get_cm_policy(self, as_class=False): + """ + get the credit mining policy. If as_class True, will return as class, + otherwise will return as policy name (str) + """ + policy_str = self.sessconfig.get(u'credit_mining', u'policy') + + if as_class: + switch_policy = { + "random": RandomPolicy, + "creation": CreationDatePolicy, + "seederratio": SeederRatioPolicy + } + + ret = switch_policy[policy_str] + else: + ret = policy_str + + return ret + + def set_cm_sources(self, source_list, key): + """ + set source list for a chosen key : + boosting_sources, boosting_enabled, boosting_disabled, or archive_sources + """ + return self.sessconfig.set(u'credit_mining', u'%s' % key, source_list) + + def get_cm_sources(self): + """ + get all the lists as list of string in the configuration + """ + ret = {"boosting_sources": self.sessconfig.get(u'credit_mining', u'boosting_sources'), + "boosting_enabled": self.sessconfig.get(u'credit_mining', u'boosting_enabled'), + "boosting_disabled": self.sessconfig.get(u'credit_mining', u'boosting_disabled'), + "archive_sources": self.sessconfig.get(u'credit_mining', u'archive_sources')} + + return ret + # # Static methods # diff --git a/Tribler/Core/defaults.py b/Tribler/Core/defaults.py index f82ea19de4b..51138b408ab 100644 --- a/Tribler/Core/defaults.py +++ b/Tribler/Core/defaults.py @@ -39,8 +39,9 @@ # Version 12: Added watch folder options. # Version 13: Added HTTP API options. # Version 14: Added option to enable/disable channel, previewchannel and tunnel community. +# Version 15: Added credit mining options -SESSDEFAULTS_VERSION = 14 +SESSDEFAULTS_VERSION = 15 sessdefaults = OrderedDict() # General Tribler settings @@ -163,6 +164,24 @@ sessdefaults['http_api']['enabled'] = False sessdefaults['http_api']['port'] = -1 +# Credit mining config +sessdefaults['credit_mining'] = OrderedDict() +sessdefaults['credit_mining']['max_torrents_per_source'] = 10 +sessdefaults['credit_mining']['max_torrents_active'] = 20 +sessdefaults['credit_mining']['source_interval'] = 100 +sessdefaults['credit_mining']['swarm_interval'] = 100 +sessdefaults['credit_mining']['share_mode_target'] = 3 +sessdefaults['credit_mining']['tracker_interval'] = 200 +sessdefaults['credit_mining']['logging_interval'] = 60 +sessdefaults['credit_mining']['boosting_sources'] = ["http://bt.etree.org/rss/bt_etree_org.rdf", + "9e1a3fac737543d36c83d54818ed77620932ff80", + "028d2b5eea5277ddc3cedf78601f9be246b29bf1"] +sessdefaults['credit_mining']['boosting_enabled'] = ["http://bt.etree.org/rss/bt_etree_org.rdf", + "9e1a3fac737543d36c83d54818ed77620932ff80"] +sessdefaults['credit_mining']['boosting_disabled'] = ["028d2b5eea5277ddc3cedf78601f9be246b29bf1"] +sessdefaults['credit_mining']['archive_sources'] = ["9e1a3fac737543d36c83d54818ed77620932ff80"] +sessdefaults['credit_mining']['policy'] = "seederratio" + # # BT per download opts # diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 925340f415c..a1c2994e14d 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H """Manage boosting of swarms""" -import ConfigParser -import json +import errno import logging import os -import random from binascii import hexlify, unhexlify import libtorrent as lt @@ -19,19 +17,17 @@ from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl from Tribler.Core.Utilities import utilities from Tribler.Core.Utilities.install_dir import determine_install_dir +from Tribler.Core.exceptions import OperationNotPossibleAtRuntimeException from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_TORRENTS, NTFY_UPDATE, NTFY_CHANNELCAST from Tribler.Main.globals import DefaultDownloadStartupConfig +from Tribler.Policies.BoostingPolicy import RandomPolicy, CreationDatePolicy, SeederRatioPolicy from Tribler.Policies.BoostingSource import ChannelSource from Tribler.Policies.BoostingSource import DirectorySource from Tribler.Policies.BoostingSource import RSSFeedSource +from Tribler.Policies.defs import NUMBER_TYPES, SAVED_ATTR, CREDIT_MINING_FOLDER_DOWNLOAD, CONFIG_KEY_ARCHIVELIST, \ + CONFIG_OP_RM, CONFIG_OP_ADD, CONFIG_KEY_SOURCELIST, CONFIG_KEY_ENABLEDLIST, CONFIG_KEY_DISABLEDLIST from Tribler.dispersy.taskmanager import TaskManager -NUMBER_TYPES = (int, long, float) - -# CONFIG_FILE = "boosting.ini" -TRIBLER_ROOT = determine_install_dir() -CONFIG_FILE = os.path.join(TRIBLER_ROOT, "boosting.ini") - def levenshtein_dist(t1_fname, t2_fname): """ @@ -63,115 +59,35 @@ def string_to_source(source_str): return source_str.decode('hex') \ if len(source_str) == 40 and not (os.path.isdir(source_str) or source_str.startswith('http://')) else source_str -class BoostingPolicy(object): - """ - Base class for determining what swarm selection policy will be applied - """ - - def __init__(self, session): - self.session = session - self.key = lambda x: None - # function that checks if key can be applied to torrent - self.key_check = lambda x: False - self.reverse = None - - self._logger = logging.getLogger(self.__class__.__name__) - - def apply(self, torrents, max_active, force=False): - """ - apply the policy to the torrents stored - """ - sorted_torrents = sorted([torrent for torrent in torrents.itervalues() - if self.key_check(torrent)], - key=self.key, reverse=self.reverse) - - torrents_start = [] - for torrent in sorted_torrents[:max_active]: - if not self.session.get_download(torrent["metainfo"].get_infohash()): - torrents_start.append(torrent) - torrents_stop = [] - for torrent in sorted_torrents[max_active:]: - if self.session.get_download(torrent["metainfo"].get_infohash()): - torrents_stop.append(torrent) - - if force: - return torrents_start, torrents_stop - - # if both results are empty for some reason (e.g, key_check too restrictive) - # or torrent started less than half available torrent (try to keep boosting alive) - # if it's already random, just let it be - if not isinstance(self, RandomPolicy) and ((not torrents_start and not torrents_stop) or - (len(torrents_start) < len(torrents) / 2 and len( - torrents_start) < max_active / 2)): - self._logger.error("Start and stop torrent list are empty. Fallback to Random") - # fallback to random policy - torrents_start, torrents_stop = RandomPolicy(self.session).apply(torrents, max_active) - - return torrents_start, torrents_stop - - -class RandomPolicy(BoostingPolicy): - """ - A credit mining policy that chooses swarm randomly - """ - def __init__(self, session): - BoostingPolicy.__init__(self, session) - self.key = lambda v: random.random() - self.key_check = lambda v: True - self.reverse = False - - -class CreationDatePolicy(BoostingPolicy): - """ - A credit mining policy that chooses swarm by its creation date - - The idea is, older swarm need to be boosted. - """ - def __init__(self, session): - BoostingPolicy.__init__(self, session) - self.key = lambda v: v['creation_date'] - self.key_check = lambda v: v['creation_date'] > 0 - self.reverse = True - - -class SeederRatioPolicy(BoostingPolicy): - """ - Default policy. Find the most underseeded swarm to boost. - """ - def __init__(self, session): - BoostingPolicy.__init__(self, session) - self.key = lambda v: v['num_seeders'] / float(v['num_seeders'] + - v['num_leechers']) - self.key_check = lambda v: isinstance(v['num_seeders'], NUMBER_TYPES) and isinstance( - v['num_leechers'], NUMBER_TYPES) and v['num_seeders'] + v['num_leechers'] > 0 - self.reverse = False - class BoostingSettings(object): """ Class contains settings on boosting manager """ - def __init__(self, session, policy=SeederRatioPolicy): + def __init__(self, session, policy=SeederRatioPolicy, load_config=True): self.session = session - self.config_file = CONFIG_FILE - self.max_torrents_active = 30 - self.max_torrents_per_source = 100 - self.source_interval = 20 - self.swarm_interval = 20 - self.initial_swarm_interval = 30 + # Configurable parameter (changeable in runtime -plus sources-) + self.max_torrents_active = 20 + self.max_torrents_per_source = 10 + self.source_interval = 100 + self.swarm_interval = 100 + + # Can't be changed in runtime + self.tracker_interval = 200 + self.logging_interval = 60 + self.share_mode_target = 3 self.policy = policy(session) - self.tracker_interval = 50 - self.initial_tracker_interval = 25 - self.logging_interval = 40 - self.initial_logging_interval = 20 + # Non-Configurable + self.initial_logging_interval = 20 + self.initial_tracker_interval = 25 + self.initial_swarm_interval = 30 self.min_connection_start = 5 self.min_channels_start = 100 - - self.share_mode_target = 2 self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), - "credit_mining") + CREDIT_MINING_FOLDER_DOWNLOAD) + self.load_config = load_config class BoostingManager(TaskManager): @@ -186,11 +102,9 @@ def __init__(self, session, settings=None): self._logger = logging.getLogger(self.__class__.__name__) BoostingManager.__single = self - - self._saved_attributes = ["max_torrents_per_source", - "max_torrents_active", "source_interval", - "swarm_interval", "share_mode_target", - "tracker_interval", "logging_interval"] + self.boosting_sources = {} + self.torrents = {} + self.running = True self.session = session assert self.session.get_libtorrent() @@ -199,24 +113,19 @@ def __init__(self, session, settings=None): self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) # use provided settings or a default one - self.settings = settings or BoostingSettings(session) + self.settings = settings or BoostingSettings(session, load_config=True) + + if self.settings.load_config: + self._logger.info("Loading config file from session configuration") + self.load_config() if not os.path.exists(self.settings.credit_mining_path): os.makedirs(self.settings.credit_mining_path) - self.boosting_sources = {} - self.torrents = {} - local_settings = {} local_settings['share_mode_target'] = self.settings.share_mode_target self.session.lm.ltmgr.get_session().set_settings(local_settings) - if os.path.exists(self.settings.config_file): - self._logger.info("Config file %s", open(self.settings.config_file).read()) - self.load_config() - else: - self._logger.warning("Initial config file missing") - self.session.add_observer(self.on_torrent_notify, NTFY_TORRENTS, [NTFY_UPDATE]) # TODO(emilon): Refactor this to use taskmanager @@ -521,34 +430,31 @@ def load_config(self): """ load config in file configuration and apply it to manager """ - config = ConfigParser.ConfigParser() - config.read(self.settings.config_file) - validate_source = lambda source_str: unhexlify(source_str) if len(source_str) == 40 and \ - not source_str.startswith("http") else source_str + validate_source = lambda s: unhexlify(s) if len(s) == 40 and not s.startswith("http") else s - def _add_sources(value): + def _add_sources(values): """ adding sources in configuration file """ - for boosting_source in json.loads(value): + for boosting_source in values: boosting_source = validate_source(boosting_source) self.add_source(boosting_source) - def _archive_sources(value): + def _archive_sources(values): """ setting archive to sources """ - for archive_source in json.loads(value): + for archive_source in values: archive_source = validate_source(archive_source) self.set_archive(archive_source, True) - def _set_enable_boosting(value, enabled): + def _set_enable_boosting(values, enabled): """ set disable/enable source """ - for boosting_source in json.loads(value): + for boosting_source in values: boosting_source = validate_source(boosting_source) - if not self.boosting_sources[boosting_source]: + if boosting_source not in self.boosting_sources.keys(): self.add_source(boosting_source) self.boosting_sources[boosting_source].enabled = enabled @@ -577,13 +483,20 @@ def _set_enable_boosting(value, enabled): "seederratio" : SeederRatioPolicy } - for k, val in config.items(__name__): + # set policy + self.settings.policy = self.session.get_credit_mining_policy(True)(self.session) + + dict_to_load = {} + dict_to_load.update(self.session.get_credit_mining_sources()) + dict_to_load.update(dict.fromkeys(SAVED_ATTR)) + + for k, val in dict_to_load.items(): try: - if k in self._saved_attributes: - object.__setattr__(self.settings, k, int(val)) - elif k == "policy": - self.settings.policy = switch_policy[val](self.session) - else: + if k in SAVED_ATTR: + # see the session configuration + object.__setattr__(self.settings, k, + getattr(self.session, "get_credit_mining_%s" %k)()) + else: #credit mining source handle switch[k]["cmd"](*((switch[k]['args'][0] or val,) + switch[k]['args'][1:])) except KeyError: self._logger.error("Key %s can't be applied", k) @@ -592,10 +505,12 @@ def save_config(self): """ save the environment parameters in config file """ - config = ConfigParser.ConfigParser() - config.add_section(__name__) - for k in self._saved_attributes: - config.set(__name__, k, BoostingSettings.__getattribute__(self.settings, k)) + for k in SAVED_ATTR: + try: + setattr(self.session, "set_credit_mining_%s" % k, getattr(self.settings, k)) + except OperationNotPossibleAtRuntimeException, err_msg: + # some of the attribute can't be changed in runtime. See lm.sessconfig_changed_callback + self._logger.debug("Cannot set attribute %s. Not permitted in runtime", k) archive_sources = [] lboosting_sources = [] @@ -615,22 +530,12 @@ def save_config(self): if boosting_source.archive: archive_sources.append(bsname) - config.set(__name__, "boosting_sources", json.dumps(lboosting_sources)) - config.set(__name__, "boosting_enabled", json.dumps(flag_enabled_sources)) - config.set(__name__, "boosting_disabled", json.dumps(flag_disabled_sources)) - if archive_sources: - config.set(__name__, "archive_sources", json.dumps(archive_sources)) - - policy = "None" - if isinstance(self.settings.policy, RandomPolicy): - policy = "random" - elif isinstance(self.settings.policy, CreationDatePolicy): - policy = "creation" - elif isinstance(self.settings.policy, SeederRatioPolicy): - policy = "seederratio" - config.set(__name__, "policy", policy) - with open(self.settings.config_file, "w") as configf: - config.write(configf) + self.session.set_credit_mining_sources(lboosting_sources, CONFIG_KEY_SOURCELIST) + self.session.set_credit_mining_sources(flag_enabled_sources, CONFIG_KEY_ENABLEDLIST) + self.session.set_credit_mining_sources(flag_disabled_sources, CONFIG_KEY_DISABLEDLIST) + self.session.set_credit_mining_sources(archive_sources, CONFIG_KEY_ARCHIVELIST) + + self.session.save_pstate_sessconfig() def log_statistics(self): """Log transfer statistics""" diff --git a/Tribler/Policies/BoostingSource.py b/Tribler/Policies/BoostingSource.py new file mode 100644 index 00000000000..27c34640260 --- /dev/null +++ b/Tribler/Policies/BoostingSource.py @@ -0,0 +1,633 @@ +# coding=utf-8 +""" +Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H +Supported boosting sources +""" +import HTMLParser +import glob +import logging +import os +import random +import re +import time +import urllib +from binascii import hexlify +from hashlib import sha1 + +import libtorrent as lt +from twisted.internet import reactor +from twisted.internet.task import LoopingCall +from twisted.web.client import Agent, readBody +from twisted.web.http_headers import Headers + +from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.simpledefs import NTFY_INSERT, NTFY_TORRENTS, NTFY_UPDATE, NTFY_CHANNELCAST, NTFY_VOTECAST +from Tribler.Core.version import version_id +from Tribler.Main.Utility.GuiDBTuples import Torrent, CollectedTorrent, RemoteTorrent, NotCollectedTorrent, Channel, \ + ChannelTorrent +from Tribler.community.allchannel.community import AllChannelCommunity +from Tribler.community.channel.community import ChannelCommunity +from Tribler.dispersy.exception import CommunityNotFoundException +from Tribler.dispersy.taskmanager import TaskManager +from Tribler.dispersy.util import call_on_reactor_thread + +def ent2chr(input_str): + """ + Function to unescape literal string in XML to symbols + source : http://www.gossamer-threads.com/lists/python/python/177423 + """ + code = input_str.group(1) + code_int = int(code) if code.isdigit() else int(code[1:], 16) + return chr(code_int) if code_int < 256 else '?' + + +class TorrentManagerCM(TaskManager): + """ + *Temporary* class to handle load torrent. + + Adapted from TorrentManager in SearchGridManager + """ + __single = None + + def __init__(self, session): + super(TorrentManagerCM, self).__init__() + TorrentManagerCM.__single = self + + self.session = session + self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) + self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) + self.votecastdb = self.session.open_dbhandler(NTFY_VOTECAST) + + self.dslist = [] + + # TODO(ardhi) : temporary function until GUI and core code are separated + def load_torrent(self, torrent, callback=None): + """ + function to load torrent dictionary to torrent object. + + From TorrentManager.loadTorrent in SearchGridManager + """ + + # session is quitting + if not (self.session and self.session.get_torrent_store() and self.session.lm.torrent_store): + return + + if not isinstance(torrent, CollectedTorrent): + if torrent.torrent_id <= 0: + torrent_id = self.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + if not self.session.has_collected_torrent(torrent.infohash): + files = [] + trackers = [] + + # see if we have most info in our tables + if isinstance(torrent, RemoteTorrent): + torrent_id = self.torrent_db.getTorrentID(torrent.infohash) + else: + torrent_id = torrent.torrent_id + + trackers.extend(self.torrent_db.getTrackerListByTorrentID(torrent_id)) + + if 'DHT' in trackers: + trackers.remove('DHT') + if 'no-DHT' in trackers: + trackers.remove('no-DHT') + + # replacement # self.downloadTorrentfileFromPeers(torrent, None) + if self.session.has_download(torrent.infohash): + return False + + if torrent.query_candidates is None or len(torrent.query_candidates) == 0: + self.session.download_torrentfile(torrent.infohash, None, 0) + else: + for candidate in torrent.query_candidates: + self.session.download_torrentfile_from_peer(candidate, torrent.infohash, None, 0) + + torrent = NotCollectedTorrent(torrent, files, trackers) + + else: + tdef = TorrentDef.load_from_memory(self.session.get_collected_torrent(torrent.infohash)) + + if torrent.torrent_id <= 0: + del torrent.torrent_id + + torrent = CollectedTorrent(torrent, tdef) + + # replacement # self.library_manager.addDownloadState(torrent) + for dl_state in self.dslist: + torrent.addDs(dl_state) + + # return + if callback is not None: + callback(torrent) + else: + return torrent + + @staticmethod + def get_instance(*args, **kw): + """ + get single instance of TorrentManagerCM + """ + if TorrentManagerCM.__single is None: + TorrentManagerCM(*args, **kw) + return TorrentManagerCM.__single + + @staticmethod + def del_instance(): + """ + resetting, then deleting single instance + """ + TorrentManagerCM.__single = None + + +class BoostingSource(object): + """ + Base class for boosting source. For now, it can be RSS, directory, and channel + """ + + def __init__(self, session, source, boost_settings, callback): + self.session = session + self.channelcast_db = session.lm.channelcast_db + + self.torrents = {} + self.source = source + self.interval = boost_settings.source_interval + self.max_torrents = boost_settings.max_torrents_per_source + self.callback = callback + self.archive = False + + self.enabled = True + + self.av_uprate = 0 + self.av_dwnrate = 0 + self.storage_used = 0 + self.ready = False + + self.min_connection = boost_settings.min_connection_start + self.min_channels = boost_settings.min_channels_start + + self.torrent_mgr = TorrentManagerCM.get_instance(session) + + # local import for handling circular import + from Tribler.Policies.BoostingManager import BoostingManager + self.boosting_manager = BoostingManager.get_instance() + self._logger = logging.getLogger(BoostingManager.__name__) + + def kill_tasks(self): + """ + kill tasks on this source + """ + + self.ready = False + self.torrent_mgr.del_instance() + + self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_load") + + self.session.lm.threadpool.cancel_pending_task(self.source) + + def _load_if_ready(self, source): + """ + load source if and only if the overall system is ready. + + it depends on #connection and #channel + + Useful to not burden the apps in startup + """ + + nr_channels = self.channelcast_db.getNrChannels() + nr_connections = 0 + + for community in self.session.lm.dispersy.get_communities(): + from Tribler.community.search.community import SearchCommunity + if isinstance(community, SearchCommunity): + nr_connections = community.get_nr_connections() + + # condition example + if nr_channels > self.min_channels and nr_connections > self.min_connection: + called_func = self._load + else: + called_func = self._load_if_ready + + self.session.lm.threadpool.add_task(lambda src=source: called_func(src), 15, task_name=str(self.source) + "_load") + + def _load(self, source): + pass + + def _update(self): + pass + + def get_source_text(self): + """ + returning 'raw' source. May be overriden + """ + return self.source + + +class ChannelSource(BoostingSource): + """ + Credit mining source from a channel. + """ + def __init__(self, session, dispersy_cid, boost_settings, callback): + BoostingSource.__init__(self, session, dispersy_cid, boost_settings, callback) + + self.channel_id = None + + self.channel = None + self.community = None + self.database_updated = True + + self.check_torrent_interval = 10 + + self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) + self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load_if_ready(cid), 0, + task_name=str(self.source) + "_load") + + self.unavail_torrent = {} + + def kill_tasks(self): + BoostingSource.kill_tasks(self) + + # cancel loading channel id + self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_get_channel_id") + + self.session.remove_observer(self._on_database_updated) + + def _load(self, dispersy_cid): + dispersy = self.session.get_dispersy_instance() + + def join_community(): + """ + find the community/channel id, then join + """ + try: + self.community = dispersy.get_community(dispersy_cid, True) + self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=str(self.source) + "_get_channel_id") + + except CommunityNotFoundException: + + allchannelcommunity = None + for community in dispersy.get_communities(): + if isinstance(community, AllChannelCommunity): + allchannelcommunity = community + break + + if allchannelcommunity: + self.community = ChannelCommunity.init_community(dispersy, dispersy.get_member(mid=dispersy_cid), + allchannelcommunity._my_member, self.session) + self._logger.info("Joined channel community %s", dispersy_cid.encode("HEX")) + self.session.lm.threadpool.add_task(get_channel_id, 0, + task_name=str(self.source) + "_get_channel_id") + else: + self._logger.error("Could not find AllChannelCommunity") + + def get_channel_id(): + """ + find channel id by looking at the network + """ + if self.community and self.community._channel_id: # before: and self.gui_util.registered: + self.channel_id = self.community._channel_id + + channel_dict = self.boosting_manager.channelcast_db.getChannel(self.channel_id) + self.channel = Channel(*channel_dict) + + if not self.boosting_manager.is_pending_task_active(str(self.source) + "_update"): + self._logger.debug("Registering update call") + self.boosting_manager.register_task(str(self.source) + "_update", LoopingCall(self._update)).start( + self.interval, now=True) + # self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") + self._logger.info("Got channel id %s", self.channel_id) + else: + self._logger.warning("Could not get channel id, retrying in 10 s") + self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=str(self.source) + "_get_channel_id") + + try: + join_community() + self.ready = True + except CommunityNotFoundException: + self._logger.info("Channel %s was not ready, waits for next interval (%d chn)", hexlify(self.source), + len(dispersy.get_communities())) + self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load_if_ready(cid), 0, + task_name=str(self.source) + "_load") + + def _check_tor(self): + """ + periodically check torrents in channel. Will return the torrent data if finished. + """ + def showtorrent(torrent): + """ + assembly torrent data, call the callback + """ + if torrent.files: + infohash = torrent.infohash + self._logger.debug("[ChannelSource] Got torrent %s", hexlify(infohash)) + self.torrents[infohash] = {} + self.torrents[infohash]['name'] = torrent.name + self.torrents[infohash]['metainfo'] = torrent.tdef + self.torrents[infohash]['creation_date'] = torrent.creation_date + self.torrents[infohash]['length'] = torrent.tdef.get_length() + self.torrents[infohash]['num_files'] = len(torrent.files) + self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] or 0 + self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] or 0 + self.torrents[infohash]['enabled'] = self.enabled + + # seeding stats from DownloadState + self.torrents[infohash]['last_seeding_stats'] = {} + + del self.unavail_torrent[infohash] + + if self.callback: + self.callback(self.source, infohash, self.torrents[infohash]) + self.database_updated = False + + self._logger.debug("Unavailable #torrents : %d from %s", len(self.unavail_torrent), hexlify(self.source)) + + if len(self.unavail_torrent) and self.enabled: + for torrent in self.unavail_torrent.values(): + self.torrent_mgr.load_torrent(torrent, showtorrent) + + def _update(self): + if len(self.torrents) < self.max_torrents: + + if self.database_updated: + CHANTOR_DB = ['ChannelTorrents.channel_id', 'Torrent.torrent_id', 'infohash', '""', 'length', + 'category', 'status', 'num_seeders', 'num_leechers', 'ChannelTorrents.id', + 'ChannelTorrents.dispersy_id', 'ChannelTorrents.name', 'Torrent.name', + 'ChannelTorrents.description', 'ChannelTorrents.time_stamp', 'ChannelTorrents.inserted'] + + torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, CHANTOR_DB, + self.max_torrents) + + # TODO(ardhi) : temporary function until GUI and core code are separated + def create_torrents(tor_values, _, channel_dict): + """ + function to create torrents from channel. Adapted from + ChannelManager in SearchGridManager + """ + + #adding new channel from the one that can't be detected from torrent values + fetch_channels = set(hit[0] for hit in tor_values if hit[0] not in channel_dict) + if len(fetch_channels) > 0: + channels_new_dict = self.channelcast_db.getChannels(fetch_channels) + channels = [] + for hit in channels_new_dict: + channel = Channel(*hit) + channels.append(channel) + + for channel in channels: + channel_dict[channel.id] = channel + + # creating torrents + torrents = [] + for hit in tor_values: + if hit: + chan_torrent = ChannelTorrent(*hit[1:] + [channel_dict.get(hit[0], None), None]) + chan_torrent.torrent_db = self.boosting_manager.torrent_db + chan_torrent.channelcast_db = self.channelcast_db + + if chan_torrent.name: + torrents.append(chan_torrent) + + return torrents + + listtor = create_torrents(torrent_values, True, + {self.channel_id: self.channelcast_db.getChannel(self.channel_id)}) + # listtor = self.gui_util.channelsearch_manager._createTorrents( + # torrent_values, True, {self.channel_id: self.channelcast_db.getChannel(self.channel_id)})[2] + + # dict {key_infohash(binary):Torrent(object-GUIDBTuple)} + self.unavail_torrent.update({t.infohash: t for t in listtor if t.infohash not in self.torrents}) + + # it's highly probable the checktor function is running at this time (if it's already running) + # if not running, start the checker + if not self.boosting_manager.is_pending_task_active(hexlify(self.source) + "_checktor"): + self._logger.debug("Registering check torrent function") + self.boosting_manager.register_task(hexlify(self.source) + "_checktor", + LoopingCall(self._check_tor)).start(self.check_torrent_interval, + now=True) + + def _on_database_updated(self, subject, change_type, infohash): + self.database_updated = True + + def get_source_text(self): + return self.channel.name if self.channel else None + + +class RSSFeedSource(BoostingSource): + """ + Credit mining source from a RSS feed. + + # supported list (tested) : + # http://bt.etree.org/rss/bt_etree_org.rdf + # http://www.mininova.org/rss.xml + # https://kat.cr (via torcache) + + # not supported (till now) + # https://eztv.ag/ezrss.xml (https link) + """ + + def __init__(self, session, rss_feed, boost_settings, callback): + BoostingSource.__init__(self, session, rss_feed, boost_settings, callback) + + self.feed_handle = None + + self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load_if_ready(feed), 0, + task_name=str(self.source) + "_load") + + self.title = "" + self.description = "" + self.total_torrents = 0 + + def _load(self, rss_feed): + self.feed_handle = self.session.lm.ltmgr.get_session().add_feed( + {'url': rss_feed, 'auto_download': False, 'auto_map_handles': False}) + + def wait_for_feed(): + """ + Wait until the RSS feed is no longer updating. + """ + feed_status = self.feed_handle.get_feed_status() + if feed_status['updating']: + self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=str(self.source) + "_wait_for_feed") + elif len(feed_status['error']) > 0: + self._logger.error("Got error for RSS feed %s : %s", feed_status['url'], feed_status['error']) + if "503" in feed_status["error"]: + def retry_task(): + self.feed_handle.update_feed() + self.session.lm.threadpool.add_task(wait_for_feed, 1, + task_name=str(self.source) + "_wait_for_feed") + + # if failed, wait for 10 second to retry + self.session.lm.threadpool.add_task(retry_task, 10, task_name=str(self.source) + "_wait_for_feed") + else: + # The feed is done updating. Now periodically start retrieving torrents. + self.boosting_manager.register_task(str(self.source) + "_update", LoopingCall(self._update), + 10, interval=self.interval) + self._logger.info("Got RSS feed %s", feed_status['url']) + self.ready = True + + wait_for_feed() + + def _update(self): + if len(self.torrents) < self.max_torrents: + + feed_status = self.feed_handle.get_feed_status() + + self.title = feed_status['title'] + self.description = feed_status['description'] + + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', + 'enabled', 'last_seeding_stats'] + + self.total_torrents = len(feed_status['items']) + + def __cb_body(body_bin, item_torrent, torrent_fname): + tdef = None + metainfo = None + try: + metainfo = lt.bdecode(body_bin) + tdef = TorrentDef.load_from_dict(metainfo) + tdef.save(torrent_fname) + except ValueError, err: + self._logger.error("Could not parse/save torrent, skipping %s. Reason: %s", + item_torrent['url'], err.message + + ", metainfo is "+("not " if metainfo else "")+"None") + except IOError, ioerr: + # can't save to disk. Ignore this swarm + self._logger.exception("IO error, check %s. Message : %s", + self.boosting_manager.settings.credit_mining_path, ioerr.message) + return + if tdef: + # Create a torrent dict. + torrent_values = [item_torrent['title'], tdef, tdef.get_creation_date(), tdef.get_length(), + len(tdef.get_files()), -1, -1, self.enabled, {}] + self.torrents[sha1(item_torrent['url']).digest()] = dict(zip(torrent_keys, torrent_values)) + + try: + # manually generate an ID and put this into DB + self.session.lm.torrent_db.addOrGetTorrentID(sha1(item_torrent['url']).digest()) + self.session.lm.torrent_db.addExternalTorrent(tdef) + + # create Torrent object and store it + self.torrent_mgr.load_torrent(Torrent.fromTorrentDef(tdef)) + except AttributeError, err: + # if we can't find torrent_db, fallback + self._logger.error("Can't do %s. Return gracely as failed.", err.message) + return + # self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) + + # Notify the BoostingManager and provide the real infohash. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[sha1( + item_torrent['url']).digest()]) + + def __success_cb(response, item_dict, torrent_filename): + return readBody(response).addCallback(__cb_body, item_dict, torrent_filename) + + regex_unescape_xml = re.compile(r"\&\#(x?[0-9a-fA-F]+);") + + for item in feed_status['items']: + # Not all RSS feeds provide us with the infohash, + # so we use a fake infohash based on the URL to identify the torrents. + url = regex_unescape_xml.sub(ent2chr, item['url']) + infohash = sha1(url).digest() + if infohash not in self.torrents: + # Store the torrents as rss-infohash_as_hex.torrent. + torrent_filename = os.path.join(self.boosting_manager.settings.credit_mining_path, + 'rss-%s.torrent' % infohash.encode('hex')) + tdef = None + if not os.path.exists(torrent_filename): + + # create Agent to download torrent file + agent = Agent(reactor) + ses_agent = agent.request( + 'GET', # http://stackoverflow.com/a/845595 + urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]"), + Headers({'User-Agent': ['Tribler ' + version_id]}), + None) + ses_agent.addCallback(__success_cb, item, torrent_filename) + + else: + # torrent already exist in our system + tdef = TorrentDef.load(torrent_filename) + + if tdef: + # Create a torrent dict. + torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), + len(tdef.get_files()), -1, -1, self.enabled, {}] + self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) + + # manually generate an ID and put this into DB + self.session.lm.torrent_db.addOrGetTorrentID(infohash) + self.session.lm.torrent_db.addExternalTorrent(tdef) + + # create Torrent object and store it + self.torrent_mgr.load_torrent(Torrent.fromTorrentDef(tdef)) + # self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) + + # Notify the BoostingManager and provide the real infohash. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) + + def kill_tasks(self): + BoostingSource.kill_tasks(self) + + self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_wait_for_feed") + + # stop updating + self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_update") + + +class DirectorySource(BoostingSource): + """ + Credit mining source from a local directory + + The directory must be exist. + """ + + def __init__(self, session, directory, boost_settings, callback): + BoostingSource.__init__(self, session, directory, boost_settings, callback) + self._load(directory) + + def _load(self, directory): + if os.path.isdir(directory): + # Wait for __init__ to finish so the source is registered with the + # BoostinManager, otherwise adding torrents won't work + self.session.lm.threadpool.add_task(self._update, 1, task_name=str(self.source) + "_update") + self._logger.info("Got directory %s", directory) + self.ready = True + else: + self._logger.error("Could not find directory %s", directory) + + def _update(self): + if len(self.torrents) < self.max_torrents: + + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', + 'enabled', 'last_seeding_stats'] + + for torrent_filename in glob.glob(self.source + '/*.torrent'): + if torrent_filename not in self.torrents: + try: + tdef = TorrentDef.load(torrent_filename) + except ValueError, verr: + self._logger.debug("Could not load torrent locally, skipping %s", torrent_filename) + self._logger.error("Could not load %s. Reason %s", torrent_filename, verr) + continue + + # Create a torrent dict. + infohash = tdef.get_infohash() + torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), + len(tdef.get_files()), -1, -1, self.enabled, {}] + self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) + # Notify the BoostingManager. + if self.callback: + self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) + + self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source) + "_update") + + def kill_tasks(self): + BoostingSource.kill_tasks(self) + + # stop updating + self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_update") diff --git a/Tribler/Policies/defs.py b/Tribler/Policies/defs.py new file mode 100644 index 00000000000..ddd886dfdbb --- /dev/null +++ b/Tribler/Policies/defs.py @@ -0,0 +1,23 @@ +""" +Definition/Constant that used in credit mining +""" + +from Tribler.Core.Utilities.install_dir import determine_install_dir + +NUMBER_TYPES = (int, long, float) + +TRIBLER_ROOT = determine_install_dir() + +SAVED_ATTR = ["max_torrents_per_source", + "max_torrents_active", "source_interval", + "swarm_interval", "share_mode_target", + "tracker_interval", "logging_interval"] + +CREDIT_MINING_FOLDER_DOWNLOAD = "credit_mining" + +CONFIG_OP_ADD = "add" +CONFIG_OP_RM = "rm" +CONFIG_KEY_SOURCELIST = "boosting_sources" +CONFIG_KEY_ARCHIVELIST = "archive_sources" +CONFIG_KEY_ENABLEDLIST = "boosting_enabled" +CONFIG_KEY_DISABLEDLIST = "boosting_disabled" diff --git a/boosting.ini b/boosting.ini deleted file mode 100644 index 8064b71f82f..00000000000 --- a/boosting.ini +++ /dev/null @@ -1,13 +0,0 @@ -[Tribler.Policies.BoostingManager] -max_torrents_per_source = 10 -max_torrents_active = 20 -source_interval = 300 -swarm_interval = 300 -share_mode_target = 3 -tracker_interval = 300 -logging_interval = 60 -boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] -boosting_enabled = ["http://bt.etree.org/rss/bt_etree_org.rdf"] -boosting_disabled = [] -policy = seederratio - diff --git a/boosting.ini.1 b/boosting.ini.1 deleted file mode 100644 index 8f0dd95ee73..00000000000 --- a/boosting.ini.1 +++ /dev/null @@ -1,11 +0,0 @@ -[Tribler.Policies.BoostingManager] -max_torrents_per_source = 100 -max_torrents_active = 13 -source_interval = 300 -swarm_interval = 300 -share_mode_target = 3 -tracker_interval = 300 -logging_interval = 60 -boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] -policy = seederratio - diff --git a/boosting.ini.2 b/boosting.ini.2 deleted file mode 100644 index d4e2539bd4e..00000000000 --- a/boosting.ini.2 +++ /dev/null @@ -1,11 +0,0 @@ -[Tribler.Policies.BoostingManager] -max_torrents_per_source = 100 -max_torrents_active = 13 -source_interval = 300 -swarm_interval = 300 -share_mode_target = 5 -tracker_interval = 300 -logging_interval = 60 -boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] -policy = seederratio - diff --git a/boosting.ini.3 b/boosting.ini.3 deleted file mode 100644 index 637ec58653f..00000000000 --- a/boosting.ini.3 +++ /dev/null @@ -1,11 +0,0 @@ -[Tribler.Policies.BoostingManager] -max_torrents_per_source = 100 -max_torrents_active = 13 -source_interval = 300 -swarm_interval = 300 -share_mode_target = 1 -tracker_interval = 300 -logging_interval = 60 -boosting_sources = ["http://bt.etree.org/rss/bt_etree_org.rdf"] -policy = seederratio - diff --git a/boosting.ini.one b/boosting.ini.one deleted file mode 100644 index 90575fd95fd..00000000000 --- a/boosting.ini.one +++ /dev/null @@ -1,11 +0,0 @@ -[Tribler.Policies.BoostingManager] -max_torrents_per_source = 100 -max_torrents_active = 13 -source_interval = 300 -swarm_interval = 300 -share_mode_target = 3 -tracker_interval = 300 -logging_interval = 60 -boosting_sources = ["/media/hdd250/Experiments/last/torrents-one"] -policy = seederratio - From a85ab3cf1cfc838250506563a43ecdb43fc22c72 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Tue, 31 May 2016 20:45:38 +0200 Subject: [PATCH 20/24] Refactor policies, constant, and function to different file Squashed commits : - Separate Credit mining policies in different file - Refactor constant and functions to different file - Removed unused part of the code in utilities - Keep BoostingSource and BoostingManager clean - Extract lambda-variable as function in policies - Refer BoostingManager object in LaunchMany - Separate instantiation and start source Because of this, creating new instance will not make mining automatically start. Calling start function is needed. - Refactor source to have its own TaskManager - Use custom RSS parser instead of libtorrent's --- Tribler/Core/Utilities/utilities.py | 8 +- Tribler/Policies/BoostingManager.py | 76 +-- Tribler/Policies/BoostingPolicy.py | 110 +++++ Tribler/Policies/BoostingSource.py | 634 +++++++++---------------- Tribler/Policies/credit_mining_util.py | 210 ++++++++ Tribler/Tools/boostchannel.py | 4 +- 6 files changed, 573 insertions(+), 469 deletions(-) create mode 100644 Tribler/Policies/BoostingPolicy.py create mode 100644 Tribler/Policies/credit_mining_util.py diff --git a/Tribler/Core/Utilities/utilities.py b/Tribler/Core/Utilities/utilities.py index c23894d1dad..5d8b18f934d 100644 --- a/Tribler/Core/Utilities/utilities.py +++ b/Tribler/Core/Utilities/utilities.py @@ -245,16 +245,12 @@ def fix_torrent(file_path): return fixed_data + def translate_peers_into_health(peer_info_dicts): """ peer_info_dicts is a peer_info dictionary from LibTorrentDownloadImpl.create_peerlist_data purpose : where we want to measure a swarm's health but no tracker can be contacted """ - num_seeders = 0 - num_leech = 0 - - num_all_peer = len(peer_info_dicts) - upload_only = 0 finished = 0 unfinished_able_dl = 0 @@ -285,6 +281,6 @@ def translate_peers_into_health(peer_info_dicts): # make sure to change those description when changing the algorithm num_seeders = max(upload_only, finished) - num_leech = max(interest_in_us, min(unfinished_able_dl, num_all_peer - finished)) + num_leech = max(interest_in_us, min(unfinished_able_dl, len(peer_info_dicts) - finished)) return num_seeders, num_leech diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index a1c2994e14d..0e75ccba020 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H """Manage boosting of swarms""" -import errno import logging import os from binascii import hexlify, unhexlify @@ -16,7 +15,6 @@ from Tribler.Core.DownloadConfig import DownloadStartupConfig from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl from Tribler.Core.Utilities import utilities -from Tribler.Core.Utilities.install_dir import determine_install_dir from Tribler.Core.exceptions import OperationNotPossibleAtRuntimeException from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_TORRENTS, NTFY_UPDATE, NTFY_CHANNELCAST from Tribler.Main.globals import DefaultDownloadStartupConfig @@ -24,45 +22,15 @@ from Tribler.Policies.BoostingSource import ChannelSource from Tribler.Policies.BoostingSource import DirectorySource from Tribler.Policies.BoostingSource import RSSFeedSource -from Tribler.Policies.defs import NUMBER_TYPES, SAVED_ATTR, CREDIT_MINING_FOLDER_DOWNLOAD, CONFIG_KEY_ARCHIVELIST, \ - CONFIG_OP_RM, CONFIG_OP_ADD, CONFIG_KEY_SOURCELIST, CONFIG_KEY_ENABLEDLIST, CONFIG_KEY_DISABLEDLIST +from Tribler.Policies.credit_mining_util import source_to_string, string_to_source, compare_torrents +from Tribler.Policies.defs import SAVED_ATTR, CREDIT_MINING_FOLDER_DOWNLOAD, CONFIG_KEY_ARCHIVELIST, \ + CONFIG_KEY_SOURCELIST, CONFIG_KEY_ENABLEDLIST, CONFIG_KEY_DISABLEDLIST from Tribler.dispersy.taskmanager import TaskManager -def levenshtein_dist(t1_fname, t2_fname): - """ - Calculates the Levenshtein distance between a and b. - """ - len_t1_fname, len_t2_fname = len(t1_fname), len(t2_fname) - if len_t1_fname > len_t2_fname: - # Make sure len_t1_fname <= len_t2_fname, to use O(min(len_t1_fname,len_t2_fname)) space - t1_fname, t2_fname = t2_fname, t1_fname - len_t1_fname, len_t2_fname = len_t2_fname, len_t1_fname - - current = range(len_t1_fname + 1) - for i in range(1, len_t2_fname + 1): - previous, current = current, [i] + [0] * len_t1_fname - for j in range(1, len_t1_fname + 1): - add, delete = previous[j] + 1, current[j - 1] + 1 - change = previous[j - 1] - if t1_fname[j - 1] != t2_fname[i - 1]: - change += 1 - current[j] = min(add, delete, change) - - return current[len_t1_fname] - -def source_to_string(source_obj): - return hexlify(source_obj) if len(source_obj) == 20 and not (source_obj.startswith('http://') - or source_obj.startswith('https://')) else source_obj - -def string_to_source(source_str): - return source_str.decode('hex') \ - if len(source_str) == 40 and not (os.path.isdir(source_str) or source_str.startswith('http://')) else source_str - - class BoostingSettings(object): """ - Class contains settings on boosting manager + This class contains settings used by the boosting manager """ def __init__(self, session, policy=SeederRatioPolicy, load_config=True): self.session = session @@ -73,7 +41,7 @@ def __init__(self, session, policy=SeederRatioPolicy, load_config=True): self.source_interval = 100 self.swarm_interval = 100 - # Can't be changed in runtime + # Can't be changed on runtime self.tracker_interval = 200 self.logging_interval = 60 self.share_mode_target = 3 @@ -154,7 +122,7 @@ def del_instance(): def shutdown(self): """ - save configuration before stopping stuffs + Shutting down boosting manager. It also stops and remove all the sources. """ self.save_config() self._logger.info("Shutting down boostingmanager") @@ -170,8 +138,9 @@ def shutdown(self): self.session.lm.threadpool.cancel_pending_task("CreditMining_log_init") self.cancel_all_pending_tasks() - # for torrent in self.torrents.itervalues(): - # self.stop_download(torrent) + + # remove credit mining downloaded data + shutil.rmtree(self.settings.credit_mining_path, ignore_errors=True) def get_source_object(self, sourcekey): return self.boosting_sources.get(sourcekey, None) @@ -205,6 +174,7 @@ def add_source(self, source): try: isdir = os.path.isdir(source) except TypeError: + # this handle binary data that has null bytes '\00' isdir = False if isdir: @@ -242,23 +212,8 @@ def remove_source(self, source_key): def on_torrent_insert(self, source, infohash, torrent): """ This function called when a source is finally determined. Fetch some torrents from it, - then insert it to our data + then insert it into our data """ - def compare_torrents(torrent_1, torrent_2): - """ - comparing swarms. We don't want to download same swarm with different infohash - :return: whether those t1 and t2 similar enough - """ - files1 = [files for files in torrent_1['metainfo'].get_files_with_length() if files[1] > 1024 * 1024] - files2 = [files for files in torrent_2['metainfo'].get_files_with_length() if files[1] > 1024 * 1024] - - if len(files1) == len(files2): - for ft1 in files1: - for ft2 in files2: - if ft1[1] != ft2[1] or levenshtein_dist(ft1[0], ft2[0]) > 5: - return False - return True - return False # Remember where we got this torrent from self._logger.debug("remember torrent %s from %s", torrent, source_to_string(source)) @@ -546,8 +501,8 @@ def log_statistics(self): if unhexlify(str(status.info_hash)) in self.torrents: self._logger.debug("Status for %s : %s %s | ul_lim : %d, max_ul %d, maxcon %d", status.info_hash, - status.all_time_download, - status.all_time_upload) + status.all_time_download, status.all_time_upload, lt_torrent.upload_limit(), + lt_torrent.max_uploads(), lt_torrent.max_connections()) # piece_priorities will fail in libtorrent 1.0.9 if lt.version == '1.0.9.0': @@ -564,7 +519,10 @@ def log_statistics(self): def update_torrent_stats(self, torrent_infohash_str, seeding_stats): """ - function to update swarm statistics + function to update swarm statistics. + + This function called when we get new Downloadstate for active torrents. + Updated downloadstate (seeding_stats) for a particular torrent is stored here. """ if 'time_seeding' in self.torrents[torrent_infohash_str]['last_seeding_stats']: if seeding_stats['time_seeding'] >= self.torrents[torrent_infohash_str][ diff --git a/Tribler/Policies/BoostingPolicy.py b/Tribler/Policies/BoostingPolicy.py new file mode 100644 index 00000000000..3def71ea11f --- /dev/null +++ b/Tribler/Policies/BoostingPolicy.py @@ -0,0 +1,110 @@ +# coding=utf-8 +""" +Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H +Supported boosting policy +""" +import logging +import random + + +class BoostingPolicy(object): + """ + Base class for determining what swarm selection policy will be applied + """ + + def __init__(self, session): + self.session = session + # function that checks if key can be applied to torrent + self.reverse = None + + self._logger = logging.getLogger(self.__class__.__name__) + + def apply(self, torrents, max_active, force=False): + """ + apply the policy to the torrents stored + """ + sorted_torrents = sorted([torrent for torrent in torrents.itervalues() + if self.key_check(torrent)], + key=self.key, reverse=self.reverse) + + torrents_start = [] + for torrent in sorted_torrents[:max_active]: + if not self.session.get_download(torrent["metainfo"].get_infohash()): + torrents_start.append(torrent) + torrents_stop = [] + for torrent in sorted_torrents[max_active:]: + if self.session.get_download(torrent["metainfo"].get_infohash()): + torrents_stop.append(torrent) + + if force: + return torrents_start, torrents_stop + + # if both results are empty for some reason (e.g, key_check too restrictive) + # or torrent started less than half available torrent (try to keep boosting alive) + # if it's already random, just let it be + if not isinstance(self, RandomPolicy) and ((not torrents_start and not torrents_stop) or + (len(torrents_start) < len(torrents) / 2 and len( + torrents_start) < max_active / 2)): + self._logger.error("Start and stop torrent list are empty. Fallback to Random") + # fallback to random policy + torrents_start, torrents_stop = RandomPolicy(self.session).apply(torrents, max_active) + + return torrents_start, torrents_stop + + def key(self, key): + """ + function to find a key of an object + """ + return None + + def key_check(self, key): + """ + function to check whether a swarm is included to download + """ + return False + +class RandomPolicy(BoostingPolicy): + """ + A credit mining policy that chooses a swarm randomly + """ + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.reverse = False + + def key_check(self, key): + return True + + def key(self, key): + return random.random() + + +class CreationDatePolicy(BoostingPolicy): + """ + A credit mining policy that chooses swarm by its creation date + + The idea is, older swarms need to be boosted. + """ + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.reverse = True + + def key_check(self, key): + return key['creation_date'] > 0 + + def key(self, key): + return key['creation_date'] + + +class SeederRatioPolicy(BoostingPolicy): + """ + Default policy. Find the most underseeded swarm to boost. + """ + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.reverse = False + + def key(self, key): + return key['num_seeders'] / float(key['num_seeders'] + key['num_leechers']) + + def key_check(self, key): + return (key['num_seeders'] + key['num_leechers']) > 0 diff --git a/Tribler/Policies/BoostingSource.py b/Tribler/Policies/BoostingSource.py index 27c34640260..7fe6dc39ec4 100644 --- a/Tribler/Policies/BoostingSource.py +++ b/Tribler/Policies/BoostingSource.py @@ -3,151 +3,42 @@ Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H Supported boosting sources """ -import HTMLParser import glob import logging import os -import random import re -import time import urllib from binascii import hexlify from hashlib import sha1 +import feedparser import libtorrent as lt +from twisted.internet import defer from twisted.internet import reactor +from twisted.internet.defer import CancelledError from twisted.internet.task import LoopingCall -from twisted.web.client import Agent, readBody +from twisted.web.client import Agent, readBody, getPage +from twisted.web.error import Error from twisted.web.http_headers import Headers from Tribler.Core.TorrentDef import TorrentDef -from Tribler.Core.simpledefs import NTFY_INSERT, NTFY_TORRENTS, NTFY_UPDATE, NTFY_CHANNELCAST, NTFY_VOTECAST +from Tribler.Core.simpledefs import NTFY_INSERT, NTFY_TORRENTS, NTFY_UPDATE from Tribler.Core.version import version_id -from Tribler.Main.Utility.GuiDBTuples import Torrent, CollectedTorrent, RemoteTorrent, NotCollectedTorrent, Channel, \ - ChannelTorrent +from Tribler.Main.Utility.GuiDBTuples import Torrent, Channel +from Tribler.Policies.credit_mining_util import TorrentManagerCM, ent2chr from Tribler.community.allchannel.community import AllChannelCommunity from Tribler.community.channel.community import ChannelCommunity from Tribler.dispersy.exception import CommunityNotFoundException from Tribler.dispersy.taskmanager import TaskManager -from Tribler.dispersy.util import call_on_reactor_thread -def ent2chr(input_str): - """ - Function to unescape literal string in XML to symbols - source : http://www.gossamer-threads.com/lists/python/python/177423 - """ - code = input_str.group(1) - code_int = int(code) if code.isdigit() else int(code[1:], 16) - return chr(code_int) if code_int < 256 else '?' - - -class TorrentManagerCM(TaskManager): - """ - *Temporary* class to handle load torrent. - - Adapted from TorrentManager in SearchGridManager - """ - __single = None - - def __init__(self, session): - super(TorrentManagerCM, self).__init__() - TorrentManagerCM.__single = self - - self.session = session - self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) - self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) - self.votecastdb = self.session.open_dbhandler(NTFY_VOTECAST) - - self.dslist = [] - - # TODO(ardhi) : temporary function until GUI and core code are separated - def load_torrent(self, torrent, callback=None): - """ - function to load torrent dictionary to torrent object. - - From TorrentManager.loadTorrent in SearchGridManager - """ - - # session is quitting - if not (self.session and self.session.get_torrent_store() and self.session.lm.torrent_store): - return - - if not isinstance(torrent, CollectedTorrent): - if torrent.torrent_id <= 0: - torrent_id = self.torrent_db.getTorrentID(torrent.infohash) - if torrent_id: - torrent.update_torrent_id(torrent_id) - - if not self.session.has_collected_torrent(torrent.infohash): - files = [] - trackers = [] - - # see if we have most info in our tables - if isinstance(torrent, RemoteTorrent): - torrent_id = self.torrent_db.getTorrentID(torrent.infohash) - else: - torrent_id = torrent.torrent_id - - trackers.extend(self.torrent_db.getTrackerListByTorrentID(torrent_id)) - - if 'DHT' in trackers: - trackers.remove('DHT') - if 'no-DHT' in trackers: - trackers.remove('no-DHT') - - # replacement # self.downloadTorrentfileFromPeers(torrent, None) - if self.session.has_download(torrent.infohash): - return False - - if torrent.query_candidates is None or len(torrent.query_candidates) == 0: - self.session.download_torrentfile(torrent.infohash, None, 0) - else: - for candidate in torrent.query_candidates: - self.session.download_torrentfile_from_peer(candidate, torrent.infohash, None, 0) - - torrent = NotCollectedTorrent(torrent, files, trackers) - else: - tdef = TorrentDef.load_from_memory(self.session.get_collected_torrent(torrent.infohash)) - - if torrent.torrent_id <= 0: - del torrent.torrent_id - - torrent = CollectedTorrent(torrent, tdef) - - # replacement # self.library_manager.addDownloadState(torrent) - for dl_state in self.dslist: - torrent.addDs(dl_state) - - # return - if callback is not None: - callback(torrent) - else: - return torrent - - @staticmethod - def get_instance(*args, **kw): - """ - get single instance of TorrentManagerCM - """ - if TorrentManagerCM.__single is None: - TorrentManagerCM(*args, **kw) - return TorrentManagerCM.__single - - @staticmethod - def del_instance(): - """ - resetting, then deleting single instance - """ - TorrentManagerCM.__single = None - - -class BoostingSource(object): +class BoostingSource(TaskManager): """ Base class for boosting source. For now, it can be RSS, directory, and channel """ - def __init__(self, session, source, boost_settings, callback): + def __init__(self, session, source, boost_settings, torrent_insert_cb): + super(BoostingSource, self).__init__() self.session = session self.channelcast_db = session.lm.channelcast_db @@ -155,7 +46,7 @@ def __init__(self, session, source, boost_settings, callback): self.source = source self.interval = boost_settings.source_interval self.max_torrents = boost_settings.max_torrents_per_source - self.callback = callback + self.torrent_insert_callback = torrent_insert_cb self.archive = False self.enabled = True @@ -168,49 +59,60 @@ def __init__(self, session, source, boost_settings, callback): self.min_connection = boost_settings.min_connection_start self.min_channels = boost_settings.min_channels_start - self.torrent_mgr = TorrentManagerCM.get_instance(session) + self.torrent_mgr = TorrentManagerCM(session) - # local import for handling circular import - from Tribler.Policies.BoostingManager import BoostingManager - self.boosting_manager = BoostingManager.get_instance() - self._logger = logging.getLogger(BoostingManager.__name__) + self._logger = logging.getLogger(BoostingSource.__name__) + + self.boosting_manager = self.session.lm.boosting_manager + + def start(self): + """ + Start operating mining for this source + """ + d = self._load_if_ready(self.source) + self.register_task(str(self.source) + "_load", d, value=self.source) + self._logger.debug("Start mining on %s", self.source) def kill_tasks(self): """ kill tasks on this source """ - self.ready = False - self.torrent_mgr.del_instance() - - self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_load") - - self.session.lm.threadpool.cancel_pending_task(self.source) + self.cancel_all_pending_tasks() def _load_if_ready(self, source): """ load source if and only if the overall system is ready. - it depends on #connection and #channel - - Useful to not burden the apps in startup + This is useful so we don't burden the application during the startup """ + def check_system(defer_param=None): + """ + function that check the system whether it's ready or not - nr_channels = self.channelcast_db.getNrChannels() - nr_connections = 0 + it depends on #connection and #channel + """ + if defer_param is None: + defer_param = defer.Deferred() - for community in self.session.lm.dispersy.get_communities(): - from Tribler.community.search.community import SearchCommunity - if isinstance(community, SearchCommunity): - nr_connections = community.get_nr_connections() + nr_channels = self.channelcast_db.getNrChannels() + nr_connections = 0 - # condition example - if nr_channels > self.min_channels and nr_connections > self.min_connection: - called_func = self._load - else: - called_func = self._load_if_ready + for community in self.session.lm.dispersy.get_communities(): + from Tribler.community.search.community import SearchCommunity + if isinstance(community, SearchCommunity): + nr_connections = community.get_nr_connections() + + if nr_channels > self.min_channels and nr_connections > self.min_connection: + defer_param.callback(source) + else: + self.register_task(str(self.source)+"_check_sys", reactor.callLater(10, check_system, defer_param)) + + return defer_param - self.session.lm.threadpool.add_task(lambda src=source: called_func(src), 15, task_name=str(self.source) + "_load") + defer_check = check_system() + defer_check.addCallbacks(self._load, self._on_err) + return defer_check def _load(self, source): pass @@ -224,13 +126,27 @@ def get_source_text(self): """ return self.source + def _on_err(self, err_msg): + self._logger.error(err_msg) + + def check_and_register_task(self, name, task, delay=None, value=None, interval=None): + """ + Helper function to avoid assertion in register task. + + It will register task if it has not already registered + """ + task_ret = None + if not self.is_pending_task_active(name): + task_ret = self.register_task(name, task, delay, value, interval) + + return task_ret class ChannelSource(BoostingSource): """ Credit mining source from a channel. """ - def __init__(self, session, dispersy_cid, boost_settings, callback): - BoostingSource.__init__(self, session, dispersy_cid, boost_settings, callback) + def __init__(self, session, dispersy_cid, boost_settings, torrent_insert_cb): + BoostingSource.__init__(self, session, dispersy_cid, boost_settings, torrent_insert_cb) self.channel_id = None @@ -239,19 +155,15 @@ def __init__(self, session, dispersy_cid, boost_settings, callback): self.database_updated = True self.check_torrent_interval = 10 + self.dispersy_cid = dispersy_cid self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) - self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load_if_ready(cid), 0, - task_name=str(self.source) + "_load") self.unavail_torrent = {} def kill_tasks(self): BoostingSource.kill_tasks(self) - # cancel loading channel id - self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_get_channel_id") - self.session.remove_observer(self._on_database_updated) def _load(self, dispersy_cid): @@ -263,7 +175,7 @@ def join_community(): """ try: self.community = dispersy.get_community(dispersy_cid, True) - self.session.lm.threadpool.add_task(get_channel_id, 0, task_name=str(self.source) + "_get_channel_id") + self.register_task(str(self.source) + "_get_id", reactor.callLater(1, get_channel_id)) except CommunityNotFoundException: @@ -277,8 +189,7 @@ def join_community(): self.community = ChannelCommunity.init_community(dispersy, dispersy.get_member(mid=dispersy_cid), allchannelcommunity._my_member, self.session) self._logger.info("Joined channel community %s", dispersy_cid.encode("HEX")) - self.session.lm.threadpool.add_task(get_channel_id, 0, - task_name=str(self.source) + "_get_channel_id") + self.register_task(str(self.source) + "_get_id", reactor.callLater(1, get_channel_id)) else: self._logger.error("Could not find AllChannelCommunity") @@ -286,30 +197,25 @@ def get_channel_id(): """ find channel id by looking at the network """ - if self.community and self.community._channel_id: # before: and self.gui_util.registered: + if self.community and self.community._channel_id: self.channel_id = self.community._channel_id - channel_dict = self.boosting_manager.channelcast_db.getChannel(self.channel_id) + channel_dict = self.channelcast_db.getChannel(self.channel_id) self.channel = Channel(*channel_dict) - if not self.boosting_manager.is_pending_task_active(str(self.source) + "_update"): + task_call = self.check_and_register_task(str(self.source) + "_update", + LoopingCall(self._update)).start(self.interval, now=True) + if task_call: self._logger.debug("Registering update call") - self.boosting_manager.register_task(str(self.source) + "_update", LoopingCall(self._update)).start( - self.interval, now=True) - # self.session.lm.threadpool.add_task(self._update, 0, task_name=str(self.source)+"_update") + self._logger.info("Got channel id %s", self.channel_id) + + self.ready = True else: self._logger.warning("Could not get channel id, retrying in 10 s") - self.session.lm.threadpool.add_task(get_channel_id, 10, task_name=str(self.source) + "_get_channel_id") + self.register_task(str(self.source) + "_get_id", reactor.callLater(10, get_channel_id)) - try: - join_community() - self.ready = True - except CommunityNotFoundException: - self._logger.info("Channel %s was not ready, waits for next interval (%d chn)", hexlify(self.source), - len(dispersy.get_communities())) - self.session.lm.threadpool.add_task(lambda cid=dispersy_cid: self._load_if_ready(cid), 0, - task_name=str(self.source) + "_load") + self.register_task(str(self.source) + "_join_comm", reactor.callLater(1, join_community)) def _check_tor(self): """ @@ -320,6 +226,11 @@ def showtorrent(torrent): assembly torrent data, call the callback """ if torrent.files: + if len(self.torrents) >= self.max_torrents: + self._logger.debug("Max torrents in source reached. Not adding %s", torrent.infohash) + del self.unavail_torrent[torrent.infohash] + return + infohash = torrent.infohash self._logger.debug("[ChannelSource] Got torrent %s", hexlify(infohash)) self.torrents[infohash] = {} @@ -337,8 +248,8 @@ def showtorrent(torrent): del self.unavail_torrent[infohash] - if self.callback: - self.callback(self.source, infohash, self.torrents[infohash]) + if self.torrent_insert_callback: + self.torrent_insert_callback(self.source, infohash, self.torrents[infohash]) self.database_updated = False self._logger.debug("Unavailable #torrents : %d from %s", len(self.unavail_torrent), hexlify(self.source)) @@ -348,66 +259,31 @@ def showtorrent(torrent): self.torrent_mgr.load_torrent(torrent, showtorrent) def _update(self): - if len(self.torrents) < self.max_torrents: - - if self.database_updated: - CHANTOR_DB = ['ChannelTorrents.channel_id', 'Torrent.torrent_id', 'infohash', '""', 'length', - 'category', 'status', 'num_seeders', 'num_leechers', 'ChannelTorrents.id', - 'ChannelTorrents.dispersy_id', 'ChannelTorrents.name', 'Torrent.name', - 'ChannelTorrents.description', 'ChannelTorrents.time_stamp', 'ChannelTorrents.inserted'] - - torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, CHANTOR_DB, - self.max_torrents) - - # TODO(ardhi) : temporary function until GUI and core code are separated - def create_torrents(tor_values, _, channel_dict): - """ - function to create torrents from channel. Adapted from - ChannelManager in SearchGridManager - """ - - #adding new channel from the one that can't be detected from torrent values - fetch_channels = set(hit[0] for hit in tor_values if hit[0] not in channel_dict) - if len(fetch_channels) > 0: - channels_new_dict = self.channelcast_db.getChannels(fetch_channels) - channels = [] - for hit in channels_new_dict: - channel = Channel(*hit) - channels.append(channel) - - for channel in channels: - channel_dict[channel.id] = channel - - # creating torrents - torrents = [] - for hit in tor_values: - if hit: - chan_torrent = ChannelTorrent(*hit[1:] + [channel_dict.get(hit[0], None), None]) - chan_torrent.torrent_db = self.boosting_manager.torrent_db - chan_torrent.channelcast_db = self.channelcast_db - - if chan_torrent.name: - torrents.append(chan_torrent) - - return torrents - - listtor = create_torrents(torrent_values, True, - {self.channel_id: self.channelcast_db.getChannel(self.channel_id)}) - # listtor = self.gui_util.channelsearch_manager._createTorrents( - # torrent_values, True, {self.channel_id: self.channelcast_db.getChannel(self.channel_id)})[2] - - # dict {key_infohash(binary):Torrent(object-GUIDBTuple)} - self.unavail_torrent.update({t.infohash: t for t in listtor if t.infohash not in self.torrents}) - - # it's highly probable the checktor function is running at this time (if it's already running) - # if not running, start the checker - if not self.boosting_manager.is_pending_task_active(hexlify(self.source) + "_checktor"): - self._logger.debug("Registering check torrent function") - self.boosting_manager.register_task(hexlify(self.source) + "_checktor", - LoopingCall(self._check_tor)).start(self.check_torrent_interval, - now=True) - - def _on_database_updated(self, subject, change_type, infohash): + if len(self.torrents) < self.max_torrents and self.database_updated: + CHANTOR_DB = ['ChannelTorrents.channel_id', 'Torrent.torrent_id', 'infohash', '""', 'length', + 'category', 'status', 'num_seeders', 'num_leechers', 'ChannelTorrents.id', + 'ChannelTorrents.dispersy_id', 'ChannelTorrents.name', 'Torrent.name', + 'ChannelTorrents.description', 'ChannelTorrents.time_stamp', 'ChannelTorrents.inserted'] + + torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, CHANTOR_DB, + self.max_torrents) + + listtor = self.torrent_mgr.create_torrents(torrent_values, True, + {self.channel_id: self.channelcast_db.getChannel( + self.channel_id)}) + + # dict {key_infohash(binary):Torrent(object-GUIDBTuple)} + self.unavail_torrent.update({t.infohash: t for t in listtor if t.infohash not in self.torrents}) + + # it's highly probable the checktor function is running at this time (if it's already running) + # if not running, start the checker + + task_call = self.check_and_register_task(hexlify(self.source) + "_checktor", LoopingCall(self._check_tor)) + if task_call: + self._logger.debug("Registering check torrent function") + task_call.start(self.check_torrent_interval, now=True) + + def _on_database_updated(self, dummy_subject, dummy_change_type, dummy_infohash): self.database_updated = True def get_source_text(self): @@ -417,217 +293,171 @@ def get_source_text(self): class RSSFeedSource(BoostingSource): """ Credit mining source from a RSS feed. - - # supported list (tested) : - # http://bt.etree.org/rss/bt_etree_org.rdf - # http://www.mininova.org/rss.xml - # https://kat.cr (via torcache) - - # not supported (till now) - # https://eztv.ag/ezrss.xml (https link) """ - def __init__(self, session, rss_feed, boost_settings, callback): - BoostingSource.__init__(self, session, rss_feed, boost_settings, callback) + def __init__(self, session, rss_feed, boost_settings, torrent_insert_cb): + BoostingSource.__init__(self, session, rss_feed, boost_settings, torrent_insert_cb) + + self.parsed_rss = None - self.feed_handle = None + self.torrent_store = self.session.lm.torrent_store - self.session.lm.threadpool.add_task(lambda feed=rss_feed: self._load_if_ready(feed), 0, - task_name=str(self.source) + "_load") + # Not all RSS feeds provide us with the infohash, + # so we use a fake infohash based on the URL (generated by sha1) to identify the torrents. + # keys : fake infohash, value : real infohash. Type : (length 20 string, binary) + self.fake_infohash_id = {} self.title = "" self.description = "" self.total_torrents = 0 + self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) + + def _on_success_rss(self, body_rss, rss_feed): + """ + function called when RSS successfully read + """ + self.register_task(str(self.source) + "_update", LoopingCall(self._update), + 10, interval=self.interval) + self.parsed_rss = feedparser.parse(body_rss) + self._logger.info("Got RSS feed %s", rss_feed) + self.ready = True + + def _on_error_rss(self, failure, rss_feed): + """ + function called when RSS failed except from 503 + aborting load the source + """ + failure.trap(CancelledError, Error) + self._logger.error("Aborting load on : %s. Reason : %s.", rss_feed, failure.getErrorMessage()) + + if "503" in failure.getErrorMessage(): + self.register_task(str(self.source)+"_load_delay", reactor.callLater(10, self._load, rss_feed)) + return + + if rss_feed in self.boosting_manager.boosting_sources: + self.boosting_manager.set_enable_mining(rss_feed, False) + def _load(self, rss_feed): - self.feed_handle = self.session.lm.ltmgr.get_session().add_feed( - {'url': rss_feed, 'auto_download': False, 'auto_map_handles': False}) - def wait_for_feed(): - """ - Wait until the RSS feed is no longer updating. - """ - feed_status = self.feed_handle.get_feed_status() - if feed_status['updating']: - self.session.lm.threadpool.add_task(wait_for_feed, 1, task_name=str(self.source) + "_wait_for_feed") - elif len(feed_status['error']) > 0: - self._logger.error("Got error for RSS feed %s : %s", feed_status['url'], feed_status['error']) - if "503" in feed_status["error"]: - def retry_task(): - self.feed_handle.update_feed() - self.session.lm.threadpool.add_task(wait_for_feed, 1, - task_name=str(self.source) + "_wait_for_feed") - - # if failed, wait for 10 second to retry - self.session.lm.threadpool.add_task(retry_task, 10, task_name=str(self.source) + "_wait_for_feed") - else: - # The feed is done updating. Now periodically start retrieving torrents. - self.boosting_manager.register_task(str(self.source) + "_update", LoopingCall(self._update), - 10, interval=self.interval) - self._logger.info("Got RSS feed %s", feed_status['url']) - self.ready = True + defer_feed = getPage(rss_feed) + defer_feed.addCallback(self._on_success_rss, rss_feed) + defer_feed.addErrback(self._on_error_rss, rss_feed) - wait_for_feed() + self.register_task(str(self.source)+"_wait_feed", defer_feed) def _update(self): - if len(self.torrents) < self.max_torrents: + if len(self.torrents) >= self.max_torrents: + return - feed_status = self.feed_handle.get_feed_status() + feed_elem = self.parsed_rss['feed'] - self.title = feed_status['title'] - self.description = feed_status['description'] + self.title = feed_elem['title'] + self.description = feed_elem['subtitle'] - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', - 'enabled', 'last_seeding_stats'] + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', + 'enabled', 'last_seeding_stats'] - self.total_torrents = len(feed_status['items']) - - def __cb_body(body_bin, item_torrent, torrent_fname): - tdef = None - metainfo = None - try: - metainfo = lt.bdecode(body_bin) - tdef = TorrentDef.load_from_dict(metainfo) - tdef.save(torrent_fname) - except ValueError, err: - self._logger.error("Could not parse/save torrent, skipping %s. Reason: %s", - item_torrent['url'], err.message + - ", metainfo is "+("not " if metainfo else "")+"None") - except IOError, ioerr: - # can't save to disk. Ignore this swarm - self._logger.exception("IO error, check %s. Message : %s", - self.boosting_manager.settings.credit_mining_path, ioerr.message) - return - if tdef: - # Create a torrent dict. - torrent_values = [item_torrent['title'], tdef, tdef.get_creation_date(), tdef.get_length(), - len(tdef.get_files()), -1, -1, self.enabled, {}] - self.torrents[sha1(item_torrent['url']).digest()] = dict(zip(torrent_keys, torrent_values)) - - try: - # manually generate an ID and put this into DB - self.session.lm.torrent_db.addOrGetTorrentID(sha1(item_torrent['url']).digest()) - self.session.lm.torrent_db.addExternalTorrent(tdef) - - # create Torrent object and store it - self.torrent_mgr.load_torrent(Torrent.fromTorrentDef(tdef)) - except AttributeError, err: - # if we can't find torrent_db, fallback - self._logger.error("Can't do %s. Return gracely as failed.", err.message) - return - # self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) - - # Notify the BoostingManager and provide the real infohash. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[sha1( - item_torrent['url']).digest()]) - - def __success_cb(response, item_dict, torrent_filename): - return readBody(response).addCallback(__cb_body, item_dict, torrent_filename) - - regex_unescape_xml = re.compile(r"\&\#(x?[0-9a-fA-F]+);") - - for item in feed_status['items']: - # Not all RSS feeds provide us with the infohash, - # so we use a fake infohash based on the URL to identify the torrents. - url = regex_unescape_xml.sub(ent2chr, item['url']) - infohash = sha1(url).digest() - if infohash not in self.torrents: - # Store the torrents as rss-infohash_as_hex.torrent. - torrent_filename = os.path.join(self.boosting_manager.settings.credit_mining_path, - 'rss-%s.torrent' % infohash.encode('hex')) - tdef = None - if not os.path.exists(torrent_filename): + def __cb_body(body_bin, item_torrent_entry): + tdef = None + metainfo = None + # tdef.get_infohash returned binary string by length 20 + try: + metainfo = lt.bdecode(body_bin) + tdef = TorrentDef.load_from_dict(metainfo) + self.session.save_collected_torrent(tdef.get_infohash(), body_bin) + except ValueError, err: + self._logger.error("Could not parse/save torrent, skipping %s. Reason: %s", + item_torrent_entry['link'], err.message + + ", metainfo is " + ("not " if metainfo else "") +"None") + + if tdef and len(self.torrents) < self.max_torrents: + # Create a torrent dict. + real_infohash = tdef.get_infohash() + torrent_values = [item_torrent_entry['title'], tdef, tdef.get_creation_date(), tdef.get_length(), + len(tdef.get_files()), -1, -1, self.enabled, {}] + + # store the real infohash to generated infohash + self.torrents[real_infohash] = dict(zip(torrent_keys, torrent_values)) + self.fake_infohash_id[sha1(item_torrent_entry['id']).digest()] = real_infohash + + # manually generate an ID and put this into DB + self.torrent_db.addOrGetTorrentID(real_infohash) + self.torrent_db.addExternalTorrent(tdef) + + # create Torrent object and store it + self.torrent_mgr.load_torrent(Torrent.fromTorrentDef(tdef)) + + # Notify the BoostingManager and provide the real infohash. + if self.torrent_insert_callback: + self.torrent_insert_callback(self.source, real_infohash, self.torrents[real_infohash]) + elif tdef: + self._logger.debug("Max torrents in source reached. Not adding %s", tdef.get_infohash()) + + def __success_cb(response, item_dict): + return readBody(response).addCallback(__cb_body, item_dict).addErrback(self._on_err) + + regex_unescape_xml = re.compile(r"\&\#(x?[0-9a-fA-F]+);") + + for item in self.parsed_rss['entries']: + f_links = item['links'] + for link in f_links: + if link['type'] == u'application/x-bittorrent': + url = regex_unescape_xml.sub(ent2chr, str(link['href'])) + fake_infohash = sha1(url).digest() + if fake_infohash not in self.fake_infohash_id.keys(): # create Agent to download torrent file + self.fake_infohash_id[fake_infohash] = None agent = Agent(reactor) ses_agent = agent.request( 'GET', # http://stackoverflow.com/a/845595 urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]"), Headers({'User-Agent': ['Tribler ' + version_id]}), None) - ses_agent.addCallback(__success_cb, item, torrent_filename) - - else: - # torrent already exist in our system - tdef = TorrentDef.load(torrent_filename) - - if tdef: - # Create a torrent dict. - torrent_values = [item['title'], tdef, tdef.get_creation_date(), tdef.get_length(), - len(tdef.get_files()), -1, -1, self.enabled, {}] - self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) - - # manually generate an ID and put this into DB - self.session.lm.torrent_db.addOrGetTorrentID(infohash) - self.session.lm.torrent_db.addExternalTorrent(tdef) - - # create Torrent object and store it - self.torrent_mgr.load_torrent(Torrent.fromTorrentDef(tdef)) - # self.gui_util.torrentsearch_manager.loadTorrent(Torrent.fromTorrentDef(tdef)) - - # Notify the BoostingManager and provide the real infohash. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) - - def kill_tasks(self): - BoostingSource.kill_tasks(self) - - self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_wait_for_feed") - - # stop updating - self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_update") + ses_agent.addCallback(__success_cb, item).addErrback(self._on_err) class DirectorySource(BoostingSource): """ Credit mining source from a local directory - The directory must be exist. + The directory must exist. """ - def __init__(self, session, directory, boost_settings, callback): - BoostingSource.__init__(self, session, directory, boost_settings, callback) - self._load(directory) - def _load(self, directory): if os.path.isdir(directory): # Wait for __init__ to finish so the source is registered with the # BoostinManager, otherwise adding torrents won't work - self.session.lm.threadpool.add_task(self._update, 1, task_name=str(self.source) + "_update") + self.register_task(str(self.source) + "_update", + LoopingCall(self._update), delay=2, interval=self.interval) self._logger.info("Got directory %s", directory) self.ready = True else: self._logger.error("Could not find directory %s", directory) def _update(self): - if len(self.torrents) < self.max_torrents: - - torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', - 'enabled', 'last_seeding_stats'] - - for torrent_filename in glob.glob(self.source + '/*.torrent'): - if torrent_filename not in self.torrents: - try: - tdef = TorrentDef.load(torrent_filename) - except ValueError, verr: - self._logger.debug("Could not load torrent locally, skipping %s", torrent_filename) - self._logger.error("Could not load %s. Reason %s", torrent_filename, verr) - continue - - # Create a torrent dict. - infohash = tdef.get_infohash() - torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), - len(tdef.get_files()), -1, -1, self.enabled, {}] - self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) - # Notify the BoostingManager. - if self.callback: - self.callback(self.source, tdef.get_infohash(), self.torrents[infohash]) - - self.session.lm.threadpool.add_task(self._update, self.interval, task_name=str(self.source) + "_update") + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', + 'enabled', 'last_seeding_stats'] - def kill_tasks(self): - BoostingSource.kill_tasks(self) + # Wait for __init__ to finish so the source is registered with the + # BoostingManager, otherwise adding torrents won't work. Although we already include delay when call this + if not self.ready: + return - # stop updating - self.session.lm.threadpool.cancel_pending_task(str(self.source) + "_update") + for torrent_filename in glob.glob(self.source + '/*.torrent'): + if torrent_filename not in self.torrents and len(self.torrents) < self.max_torrents: + try: + tdef = TorrentDef.load(torrent_filename) + except ValueError, verr: + self._logger.error("Could not load %s. Reason %s", torrent_filename, verr) + continue + + # Create a torrent dict. + infohash = tdef.get_infohash() + torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), + len(tdef.get_files()), -1, -1, self.enabled, {}] + self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) + # Notify the BoostingManager. + if self.torrent_insert_callback: + self.torrent_insert_callback(self.source, tdef.get_infohash(), self.torrents[infohash]) diff --git a/Tribler/Policies/credit_mining_util.py b/Tribler/Policies/credit_mining_util.py new file mode 100644 index 00000000000..1814668145f --- /dev/null +++ b/Tribler/Policies/credit_mining_util.py @@ -0,0 +1,210 @@ +""" +File containing function used in credit mining module. +""" + + +import os +from binascii import hexlify + +from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.simpledefs import NTFY_CHANNELCAST +from Tribler.Core.simpledefs import NTFY_TORRENTS +from Tribler.Core.simpledefs import NTFY_VOTECAST +from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent, RemoteTorrent, NotCollectedTorrent, Channel, \ + ChannelTorrent +from Tribler.dispersy.taskmanager import TaskManager + + +def levenshtein_dist(t1_fname, t2_fname): + """ + Calculates the Levenshtein distance between a and b. + + Levenshtein distance (LD) is a measure of the similarity between two strings. + (from http://people.cs.pitt.edu/~kirk/cs1501/Pruhs/Fall2006/Assignments/editdistance/Levenshtein%20Distance.htm) + """ + len_t1_fname, len_t2_fname = len(t1_fname), len(t2_fname) + if len_t1_fname > len_t2_fname: + # Make sure len_t1_fname <= len_t2_fname, to use O(min(len_t1_fname,len_t2_fname)) space + t1_fname, t2_fname = t2_fname, t1_fname + len_t1_fname, len_t2_fname = len_t2_fname, len_t1_fname + + current = range(len_t1_fname + 1) + for i in xrange(1, len_t2_fname + 1): + previous, current = current, [i] + [0] * len_t1_fname + for j in xrange(1, len_t1_fname + 1): + add, delete = previous[j] + 1, current[j - 1] + 1 + change = previous[j - 1] + if t1_fname[j - 1] != t2_fname[i - 1]: + change += 1 + current[j] = min(add, delete, change) + + return current[len_t1_fname] + + +def source_to_string(source_obj): + return hexlify(source_obj) if len(source_obj) == 20 and not (source_obj.startswith('http://') + or source_obj.startswith('https://')) else source_obj + + +def string_to_source(source_str): + # don't need to handle null byte because lazy evaluation + return source_str.decode('hex') \ + if len(source_str) == 40 and not (os.path.isdir(source_str) or source_str.startswith('http://')) else source_str + + +def compare_torrents(torrent_1, torrent_2): + """ + comparing swarms. We don't want to download the same swarm with different infohash + :return: whether those t1 and t2 similar enough + """ + files1 = [files for files in torrent_1['metainfo'].get_files_with_length() if files[1] > 1024 * 1024] + files2 = [files for files in torrent_2['metainfo'].get_files_with_length() if files[1] > 1024 * 1024] + + if len(files1) == len(files2): + for ft1 in files1: + for ft2 in files2: + if ft1[1] != ft2[1] or levenshtein_dist(ft1[0], ft2[0]) > 5: + return False + return True + return False + + +def ent2chr(input_str): + """ + Function to unescape literal string in XML to symbols + source : http://www.gossamer-threads.com/lists/python/python/177423 + """ + code = input_str.group(1) + code_int = int(code) if code.isdigit() else int(code[1:], 16) + return chr(code_int) if code_int < 256 else '?' + +# TODO(ardhi) : temporary function until GUI and core code are separated +class TorrentManagerCM(TaskManager): + """ + *Temporary* class to handle load torrent. + + Adapted from TorrentManager in SearchGridManager + """ + __single = None + + def __init__(self, session): + super(TorrentManagerCM, self).__init__() + TorrentManagerCM.__single = self + + self.session = session + self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) + self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) + self.votecastdb = self.session.open_dbhandler(NTFY_VOTECAST) + + self.dslist = [] + + def load_torrent(self, torrent, callback=None): + """ + function to load torrent dictionary to torrent object. + + From TorrentManager.loadTorrent in SearchGridManager + """ + + # session is quitting + if not (self.session and self.session.get_torrent_store() and self.session.lm.torrent_store): + return + + if not isinstance(torrent, CollectedTorrent): + if torrent.torrent_id <= 0: + torrent_id = self.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + if not self.session.has_collected_torrent(torrent.infohash): + files = [] + trackers = [] + + # see if we have most info in our tables + if isinstance(torrent, RemoteTorrent): + torrent_id = self.torrent_db.getTorrentID(torrent.infohash) + else: + torrent_id = torrent.torrent_id + + trackers.extend(self.torrent_db.getTrackerListByTorrentID(torrent_id)) + + if 'DHT' in trackers: + trackers.remove('DHT') + if 'no-DHT' in trackers: + trackers.remove('no-DHT') + + # replacement # self.downloadTorrentfileFromPeers(torrent, None) + if self.session.has_download(torrent.infohash): + return False + + if torrent.query_candidates is None or len(torrent.query_candidates) == 0: + self.session.download_torrentfile(torrent.infohash, None, 0) + else: + for candidate in torrent.query_candidates: + self.session.download_torrentfile_from_peer(candidate, torrent.infohash, None, 0) + + torrent = NotCollectedTorrent(torrent, files, trackers) + + else: + tdef = TorrentDef.load_from_memory(self.session.get_collected_torrent(torrent.infohash)) + + if torrent.torrent_id <= 0: + del torrent.torrent_id + + torrent = CollectedTorrent(torrent, tdef) + + # replacement # self.library_manager.addDownloadState(torrent) + for dl_state in self.dslist: + torrent.addDs(dl_state) + + # return + if callback is not None: + callback(torrent) + else: + return torrent + + def create_torrents(self, tor_values, _, channel_dict): + """ + function to create torrents from channel. Adapted from + ChannelManager in SearchGridManager + """ + + #adding new channel from the one that can't be detected from torrent values + fetch_channels = set(hit[0] for hit in tor_values if hit[0] not in channel_dict) + if len(fetch_channels) > 0: + channels_new_dict = self.channelcast_db.getChannels(fetch_channels) + channels = [] + for hit in channels_new_dict: + channel = Channel(*hit) + channels.append(channel) + + for channel in channels: + channel_dict[channel.id] = channel + + # creating torrents + torrents = [] + for hit in tor_values: + if hit: + chan_torrent = ChannelTorrent(*hit[1:] + [channel_dict.get(hit[0], None), None]) + chan_torrent.torrent_db = self.torrent_db + chan_torrent.channelcast_db = self.channelcast_db + + if chan_torrent.name: + torrents.append(chan_torrent) + + return torrents + + @staticmethod + def get_instance(*args, **kw): + """ + get single instance of TorrentManagerCM + """ + if TorrentManagerCM.__single is None: + TorrentManagerCM(*args, **kw) + return TorrentManagerCM.__single + + @staticmethod + def del_instance(): + """ + resetting, then deleting single instance + """ + TorrentManagerCM.__single = None diff --git a/Tribler/Tools/boostchannel.py b/Tribler/Tools/boostchannel.py index 55c0b1ada8e..5b137783f72 100644 --- a/Tribler/Tools/boostchannel.py +++ b/Tribler/Tools/boostchannel.py @@ -10,7 +10,8 @@ from Tribler.Core.Session import Session from Tribler.Core.SessionConfig import SessionStartupConfig -from Tribler.Policies.BoostingManager import BoostingManager, RandomPolicy, CreationDatePolicy, SeederRatioPolicy +from Tribler.Policies.BoostingManager import BoostingManager +from Tribler.Policies.BoostingPolicy import RandomPolicy, CreationDatePolicy, SeederRatioPolicy from Tribler.community.channel.community import ChannelCommunity from Tribler.community.channel.preview import PreviewChannelCommunity from Tribler.community.allchannel.community import AllChannelCommunity @@ -84,7 +85,6 @@ def main(): config.set_multicast_local_peer_discovery(False) config.set_megacache(True) config.set_dispersy(True) - config.set_swift_proc(False) config.set_mainline_dht(False) config.set_torrent_collecting(False) config.set_libtorrent(True) From 66eb87f7e49e9a692ee4b2702bf88508adf016f0 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 27 Jun 2016 15:45:27 +0200 Subject: [PATCH 21/24] Fix resume data to use configparser This commits also did some changes in defaults.py: - Specify default configuration - Disable credit mining by default - Remove default channel to boost By this commit, only RSS feed from etree.org left in the default configuration - Remove old write_resume_data function By this commits, use new save_resume_data function - Remove singleton usage in LaunchManyCore --- .../Core/APIImplementation/LaunchManyCore.py | 11 +- .../Core/Libtorrent/LibtorrentDownloadImpl.py | 110 +++++++-------- Tribler/Core/defaults.py | 18 ++- Tribler/Policies/BoostingManager.py | 130 +++++++----------- 4 files changed, 115 insertions(+), 154 deletions(-) diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index 540f37288a0..cfb1de7a459 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -10,9 +10,9 @@ from glob import iglob from threading import Event, enumerate as enumerate_threads from traceback import print_exc -from twisted.internet.defer import Deferred from twisted.internet import reactor +from twisted.internet.defer import Deferred from twisted.internet.defer import inlineCallbacks from Tribler.Core.APIImplementation.threadpoolmanager import ThreadPoolManager @@ -96,6 +96,8 @@ def __init__(self): self.startup_deferred = Deferred() + self.boosting_manager = None + def register(self, session, sesslock): if not self.registered: self.registered = True @@ -315,6 +317,10 @@ def load_communities(): self.watch_folder = WatchFolder(self.session) self.watch_folder.start() + if self.session.get_creditmining_enable(): + from Tribler.Policies.BoostingManager import BoostingManager + self.boosting_manager = BoostingManager(self.session) + self.version_check_manager = VersionCheckManager(self.session) self.initComplete = True @@ -672,6 +678,9 @@ def early_shutdown(self): # Note: sesslock not held self.shutdownstarttime = timemod.time() + if self.boosting_manager: + yield self.boosting_manager.shutdown() + self.boosting_manager = None if self.torrent_checker: yield self.torrent_checker.shutdown() self.torrent_checker = None diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index f2cbedaf250..3fdff94b6c1 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -11,12 +11,10 @@ from twisted.internet.defer import Deferred, CancelledError from Tribler.Core import NoDispersyRLock -from Tribler.Core.APIImplementation import maketorrent from Tribler.Core.DownloadConfig import DownloadStartupConfig, DownloadConfigInterface from Tribler.Core.DownloadState import DownloadState from Tribler.Core.Libtorrent import checkHandleAndSynchronize, waitForHandleAndSynchronize from Tribler.Core.TorrentDef import TorrentDefNoMetainfo, TorrentDef -from Tribler.Core.Utilities.torrent_utils import get_info_from_handle from Tribler.Core.Utilities import maketorrent from Tribler.Core.Utilities.torrent_utils import get_info_from_handle from Tribler.Core.osutils import fix_filebasename @@ -147,13 +145,13 @@ def __init__(self, session, tdef): self.askmoreinfo = False self.correctedinfoname = u"" - self.checkpoint_disabled = False + self._checkpoint_disabled = False self.deferreds_resume = [] def __str__(self): return "LibtorrentDownloadImpl " % \ - (self.correctedinfoname, self.get_hops(), self.checkpoint_disabled) + (self.correctedinfoname, self.get_hops(), self._checkpoint_disabled) def __repr__(self): return self.__str__() @@ -162,10 +160,10 @@ def get_def(self): return self.tdef def set_checkpoint_disabled(self, disabled=True): - self.checkpoint_disabled = disabled + self._checkpoint_disabled = disabled def get_checkpoint_disabled(self): - return self.checkpoint_disabled + return self._checkpoint_disabled def setup(self, dcfg=None, pstate=None, initialdlstatus=None, wrapperDelay=0, share_mode=False, checkpoint_disabled=False): @@ -180,6 +178,8 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None, # Called by any thread, assume sessionlock is held self.set_checkpoint_disabled(checkpoint_disabled) + self.set_share_mode(share_mode) + try: # The deferred to be returned deferred = Deferred() @@ -204,7 +204,7 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None, def schedule_create_engine(): self.cew_scheduled = True create_engine_wrapper_deferred = self.network_create_engine_wrapper( - self.pstate_for_restart, initialdlstatus, share_mode=share_mode) + self.pstate_for_restart, initialdlstatus) create_engine_wrapper_deferred.chainDeferred(deferred) @@ -256,7 +256,7 @@ def do_check(): do_check() return can_create_deferred - def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode=False, checkpoint_disabled=False): + def network_create_engine_wrapper(self, pstate, initialdlstatus=None, checkpoint_disabled=False): with self.dllock: self._logger.debug("LibtorrentDownloadImpl: network_create_engine_wrapper()") @@ -268,7 +268,7 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode atp["duplicate_is_error"] = True atp["hops"] = self.get_hops() - if share_mode: + if self.get_share_mode(): atp["flags"] = lt.add_torrent_params_flags_t.flag_share_mode self.set_checkpoint_disabled(checkpoint_disabled) @@ -296,16 +296,6 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode atp["ti"] = torrentinfo has_resume_data = resume_data and isinstance(resume_data, dict) - - # TODO(ardhi) : properly store pstate in BoostingManager - if has_resume_data is not None: - # we have resume data but somehow its not working (Credit mining case) - if not isinstance(resume_data, dict): - new_dict = pstate.get('engineresumedata', None) - if isinstance(new_dict, dict): - resume_data = new_dict - has_resume_data = resume_data and isinstance(resume_data, dict) - if has_resume_data: atp["resume_data"] = lt.bencode(resume_data) else: @@ -315,8 +305,8 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, share_mode self.handle = self.ltmgr.add_torrent(self, atp) if self.handle: - # if in share mode, don't change priority of the file - self.set_selected_files(share_mode=share_mode) + + self.set_selected_files() # If we lost resume_data always resume download in order to force checking if initialdlstatus != DLSTATUS_STOPPED or not resume_data: @@ -372,10 +362,6 @@ def get_vod_fileindex(self): return self.vod_index return -1 - @checkHandleAndSynchronize(None) - def write_resume_data(self): - return self.handle.write_resume_data() - @checkHandleAndSynchronize(0) def get_vod_filesize(self): fileindex = self.get_vod_fileindex() @@ -658,7 +644,7 @@ def set_corrected_infoname(self): self.correctedinfoname = self.get_corrected_filename() @checkHandleAndSynchronize() - def set_selected_files(self, selected_files=None, share_mode=False): + def set_selected_files(self, selected_files=None): if not isinstance(self.tdef, TorrentDefNoMetainfo): if selected_files is None: @@ -710,7 +696,8 @@ def set_selected_files(self, selected_files=None, share_mode=False): except TypeError: self.handle.rename_file(index, new_path.encode("utf-8")) - if not share_mode: + # if in share mode, don't change priority of the file + if self.get_share_mode(): self.handle.prioritize_files(filepriorities) self.unwanteddir_abs = unwanteddir_abs @@ -877,40 +864,35 @@ def create_peerlist_data(peer_info): A function to convert peer_info libtorrent object into dictionary This data is used to identify peers with combination of several flags """ - peer_dict = {} - - peer_dict['id'] = peer_info.pid - peer_dict['extended_version'] = peer_info.client - peer_dict['ip'] = peer_info.ip[0] - peer_dict['port'] = peer_info.ip[1] - # optimistic_unchoke = 0x800 seems unavailable in python bindings - peer_dict['optimistic'] = bool(peer_info.flags & 2048) - peer_dict['direction'] = 'L' if bool(peer_info.flags & peer_info.local_connection) else 'R' - peer_dict['uprate'] = peer_info.payload_up_speed - peer_dict['uinterested'] = bool(peer_info.flags & peer_info.remote_interested) - peer_dict['uchoked'] = bool(peer_info.flags & peer_info.remote_choked) - peer_dict['uhasqueries'] = peer_info.upload_queue_length > 0 - peer_dict['uflushed'] = peer_info.used_send_buffer > 0 - peer_dict['downrate'] = peer_info.payload_down_speed - peer_dict['dinterested'] = bool(peer_info.flags & peer_info.interesting) - peer_dict['dchoked'] = bool(peer_info.flags & peer_info.choked) - peer_dict['snubbed'] = bool(peer_info.flags & 4096) # snubbed = 0x1000 seems unavailable in python bindings - peer_dict['utotal'] = peer_info.total_upload - peer_dict['dtotal'] = peer_info.total_download - peer_dict['completed'] = peer_info.progress - peer_dict['have'] = peer_info.pieces - peer_dict['speed'] = peer_info.remote_dl_rate - peer_dict['country'] = peer_info.country - peer_dict['connection_type'] = peer_info.connection_type - - # add upload_only and/or seed - peer_dict['seed'] = bool(peer_info.flags & peer_info.seed) - peer_dict['upload_only'] = bool(peer_info.flags & peer_info.upload_only) - - # add read and write state (check unchoke/choke peers) - # read and write state is char with value 0, 1, 2, 4. May be empty - peer_dict['rstate'] = peer_info.read_state - peer_dict['wstate'] = peer_info.write_state + peer_dict = {'id': peer_info.pid, + 'extended_version': peer_info.client, + 'ip': peer_info.ip[0], + 'port': peer_info.ip[1], + # optimistic_unchoke = 0x800 seems unavailable in python bindings + 'optimistic': bool(peer_info.flags & 0x800), + 'direction': 'L' if bool(peer_info.flags & peer_info.local_connection) else 'R', + 'uprate': peer_info.payload_up_speed, + 'uinterested': bool(peer_info.flags & peer_info.remote_interested), + 'uchoked': bool(peer_info.flags & peer_info.remote_choked), + 'uhasqueries': peer_info.upload_queue_length > 0, + 'uflushed': peer_info.used_send_buffer > 0, + 'downrate': peer_info.payload_down_speed, + 'dinterested': bool(peer_info.flags & peer_info.interesting), + 'dchoked': bool(peer_info.flags & peer_info.choked), + 'snubbed': bool(peer_info.flags & 0x1000), + 'utotal': peer_info.total_upload, + 'dtotal': peer_info.total_download, + 'completed': peer_info.progress, + 'have': peer_info.pieces, 'speed': peer_info.remote_dl_rate, + 'country': peer_info.country, + 'connection_type': peer_info.connection_type, + # add upload_only and/or seed + 'seed': bool(peer_info.flags & peer_info.seed), + 'upload_only': bool(peer_info.flags & peer_info.upload_only), + # add read and write state (check unchoke/choke peers) + # read and write state is char with value 0, 1, 2, 4. May be empty + 'rstate': peer_info.read_state, + 'wstate': peer_info.write_state} return peer_dict @@ -1097,10 +1079,10 @@ def get_dest_files(self, exts=None): def checkpoint(self): """ Called by any thread """ - if self.checkpoint_disabled: + if self._checkpoint_disabled: self._logger.warning("Ignoring checkpoint() call as is checkpointing disabled for this download.") else: - (infohash, pstate) = self.network_checkpoint() + infohash, pstate = self.network_checkpoint() checkpoint = lambda: self.session.lm.save_download_pstate(infohash, pstate) self.session.lm.threadpool.add_task(checkpoint, 0) @@ -1191,7 +1173,7 @@ def dlconfig_changed_callback(self, section, name, new_value, old_value): def get_share_mode(self): return self.handle.status().share_mode - @checkHandleAndSynchronize + @waitForHandleAndSynchronize(True) def set_share_mode(self, share_mode): self.handle.set_share_mode(share_mode) diff --git a/Tribler/Core/defaults.py b/Tribler/Core/defaults.py index 51138b408ab..041105b8160 100644 --- a/Tribler/Core/defaults.py +++ b/Tribler/Core/defaults.py @@ -166,20 +166,19 @@ # Credit mining config sessdefaults['credit_mining'] = OrderedDict() -sessdefaults['credit_mining']['max_torrents_per_source'] = 10 -sessdefaults['credit_mining']['max_torrents_active'] = 20 +sessdefaults['credit_mining']['enabled'] = False +sessdefaults['credit_mining']['max_torrents_per_source'] = 20 +sessdefaults['credit_mining']['max_torrents_active'] = 50 sessdefaults['credit_mining']['source_interval'] = 100 sessdefaults['credit_mining']['swarm_interval'] = 100 sessdefaults['credit_mining']['share_mode_target'] = 3 sessdefaults['credit_mining']['tracker_interval'] = 200 sessdefaults['credit_mining']['logging_interval'] = 60 -sessdefaults['credit_mining']['boosting_sources'] = ["http://bt.etree.org/rss/bt_etree_org.rdf", - "9e1a3fac737543d36c83d54818ed77620932ff80", - "028d2b5eea5277ddc3cedf78601f9be246b29bf1"] -sessdefaults['credit_mining']['boosting_enabled'] = ["http://bt.etree.org/rss/bt_etree_org.rdf", - "9e1a3fac737543d36c83d54818ed77620932ff80"] -sessdefaults['credit_mining']['boosting_disabled'] = ["028d2b5eea5277ddc3cedf78601f9be246b29bf1"] -sessdefaults['credit_mining']['archive_sources'] = ["9e1a3fac737543d36c83d54818ed77620932ff80"] +# By default we want to automatically boost legal-predetermined channel. +sessdefaults['credit_mining']['boosting_sources'] = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +sessdefaults['credit_mining']['boosting_enabled'] = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +sessdefaults['credit_mining']['boosting_disabled'] = [] +sessdefaults['credit_mining']['archive_sources'] = [] sessdefaults['credit_mining']['policy'] = "seederratio" # @@ -271,5 +270,4 @@ tribler_defaults['Tribler']['mintray'] = 2 if sys.platform == 'win32' else 0 tribler_defaults['Tribler']['free_space_threshold'] = 100 * 1024 * 1024 tribler_defaults['Tribler']['version_info'] = {} -tribler_defaults['Tribler']['boosting_sources'] = "" tribler_defaults['Tribler']['last_reported_version'] = None diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 0e75ccba020..6c360a836bf 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -3,9 +3,11 @@ """Manage boosting of swarms""" import logging import os +import shutil from binascii import hexlify, unhexlify import libtorrent as lt +from twisted.internet.task import LoopingCall from twisted.internet import reactor from twisted.internet.task import LoopingCall @@ -15,10 +17,11 @@ from Tribler.Core.DownloadConfig import DownloadStartupConfig from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl from Tribler.Core.Utilities import utilities +from Tribler.Core.Utilities.configparser import CallbackConfigParser from Tribler.Core.exceptions import OperationNotPossibleAtRuntimeException from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_TORRENTS, NTFY_UPDATE, NTFY_CHANNELCAST from Tribler.Main.globals import DefaultDownloadStartupConfig -from Tribler.Policies.BoostingPolicy import RandomPolicy, CreationDatePolicy, SeederRatioPolicy +from Tribler.Policies.BoostingPolicy import SeederRatioPolicy from Tribler.Policies.BoostingSource import ChannelSource from Tribler.Policies.BoostingSource import DirectorySource from Tribler.Policies.BoostingSource import RSSFeedSource @@ -72,7 +75,6 @@ def __init__(self, session, settings=None): BoostingManager.__single = self self.boosting_sources = {} self.torrents = {} - self.running = True self.session = session assert self.session.get_libtorrent() @@ -90,19 +92,19 @@ def __init__(self, session, settings=None): if not os.path.exists(self.settings.credit_mining_path): os.makedirs(self.settings.credit_mining_path) - local_settings = {} - local_settings['share_mode_target'] = self.settings.share_mode_target - self.session.lm.ltmgr.get_session().set_settings(local_settings) + self.session.lm.ltmgr.get_session().set_settings( + {'share_mode_target': self.settings.share_mode_target}) self.session.add_observer(self.on_torrent_notify, NTFY_TORRENTS, [NTFY_UPDATE]) - # TODO(emilon): Refactor this to use taskmanager - self.session.lm.threadpool.add_task(self._select_torrent, self.settings.initial_swarm_interval, - "CreditMining_select_init") - self.session.lm.threadpool.add_task(self.scrape_trackers, - self.settings.initial_tracker_interval, "CreditMining_scrape") - self.session.lm.threadpool.add_task(self.log_statistics, - self.settings.initial_logging_interval, "CreditMining_log_init") + self.register_task("CreditMining_select", LoopingCall(self._select_torrent), + self.settings.initial_swarm_interval, interval=self.settings.swarm_interval) + + self.register_task("CreditMining_scrape", LoopingCall(self.scrape_trackers), + self.settings.initial_tracker_interval, interval=self.settings.tracker_interval) + + self.register_task("CreditMining_log", LoopingCall(self.log_statistics), + self.settings.initial_logging_interval, interval=self.settings.logging_interval) @staticmethod def get_instance(*args, **kw): @@ -130,16 +132,9 @@ def shutdown(self): for sourcekey in self.boosting_sources.keys(): self.remove_source(sourcekey) - if self.session.lm.threadpool.is_pending_task_active("CreditMining_select_init"): - self.session.lm.threadpool.cancel_pending_task("CreditMining_select_init") - if self.session.lm.threadpool.is_pending_task_active("CreditMining_scrape"): - self.session.lm.threadpool.cancel_pending_task("CreditMining_scrape") - if self.session.lm.threadpool.is_pending_task_active("CreditMining_log_init"): - self.session.lm.threadpool.cancel_pending_task("CreditMining_log_init") - self.cancel_all_pending_tasks() - # remove credit mining downloaded data + #remove credit mining data in not persistent mode shutil.rmtree(self.settings.credit_mining_path, ignore_errors=True) def get_source_object(self, sourcekey): @@ -287,8 +282,6 @@ def scrape_trackers(self): # check health(seeder/leecher) self.session.lm.torrent_checker.add_gui_request(infohash, True) - self.session.lm.threadpool.add_task(self.scrape_trackers, self.settings.tracker_interval, "CreditMining_scrape") - def set_archive(self, source, enable): """ setting archive of a particular source. Affect all the torrents in this source @@ -303,53 +296,42 @@ def start_download(self, torrent): """ Start downloading a particular torrent and add it to download list in Tribler """ - def do_start(): - """ - add the actual torrent to the manager to download it later - :return: - """ - dscfg = DownloadStartupConfig() - dscfg.set_dest_dir(self.settings.credit_mining_path) - dscfg.set_safe_seeding(False) + dscfg = DownloadStartupConfig() + dscfg.set_dest_dir(self.settings.credit_mining_path) + dscfg.set_safe_seeding(False) - preload = torrent.get('preload', False) + preload = torrent.get('preload', False) - # not using Session.start_download because we need to specify pstate - if self.session.lm.download_exists(torrent["metainfo"].get_infohash()): - self._logger.error("Already downloading %s. Cancel start_download", - hexlify(torrent["metainfo"].get_infohash())) - return + pstate = CallbackConfigParser() + pstate.add_section('state') + pstate.set('state', 'engineresumedata', torrent.get('pstate', None)) - self._logger.info("Starting %s preload %s has pstate %s", - hexlify(torrent["metainfo"].get_infohash()), preload, - True if torrent.get('pstate', None) else False) + # not using Session.start_download because we need to specify pstate + if self.session.lm.download_exists(torrent["metainfo"].get_infohash()): + self._logger.error("Already downloading %s. Cancel start_download", + hexlify(torrent["metainfo"].get_infohash())) + return - torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=torrent.get('pstate', None), - hidden=True, share_mode=not preload, checkpoint_disabled=True) - torrent['download'].set_priority(torrent.get('prio', 1)) + self._logger.info("Starting %s preload %s has pstate %s", + hexlify(torrent["metainfo"].get_infohash()), preload, + True if torrent.get('pstate', None) else False) - self.session.lm.threadpool.add_task_in_thread(do_start, 0) + torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=pstate, + hidden=True, share_mode=not preload, checkpoint_disabled=True) + torrent['download'].set_priority(torrent.get('prio', 1)) def stop_download(self, torrent): """ Stopping torrent that currently downloading """ - - def do_stop(): - """ - The actual function to stop torrent downloading - :return: - """ - ihash = lt.big_number(torrent["metainfo"].get_infohash()) - self._logger.info("Stopping %s", str(ihash)) - download = torrent.pop('download', False) - lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(ihash) - if download and lt_torrent.is_valid(): - self._logger.debug("Writing resume data") - torrent['pstate'] = {'engineresumedata': download.write_resume_data()} - self.session.remove_download(download, hidden=True) - - self.session.lm.threadpool.add_task_in_thread(do_stop, 0) + ihash = lt.big_number(torrent["metainfo"].get_infohash()) + self._logger.info("Stopping %s", str(ihash)) + download = torrent.pop('download', False) + lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(ihash) + if download and lt_torrent.is_valid(): + self._logger.info("Writing resume data for %s", str(ihash)) + torrent['pstate'] = download.write_resume_data() + self.session.remove_download(download, hidden=True) def _select_torrent(self): """ @@ -379,8 +361,6 @@ def _select_torrent(self): self._logger.info("Selecting from %s torrents %s start download", len(torrents), len(torrents_start)) - self.session.lm.threadpool.add_task(self._select_torrent, self.settings.swarm_interval) - def load_config(self): """ load config in file configuration and apply it to manager @@ -432,17 +412,11 @@ def _set_enable_boosting(values, enabled): }, } - switch_policy = { - "random" : RandomPolicy, - "creation" : CreationDatePolicy, - "seederratio" : SeederRatioPolicy - } - # set policy - self.settings.policy = self.session.get_credit_mining_policy(True)(self.session) + self.settings.policy = self.session.get_cm_policy(True)(self.session) dict_to_load = {} - dict_to_load.update(self.session.get_credit_mining_sources()) + dict_to_load.update(self.session.get_cm_sources()) dict_to_load.update(dict.fromkeys(SAVED_ATTR)) for k, val in dict_to_load.items(): @@ -450,7 +424,7 @@ def _set_enable_boosting(values, enabled): if k in SAVED_ATTR: # see the session configuration object.__setattr__(self.settings, k, - getattr(self.session, "get_credit_mining_%s" %k)()) + getattr(self.session, "get_cm_%s" %k)()) else: #credit mining source handle switch[k]["cmd"](*((switch[k]['args'][0] or val,) + switch[k]['args'][1:])) except KeyError: @@ -462,8 +436,8 @@ def save_config(self): """ for k in SAVED_ATTR: try: - setattr(self.session, "set_credit_mining_%s" % k, getattr(self.settings, k)) - except OperationNotPossibleAtRuntimeException, err_msg: + setattr(self.session, "set_cm_%s" % k, getattr(self.settings, k)) + except OperationNotPossibleAtRuntimeException: # some of the attribute can't be changed in runtime. See lm.sessconfig_changed_callback self._logger.debug("Cannot set attribute %s. Not permitted in runtime", k) @@ -485,10 +459,10 @@ def save_config(self): if boosting_source.archive: archive_sources.append(bsname) - self.session.set_credit_mining_sources(lboosting_sources, CONFIG_KEY_SOURCELIST) - self.session.set_credit_mining_sources(flag_enabled_sources, CONFIG_KEY_ENABLEDLIST) - self.session.set_credit_mining_sources(flag_disabled_sources, CONFIG_KEY_DISABLEDLIST) - self.session.set_credit_mining_sources(archive_sources, CONFIG_KEY_ARCHIVELIST) + self.session.set_cm_sources(lboosting_sources, CONFIG_KEY_SOURCELIST) + self.session.set_cm_sources(flag_enabled_sources, CONFIG_KEY_ENABLEDLIST) + self.session.set_cm_sources(flag_disabled_sources, CONFIG_KEY_DISABLEDLIST) + self.session.set_cm_sources(archive_sources, CONFIG_KEY_ARCHIVELIST) self.session.save_pstate_sessconfig() @@ -505,7 +479,7 @@ def log_statistics(self): lt_torrent.max_uploads(), lt_torrent.max_connections()) # piece_priorities will fail in libtorrent 1.0.9 - if lt.version == '1.0.9.0': + if lt.__version__ == '1.0.9.0': continue else: non_zero_values = [] @@ -515,8 +489,6 @@ def log_statistics(self): if non_zero_values: self._logger.debug("Non zero priorities for %s : %s", status.info_hash, non_zero_values) - self.session.lm.threadpool.add_task(self.log_statistics, self.settings.logging_interval) - def update_torrent_stats(self, torrent_infohash_str, seeding_stats): """ function to update swarm statistics. From 57e86368dfe5e58cfdd6c78d96daea9e30e8fa44 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama Date: Tue, 14 Jun 2016 16:53:28 +0200 Subject: [PATCH 22/24] Add logger.conf, Delete bootstraptribler.txt and boostchannel.py Note for boostchannel.py: We don't have separate GUI and Core yet. This file is standalone program to boost a channel. Will be created later if we have separate GUI and Core. --- Tribler/Tools/boostchannel.py | 123 ---------------------------------- Tribler/bootstraptribler.txt | 42 ------------ logger.conf | 14 +++- 3 files changed, 13 insertions(+), 166 deletions(-) delete mode 100644 Tribler/Tools/boostchannel.py delete mode 100644 Tribler/bootstraptribler.txt diff --git a/Tribler/Tools/boostchannel.py b/Tribler/Tools/boostchannel.py deleted file mode 100644 index 5b137783f72..00000000000 --- a/Tribler/Tools/boostchannel.py +++ /dev/null @@ -1,123 +0,0 @@ -# Written by Egbert Bouman - -import sys -import time -import shutil -import random -import getopt -import tempfile -from traceback import print_exc - -from Tribler.Core.Session import Session -from Tribler.Core.SessionConfig import SessionStartupConfig -from Tribler.Policies.BoostingManager import BoostingManager -from Tribler.Policies.BoostingPolicy import RandomPolicy, CreationDatePolicy, SeederRatioPolicy -from Tribler.community.channel.community import ChannelCommunity -from Tribler.community.channel.preview import PreviewChannelCommunity -from Tribler.community.allchannel.community import AllChannelCommunity - - -def usage(): - print "Usage: python boostchannel.py [options] dispersy_cid" - print "Options:" - print " --db_interval \tnumber of seconds between database refreshes" - print " --sw_interval \tnumber of seconds between swarm selection" - print " --max_per_source \tmaximum number of swarms per source" - print " \t\t\t\tthat should be taken into consideration" - print " --max_active \t\tmaximum number of swarms that should be" - print " \t\t\t\tactive simultaneously" - print " --policy \t\tpolicy for swarm selection" - print " \t\t\t\tpossible values: RandomPolicy" - print " \t\t\t\t CreationDatePolicy" - print " \t\t\t\t SeederRatioPolicy (default)" - print " --help\t\t\tprint this help screen" - print - print "Example:" - print " python boostchannel.py --max_active=5 3c8378fc3493b5772b1e6a25672d3889367cb7c3" - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], "hd:s:m:a:p:", ["help", "db_interval=", "sw_interval=", "max_per_source=", "max_active=", "policy="]) - except getopt.GetoptError as err: - print str(err) - usage() - sys.exit(2) - - kwargs = {} - for o, a in opts: - if o in ("-h", "--help"): - usage() - sys.exit(0) - elif o in ("-d", "--db_interval"): - kwargs['src_interval'] = int(a) - elif o in ("-s", "--sw_interval"): - kwargs['sw_interval'] = int(a) - elif o in ("-m", "--max_per_source"): - kwargs['max_eligible'] = int(a) - elif o in ("-a", "--max_active"): - kwargs['max_active'] = int(a) - elif o in ("-p", "--policy"): - if a == 'RandomPolicy': - kwargs['policy'] = RandomPolicy - elif a == 'CreationDatePolicy': - kwargs['policy'] = CreationDatePolicy - elif a == 'SeederRatioPolicy': - kwargs['policy'] = SeederRatioPolicy - else: - assert False, "Unknown policy" - else: - assert False, "Unhandled option" - - if len(args[0]) != 40: - print "Incorrect dispersy_cid" - sys.exit(2) - else: - dispersy_cid = args[0].decode('hex') - - print "Press Ctrl-C to stop boosting this channel" - - statedir = tempfile.mkdtemp() - - config = SessionStartupConfig() - config.set_state_dir(statedir) - config.set_listen_port(random.randint(10000, 60000)) - config.set_torrent_checking(False) - config.set_multicast_local_peer_discovery(False) - config.set_megacache(True) - config.set_dispersy(True) - config.set_mainline_dht(False) - config.set_torrent_collecting(False) - config.set_libtorrent(True) - config.set_dht_torrent_collecting(False) - - s = Session(config) - s.start() - - while not s.lm.initComplete: - time.sleep(1) - - def load_communities(): - dispersy.define_auto_load(AllChannelCommunity, (s.dispersy_member,), load=True) - dispersy.define_auto_load(ChannelCommunity, load=True) - dispersy.define_auto_load(PreviewChannelCommunity) - print >> sys.stderr, "Dispersy communities are ready" - - dispersy = s.get_dispersy_instance() - dispersy.callback.call(load_communities) - - bm = BoostingManager.get_instance(s, None, **kwargs) - bm.add_source(dispersy_cid) - - try: - while True: - time.sleep(sys.maxsize / 2048) - except: - print_exc() - - s.shutdown() - time.sleep(3) - shutil.rmtree(statedir) - - -if __name__ == "__main__": - main() diff --git a/Tribler/bootstraptribler.txt b/Tribler/bootstraptribler.txt deleted file mode 100644 index cff68bfa681..00000000000 --- a/Tribler/bootstraptribler.txt +++ /dev/null @@ -1,42 +0,0 @@ -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 -127.0.0.1 0 diff --git a/logger.conf b/logger.conf index af408213581..074db031cd2 100644 --- a/logger.conf +++ b/logger.conf @@ -1,5 +1,5 @@ [loggers] -keys=root,candidates,twisted,MetadataInjector,HiddenTunnelCommunity,TunnelMain,TunnelLogger,BarterCommunity,BarterCommunityCrawler, MultiChainCommunity +keys=root,candidates,twisted,MetadataInjector,HiddenTunnelCommunity,TunnelMain,TunnelLogger,BarterCommunity,BarterCommunityCrawler, BoostingManager, MultiChainCommunity, BoostingSource [handlers] keys=debugging,default @@ -64,6 +64,18 @@ qualname=MultiChainCommunity handlers=default propagate=0 +[logger_BoostingManager] +level=INFO +qualname=BoostingManager +handlers=default +propagate=0 + +[logger_BoostingSource] +level=INFO +qualname=BoostingSource +handlers=default +propagate=0 + [handler_default] class=StreamHandler level=NOTSET From b11fd657c42940ffeb5644462fb64bfe8610ed61 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Wed, 8 Jun 2016 14:00:03 +0200 Subject: [PATCH 23/24] Fix switch policy when loading config This remove complex switch policy created in previous commits Squashed commits: - Use translated seed/leech from peers if needed - Add validate source function - Refactor get_source_object to accept single type parameter - Immediately load config in credit mining startup --- Tribler/Policies/BoostingManager.py | 140 ++++++++++--------------- Tribler/Policies/credit_mining_util.py | 31 ++---- Tribler/Policies/defs.py | 2 +- 3 files changed, 68 insertions(+), 105 deletions(-) diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py index 6c360a836bf..d69b1fe4f26 100644 --- a/Tribler/Policies/BoostingManager.py +++ b/Tribler/Policies/BoostingManager.py @@ -9,23 +9,17 @@ import libtorrent as lt from twisted.internet.task import LoopingCall -from twisted.internet import reactor -from twisted.internet.task import LoopingCall -from twisted.web.client import Agent, readBody -from twisted.web.http_headers import Headers - -from Tribler.Core.DownloadConfig import DownloadStartupConfig +from Tribler.Core.DownloadConfig import DownloadStartupConfig, DefaultDownloadStartupConfig from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl from Tribler.Core.Utilities import utilities -from Tribler.Core.Utilities.configparser import CallbackConfigParser from Tribler.Core.exceptions import OperationNotPossibleAtRuntimeException from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_TORRENTS, NTFY_UPDATE, NTFY_CHANNELCAST -from Tribler.Main.globals import DefaultDownloadStartupConfig from Tribler.Policies.BoostingPolicy import SeederRatioPolicy from Tribler.Policies.BoostingSource import ChannelSource from Tribler.Policies.BoostingSource import DirectorySource from Tribler.Policies.BoostingSource import RSSFeedSource -from Tribler.Policies.credit_mining_util import source_to_string, string_to_source, compare_torrents +from Tribler.Policies.credit_mining_util import source_to_string, string_to_source, compare_torrents, \ + validate_source_string from Tribler.Policies.defs import SAVED_ATTR, CREDIT_MINING_FOLDER_DOWNLOAD, CONFIG_KEY_ARCHIVELIST, \ CONFIG_KEY_SOURCELIST, CONFIG_KEY_ENABLEDLIST, CONFIG_KEY_DISABLEDLIST from Tribler.dispersy.taskmanager import TaskManager @@ -60,14 +54,16 @@ def __init__(self, session, policy=SeederRatioPolicy, load_config=True): CREDIT_MINING_FOLDER_DOWNLOAD) self.load_config = load_config + # whether we want to check dependencies of BoostingManager + self.check_dependencies = True + self.auto_start_source = True + class BoostingManager(TaskManager): """ Class to manage all the credit mining activities """ - __single = None - def __init__(self, session, settings=None): super(BoostingManager, self).__init__() self._logger = logging.getLogger(self.__class__.__name__) @@ -77,16 +73,23 @@ def __init__(self, session, settings=None): self.torrents = {} self.session = session - assert self.session.get_libtorrent() - - self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) - self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) # use provided settings or a default one self.settings = settings or BoostingSettings(session, load_config=True) + if self.settings.check_dependencies: + assert self.session.get_libtorrent() + assert self.session.get_torrent_checking() + assert self.session.get_dispersy() + assert self.session.get_torrent_store() + assert self.session.get_enable_torrent_search() + assert self.session.get_enable_channel_search() + assert self.session.get_megacache() + + self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) + self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) + if self.settings.load_config: - self._logger.info("Loading config file from session configuration") self.load_config() if not os.path.exists(self.settings.credit_mining_path): @@ -106,22 +109,6 @@ def __init__(self, session, settings=None): self.register_task("CreditMining_log", LoopingCall(self.log_statistics), self.settings.initial_logging_interval, interval=self.settings.logging_interval) - @staticmethod - def get_instance(*args, **kw): - """ - get single instance of Boostingmanager - """ - if BoostingManager.__single is None: - BoostingManager(*args, **kw) - return BoostingManager.__single - - @staticmethod - def del_instance(): - """ - resetting, then deleting single instance - """ - BoostingManager.__single = None - def shutdown(self): """ Shutting down boosting manager. It also stops and remove all the sources. @@ -134,10 +121,13 @@ def shutdown(self): self.cancel_all_pending_tasks() - #remove credit mining data in not persistent mode + # remove credit mining downloaded data shutil.rmtree(self.settings.credit_mining_path, ignore_errors=True) def get_source_object(self, sourcekey): + """ + Get the actual object of the source key + """ return self.boosting_sources.get(sourcekey, None) def set_enable_mining(self, source, mining_bool=True, force_restart=False): @@ -182,6 +172,9 @@ def add_source(self, source): self._logger.error("Cannot add unknown source %s", source) return + if self.settings.auto_start_source: + self.boosting_sources[source].start() + self._logger.info("Added source %s", source) else: self._logger.info("Already have source %s", source) @@ -267,7 +260,7 @@ def scrape_trackers(self): Manually scrape tracker by requesting to tracker manager """ - for infohash, _ in self.torrents.iteritems(): + for infohash in list(self.torrents): # torrent handle lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(lt.big_number(infohash)) @@ -277,6 +270,13 @@ def scrape_trackers(self): peer_list.append(peer) num_seed, num_leech = utilities.translate_peers_into_health(peer_list) + + # calculate number of seeder and leecher by looking at the peers + if self.torrents[infohash]['num_seeders'] == 0: + self.torrents[infohash]['num_seeders'] = num_seed + if self.torrents[infohash]['num_leechers'] == 0: + self.torrents[infohash]['num_leechers'] = num_leech + self._logger.debug("Seeder/leecher data translated from peers : seeder %s, leecher %s", num_seed, num_leech) # check health(seeder/leecher) @@ -284,7 +284,7 @@ def scrape_trackers(self): def set_archive(self, source, enable): """ - setting archive of a particular source. Affect all the torrents in this source + setting archive of a particular source. This affects all the torrents in this source """ if source in self.boosting_sources: self.boosting_sources[source].archive = enable @@ -302,22 +302,16 @@ def start_download(self, torrent): preload = torrent.get('preload', False) - pstate = CallbackConfigParser() - pstate.add_section('state') - pstate.set('state', 'engineresumedata', torrent.get('pstate', None)) - - # not using Session.start_download because we need to specify pstate if self.session.lm.download_exists(torrent["metainfo"].get_infohash()): self._logger.error("Already downloading %s. Cancel start_download", hexlify(torrent["metainfo"].get_infohash())) return - self._logger.info("Starting %s preload %s has pstate %s", - hexlify(torrent["metainfo"].get_infohash()), preload, - True if torrent.get('pstate', None) else False) + self._logger.info("Starting %s preload %s", + hexlify(torrent["metainfo"].get_infohash()), preload) - torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, pstate=pstate, - hidden=True, share_mode=not preload, checkpoint_disabled=True) + torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, hidden=True, + share_mode=not preload, checkpoint_disabled=True) torrent['download'].set_priority(torrent.get('prio', 1)) def stop_download(self, torrent): @@ -330,7 +324,7 @@ def stop_download(self, torrent): lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(ihash) if download and lt_torrent.is_valid(): self._logger.info("Writing resume data for %s", str(ihash)) - torrent['pstate'] = download.write_resume_data() + download.save_resume_data() self.session.remove_download(download, hidden=True) def _select_torrent(self): @@ -365,14 +359,14 @@ def load_config(self): """ load config in file configuration and apply it to manager """ - validate_source = lambda s: unhexlify(s) if len(s) == 40 and not s.startswith("http") else s + self._logger.info("Loading config file from session configuration") def _add_sources(values): """ adding sources in configuration file """ for boosting_source in values: - boosting_source = validate_source(boosting_source) + boosting_source = validate_source_string(boosting_source) self.add_source(boosting_source) def _archive_sources(values): @@ -380,7 +374,7 @@ def _archive_sources(values): setting archive to sources """ for archive_source in values: - archive_source = validate_source(archive_source) + archive_source = validate_source_string(archive_source) self.set_archive(archive_source, True) def _set_enable_boosting(values, enabled): @@ -388,47 +382,27 @@ def _set_enable_boosting(values, enabled): set disable/enable source """ for boosting_source in values: - boosting_source = validate_source(boosting_source) + boosting_source = validate_source_string(boosting_source) if boosting_source not in self.boosting_sources.keys(): self.add_source(boosting_source) self.boosting_sources[boosting_source].enabled = enabled - switch = { - "boosting_sources": { - "cmd": _add_sources, - "args": (None,) - }, - "archive_sources": { - "cmd": _archive_sources, - "args": (None,) - }, - "boosting_enabled": { - "cmd": _set_enable_boosting, - "args": (None, True) - }, - "boosting_disabled": { - "cmd": _set_enable_boosting, - "args": (None, False) - }, - } - # set policy self.settings.policy = self.session.get_cm_policy(True)(self.session) - dict_to_load = {} - dict_to_load.update(self.session.get_cm_sources()) - dict_to_load.update(dict.fromkeys(SAVED_ATTR)) - - for k, val in dict_to_load.items(): - try: - if k in SAVED_ATTR: - # see the session configuration - object.__setattr__(self.settings, k, - getattr(self.session, "get_cm_%s" %k)()) - else: #credit mining source handle - switch[k]["cmd"](*((switch[k]['args'][0] or val,) + switch[k]['args'][1:])) - except KeyError: - self._logger.error("Key %s can't be applied", k) + for k in SAVED_ATTR: + # see the session configuration + object.__setattr__(self.settings, k, getattr(self.session, "get_cm_%s" %k)()) + + for k, val in self.session.get_cm_sources().items(): + if k is "boosting_sources": + _add_sources(val) + elif k is "archive_sources": + _archive_sources(val) + elif k is "boosting_enabled": + _set_enable_boosting(val, True) + elif k is "boosting_disabled": + _set_enable_boosting(val, False) def save_config(self): """ diff --git a/Tribler/Policies/credit_mining_util.py b/Tribler/Policies/credit_mining_util.py index 1814668145f..fa6f3433670 100644 --- a/Tribler/Policies/credit_mining_util.py +++ b/Tribler/Policies/credit_mining_util.py @@ -4,7 +4,7 @@ import os -from binascii import hexlify +from binascii import hexlify, unhexlify from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.simpledefs import NTFY_CHANNELCAST @@ -12,9 +12,17 @@ from Tribler.Core.simpledefs import NTFY_VOTECAST from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent, RemoteTorrent, NotCollectedTorrent, Channel, \ ChannelTorrent +from Tribler.Policies.defs import SIMILARITY_TRESHOLD from Tribler.dispersy.taskmanager import TaskManager +def validate_source_string(source): + """ + Function to check whether a source string is a valid source or not + """ + return unhexlify(source) if len(source) == 40 and not source.startswith("http") else source + + def levenshtein_dist(t1_fname, t2_fname): """ Calculates the Levenshtein distance between a and b. @@ -63,7 +71,7 @@ def compare_torrents(torrent_1, torrent_2): if len(files1) == len(files2): for ft1 in files1: for ft2 in files2: - if ft1[1] != ft2[1] or levenshtein_dist(ft1[0], ft2[0]) > 5: + if ft1[1] != ft2[1] or levenshtein_dist(ft1[0], ft2[0]) > SIMILARITY_TRESHOLD: return False return True return False @@ -85,11 +93,8 @@ class TorrentManagerCM(TaskManager): Adapted from TorrentManager in SearchGridManager """ - __single = None - def __init__(self, session): super(TorrentManagerCM, self).__init__() - TorrentManagerCM.__single = self self.session = session self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) @@ -192,19 +197,3 @@ def create_torrents(self, tor_values, _, channel_dict): torrents.append(chan_torrent) return torrents - - @staticmethod - def get_instance(*args, **kw): - """ - get single instance of TorrentManagerCM - """ - if TorrentManagerCM.__single is None: - TorrentManagerCM(*args, **kw) - return TorrentManagerCM.__single - - @staticmethod - def del_instance(): - """ - resetting, then deleting single instance - """ - TorrentManagerCM.__single = None diff --git a/Tribler/Policies/defs.py b/Tribler/Policies/defs.py index ddd886dfdbb..d37c83d6ec4 100644 --- a/Tribler/Policies/defs.py +++ b/Tribler/Policies/defs.py @@ -4,7 +4,7 @@ from Tribler.Core.Utilities.install_dir import determine_install_dir -NUMBER_TYPES = (int, long, float) +SIMILARITY_TRESHOLD = 5 TRIBLER_ROOT = determine_install_dir() From b9d1940ff4c6991d071707c31d0f81b178682f84 Mon Sep 17 00:00:00 2001 From: Ardhi Putra Pratama H Date: Mon, 27 Jun 2016 15:28:13 +0200 Subject: [PATCH 24/24] Credit mining testcase This is a combination of 12 commits. Squashed commits : - modify testcase to fit new preferences - Remove singleton in test - Change test to adapt TaskManager in source - Add test in dependencies and default load - Move rss xml creation to separate method - Refer channel and torrent creation to RestAPI test - Add more Levenshtein distance test - Add test on custom RSS parser - Refactor test on single type parameter in BoostingManager - Remove threading event usage in test - Reduce timeout in RSS test - Test cleaning pylint --- Tribler/Test/Core/CreditMining/__init__.py | 3 + .../Core/CreditMining/mock_creditmining.py | 138 +++++ .../Core/CreditMining/test_creditmining.py | 405 ++++++++++++++ .../CreditMining/test_creditmining_sys.py | 517 ++++++++++++++++++ .../RestApi/test_channels_endpoints.py | 51 +- Tribler/Test/data/test_rss_cm.xml | 31 ++ Tribler/Test/test_BoostingManager.py | 53 -- Tribler/Test/test_as_server.py | 1 + Tribler/Test/test_my_channel.py | 20 +- Tribler/Test/util.py | 21 + 10 files changed, 1151 insertions(+), 89 deletions(-) create mode 100644 Tribler/Test/Core/CreditMining/__init__.py create mode 100644 Tribler/Test/Core/CreditMining/mock_creditmining.py create mode 100644 Tribler/Test/Core/CreditMining/test_creditmining.py create mode 100644 Tribler/Test/Core/CreditMining/test_creditmining_sys.py create mode 100644 Tribler/Test/data/test_rss_cm.xml delete mode 100644 Tribler/Test/test_BoostingManager.py diff --git a/Tribler/Test/Core/CreditMining/__init__.py b/Tribler/Test/Core/CreditMining/__init__.py new file mode 100644 index 00000000000..3dc228c3f6d --- /dev/null +++ b/Tribler/Test/Core/CreditMining/__init__.py @@ -0,0 +1,3 @@ +""" +This package contains tests for the credit mining-related code of Tribler. +""" diff --git a/Tribler/Test/Core/CreditMining/mock_creditmining.py b/Tribler/Test/Core/CreditMining/mock_creditmining.py new file mode 100644 index 00000000000..525abae90d5 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/mock_creditmining.py @@ -0,0 +1,138 @@ +""" +Module of Credit mining mock classes + +Written by Ardhi Putra Pratama H +""" +from twisted.web.resource import Resource + + +class MockLtTorrent(object): + """ + Class representing libtorrent handle for getting peer info + """ + def __init__(self, infohash="12345"): + self.info_hash = infohash + self.all_time_download = 0 + self.all_time_upload = 0 + + def upload_limit(self): + return 12 + + def max_uploads(self): + return 13 + + def max_connections(self): + return 14 + + def piece_priorities(self): + return [0, 1, 1, 0, 1, 1, 1] + + def get_peer_info(self): + """ + class returning peer info for a particular handle + """ + peer = [None] * 6 + peer[0] = MockLtPeer(1, "ip1") + peer[0].setvalue(True, True, True) + peer[1] = MockLtPeer(2, "ip2") + peer[1].setvalue(False, False, True) + peer[2] = MockLtPeer(3, "ip3") + peer[2].setvalue(True, False, True) + peer[3] = MockLtPeer(4, "ip4") + peer[3].setvalue(False, True, False) + peer[4] = MockLtPeer(5, "ip5") + peer[4].setvalue(False, True, True) + peer[5] = MockLtPeer(6, "ip6") + peer[5].setvalue(False, False, False) + return peer + + def is_valid(self): + """ + check whether the handle is valid or not + """ + return True + + def status(self): + return self + +class MockLtPeer(object): + """ + Dummy peer object returned by libtorrent python binding + """ + def __init__(self, pid, ip): + self.pid = pid + self.client = 2 + self.ip = [ip, "port"] + self.flags = 1 + self.local_connection = True + self.payload_up_speed = 0 + self.remote_interested = 1 + self.remote_choked = 1 + self.upload_queue_length = 0 + self.used_send_buffer = 0 + self.payload_down_speed = 0 + self.interesting = True + self.choked = True + self.total_upload = 0 + self.total_download = 0 + self.progress = 0 + self.pieces = 0 + self.remote_dl_rate = 0 + self.country = "ID" + self.connection_type = 0 + self.seed = 1 + self.upload_only = 1 + self.read_state = False + self.write_state = False + + def setvalue(self, upload_only, uinterested, completed): + self.upload_only = upload_only + self.remote_interested = uinterested + self.progress = 1 if completed else 0 + + +class MockMeta(object): + """ + class for mocking the torrent metainfo + """ + + def __init__(self, id_hash): + self.infohash = id_hash + + def get_infohash(self): + """ + returning infohash of torrents + """ + return self.infohash + + +class MockLtSession(object): + """ + Mock for session and LibTorrentMgr + """ + def __init__(self): + pass + + def get_session(self): + """ + supposed to get libtorrent session + """ + return self + + def set_settings(self, _): + """ + set settings (don't do anything) + """ + pass + + def shutdown(self): + """ + obligatory shutdown function + """ + pass + + +class ResourceFailClass(Resource): + def render_GET(self, request): + request.setResponseCode(503) + return "Error 503." diff --git a/Tribler/Test/Core/CreditMining/test_creditmining.py b/Tribler/Test/Core/CreditMining/test_creditmining.py new file mode 100644 index 00000000000..eeb2d85b195 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/test_creditmining.py @@ -0,0 +1,405 @@ +# coding=utf-8 +""" +Module of Credit mining function testing + +Written by Mihai Capotă and Ardhi Putra Pratama H +""" + +import binascii +import random +import re + +import Tribler.Policies.BoostingManager as bm +from Tribler.Core.DownloadConfig import DefaultDownloadStartupConfig +from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl +from Tribler.Core.SessionConfig import SessionConfigInterface +from Tribler.Core.Utilities import utilities +from Tribler.Core.defaults import sessdefaults +from Tribler.Policies.BoostingPolicy import CreationDatePolicy, SeederRatioPolicy, RandomPolicy +from Tribler.Policies.BoostingSource import ent2chr +from Tribler.Policies.credit_mining_util import levenshtein_dist, source_to_string +from Tribler.Test.Core.CreditMining.mock_creditmining import MockMeta, MockLtPeer, MockLtSession, MockLtTorrent +from Tribler.Test.test_as_server import TestAsServer + + +class TestBoostingManagerPolicies(TestAsServer): + """ + The class to test core function of credit mining policies + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerPolicies, self).__init__(*argv, **kwargs) + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerPolicies, self).setUp() + self.session.get_download = lambda x: x % 2 + random.seed(0) + self.torrents = dict() + for i in xrange(1, 11): + mock_metainfo = MockMeta(i) + + self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": i, + "num_leechers": i-1, "creation_date": i} + + def test_random_policy(self): + """ + testing random policy + """ + policy = RandomPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(3, len(ids_start), "Start failed %s vs %s" % (ids_start, torrents_start)) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(2, len(ids_stop), "Stop failed %s vs %s" % (ids_stop, torrents_stop)) + + def test_seederratio_policy(self): + """ + testing seeder ratio policy + """ + policy = SeederRatioPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(ids_start, [10, 8, 6]) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(ids_stop, [3, 1]) + + def test_fallback_policy(self): + """ + testing policy (seederratio) and then fallback + """ + + for i in xrange(1, 11): + mock_metainfo = MockMeta(i) + self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": -i, + "num_leechers": -i, "creation_date": i} + + policy = SeederRatioPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(3, len(ids_start), "Start failed %s vs %s" % (ids_start, torrents_start)) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(2, len(ids_stop), "Stop failed %s vs %s" % (ids_stop, torrents_stop)) + + def test_creationdate_policy(self): + """ + test policy based on creation date + """ + policy = CreationDatePolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 5, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(ids_start, [10, 8, 6]) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(ids_stop, [5, 3, 1]) + + +class TestBoostingManagerUtilities(TestAsServer): + """ + Test several utilities used in credit mining + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerUtilities, self).__init__(*argv, **kwargs) + + self.peer = [None] * 6 + self.peer[0] = MockLtPeer(1, "ip1") + self.peer[0].setvalue(True, True, True) + self.peer[1] = MockLtPeer(2, "ip2") + self.peer[1].setvalue(False, False, True) + self.peer[2] = MockLtPeer(3, "ip3") + self.peer[2].setvalue(True, False, True) + self.peer[3] = MockLtPeer(4, "ip4") + self.peer[3].setvalue(False, True, False) + self.peer[4] = MockLtPeer(5, "ip5") + self.peer[4].setvalue(False, True, True) + self.peer[5] = MockLtPeer(6, "ip6") + self.peer[5].setvalue(False, False, False) + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerUtilities, self).setUp() + + self.session.get_libtorrent = lambda: True + + self.bsettings = bm.BoostingSettings(self.session) + self.bsettings.credit_mining_path = self.session_base_dir + self.bsettings.load_config = False + self.bsettings.check_dependencies = False + self.bsettings.initial_logging_interval = 900 + + def tearDown(self): + # TODO(ardhi) : remove it when Tribler free of singleton + # and 1 below + DefaultDownloadStartupConfig.delInstance() + + super(TestBoostingManagerUtilities, self).tearDown() + + def test_boosting_dependencies(self): + """ + Test whether boosting manager dependencies works or not. + + In all test, check dependencies always off. In production, it is on by default. + """ + self.bsettings.check_dependencies = True + self.bsettings.initial_swarm_interval = 9000 + self.bsettings.initial_tracker_interval = 9000 + self.bsettings.initial_logging_interval = 9000 + self.session.open_dbhandler = lambda _: None + self.session.lm.ltmgr = MockLtSession() + + self.session.get_torrent_checking = lambda: True + self.session.get_dispersy = lambda: True + self.session.get_torrent_store = lambda: True + self.session.get_enable_torrent_search = lambda: True + self.session.get_enable_channel_search = lambda: True + self.session.get_megacache = lambda: False + + self.assertRaises(AssertionError, bm.BoostingManager, self.session, self.bsettings) + + def test_load_default(self): + """ + Test load default configuration in BoostingManager + """ + self.bsettings.load_config = True + self.bsettings.auto_start_source = False + self.bsettings.initial_swarm_interval = 9000 + self.bsettings.initial_tracker_interval = 9000 + self.bsettings.initial_logging_interval = 9000 + self.session.open_dbhandler = lambda _: None + self.session.lm.ltmgr = MockLtSession() + + # it will automatically load the default configuration + boost_man = bm.BoostingManager(self.session, self.bsettings) + + # def validate(d_defer): + self.assertEqual(sessdefaults['credit_mining']['source_interval'], boost_man.settings.source_interval) + self.assertEqual(sessdefaults['credit_mining']['archive_sources'], + [source_to_string(src.source) for src in boost_man.boosting_sources.values() + if src.archive]) + + boost_man.cancel_all_pending_tasks() + + def test_sessionconfig(self): + """ + test basic credit mining preferences + """ + sci = SessionConfigInterface() + + sci.set_cm_logging_interval(100) + self.assertEqual(sci.get_cm_logging_interval(), 100) + + sci.set_cm_max_torrents_active(20) + self.assertEqual(sci.get_cm_max_torrents_active(), 20) + + sci.set_cm_max_torrents_per_source(10) + self.assertEqual(sci.get_cm_max_torrents_per_source(), 10) + + sci.set_cm_source_interval(100) + self.assertEqual(sci.get_cm_source_interval(), 100) + + sci.set_cm_policy("random") + self.assertIs(sci.get_cm_policy(as_class=True), RandomPolicy) + + sci.set_cm_policy(SeederRatioPolicy(self.session)) + self.assertEqual(sci.get_cm_policy(as_class=False), "seederratio") + + sci.set_cm_share_mode_target(2) + self.assertEqual(sci.get_cm_share_mode_target(), 2) + + sci.set_cm_swarm_interval(200) + self.assertEqual(sci.get_cm_swarm_interval(), 200) + + sci.set_cm_tracker_interval(300) + self.assertEqual(sci.get_cm_tracker_interval(), 300) + + def test_translate_peer_info(self): + """ + test - predict number of seeder and leecher only based on peer discovered and + their activities + """ + peerlist_dict = [] + for peer in self.peer: + peerlist_dict.append(LibtorrentDownloadImpl.create_peerlist_data(peer)) + + num_seed, num_leech = utilities.translate_peers_into_health(peerlist_dict) + self.assertEqual(num_seed, 4, "Seeder number don't match") + self.assertEqual(num_leech, 3, "Leecher number don't match") + + def test_levenshtein(self): + """ + test levenshtein between two string (in this case, file name) + + source : + http://people.cs.pitt.edu/~kirk/cs1501/Pruhs/Fall2006/Assignments/editdistance/Levenshtein%20Distance.htm + """ + string1 = "GUMBO" + string2 = "GAMBOL" + dist = levenshtein_dist(string1, string2) + dist_swap = levenshtein_dist(string2, string1) + + # random string check + self.assertEqual(dist, 2, "Wrong levenshtein distance") + self.assertEqual(dist_swap, 2, "Wrong levenshtein distance") + + string1 = "ubuntu-15.10-desktop-i386.iso" + string2 = "ubuntu-15.10-desktop-amd64.iso" + dist = levenshtein_dist(string1, string2) + + # similar filename check + self.assertEqual(dist, 4, "Wrong levenshtein distance") + + dist = levenshtein_dist(string1, string1) + # equal filename check + self.assertEqual(dist, 0, "Wrong levenshtein distance") + + string2 = "Learning-Ubuntu-Linux-Server.tgz" + dist = levenshtein_dist(string1, string2) + # equal filename check + self.assertEqual(dist, 28, "Wrong levenshtein distance") + + def test_update_statistics(self): + """ + test updating statistics of a torrent (pick a new one) + """ + self.session.open_dbhandler = lambda _: None + + infohash_1 = "a"*20 + infohash_2 = "b"*20 + torrents = { + infohash_1: { + "last_seeding_stats": { + "time_seeding": 100, + "value": 5 + } + }, + infohash_2: { + "last_seeding_stats": {} + } + } + + new_seeding_stats = { + "time_seeding": 110, + "value": 1 + } + new_seeding_stats_unexist = { + "time_seeding": 10, + "value": 8 + } + + self.session.lm.ltmgr = MockLtSession() + + boost_man = bm.BoostingManager(self.session, self.bsettings) + boost_man.torrents = torrents + + boost_man.update_torrent_stats(infohash_1, new_seeding_stats) + self.assertEqual(boost_man.torrents[infohash_1]['last_seeding_stats'] + ['value'], 1) + + boost_man.update_torrent_stats(infohash_2, new_seeding_stats_unexist) + self.assertEqual(boost_man.torrents[infohash_2]['last_seeding_stats'] + ['value'], 8) + + boost_man.cancel_all_pending_tasks() + + def test_escape_xml(self): + """ + testing escape symbols occured in xml/rss document file. + """ + re_symbols = re.compile(r'\&\#(x?[0-9a-fA-F]+);') + + ampersand_str = re_symbols.sub(ent2chr, '&') + self.assertEqual(ampersand_str, "&", "wrong ampersand conversion %s" % ampersand_str) + + str_123 = re_symbols.sub(ent2chr, "123") + self.assertEqual(str_123, "123", "wrong number conversion %s" % str_123) + + def test_logging(self): + self.session.open_dbhandler = lambda _: None + + infohash_1 = "a"*20 + infohash_2 = "b"*20 + torrents = { + infohash_1: { + "last_seeding_stats": { + "time_seeding": 100, + "value": 5 + } + }, + infohash_2: { + "last_seeding_stats": {} + } + } + self.session.lm.ltmgr = MockLtSession() + + boost_man = bm.BoostingManager(self.session, self.bsettings) + boost_man.torrents = torrents + + boost_man.session.lm.ltmgr.get_session().get_torrents = \ + lambda: [MockLtTorrent(binascii.hexlify(infohash_1)), + MockLtTorrent(binascii.hexlify(infohash_2))] + + boost_man.log_statistics() + boost_man.cancel_all_pending_tasks() + + +class TestBoostingManagerError(TestAsServer): + """ + Class to test a bunch of credit mining error handle + """ + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerError, self).setUp() + + self.session.open_dbhandler = lambda _: True + self.session.get_libtorrent = lambda: True + self.session.lm.ltmgr = MockLtSession() + + self.boost_setting = bm.BoostingSettings(self.session) + self.boost_setting.load_config = False + self.boost_setting.initial_logging_interval = 900 + self.boost_setting.check_dependencies = False + self.boosting_manager = bm.BoostingManager(self.session, self.boost_setting) + self.session.lm.boosting_manager = self.boosting_manager + + def tearDown(self): + DefaultDownloadStartupConfig.delInstance() + super(TestBoostingManagerError, self).tearDown() + + def test_insert_torrent_unknown_source(self): + """ + testing insert torrent on unknown source + """ + torrent = { + 'preload': False, + 'metainfo': MockMeta("1234"), + 'infohash': '12345' + } + + self.boosting_manager.on_torrent_insert(binascii.unhexlify("abcd" * 10), '12345', torrent) + self.assertNotIn('12345', self.boosting_manager.torrents) + + def test_unknown_source(self): + """ + testing uknkown source added to boosting source, and try to apply archive + on top of that + """ + unknown_key = "1234567890" + + sources = len(self.boosting_manager.boosting_sources.keys()) + self.boosting_manager.add_source(unknown_key) + self.boosting_manager.set_archive(unknown_key, False) + self.assertEqual(sources, len(self.boosting_manager.boosting_sources.keys()), "unknown source added") + + def test_failed_start_download(self): + """ + test assertion error then not download the actual torrent + """ + torrent = { + 'preload': False, + 'metainfo': MockMeta("1234") + } + self.session.lm.download_exists = lambda _: True + self.boosting_manager.start_download(torrent) + + self.assertNotIn('download', torrent, "%s downloading despite error" % torrent) diff --git a/Tribler/Test/Core/CreditMining/test_creditmining_sys.py b/Tribler/Test/Core/CreditMining/test_creditmining_sys.py new file mode 100644 index 00000000000..d79790f7826 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/test_creditmining_sys.py @@ -0,0 +1,517 @@ +# coding=utf-8 +""" +Module of Credit mining function testing + +Written by Ardhi Putra Pratama H +""" +import binascii +import os +import shutil + +from twisted.internet import defer +from twisted.web.server import Site +from twisted.web.static import File + +from Tribler.Core.DownloadConfig import DefaultDownloadStartupConfig +from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.Utilities.twisted_thread import deferred, reactor +from Tribler.Core.simpledefs import NTFY_TORRENTS, NTFY_UPDATE +from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent +from Tribler.Policies.BoostingManager import BoostingManager, BoostingSettings +from Tribler.Test.Core.CreditMining.mock_creditmining import MockLtTorrent, ResourceFailClass +from Tribler.Test.Core.Modules.RestApi.test_channels_endpoints import AbstractTestChannelsEndpoint +from Tribler.Test.test_as_server import TestAsServer, TESTS_DATA_DIR +from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_FILE_INFOHASH +from Tribler.Test.util import prepare_xml_rss +from Tribler.community.channel.community import ChannelCommunity +from Tribler.dispersy.dispersy import Dispersy +from Tribler.dispersy.endpoint import ManualEnpoint +from Tribler.dispersy.util import blocking_call_on_reactor_thread + + +class TestBoostingManagerSys(TestAsServer): + """ + base class to test base credit mining function + """ + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSys, self).setUp() + + self.set_boosting_settings() + + self.session.lm.ltmgr.get_session().find_torrent = lambda _: MockLtTorrent() + + self.boosting_manager = BoostingManager(self.session, self.bsettings) + + self.session.lm.boosting_manager = self.boosting_manager + + def set_boosting_settings(self): + """ + set settings in credit mining + """ + self.bsettings = BoostingSettings(self.session) + self.bsettings.credit_mining_path = os.path.join(self.session_base_dir, "credit_mining") + self.bsettings.load_config = False + self.bsettings.check_dependencies = False + self.bsettings.min_connection_start = -1 + self.bsettings.min_channels_start = -1 + + self.bsettings.max_torrents_active = 8 + self.bsettings.max_torrents_per_source = 5 + + self.bsettings.tracker_interval = 5 + self.bsettings.initial_tracker_interval = 5 + self.bsettings.logging_interval = 30 + self.bsettings.initial_logging_interval = 3 + + def setUpPreSession(self): + super(TestBoostingManagerSys, self).setUpPreSession() + + self.config.set_torrent_checking(True) + self.config.set_megacache(True) + self.config.set_dispersy(True) + self.config.set_torrent_store(True) + self.config.set_enable_torrent_search(True) + self.config.set_enable_channel_search(True) + self.config.set_libtorrent(True) + + def tearDown(self): + DefaultDownloadStartupConfig.delInstance() + self.boosting_manager.shutdown() + + super(TestBoostingManagerSys, self).tearDown() + + def check_torrents(self, src, defer_param=None, target=1): + """ + function to check if a torrent is already added to the source + + In this function, + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + if len(src_obj.torrents) < target: + reactor.callLater(1, self.check_torrents, src, defer_param, target=target) + else: + # notify torrent (emulate scraping) + self.boosting_manager.scrape_trackers() + + def _get_tor_dummy(_, keys=123, include_mypref=True): + """ + function to emulate get_torrent in torrent_db + """ + return {'C.torrent_id': 93, 'category': u'Compressed', 'torrent_id': 41, + 'infohash': src_obj.torrents.keys()[0], 'length': 1150844928, 'last_tracker_check': 10001, + 'myDownloadHistory': False, 'name': u'ubuntu-15.04-desktop-amd64.iso', + 'num_leechers': 999, 'num_seeders': 123, 'status': u'unknown', 'tracker_check_retries': 0} + self.boosting_manager.torrent_db.getTorrent = _get_tor_dummy + self.session.notifier.notify(NTFY_TORRENTS, NTFY_UPDATE, src_obj.torrents.keys()[0]) + + # log it + self.boosting_manager.log_statistics() + + defer_param.callback(src) + return defer_param + + def check_source(self, src, defer_param=None, ready=True): + """ + function to check if a source is ready initializing + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + + if not ready: + defer_param.callback(src) + elif not src_obj or not src_obj.ready: + reactor.callLater(1, self.check_source, src, defer_param) + else: + defer_param.callback(src) + + return defer_param + + +class TestBoostingManagerSysRSS(TestBoostingManagerSys): + """ + testing class for RSS (dummy) source + """ + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSysRSS, self).setUp() + + files_path, self.file_server_port = prepare_xml_rss(self.session_base_dir, 'test_rss_cm.xml') + + shutil.copyfile(TORRENT_FILE, os.path.join(files_path, 'ubuntu.torrent')) + self.setUpFileServer(self.file_server_port, self.session_base_dir) + + self.rss_error_deferred = defer.Deferred() + # now the rss should be at : + # http://localhost:port/test_rss_cm.xml + # which resides in sessiondir/http_torrent_files + + def set_boosting_settings(self): + super(TestBoostingManagerSysRSS, self).set_boosting_settings() + self.bsettings.auto_start_source = False + + def setUpFileServer(self, port, path): + resource = File(path) + resource.putChild("err503", ResourceFailClass()) + factory = Site(resource) + self._logger.debug("Listen to port %s, factory %s", port, factory) + self.file_server = reactor.listenTCP(port, factory) + + @deferred(timeout=15) + def test_rss(self): + """ + test rss source + """ + url = 'http://localhost:%s/test_rss_cm.xml' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj.start() + + d = self.check_source(url) + d.addCallback(self.check_torrents, target=1) + return d + + def _on_error_rss(self, dummy_1, dummy_2): + """ + dummy errback when RSS source produces an error + """ + self.rss_error_deferred.callback(True) + + @deferred(timeout=8) + def test_rss_unexist(self): + """ + Testing an unexisting RSS feed + """ + url = 'http://localhost:%s/nothingness' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj._on_error_rss = self._on_error_rss + rss_obj.start() + + defer_err_rss = self.check_source(url, ready=False) + defer_err_rss.chainDeferred(self.rss_error_deferred) + return defer_err_rss + + @deferred(timeout=8) + def test_rss_unavailable(self): + """ + Testing an unavailable RSS feed + """ + url = 'http://localhost:%s/err503' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj._on_error_rss = self._on_error_rss + rss_obj.start() + + defer_err_rss = self.check_source(url, ready=False) + defer_err_rss.chainDeferred(self.rss_error_deferred) + return defer_err_rss + + +class TestBoostingManagerSysDir(TestBoostingManagerSys): + """ + testing class for directory source + """ + + @deferred(timeout=10) + def test_dir(self): + """ + test directory filled with .torrents + """ + self.boosting_manager.add_source(TESTS_DATA_DIR) + len_source = len(self.boosting_manager.boosting_sources) + + # deliberately try to add the same source + self.boosting_manager.add_source(TESTS_DATA_DIR) + self.assertEqual(len(self.boosting_manager.boosting_sources), len_source, "identical source added") + + dir_obj = self.boosting_manager.get_source_object(TESTS_DATA_DIR) + self.assertTrue(dir_obj.ready, "Not Ready") + + d = self.check_torrents(TESTS_DATA_DIR, target=2) + d.addCallback(lambda _: True) + return d + + @deferred(timeout=10) + def test_dir_archive_example(self): + """ + test archive mode. Use diretory because easier to fetch torrent + """ + self.boosting_manager.add_source(TESTS_DATA_DIR) + self.boosting_manager.set_archive(TESTS_DATA_DIR, True) + + dir_obj = self.boosting_manager.get_source_object(TESTS_DATA_DIR) + self.assertTrue(dir_obj.ready, "Not Ready") + + d = self.check_torrents(TESTS_DATA_DIR, target=2) + d.addCallback(lambda _: self.boosting_manager._select_torrent()) + return d + + +class TestBoostingManagerSysChannel(AbstractTestChannelsEndpoint, TestBoostingManagerSys): + """ + testing class for channel source + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerSysChannel, self).__init__(*argv, **kwargs) + self.tdef = TorrentDef.load(TORRENT_FILE) + self.channel_id = 0 + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSysChannel, self).setUp() + + def set_boosting_settings(self): + super(TestBoostingManagerSysChannel, self).set_boosting_settings() + self.bsettings.swarm_interval = 1 + self.bsettings.initial_swarm_interval = 1 + self.bsettings.max_torrents_active = 1 + self.bsettings.max_torrents_per_source = 1 + + def setUpPreSession(self): + super(TestBoostingManagerSysChannel, self).setUpPreSession() + + # we use dummy dispersy here + self.config.set_dispersy(False) + + @blocking_call_on_reactor_thread + def create_torrents_in_channel(self, dispersy_cid_hex): + """ + Helper function to insert 10 torrent into designated channel + """ + for i in xrange(0, 10): + self.insert_channel_in_db('rand%d' % i, 42 + i, 'Test channel %d' % i, 'Test description %d' % i) + + self.channel_id = self.insert_channel_in_db(dispersy_cid_hex.decode('hex'), 42, + 'Simple Channel', 'Channel description') + + torrent_list = [[self.channel_id, 1, 1, TORRENT_FILE_INFOHASH, 1460000000, TORRENT_FILE, + self.tdef.get_files_as_unicode_with_length(), self.tdef.get_trackers_as_single_tuple()]] + + self.insert_torrents_into_channel(torrent_list) + + @deferred(timeout=20) + def test_chn_lookup(self): + """ + testing channel source. + + It includes finding and downloading actual torrent + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + torrent = CollectedTorrent(torrent, self.tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + def check_torrents_channel(src, defer_param=None, target=1): + """ + check if a torrent already in channel and ready to download + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + success = True + if not src_obj or len(src_obj.torrents) < target: + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + elif not self.boosting_manager.torrents.get(TORRENT_FILE_INFOHASH, None): + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + elif not self.boosting_manager.torrents[TORRENT_FILE_INFOHASH].get('download', None): + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + + if success: + self.boosting_manager.set_enable_mining(src, False, force_restart=True) + if src_obj.community: + src_obj.community.cancel_all_pending_tasks() + + defer_param.callback(src) + + return defer_param + + chn_obj.torrent_mgr.load_torrent = _load + + d = self.check_source(dispersy_cid) + d.addCallback(check_torrents_channel, target=1) + return d + + @deferred(timeout=20) + def test_chn_exist_lookup(self): + """ + testing existing channel as a source. + + It also tests how boosting manager cope with unknown channel with retrying + the lookup + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + # channel is exist + community = ChannelCommunity.init_community(self.session.lm.dispersy, + self.session.lm.dispersy.get_member(mid=dispersy_cid), + self.session.lm.dispersy._communities['allchannel']._my_member, + self.session) + + # make the id unknown so boosting manager can test repeating search + id_tmp = community._channel_id + community._channel_id = 0 + + def _set_id_channel(channel_id): + """ + set channel id manually (emulate finding) + """ + community._channel_id = channel_id + + reactor.callLater(5, _set_id_channel, id_tmp) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + torrent = CollectedTorrent(torrent, self.tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + chn_obj.torrent_mgr.load_torrent = _load + + def clean_community(_): + """ + cleanly exit the community we are in + """ + if chn_obj.community: + chn_obj.community.cancel_all_pending_tasks() + + chn_obj.kill_tasks() + + + d = self.check_source(dispersy_cid) + d.addCallback(clean_community) + return d + + @deferred(timeout=20) + def test_chn_max_torrents(self): + """ + Test the restriction of max_torrents in a source. + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + pioneer_file = os.path.join(TESTS_DATA_DIR, "Pioneer.One.S01E06.720p.x264-VODO.torrent") + pioneer_tdef = TorrentDef.load(pioneer_file) + pioneer_ihash = binascii.unhexlify("66ED7F30E3B30FA647ABAA19A36E7503AA071535") + + torrent_list = [[self.channel_id, 1, 1, pioneer_ihash, 1460000001, pioneer_file, + pioneer_tdef.get_files_as_unicode_with_length(), pioneer_tdef.get_trackers_as_single_tuple()]] + self.insert_torrents_into_channel(torrent_list) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + chn_obj.max_torrents = 2 + chn_obj.torrent_mgr.load_torrent = lambda dummy_1, dummy_2: None + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + infohash_str = binascii.hexlify(torrent.infohash) + torrent = CollectedTorrent(torrent, self.tdef if infohash_str.startswith("fc") else pioneer_tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + def activate_mgr(): + """ + activate ltmgr and adjust max torrents to emulate overflow torrents + """ + chn_obj.max_torrents = 1 + chn_obj.torrent_mgr.load_torrent = _load + + reactor.callLater(5, activate_mgr) + + def check_torrents_channel(src, defer_param=None): + """ + check if a torrent already in channel and ready to download + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + success = True + if len(src_obj.unavail_torrent) == 0: + self.assertLessEqual(len(src_obj.torrents), src_obj.max_torrents) + else: + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param) + + if success: + src_obj.community.cancel_all_pending_tasks() + src_obj.kill_tasks() + defer_param.callback(src) + + return defer_param + + d = self.check_source(dispersy_cid) + d.addCallback(check_torrents_channel) + return d + + def tearDown(self): + self.session.lm.dispersy._communities['allchannel'].cancel_all_pending_tasks() + self.session.lm.dispersy.cancel_all_pending_tasks() + self.session.lm.dispersy = None + super(TestBoostingManagerSysChannel, self).tearDown() diff --git a/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py b/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py index bd441d4f3b7..3215e85a9b6 100644 --- a/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py +++ b/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py @@ -20,6 +20,8 @@ def setUp(self, autoload_discovery=True): self.votecast_db_handler = self.session.open_dbhandler(NTFY_VOTECAST) self.channel_db_handler._get_my_dispersy_cid = lambda: "myfakedispersyid" + self.create_votecast_called = False + def insert_channel_in_db(self, dispersy_cid, peer_id, name, description): return self.channel_db_handler.on_channel_from_dispersy(dispersy_cid, peer_id, name, description) @@ -29,6 +31,26 @@ def vote_for_channel(self, cid, vote_time): def insert_torrents_into_channel(self, torrent_list): self.channel_db_handler.on_torrents_from_dispersy(torrent_list) + @blocking_call_on_reactor_thread + def create_fake_allchannel_community(self): + """ + This method creates a fake AllChannel community so we can check whether a request is made in the community + when doing stuff with a channel. + """ + self.session.lm.dispersy._database.open() + fake_member = DummyMember(self.session.lm.dispersy, 1, "a" * 20) + member = self.session.lm.dispersy.get_new_member(u"curve25519") + fake_community = AllChannelCommunity.init_community(self.session.lm.dispersy, fake_member, member) + fake_community.disp_create_votecast = self.on_dispersy_create_votecast + self.session.lm.dispersy._communities = {"allchannel": fake_community} + return fake_community + + def on_dispersy_create_votecast(self, cid, vote, _): + """ + Check whether we have the expected parameters when this method is called. + """ + self.create_votecast_called = True + class TestChannelsEndpoint(AbstractTestChannelsEndpoint): @@ -132,39 +154,19 @@ def setUp(self, autoload_discovery=True): super(TestChannelsSubscriptionEndpoint, self).setUp(autoload_discovery) self.expected_votecast_cid = None self.expected_votecast_vote = None - self.create_votecast_called = False self.session.get_dispersy = lambda: True self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + self.create_fake_allchannel_community() for i in xrange(0, 10): self.insert_channel_in_db('rand%d' % i, 42 + i, 'Test channel %d' % i, 'Test description %d' % i) def on_dispersy_create_votecast(self, cid, vote, _): - """ - Check whether we have the expected parameters when this method is called. - """ + super(TestChannelsSubscriptionEndpoint, self).on_dispersy_create_votecast(cid, vote, _) self.assertEqual(cid, self.expected_votecast_cid) self.assertEqual(vote, self.expected_votecast_vote) - self.create_votecast_called = True - - @blocking_call_on_reactor_thread - def create_fake_allchannel_community(self): - """ - This method creates a fake AllChannel community so we can check whether a request is made in the community - when doing stuff with a channel. - """ - self.session.lm.dispersy._database.open() - fake_member = DummyMember(self.session.lm.dispersy, 1, "a" * 20) - member = self.session.lm.dispersy.get_new_member(u"curve25519") - fake_community = AllChannelCommunity(self.session.lm.dispersy, fake_member, member) - fake_community.disp_create_votecast = self.on_dispersy_create_votecast - self.session.lm.dispersy._communities = {"allchannel": fake_community} - - def tearDown(self): - self.session.lm.dispersy = None - super(TestChannelsSubscriptionEndpoint, self).tearDown() @deferred(timeout=10) def test_subscribe_channel_not_exist(self): @@ -229,3 +231,8 @@ def verify_votecast_made(_): self.expected_votecast_vote = VOTE_UNSUBSCRIBE return self.do_request('channels/subscribed/%s' % 'rand1'.encode('hex'), expected_code=200, expected_json=expected_json, request_type='DELETE').addCallback(verify_votecast_made) + + def tearDown(self): + self.session.lm.dispersy._communities['allchannel'].cancel_all_pending_tasks() + self.session.lm.dispersy = None + super(TestChannelsSubscriptionEndpoint, self).tearDown() diff --git a/Tribler/Test/data/test_rss_cm.xml b/Tribler/Test/data/test_rss_cm.xml new file mode 100644 index 00000000000..18044929497 --- /dev/null +++ b/Tribler/Test/data/test_rss_cm.xml @@ -0,0 +1,31 @@ + + + + + Test RSS for Credit Mining + test_rss_cm.xml + Test RSS CM feed. + + en + Tue, 12 May 2015 08:39:23 GMT + Tue, 12 May 2015 08:39:23 GMT + + + + + ubuntu-15.04-desktop-amd64.iso + http://localhost:RANDOMPORT/ubuntu.torrent + + http://localhost:RANDOMPORT.ubuntu.torrent + + + + Pioneer.One.S01E06.720p.x264-VODO.torrent + http://localhost:RANDOMPORT/pioneer.torrent + + http://localhost:RANDOMPORT.pioneer.torrent + + + diff --git a/Tribler/Test/test_BoostingManager.py b/Tribler/Test/test_BoostingManager.py deleted file mode 100644 index 48f47fc41c8..00000000000 --- a/Tribler/Test/test_BoostingManager.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# Written by Mihai Capotă -# pylint: disable=too-many-public-methods -"""Test Tribler.Policies.BoostingManager""" - -import mock -import random -import unittest - -import Tribler.Policies.BoostingManager as bm - -class TestBoostingManagerPolicies(unittest.TestCase): - - def setUp(self): - random.seed(0) - self.session = mock.Mock() - self.session.get_download = lambda i: i % 2 - self.torrents = dict() - for i in range(1, 11): - mock_metainfo = mock.Mock() - mock_metainfo.get_id.return_value = i - self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": i, - "num_leechers": i-1, "creation_date": i} - - def test_RandomPolicy(self): - policy = bm.RandomPolicy(self.session) - torrents_start, torrents_stop = policy.apply(self.torrents, 2) - ids_start = [torrent["metainfo"].get_infohash() for torrent in - torrents_start] - self.assertEqual(ids_start, [4, 8]) - ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] - self.assertEqual(ids_stop, [3, 9, 5, 7, 1]) - - def test_SeederRatioPolicy(self): - policy = bm.SeederRatioPolicy(self.session) - torrents_start, torrents_stop = policy.apply(self.torrents, 6) - ids_start = [torrent["metainfo"].get_infohash() for torrent in - torrents_start] - self.assertEqual(ids_start, [10, 8, 6]) - ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] - self.assertEqual(ids_stop, [3, 1]) - - def test_CreationDatePolicy(self): - policy = bm.CreationDatePolicy(self.session) - torrents_start, torrents_stop = policy.apply(self.torrents, 5) - ids_start = [torrent["metainfo"].get_infohash() for torrent in - torrents_start] - self.assertEqual(ids_start, [10, 8, 6]) - ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] - self.assertEqual(ids_stop, [5, 3, 1]) - -if __name__ == "__main__": - unittest.main() diff --git a/Tribler/Test/test_as_server.py b/Tribler/Test/test_as_server.py index 982abd49029..4eaf84ac38e 100644 --- a/Tribler/Test/test_as_server.py +++ b/Tribler/Test/test_as_server.py @@ -262,6 +262,7 @@ def setUpPreSession(self): self.config.set_upgrader_enabled(False) self.config.set_http_api_enabled(False) self.config.set_tunnel_community_enabled(False) + self.config.set_creditmining_enable(False) def tearDown(self): self.annotate(self._testMethodName, start=False) diff --git a/Tribler/Test/test_my_channel.py b/Tribler/Test/test_my_channel.py index b0106e95487..1b67946feaa 100644 --- a/Tribler/Test/test_my_channel.py +++ b/Tribler/Test/test_my_channel.py @@ -1,16 +1,15 @@ # Written by Niels Zeilemaker # see LICENSE.txt for license information -from binascii import hexlify import os import shutil -from Tribler.Core.Utilities.network_utils import get_random_port +from binascii import hexlify +from Tribler.Core.TorrentDef import TorrentDef from Tribler.Test.common import UBUNTU_1504_INFOHASH -from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_VIDEO_FILE from Tribler.Test.test_as_server import TestGuiAsServer, TESTS_DATA_DIR - -from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_VIDEO_FILE +from Tribler.Test.util import prepare_xml_rss DEBUG = True @@ -21,15 +20,8 @@ def setUp(self): super(TestMyChannel, self).setUp() # Prepare test_rss.xml file, replace the port with a random one - self.file_server_port = get_random_port() - with open(os.path.join(TESTS_DATA_DIR, 'test_rss.xml'), 'r') as source_xml,\ - open(os.path.join(self.session_base_dir, 'test_rss.xml'), 'w') as destination_xml: - for line in source_xml: - destination_xml.write(line.replace('RANDOMPORT', str(self.file_server_port))) - - # Setup file server to serve torrent file and thumbnails - files_path = os.path.join(self.session_base_dir, 'http_torrent_files') - os.mkdir(files_path) + files_path, self.file_server_port = prepare_xml_rss(self.session_base_dir, 'test_rss.xml') + shutil.copyfile(TORRENT_FILE, os.path.join(files_path, 'ubuntu.torrent')) shutil.copyfile(TORRENT_VIDEO_FILE, os.path.join(files_path, 'video.torrent')) shutil.copyfile(os.path.join(TESTS_DATA_DIR, 'ubuntu-logo14.png'), diff --git a/Tribler/Test/util.py b/Tribler/Test/util.py index 6e3f8e20ad1..e4e629724f6 100644 --- a/Tribler/Test/util.py +++ b/Tribler/Test/util.py @@ -36,11 +36,15 @@ import logging +import os import sys # logging.basicConfig() from twisted.python.log import addObserver +from Tribler.Core.Utilities.network_utils import get_random_port + + __all__ = ["process_unhandled_exceptions"] @@ -130,6 +134,23 @@ def check_exceptions(self): % (num_twisted_exceptions, self._twisted_exceptions[-1]['log_text'])) +def prepare_xml_rss(target_path, filename): + """ + Function to prepare test_rss.xml file, replace the port with a random one + """ + files_path = os.path.join(target_path, 'http_torrent_files') + os.mkdir(files_path) + + port = get_random_port() + + from Tribler.Test.test_as_server import TESTS_DATA_DIR + with open(os.path.join(TESTS_DATA_DIR, filename), 'r') as source_xml,\ + open(os.path.join(target_path, filename), 'w') as destination_xml: + for line in source_xml: + destination_xml.write(line.replace('RANDOMPORT', str(port))) + + return files_path, port + _catcher = UnhandledExceptionCatcher() _twisted_catcher = UnhandledTwistedExceptionCatcher()