diff --git a/Tribler/Core/APIImplementation/LaunchManyCore.py b/Tribler/Core/APIImplementation/LaunchManyCore.py index 24e2e1fc973..cfb1de7a459 100644 --- a/Tribler/Core/APIImplementation/LaunchManyCore.py +++ b/Tribler/Core/APIImplementation/LaunchManyCore.py @@ -10,9 +10,9 @@ from glob import iglob from threading import Event, enumerate as enumerate_threads from traceback import print_exc -from twisted.internet.defer import Deferred from twisted.internet import reactor +from twisted.internet.defer import Deferred from twisted.internet.defer import inlineCallbacks from Tribler.Core.APIImplementation.threadpoolmanager import ThreadPoolManager @@ -96,6 +96,8 @@ def __init__(self): self.startup_deferred = Deferred() + self.boosting_manager = None + def register(self, session, sesslock): if not self.registered: self.registered = True @@ -315,11 +317,16 @@ def load_communities(): self.watch_folder = WatchFolder(self.session) self.watch_folder.start() + if self.session.get_creditmining_enable(): + from Tribler.Policies.BoostingManager import BoostingManager + self.boosting_manager = BoostingManager(self.session) + self.version_check_manager = VersionCheckManager(self.session) self.initComplete = True - def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidden=False): + def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidden=False, + share_mode=False, checkpoint_disabled=False): """ Called by any thread """ d = None with self.sesslock: @@ -343,7 +350,8 @@ def add(self, tdef, dscfg, pstate=None, initialdlstatus=None, setupDelay=0, hidd # Store in list of Downloads, always. self.downloads[infohash] = d - setup_deferred = d.setup(dscfg, pstate, initialdlstatus, wrapperDelay=setupDelay) + setup_deferred = d.setup(dscfg, pstate, initialdlstatus, wrapperDelay=setupDelay, + share_mode=share_mode, checkpoint_disabled=checkpoint_disabled) setup_deferred.addCallback(self.on_download_wrapper_created) if d and not hidden and self.session.get_megacache(): @@ -368,7 +376,7 @@ def write_my_pref(): def on_download_wrapper_created(self, (d, pstate)): """ Called by network thread """ try: - if pstate is None: + if pstate is None and not d.get_checkpoint_disabled(): # Checkpoint at startup (infohash, pstate) = d.network_checkpoint() self.save_download_pstate(infohash, pstate) @@ -670,6 +678,9 @@ def early_shutdown(self): # Note: sesslock not held self.shutdownstarttime = timemod.time() + if self.boosting_manager: + yield self.boosting_manager.shutdown() + self.boosting_manager = None if self.torrent_checker: yield self.torrent_checker.shutdown() self.torrent_checker = None @@ -800,7 +811,10 @@ def sessconfig_changed_callback(self, section, name, new_value, old_value): 'anon_listen_port']) or \ (section == 'torrent_collecting' and name in ['stop_collecting_threshold']) or \ (section == 'watch_folder') or \ - (section == 'tunnel_community' and name in ['socks5_listen_port']): + (section == 'tunnel_community' and name in ['socks5_listen_port']) or \ + (section == 'credit_mining' and name in ['max_torrents_per_source', 'max_torrents_active', + 'source_interval', 'swarm_interval', 'boosting_sources', + 'boosting_enabled', 'boosting_disabled', 'archive_sources']): return True else: return False diff --git a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py index 4b678c15ef7..3fdff94b6c1 100644 --- a/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py +++ b/Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py @@ -145,11 +145,13 @@ def __init__(self, session, tdef): self.askmoreinfo = False self.correctedinfoname = u"" + self._checkpoint_disabled = False self.deferreds_resume = [] def __str__(self): - return "LibtorrentDownloadImpl " % (self.correctedinfoname, self.get_hops()) + return "LibtorrentDownloadImpl " % \ + (self.correctedinfoname, self.get_hops(), self._checkpoint_disabled) def __repr__(self): return self.__str__() @@ -157,7 +159,14 @@ def __repr__(self): def get_def(self): return self.tdef - def setup(self, dcfg=None, pstate=None, initialdlstatus=None, wrapperDelay=0): + def set_checkpoint_disabled(self, disabled=True): + self._checkpoint_disabled = disabled + + def get_checkpoint_disabled(self): + return self._checkpoint_disabled + + def setup(self, dcfg=None, pstate=None, initialdlstatus=None, + wrapperDelay=0, share_mode=False, checkpoint_disabled=False): """ Create a Download object. Used internally by Session. @param dcfg DownloadStartupConfig or None (in which case @@ -167,6 +176,10 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None, wrapperDelay=0): network_create_engine_wrapper. """ # Called by any thread, assume sessionlock is held + self.set_checkpoint_disabled(checkpoint_disabled) + + self.set_share_mode(share_mode) + try: # The deferred to be returned deferred = Deferred() @@ -190,7 +203,8 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None, wrapperDelay=0): def schedule_create_engine(): self.cew_scheduled = True - create_engine_wrapper_deferred = self.network_create_engine_wrapper(self.pstate_for_restart, initialdlstatus) + create_engine_wrapper_deferred = self.network_create_engine_wrapper( + self.pstate_for_restart, initialdlstatus) create_engine_wrapper_deferred.chainDeferred(deferred) @@ -242,7 +256,7 @@ def do_check(): do_check() return can_create_deferred - def network_create_engine_wrapper(self, pstate, initialdlstatus=None): + def network_create_engine_wrapper(self, pstate, initialdlstatus=None, checkpoint_disabled=False): with self.dllock: self._logger.debug("LibtorrentDownloadImpl: network_create_engine_wrapper()") @@ -254,6 +268,11 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None): atp["duplicate_is_error"] = True atp["hops"] = self.get_hops() + if self.get_share_mode(): + atp["flags"] = lt.add_torrent_params_flags_t.flag_share_mode + + self.set_checkpoint_disabled(checkpoint_disabled) + resume_data = pstate.get('state', 'engineresumedata') if pstate else None if not isinstance(self.tdef, TorrentDefNoMetainfo): metainfo = self.tdef.get_metainfo() @@ -279,8 +298,6 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None): has_resume_data = resume_data and isinstance(resume_data, dict) if has_resume_data: atp["resume_data"] = lt.bencode(resume_data) - self._logger.info("%s %s", self.tdef.get_name_as_unicode(), dict((k, v) - for k, v in resume_data.iteritems() if k not in ['pieces', 'piece_priority', 'peers']) if has_resume_data else None) else: atp["url"] = self.tdef.get_url() or "magnet:?xt=urn:btih:" + hexlify(self.tdef.get_infohash()) atp["name"] = self.tdef.get_name_as_unicode() @@ -288,6 +305,7 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None): self.handle = self.ltmgr.add_torrent(self, atp) if self.handle: + self.set_selected_files() # If we lost resume_data always resume download in order to force checking @@ -653,7 +671,7 @@ def set_selected_files(self, selected_files=None): torrent_storage = get_info_from_handle(self.handle).files() - # TODO(ardhi) : as from 1.0, files returning file_storage (lazy-iterable) + # as from libtorrent 1.0, files returning file_storage (lazy-iterable) if hasattr(lt, 'file_storage') and isinstance(torrent_storage, lt.file_storage): cur_path = torrent_storage.at(index).path.decode('utf-8') else: @@ -678,7 +696,9 @@ def set_selected_files(self, selected_files=None): except TypeError: self.handle.rename_file(index, new_path.encode("utf-8")) - self.handle.prioritize_files(filepriorities) + # if in share mode, don't change priority of the file + if self.get_share_mode(): + self.handle.prioritize_files(filepriorities) self.unwanteddir_abs = unwanteddir_abs @@ -838,42 +858,56 @@ def network_get_vod_stats(self): d['npieces'] = ((self.length + 1023) / 1024) return d + @staticmethod + def create_peerlist_data(peer_info): + """ + A function to convert peer_info libtorrent object into dictionary + This data is used to identify peers with combination of several flags + """ + peer_dict = {'id': peer_info.pid, + 'extended_version': peer_info.client, + 'ip': peer_info.ip[0], + 'port': peer_info.ip[1], + # optimistic_unchoke = 0x800 seems unavailable in python bindings + 'optimistic': bool(peer_info.flags & 0x800), + 'direction': 'L' if bool(peer_info.flags & peer_info.local_connection) else 'R', + 'uprate': peer_info.payload_up_speed, + 'uinterested': bool(peer_info.flags & peer_info.remote_interested), + 'uchoked': bool(peer_info.flags & peer_info.remote_choked), + 'uhasqueries': peer_info.upload_queue_length > 0, + 'uflushed': peer_info.used_send_buffer > 0, + 'downrate': peer_info.payload_down_speed, + 'dinterested': bool(peer_info.flags & peer_info.interesting), + 'dchoked': bool(peer_info.flags & peer_info.choked), + 'snubbed': bool(peer_info.flags & 0x1000), + 'utotal': peer_info.total_upload, + 'dtotal': peer_info.total_download, + 'completed': peer_info.progress, + 'have': peer_info.pieces, 'speed': peer_info.remote_dl_rate, + 'country': peer_info.country, + 'connection_type': peer_info.connection_type, + # add upload_only and/or seed + 'seed': bool(peer_info.flags & peer_info.seed), + 'upload_only': bool(peer_info.flags & peer_info.upload_only), + # add read and write state (check unchoke/choke peers) + # read and write state is char with value 0, 1, 2, 4. May be empty + 'rstate': peer_info.read_state, + 'wstate': peer_info.write_state} + + return peer_dict + def network_create_spew_from_peerlist(self): plist = [] with self.dllock: peer_infos = self.handle.get_peer_info() for peer_info in peer_infos: - # Only consider fully connected peers. # Disabling for now, to avoid presenting the user with conflicting information # (partially connected peers are included in seeder/leecher stats). # if peer_info.flags & peer_info.connecting or peer_info.flags & peer_info.handshake: # continue + peer_dict = LibtorrentDownloadImpl.create_peerlist_data(peer_info) - peer_dict = {} - peer_dict['id'] = peer_info.pid - peer_dict['extended_version'] = peer_info.client - peer_dict['ip'] = peer_info.ip[0] - peer_dict['port'] = peer_info.ip[1] - # optimistic_unchoke = 0x800 seems unavailable in python bindings - peer_dict['optimistic'] = bool(peer_info.flags & 2048) - peer_dict['direction'] = 'L' if bool(peer_info.flags & peer_info.local_connection) else 'R' - peer_dict['uprate'] = peer_info.payload_up_speed - peer_dict['uinterested'] = bool(peer_info.flags & peer_info.remote_interested) - peer_dict['uchoked'] = bool(peer_info.flags & peer_info.remote_choked) - peer_dict['uhasqueries'] = peer_info.upload_queue_length > 0 - peer_dict['uflushed'] = peer_info.used_send_buffer > 0 - peer_dict['downrate'] = peer_info.payload_down_speed - peer_dict['dinterested'] = bool(peer_info.flags & peer_info.interesting) - peer_dict['dchoked'] = bool(peer_info.flags & peer_info.choked) - peer_dict['snubbed'] = bool(peer_info.flags & 4096) # snubbed = 0x1000 seems unavailable in python bindings - peer_dict['utotal'] = peer_info.total_upload - peer_dict['dtotal'] = peer_info.total_download - peer_dict['completed'] = peer_info.progress - peer_dict['have'] = peer_info.pieces - peer_dict['speed'] = peer_info.remote_dl_rate - peer_dict['country'] = peer_info.country - peer_dict['connection_type'] = peer_info.connection_type plist.append(peer_dict) return plist @@ -1045,9 +1079,12 @@ def get_dest_files(self, exts=None): def checkpoint(self): """ Called by any thread """ - (infohash, pstate) = self.network_checkpoint() - checkpoint = lambda: self.session.lm.save_download_pstate(infohash, pstate) - self.session.lm.threadpool.add_task(checkpoint, 0) + if self._checkpoint_disabled: + self._logger.warning("Ignoring checkpoint() call as is checkpointing disabled for this download.") + else: + infohash, pstate = self.network_checkpoint() + checkpoint = lambda: self.session.lm.save_download_pstate(infohash, pstate) + self.session.lm.threadpool.add_task(checkpoint, 0) def network_checkpoint(self): """ Called by network thread """ @@ -1081,6 +1118,9 @@ def network_get_persistent_state(self): else: pstate.set('state', 'metainfo', self.tdef.get_metainfo()) + if self.get_share_mode(): + pstate.set('state', 'share_mode', True) + ds = self.network_get_state(None, False) dlstate = {'status': ds.get_status(), 'progress': ds.get_progress(), 'swarmcache': None} pstate.set('state', 'dlstate', dlstate) @@ -1115,6 +1155,10 @@ def add_peer(self, addr): """ self.handle.connect_peer(addr, 0) + @waitForHandleAndSynchronize() + def set_priority(self, prio): + self.handle.set_priority(prio) + @waitForHandleAndSynchronize(True) def dlconfig_changed_callback(self, section, name, new_value, old_value): if section == 'downloadconfig' and name == 'max_upload_rate': @@ -1125,6 +1169,14 @@ def dlconfig_changed_callback(self, section, name, new_value, old_value): return False return True + @checkHandleAndSynchronize + def get_share_mode(self): + return self.handle.status().share_mode + + @waitForHandleAndSynchronize(True) + def set_share_mode(self, share_mode): + self.handle.set_share_mode(share_mode) + class LibtorrentStatisticsResponse: diff --git a/Tribler/Core/Libtorrent/LibtorrentMgr.py b/Tribler/Core/Libtorrent/LibtorrentMgr.py index 3fc0c1c3347..a0d162d29b6 100644 --- a/Tribler/Core/Libtorrent/LibtorrentMgr.py +++ b/Tribler/Core/Libtorrent/LibtorrentMgr.py @@ -320,9 +320,9 @@ def process_alert(self, alert): if isinstance(alert, lt.metadata_received_alert): self.got_metainfo(infohash) else: - self._logger.debug("could not find torrent %s", infohash) + self._logger.debug("LibtorrentMgr: could not find torrent %s", infohash) else: - self._logger.debug("alert for invalid torrent") + self._logger.debug("Alert for invalid torrent") def get_metainfo(self, infohash_or_magnet, callback, timeout=30, timeout_callback=None, notify=True): if not self.is_dht_ready() and timeout > 5: diff --git a/Tribler/Core/SessionConfig.py b/Tribler/Core/SessionConfig.py index a3e0f9d9fca..bbad991eb59 100644 --- a/Tribler/Core/SessionConfig.py +++ b/Tribler/Core/SessionConfig.py @@ -23,6 +23,9 @@ from Tribler.Core.defaults import sessdefaults from Tribler.Core.osutils import get_appstate_dir, is_android from Tribler.Core.simpledefs import STATEDIR_SESSCONFIG +from Tribler.Policies.BoostingPolicy import CreationDatePolicy, BoostingPolicy +from Tribler.Policies.BoostingPolicy import RandomPolicy +from Tribler.Policies.BoostingPolicy import SeederRatioPolicy class SessionConfigInterface(object): @@ -842,6 +845,161 @@ def get_http_api_port(self): """ return self._obtain_port(u'http_api', u'port') + # + # Credit Mining + # + + def set_creditmining_enable(self, value): + """ + Sets to enable credit mining + """ + self.sessconfig.set(u'credit_mining', u'enabled', value) + + def get_creditmining_enable(self): + """ + Gets if credit mining is enabled + :return: (bool) True or False + """ + return self.sessconfig.get(u'credit_mining', u'enabled') + + def set_cm_max_torrents_active(self, max_torrents_active): + """ + Set credit mining max active torrents in a single session + """ + return self.sessconfig.set(u'credit_mining', u'max_torrents_active', max_torrents_active) + + def get_cm_max_torrents_active(self): + """ + get max number of torrents active in a single session + """ + return self.sessconfig.get(u'credit_mining', u'max_torrents_active') + + def set_cm_max_torrents_per_source(self, max_torrents_per_source): + """ + set a number of torrent that can be stored in a single source + """ + return self.sessconfig.set(u'credit_mining', u'max_torrents_per_source', max_torrents_per_source) + + def get_cm_max_torrents_per_source(self): + """ + get max number of torrent that can be stored in a single source + """ + return self.sessconfig.get(u'credit_mining', u'max_torrents_per_source') + + def set_cm_source_interval(self, source_interval): + """ + set interval of looking up new torrent in a swarm + """ + return self.sessconfig.set(u'credit_mining', u'source_interval', source_interval) + + def get_cm_source_interval(self): + """ + get interval of looking up new torrent in a swarm + """ + return self.sessconfig.get(u'credit_mining', u'source_interval') + + def set_cm_swarm_interval(self, swarm_interval): + """ + set the interval of choosing activity which swarm will be downloaded + """ + return self.sessconfig.set(u'credit_mining', u'swarm_interval', swarm_interval) + + def get_cm_swarm_interval(self): + """ + getting the interval of choosing activity which swarm will be downloaded + """ + return self.sessconfig.get(u'credit_mining', u'swarm_interval') + + def set_cm_tracker_interval(self, tracker_interval): + """ + set the manual (force) scraping interval. + """ + return self.sessconfig.set(u'credit_mining', u'tracker_interval', tracker_interval) + + def get_cm_tracker_interval(self): + """ + get the manual (force) scraping interval. + """ + return self.sessconfig.get(u'credit_mining', u'tracker_interval') + + def set_cm_logging_interval(self, logging_interval): + """ + set the credit mining logging interval (INFO,DEBUG) + """ + return self.sessconfig.set(u'credit_mining', u'logging_interval', logging_interval) + + def get_cm_logging_interval(self): + """ + get the credit mining logging interval (INFO,DEBUG) + """ + return self.sessconfig.get(u'credit_mining', u'logging_interval') + + def set_cm_share_mode_target(self, share_mode_target): + """ + set the share mode target in credit mining. Value can be referenced at : + http://www.libtorrent.org/reference-Settings.html#share_mode_target + """ + return self.sessconfig.set(u'credit_mining', u'share_mode_target', share_mode_target) + + def get_cm_share_mode_target(self): + """ + get the current share mode target that applies in all the swarm + """ + return self.sessconfig.get(u'credit_mining', u'share_mode_target') + + def set_cm_policy(self, policy_str): + """ + set the credit mining policy. Input can be policy name or class + """ + switch_policy = { + RandomPolicy: "random", + CreationDatePolicy: "creation", + SeederRatioPolicy: "seederratio" + } + + if isinstance(policy_str, BoostingPolicy): + policy_str = switch_policy[type(policy_str)] + + return self.sessconfig.set(u'credit_mining', u'policy', policy_str) + + def get_cm_policy(self, as_class=False): + """ + get the credit mining policy. If as_class True, will return as class, + otherwise will return as policy name (str) + """ + policy_str = self.sessconfig.get(u'credit_mining', u'policy') + + if as_class: + switch_policy = { + "random": RandomPolicy, + "creation": CreationDatePolicy, + "seederratio": SeederRatioPolicy + } + + ret = switch_policy[policy_str] + else: + ret = policy_str + + return ret + + def set_cm_sources(self, source_list, key): + """ + set source list for a chosen key : + boosting_sources, boosting_enabled, boosting_disabled, or archive_sources + """ + return self.sessconfig.set(u'credit_mining', u'%s' % key, source_list) + + def get_cm_sources(self): + """ + get all the lists as list of string in the configuration + """ + ret = {"boosting_sources": self.sessconfig.get(u'credit_mining', u'boosting_sources'), + "boosting_enabled": self.sessconfig.get(u'credit_mining', u'boosting_enabled'), + "boosting_disabled": self.sessconfig.get(u'credit_mining', u'boosting_disabled'), + "archive_sources": self.sessconfig.get(u'credit_mining', u'archive_sources')} + + return ret + # # Static methods # diff --git a/Tribler/Core/TorrentChecker/session.py b/Tribler/Core/TorrentChecker/session.py index f884d0f4c0d..21322d2233a 100644 --- a/Tribler/Core/TorrentChecker/session.py +++ b/Tribler/Core/TorrentChecker/session.py @@ -95,7 +95,11 @@ def can_add_request(self): Checks if we still can add requests to this session. :return: True or False. """ - return not self._is_initiated and len(self._infohash_list) < MAX_TRACKER_MULTI_SCRAPE + + #TODO(ardhi) : quickfix for etree.org can't handle multiple infohash in single call + etree_condition = "etree" not in self.tracker_url + + return not self._is_initiated and len(self._infohash_list) < MAX_TRACKER_MULTI_SCRAPE and etree_condition def has_request(self, infohash): return infohash in self._infohash_list diff --git a/Tribler/Core/TorrentChecker/torrent_checker.py b/Tribler/Core/TorrentChecker/torrent_checker.py index b6cc20682ce..1a00bc5d2d7 100644 --- a/Tribler/Core/TorrentChecker/torrent_checker.py +++ b/Tribler/Core/TorrentChecker/torrent_checker.py @@ -140,10 +140,11 @@ def _task_select_torrents(self): self._logger.debug(u"Selected %d new torrents to check on tracker: %s", scheduled_torrents, tracker_url) @call_on_reactor_thread - def add_gui_request(self, infohash): + def add_gui_request(self, infohash, scrape_now=False): """ Public API for adding a GUI request. :param infohash: Torrent infohash. + :param scrape_now: Flag whether we want to force scraping immediately """ result = self._torrent_db.getTorrent(infohash, (u'torrent_id', u'last_tracker_check'), False) if result is None: @@ -153,7 +154,7 @@ def add_gui_request(self, infohash): torrent_id = result[u'torrent_id'] last_check = result[u'last_tracker_check'] time_diff = time.time() - last_check - if time_diff < self._torrent_check_interval: + if time_diff < self._torrent_check_interval and not scrape_now: self._logger.debug(u"time interval too short, skip GUI request. infohash: %s", hexlify(infohash)) return @@ -357,7 +358,8 @@ def _create_session_for_request(self, infohash, tracker_url): # before creating a new session, check if the tracker is alive if not self._session.lm.tracker_manager.should_check_tracker(tracker_url): - self._logger.warn(u"skipping recently failed tracker %s", tracker_url) + self._logger.warn(u"skipping recently failed tracker %s by %d times", tracker_url, + self._session.lm.tracker_manager.get_tracker_info(tracker_url)['failures']) return session = create_tracker_session(tracker_url, self._on_result_from_session) @@ -372,7 +374,6 @@ def _create_session_for_request(self, infohash, tracker_url): self._logger.debug(u"Session created for infohash %s", hexlify(infohash)) - def _update_pending_response(self, infohash): if infohash in self._pending_response_dict: self._pending_response_dict[infohash][u'remaining_responses'] += 1 diff --git a/Tribler/Core/Utilities/utilities.py b/Tribler/Core/Utilities/utilities.py index deeb5c58c4f..5d8b18f934d 100644 --- a/Tribler/Core/Utilities/utilities.py +++ b/Tribler/Core/Utilities/utilities.py @@ -244,3 +244,43 @@ def fix_torrent(file_path): fixed_data = bencode(fixed_data) return fixed_data + + +def translate_peers_into_health(peer_info_dicts): + """ + peer_info_dicts is a peer_info dictionary from LibTorrentDownloadImpl.create_peerlist_data + purpose : where we want to measure a swarm's health but no tracker can be contacted + """ + upload_only = 0 + finished = 0 + unfinished_able_dl = 0 + interest_in_us = 0 + + # collecting some statistics + for p_info in peer_info_dicts: + upload_only_b = False + + if p_info['upload_only']: + upload_only += 1 + upload_only_b = True + if p_info['uinterested']: + interest_in_us += 1 + if p_info['completed'] == 1: + finished += 1 + else: + unfinished_able_dl += 1 if upload_only_b else 0 + + # seeders potentials: + # 1. it's only want uploading right now (upload only) + # 2. it's finished (we don't know whether it want to upload or not) + # leecher potentials: + # 1. it's interested in our piece + # 2. it's unfinished but it's not 'upload only' (it can't leech for some reason) + # 3. it's unfinished (less restrictive) + + # make sure to change those description when changing the algorithm + + num_seeders = max(upload_only, finished) + num_leech = max(interest_in_us, min(unfinished_able_dl, len(peer_info_dicts) - finished)) + return num_seeders, num_leech + diff --git a/Tribler/Core/defaults.py b/Tribler/Core/defaults.py index 0c1d68c3a1b..041105b8160 100644 --- a/Tribler/Core/defaults.py +++ b/Tribler/Core/defaults.py @@ -39,8 +39,9 @@ # Version 12: Added watch folder options. # Version 13: Added HTTP API options. # Version 14: Added option to enable/disable channel, previewchannel and tunnel community. +# Version 15: Added credit mining options -SESSDEFAULTS_VERSION = 14 +SESSDEFAULTS_VERSION = 15 sessdefaults = OrderedDict() # General Tribler settings @@ -163,6 +164,23 @@ sessdefaults['http_api']['enabled'] = False sessdefaults['http_api']['port'] = -1 +# Credit mining config +sessdefaults['credit_mining'] = OrderedDict() +sessdefaults['credit_mining']['enabled'] = False +sessdefaults['credit_mining']['max_torrents_per_source'] = 20 +sessdefaults['credit_mining']['max_torrents_active'] = 50 +sessdefaults['credit_mining']['source_interval'] = 100 +sessdefaults['credit_mining']['swarm_interval'] = 100 +sessdefaults['credit_mining']['share_mode_target'] = 3 +sessdefaults['credit_mining']['tracker_interval'] = 200 +sessdefaults['credit_mining']['logging_interval'] = 60 +# By default we want to automatically boost legal-predetermined channel. +sessdefaults['credit_mining']['boosting_sources'] = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +sessdefaults['credit_mining']['boosting_enabled'] = ["http://bt.etree.org/rss/bt_etree_org.rdf"] +sessdefaults['credit_mining']['boosting_disabled'] = [] +sessdefaults['credit_mining']['archive_sources'] = [] +sessdefaults['credit_mining']['policy'] = "seederratio" + # # BT per download opts # diff --git a/Tribler/Core/simpledefs.py b/Tribler/Core/simpledefs.py index 7fcd13e2e20..d6dfc0ff793 100644 --- a/Tribler/Core/simpledefs.py +++ b/Tribler/Core/simpledefs.py @@ -98,6 +98,7 @@ NTFY_INSERT = 'insert' # new data is inserted NTFY_DELETE = 'delete' # data is deleted NTFY_CREATE = 'create' # new data is created, meaning in the case of Channels your own channel is created +NTFY_SCRAPE = 'scrape' NTFY_STARTED = 'started' NTFY_STATE = 'state' NTFY_MODIFIED = 'modified' diff --git a/Tribler/Main/Dialogs/BoostingDialogs.py b/Tribler/Main/Dialogs/BoostingDialogs.py new file mode 100644 index 00000000000..1627861b5a0 --- /dev/null +++ b/Tribler/Main/Dialogs/BoostingDialogs.py @@ -0,0 +1,157 @@ +""" +This module contains wx dialog gui for adding/removing boosting source + +Written by Egbert Bouman and Ardhi Putra Pratama H +""" + +import wx + +from Tribler.Main.vwxGUI.GuiUtility import GUIUtility +from Tribler.Policies.BoostingManager import ChannelSource + + +class AddBoostingSource(wx.Dialog): + """ + Class for adding the source for credit mining + """ + + def __init__(self, parent): + wx.Dialog.__init__(self, parent, -1, 'Add boosting source', size=(475, 275), name="AddBoostingSourceDialog") + + self.channels = [] + self.source = '' + + text = wx.StaticText(self, -1, 'Please enter a RSS feed URL or directory to start boosting swarms:') + + self.rss_feed_radio = wx.RadioButton(self, -1, 'RSS feed:') + self.rss_feed_edit = wx.TextCtrl(self, -1) + self.rss_feed_edit.Bind(wx.EVT_TEXT, lambda evt: self.rss_feed_radio.SetValue(True)) + + self.rss_dir_radio = wx.RadioButton(self, -1, 'Torrents local directory:') + self.rss_dir_edit = wx.TextCtrl(self, -1) + self.rss_dir_edit.Bind(wx.EVT_TEXT, lambda evt: self.rss_dir_radio.SetValue(True)) + self.rss_dir_edit.Bind(wx.EVT_LEFT_DOWN, self.on_open_dir) + + self.archive_check = wx.CheckBox(self, -1, "Archive mode") + ok_btn = wx.Button(self, -1, "OK") + ok_btn.Bind(wx.EVT_BUTTON, self.on_added_source) + cancel_btn = wx.Button(self, -1, "Cancel") + cancel_btn.Bind(wx.EVT_BUTTON, self.on_cancel) + + source_grid = wx.FlexGridSizer(2, 2, 0, 0) + source_grid.AddGrowableCol(1) + source_grid.Add(self.rss_feed_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) + source_grid.Add(self.rss_feed_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + source_grid.Add(self.rss_dir_radio, 0, wx.ALIGN_CENTER_VERTICAL | wx.LEFT | wx.RIGHT | wx.TOP, 5) + source_grid.Add(self.rss_dir_edit, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + btn_sizer = wx.BoxSizer(wx.HORIZONTAL) + btn_sizer.Add(ok_btn, 0, wx.RIGHT | wx.TOP | wx.BOTTOM, 5) + btn_sizer.Add(cancel_btn, 0, wx.ALL, 5) + v_sizer = wx.BoxSizer(wx.VERTICAL) + v_sizer.Add(text, 0, wx.EXPAND | wx.ALL, 5) + v_sizer.Add(source_grid, 0, wx.EXPAND | wx.ALL, 5) + v_sizer.AddSpacer((-1, 5)) + v_sizer.Add(self.archive_check, 0, wx.LEFT | wx.RIGHT, 10) + v_sizer.AddStretchSpacer() + v_sizer.Add(btn_sizer, 0, wx.EXPAND | wx.ALL, 5) + self.SetSizer(v_sizer) + + def on_added_source(self, _): + """ + this function called when user clicked 'OK' button for adding source + """ + if self.rss_feed_radio.GetValue(): + self.source = self.rss_feed_edit.GetValue() + else: + self.source = self.rss_dir_edit.GetValue() + + GUIUtility.getInstance().Notify( + "Successfully add source for credit mining %s" % self.source) + + self.EndModal(wx.ID_OK) + + def on_cancel(self, _): + """ + this function called when user clicked 'Cancel' button when adding source + thus, cancelled + """ + self.EndModal(wx.ID_CANCEL) + + def get_value(self): + """ + get the value of new source + """ + return self.source, self.archive_check.GetValue() + + def on_open_dir(self, event): + """ + opening local directory for choosing directory source + """ + rss_dir_dialog = wx.DirDialog(self, "Choose a directory:", + style=wx.DD_DEFAULT_STYLE | wx.DD_DIR_MUST_EXIST) + + if rss_dir_dialog.ShowModal() == wx.ID_OK: + self.rss_dir_edit.SetValue(rss_dir_dialog.GetPath()) + + +class RemoveBoostingSource(wx.Dialog): + """ + Class for adding the source for credit mining + """ + def __init__(self, parent): + wx.Dialog.__init__(self, parent, -1, 'Remove boosting source', size=(475, 135), + name="RemoveBoostingSourceDialog") + + self.guiutility = GUIUtility.getInstance() + self.boosting_manager = self.guiutility.utility.session.lm.boosting_manager + self.sources = [] + self.source = '' + + text = wx.StaticText(self, -1, 'Please select the boosting source you wish to remove:') + self.source_label = wx.StaticText(self, -1, 'Source:') + self.source_choice = wx.Choice(self, -1) + ok_btn = wx.Button(self, -1, "OK") + ok_btn.Bind(wx.EVT_BUTTON, self.on_remove_source) + cancel_btn = wx.Button(self, -1, "Cancel") + cancel_btn.Bind(wx.EVT_BUTTON, self.on_cancel) + + sourcesizer = wx.BoxSizer(wx.HORIZONTAL) + sourcesizer.Add(self.source_label, 0, wx.ALIGN_CENTER_VERTICAL | wx.RIGHT | wx.TOP, 5) + sourcesizer.Add(self.source_choice, 1, wx.EXPAND | wx.LEFT | wx.RIGHT | wx.TOP, 5) + btnsizer = wx.BoxSizer(wx.HORIZONTAL) + btnsizer.Add(ok_btn, 0, wx.RIGHT | wx.TOP | wx.BOTTOM, 5) + btnsizer.Add(cancel_btn, 0, wx.ALL, 5) + vsizer = wx.BoxSizer(wx.VERTICAL) + vsizer.Add(text, 0, wx.EXPAND | wx.ALL, 5) + vsizer.Add(sourcesizer, 0, wx.EXPAND | wx.ALL, 5) + vsizer.AddStretchSpacer() + vsizer.Add(btnsizer, 0, wx.EXPAND | wx.ALL, 5) + self.SetSizer(vsizer) + + # retrieve all source except channel source + self.sources = [s.get_source_text() for s in self.boosting_manager.boosting_sources.values() + if not isinstance(s, ChannelSource)] + + self.source_choice.SetItems(self.sources) + + def on_remove_source(self, _): + """ + this function called when user clicked 'OK' button for removing source + """ + selection = self.source_choice.GetSelection() + if selection < len(self.sources): + self.source = self.sources[selection] + self.EndModal(wx.ID_OK) + + def on_cancel(self, _): + """ + this function called when user clicked 'Cancel' button when adding source + thus, cancelled + """ + self.EndModal(wx.ID_CANCEL) + + def get_value(self): + """ + get value when removing source + """ + return self.source diff --git a/Tribler/Main/Utility/GuiDBTuples.py b/Tribler/Main/Utility/GuiDBTuples.py index 74ccb304c42..aead355122b 100644 --- a/Tribler/Main/Utility/GuiDBTuples.py +++ b/Tribler/Main/Utility/GuiDBTuples.py @@ -101,7 +101,7 @@ def __init__(self, torrent_id, infohash, name, length, category, status, num_see Helper.__init__(self) assert isinstance(infohash, str), type(infohash) - assert isinstance(name, basestring), type(name) + # assert isinstance(name, basestring), type(name) self.infohash = infohash self.name = name @@ -300,7 +300,7 @@ def __init__(self, torrent_id, infohash, name, length=0, category=None, status=N class CollectedTorrent(Helper): - __slots__ = ('comment', 'trackers', 'creation_date', 'files', 'last_check', 'torrent') + __slots__ = ('comment', 'trackers', 'creation_date', 'files', 'last_check', 'torrent', 'tdef') def __init__(self, torrent, torrentdef): assert isinstance(torrent, Torrent) @@ -314,6 +314,7 @@ def __init__(self, torrent, torrentdef): self.creation_date = min(long(time()), torrentdef.get_creation_date()) self.files = torrentdef.get_files_as_unicode_with_length() self.last_check = -1 + self.tdef = torrentdef def __getattr__(self, name): return getattr(self.torrent, name) diff --git a/Tribler/Main/tribler_main.py b/Tribler/Main/tribler_main.py index f013d254338..9941b9c493c 100755 --- a/Tribler/Main/tribler_main.py +++ b/Tribler/Main/tribler_main.py @@ -682,6 +682,10 @@ def sesscb_ntfy_torrentupdates(self, events): manager = self.frame.librarylist.GetManager() manager.torrentsUpdated(infohashes) + if self.utility.session.get_creditmining_enable(): + manager = self.frame.creditminingpanel.cmlist.GetManager() + manager.torrents_updated(infohashes) + from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent if self.frame.torrentdetailspanel.torrent and self.frame.torrentdetailspanel.torrent.infohash in infohashes: @@ -703,7 +707,10 @@ def sesscb_ntfy_torrentfinished(self, subject, changeType, objectID, *args): if self._frame_and_ready(): infohash = objectID torrent = self.guiUtility.torrentsearch_manager.getTorrentByInfohash(infohash) - self.guiUtility.library_manager.addDownloadState(torrent) + # Check if we got the actual torrent as the bandwidth investor + # downloads aren't going to be there. + if torrent: + self.guiUtility.library_manager.addDownloadState(torrent) @forceWxThread def sesscb_ntfy_newversion(self, subject, changeType, objectID, *args): diff --git a/Tribler/Main/vwxGUI/CreditMiningPanel.py b/Tribler/Main/vwxGUI/CreditMiningPanel.py new file mode 100644 index 00000000000..b6b5e64afb8 --- /dev/null +++ b/Tribler/Main/vwxGUI/CreditMiningPanel.py @@ -0,0 +1,535 @@ +""" +This module contains credit mining panel and list in wx + +Written by Ardhi Putra Pratama H +""" + +import logging +from binascii import hexlify + +# pylint complaining if wx imported before binascii +import wx + +from wx.lib.agw import ultimatelistctrl as ULC +from wx.lib.agw.ultimatelistctrl import EVT_LIST_ITEM_CHECKED + +from Tribler.Core.exceptions import NotYetImplementedException +from Tribler.Core.simpledefs import NTFY_TORRENTS +from Tribler.Main.Dialogs.BoostingDialogs import RemoveBoostingSource, AddBoostingSource +from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY +from Tribler.Main.Utility.GuiDBTuples import Channel +from Tribler.Main.vwxGUI import SEPARATOR_GREY, GRADIENT_LGREY, GRADIENT_DGREY, format_time +from Tribler.Main.vwxGUI.GuiUtility import GUIUtility +from Tribler.Main.vwxGUI.list import CreditMiningList +from Tribler.Main.vwxGUI.widgets import FancyPanel, LinkStaticText, _set_font +from Tribler.Policies.BoostingSource import RSSFeedSource, DirectorySource, ChannelSource, BoostingSource + +RETURNED_CHANNELS = 30 + + +class CpanelCheckListCtrl(wx.ScrolledWindow, ULC.UltimateListCtrl): + """ + The checklist of credit mining sources. Check to enable, uncheck to disable. + It is grouped by type : RSS, directory, and Channels + """ + def __init__(self, parent, wxid=wx.ID_ANY, style=0, agwStyle=0): + ULC.UltimateListCtrl.__init__(self, parent, wxid, wx.DefaultPosition, wx.DefaultSize, style, agwStyle) + + self.guiutility = GUIUtility.getInstance() + self.boosting_manager = self.guiutility.utility.session.lm.boosting_manager + + self.channel_list = {} + + self._logger = logging.getLogger(self.__class__.__name__) + + self.InsertColumn(0, 'col') + self.SetColumnWidth(0, -3) + + # index holder for labels. 0 for RSS, 1 for directory, 2 for channel + self.labels = [0, 1, 2] + + self.InsertStringItem(self.labels[0], "RSS") + item = self.GetItem(self.labels[0]) + item.Enable(False) + item.SetData("RSS") + self.SetItem(item) + + self.InsertStringItem(self.labels[1], "Directory") + item = self.GetItem(self.labels[1]) + item.Enable(False) + item.SetData("Directory") + self.SetItem(item) + + self.InsertStringItem(self.labels[2], "Channel") + item = self.GetItem(self.labels[2]) + item.Enable(False) + item.SetData("Channel") + self.SetItem(item) + + self.Bind(EVT_LIST_ITEM_CHECKED, self.OnGetItemCheck) + + self.getting_channels = False + self._mainWin.Bind(wx.EVT_SCROLLWIN, self.on_scroll) + + def on_scroll(self, evt): + """ + scroller watcher. Might be useful for unlimited load channels + """ + vpos = self._mainWin.GetScrollPos(wx.VERTICAL) + list_total = self.GetItemCount() + list_pp = self.GetCountPerPage() + topitem_idx, _ = self._mainWin.GetVisibleLinesRange() + + total_page = list_total / list_pp + + # print "vpos %d totlist %d topidx %d pp %d" "btmidx %s" + # %(vpos, total_page*list_pp, topitem_idx, list_pp, bottomitem_idx) + if (vpos >= list_total and total_page * list_pp < vpos and + vpos > topitem_idx + list_pp) or vpos == 0: + # not so accurate but this will do + + if self.getting_channels: + evt.Skip() + return + + self.load_more() + + evt.Skip() + + def load_more(self): + """ + load more channels to the list + """ + self._logger.info("getting new channels..") + self.getting_channels = True + + def do_query_channels(): + """ + querying channels in the background. Only return as much as RETURNED_CHANNELS + """ + _, channels = self.guiutility.channelsearch_manager.getPopularChannels(20) + dict_channels = {channel.dispersy_cid: channel for channel in channels} + new_channels_ids = list(set(dict_channels.keys()) - set(self.channel_list.keys())) + + return_list = [dict_channels.get(new_channels_ids[i]) + for i in xrange(0, min(len(new_channels_ids), RETURNED_CHANNELS))] + + if return_list: + return [l for l in sorted(return_list, key=lambda x: x.nr_favorites, + reverse=True)] + + def do_update_gui(delayed_result): + """ + add fetched channels to GUI + """ + channels = delayed_result.get() + + if channels: + for channel in channels: + # s is channel object + self.channel_list[channel.dispersy_cid] = channel + self.create_source_item(channel) + + self.getting_channels = False + self.refresh_sourcelist_data() + self.Layout() + + startWorker(do_update_gui, do_query_channels, retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def create_source_item(self, source): + """ + put the source object in the sourcelist available for enable/disable + """ + item_count = self.GetItemCount() + + if isinstance(source, RSSFeedSource): + + # update label for directory as we pushed it down + self.InsertStringItem(self.labels[1], source.get_source_text(), 1) + item = self.GetItem(self.labels[1]) + item.Check(source.enabled) + item.SetData(source) + self.SetItem(item) + self.labels[1] += 1 + self.labels[2] += 1 + elif isinstance(source, DirectorySource): + self.InsertStringItem(self.labels[2], source.get_source_text(), 1) + item = self.GetItem(self.labels[2]) + item.Check(source.enabled) + item.SetData(source) + self.SetItem(item) + self.labels[2] += 1 + elif isinstance(source, ChannelSource): + self.InsertStringItem(self.labels[2] + 1, source.get_source_text() or "Loading..", 1) + item = self.GetItem(self.labels[2] + 1) + item.Check(source.enabled) + item.SetData(source) + self.SetItem(item) + + self.channel_list[source.source] = source + elif isinstance(source, Channel): + # channel can't be 'added'. Initialization only + self.InsertStringItem(item_count, source.name, 1) + self.SetItemData(item_count, source) + else: + raise NotYetImplementedException('Source type unknown') + + def OnGetItemCheck(self, evt): + item = evt.GetItem() + data = item.GetData() + flag = item.IsChecked() + + # if it was channel that not stored in cm variables + if not isinstance(data, BoostingSource) and flag: + source = data.dispersy_cid + self.boosting_manager.add_source(source) + self.boosting_manager.set_archive(source, False) + + self.boosting_manager.set_enable_mining( + data.dispersy_cid if not isinstance(data, BoostingSource) + else data.source, flag, True) + + if isinstance(data, Channel): + # channel -> channel source + channel_src = self.boosting_manager.boosting_sources.get(data.dispersy_cid) + channel_src.channel = data + item.SetData(channel_src) + self.SetItem(item) + + def refresh_sourcelist_data(self, rerun=True): + """ + delete all the source in the list and adding a new one + """ + + # don't refresh if we are quitting + if GUIUtility.getInstance().utility.abcquitting: + return + + for i in xrange(0, self.GetItemCount()): + item = self.GetItem(i) + data = item.GetData() + + if isinstance(data, ChannelSource): + if item.GetText() == "Loading..": + item.SetText(data.get_source_text() or "Loading..") + self.SetItem(item) + + if rerun and not self.guiutility.utility.session.lm.threadpool.is_pending_task_active( + str(self) + "_refresh_data_ULC"): + self.guiutility.utility.session.lm.threadpool.add_task(self.refresh_sourcelist_data, 30, + task_name=str(self) + "_refresh_data_ULC") + + def fix_channel_position(self, source): + """ + This function called when new checked channel want to pushed above + """ + chn_source = self.boosting_manager.boosting_sources[source] + + chn = self.channel_list[source] + + idx = self.FindItemData(-1, chn) + self.DeleteItem(idx) + + self.InsertStringItem(self.labels[2] + 1, chn_source.get_source_text() or "Loading..", 1) + item = self.GetItem(self.labels[2] + 1) + item.Check(chn_source.enabled) + item.SetData(chn_source) + self.SetItem(item) + + self.channel_list[source] = chn_source + + +class CreditMiningPanel(FancyPanel): + """ + A class representing panel control for credit mining + """ + def __init__(self, parent): + self._logger = logging.getLogger(self.__class__.__name__) + + self._logger.debug("CreditMiningPanel: __init__") + + self.guiutility = GUIUtility.getInstance() + self.utility = self.guiutility.utility + self.installdir = self.utility.getPath() + + FancyPanel.__init__(self, parent, border=wx.BOTTOM) + + self.SetBorderColour(SEPARATOR_GREY) + self.SetBackgroundColour(GRADIENT_LGREY, GRADIENT_DGREY) + + if not self.utility.session.get_creditmining_enable(): + wx.StaticText(self, -1, 'Credit mining inactive') + return + + self.tdb = self.utility.session.open_dbhandler(NTFY_TORRENTS) + self.boosting_manager = self.utility.session.lm.boosting_manager + + self.main_sizer = wx.BoxSizer(wx.VERTICAL) + + self.header = self.create_header_info(self) + if self.header: + self.main_sizer.Add(self.header, 0, wx.EXPAND) + + self.main_splitter = wx.SplitterWindow(self, style=wx.SP_BORDER) + self.main_splitter.SetMinimumPaneSize(300) + + self.sourcelist = CpanelCheckListCtrl(self.main_splitter, -1, + agwStyle=wx.LC_REPORT | wx.LC_NO_HEADER | wx.LC_VRULES | wx.LC_HRULES + | wx.LC_SINGLE_SEL | ULC.ULC_HAS_VARIABLE_ROW_HEIGHT) + + self.add_components(self.main_splitter) + self.SetSizer(self.main_sizer) + + self.guiutility.utility.session.lm.threadpool.add_task(self._post_init, 2, + task_name=str(self) + "_post_init") + + def add_components(self, parent): + """ + adding GUI components to the control panel + """ + self.info_panel = FancyPanel(parent, style=wx.BORDER_SUNKEN) + + if_sizer = wx.BoxSizer(wx.VERTICAL) + self.top_info_p = FancyPanel(self.info_panel, border=wx.ALL, style=wx.BORDER_SUNKEN, name="top_info_p") + tinfo_sizer = wx.BoxSizer(wx.VERTICAL) + + self.tnfo_subpanel_top = FancyPanel(self.top_info_p, border=wx.ALL) + tinfo_spanel_sizer = wx.BoxSizer(wx.HORIZONTAL) + + stat_sizer = wx.BoxSizer(wx.VERTICAL) + + self.source_label = wx.StaticText(self.tnfo_subpanel_top, -1, 'Source : -') + stat_sizer.Add(self.source_label, 1) + self.source_name = wx.StaticText(self.tnfo_subpanel_top, -1, 'Name : -') + stat_sizer.Add(self.source_name, 1) + self.torrent_num = wx.StaticText(self.tnfo_subpanel_top, -1, '# Torrents : -') + stat_sizer.Add(self.torrent_num, 1) + + # channels only + self.last_updt = wx.StaticText(self.tnfo_subpanel_top, -1, 'Latest update : -') + stat_sizer.Add(self.last_updt, 1) + self.votes_num = wx.StaticText(self.tnfo_subpanel_top, -1, 'Favorite votes : -') + stat_sizer.Add(self.votes_num, 1) + + # rss only + self.rss_title = wx.StaticText(self.tnfo_subpanel_top, -1, 'Title : -') + stat_sizer.Add(self.rss_title, 1) + self.rss_desc = wx.StaticText(self.tnfo_subpanel_top, -1, 'Description : -') + stat_sizer.Add(self.rss_desc, 1) + + self.debug_info = wx.StaticText(self.tnfo_subpanel_top, -1, 'Debug Info : -') + stat_sizer.Add(self.debug_info) + + tinfo_spanel_sizer.Add(stat_sizer, -1) + tinfo_spanel_sizer.Add(wx.StaticText(self.tnfo_subpanel_top, -1, 'Credit Mining Status: ')) + self.status_cm = wx.StaticText(self.tnfo_subpanel_top, -1, '-') + tinfo_spanel_sizer.Add(self.status_cm) + self.tnfo_subpanel_top.SetSizer(tinfo_spanel_sizer) + + tinfo_sizer.Add(self.tnfo_subpanel_top, 1, wx.EXPAND) + tinfo_sizer.Add(wx.StaticLine(self.top_info_p), 0, wx.ALL | wx.EXPAND, 5) + + self.up_rate = wx.StaticText(self.top_info_p, -1, 'Upload rate : -', name="up_rate") + tinfo_sizer.Add(self.up_rate) + self.dwn_rate = wx.StaticText(self.top_info_p, -1, 'Download rate : -', name="dwn_rate") + tinfo_sizer.Add(self.dwn_rate) + self.storage_used = wx.StaticText(self.top_info_p, -1, 'Storage Used : -', name="storage_used") + tinfo_sizer.Add(self.storage_used) + + self.top_info_p.SetSizer(tinfo_sizer) + + if_sizer.Add(self.top_info_p, 1, wx.EXPAND) + + self.cmlist = CreditMiningList(self.info_panel) + self.cmlist.do_or_schedule_refresh(True) + self.cmlist.library_manager.add_download_state_callback(self.cmlist.RefreshItems) + + if_sizer.Add(self.cmlist, 1, wx.EXPAND) + self.info_panel.SetSizer(if_sizer) + + self.sourcelist.Hide() + self.loading_holder = wx.StaticText(self.main_splitter, -1, 'Loading..') + + parent.SplitVertically(self.loading_holder, self.info_panel, 1) + parent.SetSashGravity(0.3) + self.main_sizer.Add(parent, 1, wx.EXPAND) + + def on_sourceitem_selected(self, event): + """ + This function is called when a user select 'source' in the list. + The credit mining list will only show this particular source + """ + idx = event.m_itemIndex + data = self.sourcelist.GetItem(idx).GetData() + + if isinstance(data, ChannelSource): + self.cmlist.GotFilter(data.source) + else: + self.cmlist.GotFilter(data.get_source_text() if isinstance(data, BoostingSource) else '') + + self.show_source_info(data) + + def show_source_info(self, data): + """ + shows information about selected source (not necessarily activated/enabled) in the panel + """ + + if isinstance(data, ChannelSource): + self.last_updt.Show() + self.votes_num.Show() + self.rss_title.Hide() + self.rss_desc.Hide() + + self.source_label.SetLabel("Source : Channel (stored)") + self.source_name.SetLabel("Name : " + data.get_source_text()) + self.torrent_num.SetLabel("# Torrents : " + str(data.channel.nr_torrents)) + self.last_updt.SetLabel("Latest update : " + format_time(data.channel.modified)) + self.votes_num.SetLabel('Favorite votes : ' + str(data.channel.nr_favorites)) + self.status_cm.SetLabel("Active" if data.enabled else "Inactive") + + debug_str = hexlify(data.source) + self.debug_info.SetLabel("Debug Info : \n" + debug_str) + + elif isinstance(data, Channel): + self.last_updt.Show() + self.votes_num.Show() + self.rss_title.Hide() + self.rss_desc.Hide() + + self.source_label.SetLabel("Source : Channel") + self.source_name.SetLabel("Name : " + data.name) + self.torrent_num.SetLabel("# Torrents : " + str(data.nr_torrents)) + self.last_updt.SetLabel("Latest update : " + format_time(data.modified)) + self.votes_num.SetLabel('Favorite votes : ' + str(data.nr_favorites)) + self.status_cm.SetLabel("Inactive") + + debug_str = hexlify(data.dispersy_cid) + self.debug_info.SetLabel("Debug Info : \n" + debug_str) + + elif isinstance(data, RSSFeedSource): + self.last_updt.Hide() + self.votes_num.Hide() + self.rss_title.Show() + self.rss_desc.Show() + + self.source_label.SetLabel("Source : RSS Web Feed") + self.source_name.SetLabel("Source URL : " + data.get_source_text()) + self.torrent_num.SetLabel("# Torrents : %s" % len(data.torrents)) + self.rss_title.SetLabel("Title : " + data.title) + self.rss_desc.SetLabel("Description : " + data.description) + self.status_cm.SetLabel("Active" if data.enabled else "Inactive") + + debug_str = "-" + self.debug_info.SetLabel("Debug Info : \n" + debug_str) + + elif isinstance(data, DirectorySource): + self.last_updt.Hide() + self.votes_num.Hide() + self.rss_title.Hide() + self.rss_desc.Hide() + + self.source_label.SetLabel("Source : Directory") + self.source_name.SetLabel("Name : " + data.get_source_text()) + self.torrent_num.SetLabel("# Torrents : %d" % len(data.torrents)) + self.status_cm.SetLabel("Active" if data.enabled else "Inactive") + + debug_str = "-" + self.debug_info.SetLabel("Debug Info : \n" + debug_str) + + else: + self._logger.debug("Not implemented yet") + + # show/hide items + self.tnfo_subpanel_top.Layout() + + def create_header_info(self, parent): + """ + function to create wx header/info panel above the credit mining list + """ + if self.guiutility.frame.top_bg: + header = FancyPanel(parent, border=wx.BOTTOM, name="cm_header") + text = wx.StaticText(header, -1, 'Investment overview') + + def on_add_source(_): + """ + callback when a user wants to add new source + """ + dlg = AddBoostingSource(None) + if dlg.ShowModal() == wx.ID_OK: + source, archive = dlg.get_value() + if source: + self.boosting_manager.add_source(source) + self.boosting_manager.set_archive(source, archive) + + self.sourcelist.create_source_item(self.boosting_manager.boosting_sources[source]) + + dlg.Destroy() + + def on_remove_source(_): + """ + callback when a user wants to remove source + """ + dlg = RemoveBoostingSource(None) + if dlg.ShowModal() == wx.ID_OK and dlg.get_value(): + self.boosting_manager.remove_source(dlg.get_value()) + self.sourcelist.refresh_sourcelist_data() + dlg.Destroy() + + addsource = LinkStaticText(header, 'Add', icon=None) + addsource.Bind(wx.EVT_LEFT_UP, on_add_source) + removesource = LinkStaticText(header, 'Remove', icon=None) + removesource.Bind(wx.EVT_LEFT_UP, on_remove_source) + + self.b_up = wx.StaticText(header, -1, 'Total bytes up: -', name="b_up") + self.b_down = wx.StaticText(header, -1, 'Total bytes down: -', name="b_down") + self.s_up = wx.StaticText(header, -1, 'Total speed up: -', name="s_up") + self.s_down = wx.StaticText(header, -1, 'Total speed down: -', name="s_down") + self.iv_sum = wx.StaticText(header, -1, 'Investment summary: -', name="iv_sum") + _set_font(text, size_increment=2, fontweight=wx.FONTWEIGHT_BOLD) + sizer = wx.BoxSizer(wx.VERTICAL) + sizer.AddStretchSpacer() + titlesizer = wx.BoxSizer(wx.HORIZONTAL) + titlesizer.Add(text, 0, wx.ALIGN_BOTTOM | wx.RIGHT, 5) + titlesizer.Add(wx.StaticText(header, -1, '('), 0, wx.ALIGN_BOTTOM) + titlesizer.Add(addsource, 0, wx.ALIGN_BOTTOM) + titlesizer.Add(wx.StaticText(header, -1, '/'), 0, wx.ALIGN_BOTTOM) + titlesizer.Add(removesource, 0, wx.ALIGN_BOTTOM) + titlesizer.Add(wx.StaticText(header, -1, ' boosting source)'), 0, wx.ALIGN_BOTTOM) + sizer.Add(titlesizer, 0, wx.LEFT | wx.BOTTOM, 5) + sizer.Add(self.b_up, 0, wx.LEFT, 5) + sizer.Add(self.b_down, 0, wx.LEFT, 5) + sizer.Add(self.s_up, 0, wx.LEFT, 5) + sizer.Add(self.s_down, 0, wx.LEFT, 5) + sizer.Add(self.iv_sum, 0, wx.LEFT, 5) + sizer.AddStretchSpacer() + header.SetSizer(sizer) + header.SetMinSize((-1, 100)) + else: + raise NotYetImplementedException('') + + return header + + def _post_init(self): + if GUIUtility.getInstance().utility.abcquitting: + return + + some_ready = any([i.ready for i in self.boosting_manager.boosting_sources.values()]) + + # if none are ready, keep waiting or If no source available + if not some_ready and len(self.boosting_manager.boosting_sources.values()): + self.guiutility.utility.session.lm.threadpool.add_task(self._post_init, 2, + task_name=str(self) + "_post_init") + return + + for _, source_obj in self.boosting_manager.boosting_sources.items(): + self.sourcelist.create_source_item(source_obj) + + self.sourcelist.Show() + self.main_splitter.ReplaceWindow(self.loading_holder, self.sourcelist) + self.loading_holder.Close() + + self.Bind(ULC.EVT_LIST_ITEM_SELECTED, self.on_sourceitem_selected, self.sourcelist) + + self.guiutility.utility.session.lm.threadpool.add_task(self.sourcelist.load_more, 2, + task_name=str(self) + "load_more") + self.Layout() diff --git a/Tribler/Main/vwxGUI/GuiUtility.py b/Tribler/Main/vwxGUI/GuiUtility.py index 1e3dbd28db7..b59b0f002e8 100644 --- a/Tribler/Main/vwxGUI/GuiUtility.py +++ b/Tribler/Main/vwxGUI/GuiUtility.py @@ -250,6 +250,12 @@ def ShowPage(self, page, *args): # Hide list self.frame.librarylist.Show(False) + if page == 'creditmining': + self.frame.creditminingpanel.Show(True) + + elif self.guiPage == 'creditmining': + self.frame.creditminingpanel.Show(False) + if page == 'home': self.frame.home.ResetSearchBox() self.frame.home.Show() @@ -356,6 +362,9 @@ def GetSelectedPage(self): if self.guiPage == 'my_files': return self.frame.librarylist + if self.guiPage == 'creditmining': + return self.frame.creditminingpanel + def SetTopSplitterWindow(self, window=None, show=True): while self.frame.splitter_top.GetChildren(): self.frame.splitter_top.Detach(0) @@ -595,7 +604,8 @@ def OnList(self, goto_end, event=None): 'selectedchannel': self.frame.selectedchannellist, 'mychannel': self.frame.managechannel, 'search_results': self.frame.searchlist, - 'my_files': self.frame.librarylist} + 'my_files': self.frame.librarylist, + 'creditmining': self.frame.creditminingpanel} if self.guiPage in lists and lists[self.guiPage].HasFocus(): lists[self.guiPage].ScrollToEnd(goto_end) elif event: @@ -607,7 +617,8 @@ def ScrollTo(self, id): 'selectedchannel': self.frame.selectedchannellist, 'mychannel': self.frame.managechannel, 'search_results': self.frame.searchlist, - 'my_files': self.frame.librarylist} + 'my_files': self.frame.librarylist, + 'creditmining': self.frame.creditminingpanel} if self.guiPage in lists: lists[self.guiPage].ScrollToId(id) diff --git a/Tribler/Main/vwxGUI/MainFrame.py b/Tribler/Main/vwxGUI/MainFrame.py index 16a6d99d655..5ab07a92080 100644 --- a/Tribler/Main/vwxGUI/MainFrame.py +++ b/Tribler/Main/vwxGUI/MainFrame.py @@ -34,6 +34,7 @@ from Tribler.Core.exceptions import DuplicateDownloadException from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo from Tribler.Core.Utilities.utilities import parse_magnetlink, fix_torrent +from Tribler.Main.Dialogs.SaveAs import SaveAs from Tribler.Core.DownloadConfig import DefaultDownloadStartupConfig from Tribler.Main.Utility.GuiDBHandler import startWorker @@ -41,8 +42,8 @@ from Tribler.Main.Dialogs.ConfirmationDialog import ConfirmationDialog from Tribler.Main.Dialogs.FeedbackWindow import FeedbackWindow from Tribler.Main.Dialogs.systray import ABCTaskBarIcon -from Tribler.Main.Dialogs.SaveAs import SaveAs - +from Tribler.Main.vwxGUI import DEFAULT_BACKGROUND, SEPARATOR_GREY +from Tribler.Main.vwxGUI.CreditMiningPanel import CreditMiningPanel from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread from Tribler.Main.vwxGUI import DEFAULT_BACKGROUND, SEPARATOR_GREY from Tribler.Main.vwxGUI.list import SearchList, ChannelList, LibraryList, ActivitiesList @@ -194,6 +195,10 @@ def __init__(self, abc, parent, internalvideo): self.playlist = Playlist(self.splitter_top_window) self.playlist.Show(False) + + self.creditminingpanel = CreditMiningPanel(self) + self.creditminingpanel.Show(False) + # Populate the bottom window self.splitter_bottom = wx.BoxSizer(wx.HORIZONTAL) self.torrentdetailspanel = TorrentDetails(self.splitter_bottom_window) @@ -265,6 +270,8 @@ def OnShowSplitter(event): if self.videoparentpanel: hSizer.Add(self.videoparentpanel, 1, wx.EXPAND) + hSizer.Add(self.creditminingpanel, 1, wx.EXPAND) + self.SetSizer(vSizer) # set sizes diff --git a/Tribler/Main/vwxGUI/SearchGridManager.py b/Tribler/Main/vwxGUI/SearchGridManager.py index f7570926b74..e65e269b9e7 100644 --- a/Tribler/Main/vwxGUI/SearchGridManager.py +++ b/Tribler/Main/vwxGUI/SearchGridManager.py @@ -1003,8 +1003,8 @@ def getMySubscriptions(self): subscriptions = self.channelcast_db.getMySubscribedChannels(include_dispersy=True) return self._createChannels(subscriptions) - def getPopularChannels(self): - pchannels = self.channelcast_db.getMostPopularChannels() + def getPopularChannels(self, nr_top_popular=20): + pchannels = self.channelcast_db.getMostPopularChannels(nr_top_popular) return self._createChannels(pchannels) def getUpdatedChannels(self): diff --git a/Tribler/Main/vwxGUI/home.py b/Tribler/Main/vwxGUI/home.py index 7fc44d402e8..dbedc9498b7 100644 --- a/Tribler/Main/vwxGUI/home.py +++ b/Tribler/Main/vwxGUI/home.py @@ -1,41 +1,50 @@ # Written by Niels Zeilemaker -import wx -import sys -import os import datetime - -from Tribler.community.tunnel.hidden_community import HiddenTunnelCommunity -from Tribler.community.tunnel.routing import Hop -from Tribler.community.multichain.community import MultiChainCommunity - -import random import logging +import os +import random +import sys import binascii + from time import strftime, time from traceback import print_exc +# pylint complaining if wx imported before those three +import wx + from Tribler.Category.Category import Category +from Tribler.Core.CacheDB.sqlitecachedb import bin2str +from Tribler.Core.Session import Session +from Tribler.Core.Video.VideoUtility import considered_xxx from Tribler.Core.simpledefs import (NTFY_TORRENTS, NTFY_CHANNELCAST, NTFY_INSERT, NTFY_TUNNEL, NTFY_CREATED, NTFY_EXTENDED, NTFY_BROKEN, NTFY_SELECT, NTFY_JOINED, NTFY_EXTENDED_FOR, NTFY_IP_REMOVED, NTFY_RP_REMOVED, NTFY_IP_RECREATE, NTFY_DHT_LOOKUP, NTFY_KEY_REQUEST, NTFY_KEY_RESPOND, NTFY_KEY_RESPONSE, NTFY_CREATE_E2E, NTFY_ONCREATED_E2E, NTFY_IP_CREATED, NTFY_RP_CREATED, NTFY_REMOVE) -from Tribler.Core.Session import Session - +from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY +from Tribler.Main.Utility.utility import size_format from Tribler.Main.vwxGUI import SEPARATOR_GREY, DEFAULT_BACKGROUND, LIST_BLUE, THUMBNAIL_FILETYPES +from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread -from Tribler.Main.Utility.GuiDBHandler import startWorker, GUI_PRI_DISPERSY -from Tribler.Main.vwxGUI.list_header import DetailHeader from Tribler.Main.vwxGUI.list_body import ListBody -from Tribler.Main.vwxGUI.list_item import ThumbnailListItemNoTorrent from Tribler.Main.vwxGUI.list_footer import ListFooter +from Tribler.Main.vwxGUI.list_header import DetailHeader +from Tribler.Main.vwxGUI.list_item import ThumbnailListItemNoTorrent from Tribler.Main.vwxGUI.widgets import (SelectableListCtrl, TextCtrlAutoComplete, BetterText as StaticText, - LinkStaticText, ActionButton) -from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager -from Tribler.Core.CacheDB.sqlitecachedb import bin2str -from Tribler.Core.Video.VideoUtility import considered_xxx + LinkStaticText, ActionButton, HorizontalGauge, TagText) +from Tribler.Policies.credit_mining_util import string_to_source +from Tribler.community.multichain.community import MultiChainCommunity +from Tribler.community.tunnel.hidden_community import HiddenTunnelCommunity +from Tribler.community.tunnel.routing import Hop -from Tribler.Main.Utility.utility import size_format +# width size of channel grid +COLUMN_SIZE = 3 +# how long the string before it cut +CHANNEL_STRING_LENGTH = 35 +# number of popular torrent fetched to know the 'content' of channels +TORRENT_FETCHED = 5 +# max number of channel shown in the panel +MAX_CHANNEL_SHOW = 9 class Home(wx.Panel): @@ -43,8 +52,15 @@ class Home(wx.Panel): def __init__(self, parent): wx.Panel.__init__(self, parent) self.guiutility = GUIUtility.getInstance() + self.gui_image_manager = GuiImageManager.getInstance() + self.session = self.guiutility.utility.session + self.boosting_manager = self.session.lm.boosting_manager - self.SetBackgroundColour(DEFAULT_BACKGROUND) + #dispersy_cid:Channel + self.channels = {} + + #dispersy_cid:Popular Torrents + self.chn_torrents = {} vSizer = wx.BoxSizer(wx.VERTICAL) vSizer.AddStretchSpacer() @@ -78,7 +94,7 @@ def __init__(self, parent): search_button = ActionButton(self, -1, search_img) search_button.Bind(wx.EVT_LEFT_UP, self.OnClick) - scalingSizer.Add(self.searchBox) + scalingSizer.Add(self.searchBox, 0, wx.ALIGN_CENTER_VERTICAL) scalingSizer.AddSpacer(3, -1) scalingSizer.Add(search_button, 0, wx.ALIGN_CENTER_VERTICAL, 3) @@ -91,29 +107,68 @@ def __init__(self, parent): hSizer.Add(StaticText(self, -1, " to see what others are sharing.")) vSizer.Add(text, 0, wx.ALIGN_CENTER) - vSizer.AddSpacer(10) - vSizer.Add(scalingSizer, 0, wx.ALIGN_CENTER) - vSizer.AddSpacer(10) + vSizer.Add(scalingSizer, 1, wx.ALIGN_CENTER_HORIZONTAL | wx.ALIGN_TOP) vSizer.Add(hSizer, 0, wx.ALIGN_CENTER) vSizer.AddStretchSpacer() + # channel panel is for popular channel + self.channel_panel = wx.lib.scrolledpanel.ScrolledPanel(self, 1) + self.channel_panel.SetBackgroundColour(wx.WHITE) + self.channel_panel.SetForegroundColour(parent.GetForegroundColour()) + + v_chn_sizer = wx.BoxSizer(wx.VERTICAL) + v_chn_sizer.Add( + DetailHeader(self.channel_panel, "Select popular channels to mine"), + 0, wx.EXPAND, 5) + + self.loading_channel_txt = wx.StaticText(self.channel_panel, 1, + 'Loading, please wait.' + if self.boosting_manager else "Credit Mining inactive") + + v_chn_sizer.Add(self.loading_channel_txt, 1, wx.TOP | wx.ALIGN_CENTER_HORIZONTAL, 10) + + self.chn_sizer = wx.FlexGridSizer(0, COLUMN_SIZE, 5, 5) + + for i in xrange(0, COLUMN_SIZE): + if wx.MAJOR_VERSION > 2: + if self.chn_sizer.IsColGrowable(i): + self.chn_sizer.AddGrowableCol(i, 1) + else: + self.chn_sizer.AddGrowableCol(i, 1) + + v_chn_sizer.Add(self.chn_sizer, 0, wx.EXPAND, 5) + + self.channel_panel.SetSizer(v_chn_sizer) + self.channel_panel.SetupScrolling() + + vSizer.Add(self.channel_panel, 5, wx.EXPAND) + + # video thumbnail panel self.aw_panel = ArtworkPanel(self) self.aw_panel.SetMinSize((-1, 275)) - self.aw_panel.Hide() + self.aw_panel.Show(self.guiutility.ReadGuiSetting('show_artwork', False)) vSizer.Add(self.aw_panel, 0, wx.EXPAND) self.SetSizer(vSizer) self.Layout() - self.Bind(wx.EVT_RIGHT_UP, self.OnRightClick) - self.SearchFocus() + self.channel_list_ready = False + + if self.boosting_manager: + self.session.lm.threadpool.add_task(self.refresh_channels_home, 10, + task_name=str(self.__class__)+"_refreshchannel") + def OnRightClick(self, event): menu = wx.Menu() - itemid = wx.NewId() - menu.AppendCheckItem(itemid, 'Show recent videos') - menu.Check(itemid, self.aw_panel.IsShown()) + itemid_rcvid = wx.NewId() + itemid_popchn = wx.NewId() + menu.AppendCheckItem(itemid_rcvid, 'Show recent videos') + menu.AppendCheckItem(itemid_popchn, 'Show popular channels') + + menu.Check(itemid_rcvid, self.aw_panel.IsShown()) + menu.Check(itemid_popchn, self.channel_panel.IsShown()) def toggleArtwork(event): show = not self.aw_panel.IsShown() @@ -121,7 +176,13 @@ def toggleArtwork(event): self.guiutility.WriteGuiSetting("show_artwork", show) self.Layout() - menu.Bind(wx.EVT_MENU, toggleArtwork, id=itemid) + def togglechannels(_): + show = not self.channel_panel.IsShown() + self.channel_panel.Show(show) + self.Layout() + + menu.Bind(wx.EVT_MENU, togglechannels, id=itemid_popchn) + menu.Bind(wx.EVT_MENU, toggleArtwork, id=itemid_rcvid) if menu: self.PopupMenu(menu, self.ScreenToClient(wx.GetMousePosition())) @@ -144,6 +205,169 @@ def SearchFocus(self): self.searchBox.SetFocus() self.searchBox.SelectAll() + def create_channel_item(self, parent, channel, torrents, max_fav): + """ + Function to create channel (and its torrents) checkbox on home panel + """ + + from Tribler.Main.Utility.GuiDBTuples import Channel as ChannelObj + assert isinstance(channel, ChannelObj), "Type channel should be ChannelObj %s" % channel + + vsizer = wx.BoxSizer(wx.VERTICAL) + hsizer = wx.BoxSizer(wx.HORIZONTAL) + + chn_pn = wx.Panel(parent, -1, style=wx.SUNKEN_BORDER) + + cb_chn = wx.CheckBox(chn_pn, 1, '', name=binascii.hexlify(channel.dispersy_cid)) + obj = self.boosting_manager.get_source_object(channel.dispersy_cid) + + cb_chn.SetValue(False if not obj else obj.enabled) + + control = HorizontalGauge(chn_pn, self.gui_image_manager.getImage(u"ministar.png"), + self.gui_image_manager.getImage(u"ministarEnabled.png"), 5) + + # count popularity + pop = channel.nr_favorites + if pop <= 0 or max_fav == 0: + control.SetPercentage(0) + else: + control.SetPercentage(pop/float(max_fav)) + + control.SetToolTipString('%s users marked this channel as one of their favorites.' % pop) + hsizer.Add(cb_chn, 0, wx.ALIGN_LEFT) + hsizer.Add(TagText(chn_pn, -1, label='channel', fill_colour=wx.Colour(210, 252, 120)), 0, + wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL) + hsizer.AddSpacer(5) + hsizer.Add(wx.StaticText(chn_pn, -1, channel.name.encode('utf-8')), 0, wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL) + hsizer.AddSpacer(30) + hsizer.AddStretchSpacer() + hsizer.Add(control, 0, wx.ALIGN_RIGHT | wx.ALIGN_CENTER_VERTICAL) + + vsizer.Add(hsizer, 0, wx.EXPAND) + + for trnts in torrents: + trnts = wx.StaticText(chn_pn, 1, trnts.name[:CHANNEL_STRING_LENGTH] + (trnts.name[CHANNEL_STRING_LENGTH:] + and '...')) + vsizer.Add(trnts, 0, wx.EXPAND | wx.LEFT, 25) + + chn_pn.SetSizer(vsizer) + self.Bind(wx.EVT_CHECKBOX, self.on_check_channels_cm, cb_chn) + return chn_pn + + def refresh_channels_home(self): + """ + This function will be called to get popular channel list in Home + """ + def do_query(): + """ + querying channels to show at home page. Blocking + :return: dict_channels, dict_torrents, new_channels_ids + """ + _, channels = self.guiutility.channelsearch_manager.getPopularChannels(2 * MAX_CHANNEL_SHOW) + + dict_channels = {channel.dispersy_cid: channel for channel in channels} + dict_torrents = {} + new_channels_ids = list(set(dict_channels.keys()) - + set(self.channels.keys() if not self.channel_list_ready else [])) + + for chan_id in new_channels_ids: + channel = dict_channels.get(chan_id) + torrents = self.guiutility.channelsearch_manager.getRecentReceivedTorrentsFromChannel( + channel, limit=TORRENT_FETCHED)[2] + dict_torrents[chan_id] = torrents + return dict_channels, dict_torrents, new_channels_ids + + def do_gui(delayed_result): + """ + put those new channels in the GUI + """ + (dict_channels, dict_torrents, new_channels_ids) = delayed_result.get() + count = 0 + + if self.channel_list_ready: + # reset it. Not reseting torrent_dict because it dynamically added anyway + self.channels = {} + + for chn_id in new_channels_ids: + channel = dict_channels.get(chn_id) + self.channels[chn_id] = channel + self.chn_torrents.update(dict_torrents) + + self.chn_sizer.Clear(True) + self.chn_sizer.Layout() + self.loading_channel_txt.Show() + for i in xrange(0, COLUMN_SIZE): + if wx.MAJOR_VERSION > 2: + if self.chn_sizer.IsColGrowable(i): + self.chn_sizer.AddGrowableCol(i, 1) + else: + self.chn_sizer.AddGrowableCol(i, 1) + + sortedchannels = sorted(self.channels.values(), + key=lambda z: z.nr_favorites if z else 0, reverse=True) + + max_favourite = sortedchannels[0].nr_favorites if sortedchannels else 0 + + for chn_id in [x for x in sortedchannels]: + d = chn_id.dispersy_cid + # if we can't find channel details, ignore it, or + # if no torrent available for that channel + if not dict_channels.get(d) or not len(self.chn_torrents.get(d)): + continue + + if self.session.get_creditmining_enable(): + self.chn_sizer.Add( + self.create_channel_item(self.channel_panel, dict_channels.get(d), self.chn_torrents.get(d), + max_favourite), 0, wx.ALL | wx.EXPAND) + + self.loading_channel_txt.Hide() + + count += 1 + if count >= MAX_CHANNEL_SHOW: + break + + if new_channels_ids: + self.chn_sizer.Layout() + self.channel_panel.SetupScrolling() + + # quit refreshing if Tribler quitting + if GUIUtility.getInstance().utility.abcquitting: + return + + if self.guiutility.frame.ready and isinstance(self.guiutility.GetSelectedPage(), Home): + startWorker(do_gui, do_query, retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + repeat = len(self.channels) < MAX_CHANNEL_SHOW + self.channel_list_ready = not repeat + + # try to update the popular channel once in a while + self.session.lm.threadpool.add_task_in_thread(self.refresh_channels_home, 10, + task_name=str(self.__class__)+"_refreshchannel") + + def on_check_channels_cm(self, evt): + """ + this callback called if a channel in home was checked/unchecked + """ + cbox = evt.GetEventObject() + source_str = cbox.GetName() + + # if we don't have the channel in boosting source, and its checked for the first time + if not self.boosting_manager.get_source_object(string_to_source(source_str)) and evt.IsChecked(): + source = binascii.unhexlify(source_str) + self.boosting_manager.add_source(source) + self.boosting_manager.set_archive(source, False) + + self.boosting_manager.set_enable_mining(binascii.unhexlify(source_str), evt.IsChecked()) + + if evt.IsChecked(): + chn_src = self.boosting_manager.boosting_sources[binascii.unhexlify(cbox.GetName())] + sourcelist = self.guiutility.frame.creditminingpanel.sourcelist + + if binascii.unhexlify(cbox.GetName()) in sourcelist.channel_list: + sourcelist.fix_channel_position(binascii.unhexlify(cbox.GetName())) + else: + sourcelist.create_source_item(chn_src) + class Stats(wx.Panel): diff --git a/Tribler/Main/vwxGUI/list.py b/Tribler/Main/vwxGUI/list.py index 775c893ec84..de840da03be 100644 --- a/Tribler/Main/vwxGUI/list.py +++ b/Tribler/Main/vwxGUI/list.py @@ -1,8 +1,10 @@ -# Written by Niels Zeilemaker +# Written by Niels Zeilemaker and Ardhi Putra Pratama H import copy import logging import re import sys + +from binascii import hexlify, unhexlify from colorsys import hsv_to_rgb, rgb_to_hsv from math import log from time import time @@ -15,10 +17,11 @@ from Tribler.Core.simpledefs import (DLSTATUS_HASHCHECKING, DLSTATUS_STOPPED, DLSTATUS_STOPPED_ON_ERROR, DLSTATUS_WAITING4HASHCHECK, DLSTATUS_SEEDING, DLSTATUS_DOWNLOADING) from Tribler.Main.Utility.GuiDBHandler import GUI_PRI_DISPERSY, cancelWorker, startWorker -from Tribler.Main.Utility.GuiDBTuples import Channel, ChannelTorrent, CollectedTorrent, Torrent +from Tribler.Main.Utility.GuiDBTuples import Channel, ChannelTorrent, CollectedTorrent, Torrent, LibraryTorrent from Tribler.Main.Utility.utility import eta_value, size_format, speed_format from Tribler.Main.vwxGUI import (DEFAULT_BACKGROUND, GRADIENT_DGREY, GRADIENT_LGREY, LIST_DESELECTED, LIST_GREEN, - LIST_GREY, LIST_ORANGE, SEPARATOR_GREY, TRIBLER_RED, format_time, warnWxThread) + LIST_GREY, LIST_ORANGE, SEPARATOR_GREY, TRIBLER_RED, format_time, warnWxThread, + LIST_SELECTED, LIST_EXPANDED, LIST_DARKBLUE) from Tribler.Main.vwxGUI.GuiImageManager import GuiImageManager from Tribler.Main.vwxGUI.GuiUtility import GUIUtility, forceWxThread from Tribler.Main.vwxGUI.list_body import FixedListBody, ListBody @@ -27,10 +30,11 @@ from Tribler.Main.vwxGUI.list_footer import ListFooter from Tribler.Main.vwxGUI.list_header import ChannelFilter, DownloadFilter, ListHeader, TorrentFilter from Tribler.Main.vwxGUI.list_item import (ActivityListItem, ChannelListItem, ChannelListItemAssociatedTorrents, - ColumnsManager, DragItem, LibraryListItem, TorrentListItem) + ColumnsManager, DragItem, LibraryListItem, TorrentListItem, + CreditMiningListItem) from Tribler.Main.vwxGUI.widgets import (BetterText, FancyPanel, HorizontalGauge, LinkStaticText, SwarmHealth, TagText, TorrentStatus, TransparentStaticBitmap, TransparentText, _set_font) - +from Tribler.Policies.credit_mining_util import string_to_source DEBUG_RELEVANCE = False MAX_REFRESH_PARTIAL = 5 @@ -301,6 +305,79 @@ def downloadStarted(self, _): self.refresh() +class CreditMiningSearchManager(BaseManager): + """ + search manager for credit mining purpose + """ + def __init__(self, llist): + BaseManager.__init__(self, llist) + self.boosting_manager = self.guiutility.utility.session.lm.boosting_manager + self.library_manager = self.guiutility.library_manager + + def refresh(self): + startWorker(self._on_data, self.get_torrents_list_boosting, uId=u"CreditMiningSearchManager_refresh", + retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def get_torrent_from_infohash(self, infohash): + """ + get LibraryTorrent object from torrent infohash (byte) + """ + torrent = self.boosting_manager.torrents.get(infohash, None) + if torrent: + ltorrent = LibraryTorrent('', infohash, name=torrent['name'], length=torrent['length'], category='', + status='', num_seeders=torrent['num_seeders'], + num_leechers=torrent['num_leechers']) + ltorrent.torrent_db = self.library_manager.torrent_db + ltorrent.channelcast_db = self.library_manager.channelcast_db + + #touch channel instance + ltorrent.channel #pylint: disable=pointless-statement + self.library_manager.addDownloadState(ltorrent) + return ltorrent + + def get_torrents_list_boosting(self): + """ + Get a list of LibraryTorrent(s) in Boosting Manager + """ + hits = [self.get_torrent_from_infohash(infohash) for infohash in self.boosting_manager.torrents.keys()] + return [len(hits), hits] + + def refresh_partial(self, ids): + for infohash in ids: + startWorker(self.list.RefreshDelayedData, self.get_torrent_from_infohash, cargs=(infohash,), + wargs=(infohash, ), retryOnBusy=True, priority=GUI_PRI_DISPERSY) + + def refresh_if_exists(self, infohashes): + """ + refresh list if there's a swarm available to mine + """ + if any([infohash in self.boosting_manager.torrents for infohash in infohashes]): + self._logger.info("Scheduling a refresh, missing some infohashes in the Credit Mining overview") + self.refresh() + else: + self._logger.debug("Not scheduling a refresh") + + @forceWxThread + def _on_data(self, delayed_result): + _, data = delayed_result.get() + self.list.SetData(data) + self.list.Layout() + + def torrent_updated(self, infohash): + """ + function to handle single updated torrent information + """ + if self.list.InList(infohash): + self.do_or_schedule_partial([infohash]) + + def torrents_updated(self, infohashes): + """ + function when many torrents updated + """ + infohashes = [infohash for infohash in infohashes if self.list.InList(infohash)] + self.do_or_schedule_partial(infohashes) + + class ChannelSearchManager(BaseManager): def __init__(self, list): @@ -342,7 +419,6 @@ def refreshDirty(self): def refresh(self, search_results=None): self._logger.debug("ChannelManager complete refresh") - if self.category != 'searchresults': category = self.category @@ -462,7 +538,7 @@ def joinChannel(self, cid): class List(wx.BoxSizer): def __init__(self, columns, background, spacers=[0, 0], singleSelect=False, - showChange=False, borders=True, parent=None): + showChange=False, borders=True, parent=None, list_item_max=None): """ Column alignment: @@ -491,6 +567,7 @@ def __init__(self, columns, background, spacers=[0, 0], singleSelect=False, self.singleSelect = singleSelect self.borders = borders self.showChange = showChange + self.list_item_max = list_item_max self.dirty = False self.hasData = False self.rawfilter = '' @@ -551,7 +628,8 @@ def CreateHeader(self, parent): def CreateList(self, parent=None, listRateLimit=1): if not parent: parent = self - return ListBody(parent, self, self.columns, self.spacers[0], self.spacers[1], self.singleSelect, self.showChange, listRateLimit=listRateLimit) + return ListBody(parent, self, self.columns, self.spacers[0], self.spacers[1], self.singleSelect, + self.showChange, listRateLimit=listRateLimit, list_item_max=self.list_item_max) def CreateFooter(self, parent): return ListFooter(parent) @@ -854,6 +932,7 @@ def MatchFilter(self, item): return re.search(self.filter, item[1][0].lower()) and ff def GetFilterMessage(self, empty=False): + if self.rawfilter: if empty: message = '0 items' @@ -877,8 +956,8 @@ def SetupScrolling(self, *args, **kwargs): class SizeList(List): def __init__(self, columns, background, spacers=[0, 0], singleSelect=False, - showChange=False, borders=True, parent=None): - List.__init__(self, columns, background, spacers, singleSelect, showChange, borders, parent) + showChange=False, borders=True, parent=None, list_item_max=None): + List.__init__(self, columns, background, spacers, singleSelect, showChange, borders, parent, list_item_max=None) self.prevStates = {} self.library_manager = self.guiutility.library_manager @@ -1958,6 +2037,309 @@ def GetFilterMessage(self, empty=False): return header, message +class CreditMiningList(SizeList): + """ + List of all swarms that are available for mining. + """ + def __init__(self, parent): + self.guiutility = GUIUtility.getInstance() + self.boosting_manager = self.guiutility.utility.session.lm.boosting_manager + self.utility = self.guiutility.utility + + self.statefilter = None + self.newfilter = False + self.prevStates = {} + self.old_dlstate = {} + self.old_keys = [] + + self.initnumitems = False + self.tot_bytes_up = 0 + self.tot_bytes_dwn = 0 + self.channels = [] + self.manager = None + + self.top_info_p = parent.FindWindowByName('top_info_p') or None + + columns = [{'name': 'Speed up/down', 'width': '32em', 'autoRefresh': False}, + {'name': 'Bytes up/down', 'width': '32em', 'autoRefresh': False}, + {'name': 'Seeders/leechers', 'width': '27em', 'autoRefresh': False}, + {'name': 'Duplicate', 'showColumname': False, 'width': '2em'}, + {'name': 'Hash', 'width': '27em', 'fmt': lambda ih: ih.encode('hex')[:10]}, + {'name': 'Source', 'width': '40em', 'type': 'method', 'method': self.create_source_txt}, + {'name': 'Investment status', 'width': '32em', 'autoRefresh': False}] + + columns = self.guiutility.SetColumnInfo(CreditMiningListItem, columns) + ColumnsManager.getInstance().setColumns(CreditMiningListItem, columns) + + SizeList.__init__(self, None, LIST_GREY, [0, 0], False, parent=parent) + + def GetManager(self): + if getattr(self, 'manager', None) is None: + self.manager = CreditMiningSearchManager(self) + return self.manager + + @warnWxThread + def CreateHeader(self, parent): + return None + + @warnWxThread + def CreateFooter(self, parent): + self.list.ShowMessage("No credit mining data available.") + footer = ListFooter(parent, radius=0) + footer.SetMinSize((-1, 0)) + return footer + + @warnWxThread + def create_source_txt(self, parent, item): + """ + create a source string, cut it if the length is longer than 30 + """ + torrent = self.boosting_manager.torrents.get(item.original_data.infohash, None) + text = torrent.get('source', '') + text = text[:30] + '..' if len(text) > 32 else text + return wx.StaticText(parent, -1, text) + + def OnExpand(self, item): + List.OnExpand(self, item) + return True + + @warnWxThread + def RefreshItems(self, dslist, magnetlist, rawdata=True): + didstatechange, _, _ = SizeList.RefreshItems(self, dslist, magnetlist, rawdata=True) + + newfilter = self.newfilter + + new_keys = self.boosting_manager.torrents.keys() + old_keys = getattr(self, 'old_keys', []) + if len(new_keys) != len(old_keys): + self.GetManager().refresh_if_exists(new_keys) + self.old_keys = new_keys + + if didstatechange: + if self.statefilter: + self.list.SetData() # basically this means execute filter again + + boosting_dslist = [dl_state for dl_state in dslist if dl_state.get_download().get_def().get_infohash() + in new_keys] + + # init source statistics + for _, src in self.boosting_manager.boosting_sources.items(): + src.storage_used = 0 + src.av_uprate = 0 + src.av_dwnrate = 0 + + # update torrent stats in boosting manager + for dl_state in boosting_dslist: + torrent_infohash = dl_state.get_download().get_def().get_infohash() + + if dl_state.get_seeding_statistics(): + self.boosting_manager.update_torrent_stats(torrent_infohash, dl_state.get_seeding_statistics()) + + for item in self.list.items.itervalues(): + dl_state = item.original_data.ds + torrent_infohash = item.original_data.infohash + source_str = self.boosting_manager.torrents[torrent_infohash]['source'] + + source = self.boosting_manager.get_source_object(string_to_source(source_str)) + + # ds = DownloadState + if dl_state: + source.av_uprate += dl_state.get_current_speed('up') + source.av_dwnrate += dl_state.get_current_speed('down') + + if dl_state not in boosting_dslist: + continue + + # look for the current active stats to update visible list + if dl_state.get_seeding_statistics(): + seeding_stats_i = dl_state.get_seeding_statistics() + + bytes_up = bytes_down = 0 + + if self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']: + bytes_up = self.boosting_manager.torrents[torrent_infohash]['last_seeding_stats']['total_up'] + bytes_down = self.boosting_manager.torrents[torrent_infohash][ + 'last_seeding_stats']['total_down'] + + # we may have different value of total. Find the maximum gathered + bytes_up = max(seeding_stats_i['total_up'], bytes_up) + bytes_down = max(seeding_stats_i['total_down'], bytes_down) + + item.RefreshColumn(1, size_format(bytes_up) + ' / ' + size_format(bytes_down)) + if bytes_down: + item.RefreshColumn(6, '%f' %(float(bytes_up)/float(bytes_down))) + + #refresh seeder/leecher + it_seeder = self.boosting_manager.torrents[torrent_infohash]['num_seeders'] + it_leecher = self.boosting_manager.torrents[torrent_infohash]['num_leechers'] + item.RefreshColumn(2, '%d / %d' %(it_seeder, it_leecher)) + + item.SetSelectedColour(wx.Colour(255, 175, 175)) + item.SetDeselectedColour(wx.Colour(255, 200, 200)) + item.SetExpandedColour(wx.Colour(255, 150, 150)) + item.SetExpandedAndSelectedColour(wx.Colour(255, 125, 125)) + + speed_up = dl_state.get_current_speed('up') if dl_state else 0 + speed_down = dl_state.get_current_speed('down') if dl_state else 0 + + item.RefreshColumn(0, speed_format(speed_up) + ' / ' + speed_format(speed_down)) + + else: + item.SetSelectedColour(LIST_SELECTED) + item.SetDeselectedColour(LIST_DESELECTED) + item.SetExpandedColour(LIST_EXPANDED) + item.SetExpandedAndSelectedColour(LIST_DARKBLUE) + + item.RefreshColumn(0, '- / -') + + if torrent_infohash in self.boosting_manager.torrents: + is_dup = self.boosting_manager.torrents[torrent_infohash].get('is_duplicate', None) + item.RefreshColumn(3, ('*' if is_dup else '**') if is_dup != None else '') + + # compilation of all torrents seeding stats + seeding_stats = [] + + for _, torrent_dict in self.boosting_manager.torrents.items(): + seeding_stat_t = torrent_dict['last_seeding_stats'] + + if seeding_stat_t: + seeding_stats.append(seeding_stat_t) + + self.tot_bytes_up = sum([stat['total_up'] for stat in seeding_stats]) + self.tot_bytes_dwn = sum([stat['total_down'] for stat in seeding_stats]) + + if self.top_info_p: + up_rate_txt = self.top_info_p.FindWindowByName('up_rate') + dwn_rate_txt = self.top_info_p.FindWindowByName('dwn_rate') + storage_used_txt = self.top_info_p.FindWindowByName('storage_used') + + try: + filter_src = self.rawfilter if self.rawfilter in self.boosting_manager.boosting_sources \ + else unhexlify(self.rawfilter) + except TypeError: + # can't unhex the string, go with RAW as default + filter_src = self.rawfilter + + # filter_src is RAW format + if filter_src: + active_source = self.boosting_manager.get_source_object(filter_src) + + seed_sp_list = [tr['last_seeding_stats']['total_down'] for tr in self.boosting_manager.torrents.values() + if self.boosting_manager.get_source_object(string_to_source(tr['source'])).source + == active_source.source and tr['last_seeding_stats']] + + total_dl_source = sum(seed_sp_list) + up_rate_txt.SetLabel('Active upload rate : '+speed_format(active_source.av_uprate)) + dwn_rate_txt.SetLabel('Active download rate : '+speed_format(active_source.av_dwnrate)) + storage_used_txt.SetLabel('Storage Used : '+size_format(total_dl_source)) + + header = self.parent.GetGrandParent().FindWindowByName('cm_header') + header.FindWindowByName('b_up').SetLabel('Total bytes up: ' + size_format(self.tot_bytes_up)) + header.FindWindowByName('b_down').SetLabel('Total bytes down: ' + size_format(self.tot_bytes_dwn)) + + if self.tot_bytes_dwn: + header.FindWindowByName('iv_sum').SetLabel(' Investment summary: %f' + %(float(self.tot_bytes_up)/float(self.tot_bytes_dwn))) + + header.FindWindowByName('s_up').SetLabel('Current total speed up: ' + speed_format( + sum([dl_state.get_current_speed('up') for dl_state in boosting_dslist]))) + header.FindWindowByName('s_down').SetLabel('Current total speed down: ' + speed_format( + sum([dl_state.get_current_speed('down') for dl_state in boosting_dslist]))) + + if newfilter: + self.newfilter = False + + self.old_dlstate = dict([(infohash, item.original_data.ds) for infohash, item in self.list.items.iteritems()]) + + @warnWxThread + def SetData(self, data): + SizeList.SetData(self, data) + + data_new = [] + + if len(data) > 0: + for ffile in data: + torrent_cm_dict = self.boosting_manager.torrents.get(ffile.infohash, None) + current_speed = "- / -" + if torrent_cm_dict and torrent_cm_dict["last_seeding_stats"]: + bytes_up_dwn = "%s / %s" %(size_format(torrent_cm_dict['last_seeding_stats']['total_up']), + size_format(torrent_cm_dict['last_seeding_stats']['total_down'])) + if torrent_cm_dict['last_seeding_stats']['total_down']: + ratio = '%f' % (float(torrent_cm_dict['last_seeding_stats']['total_up']) / float( + torrent_cm_dict['last_seeding_stats']['total_down'])) + else: + ratio = '%f' % 0 + else: + bytes_up_dwn = "- / -" + ratio = "-1" + + seeder_leecher = '%d / %d' %(ffile.num_seeders, ffile.num_leechers) + + init_data = [current_speed, bytes_up_dwn, seeder_leecher, '', ffile.infohash, + torrent_cm_dict.get('source', ''), ratio] + + data_new.append((ffile.infohash, init_data, ffile, CreditMiningListItem)) + else: + self.list.ShowMessage("No credit mining data available.") + self.SetNrResults(0) + + self.list.SetData(data_new) + + @warnWxThread + def RefreshData(self, key, data): + List.RefreshData(self, key, data) + + data = (data.infohash, ['-', '-', '%d / %d' % (data.num_seeders, data.num_leechers), '', data.infohash, + self.boosting_manager.torrents.get(data.infohash, None).get('source', ''), '-1'], data) + self.list.RefreshData(key, data) + + def SetNrResults(self, nr): + highlight = nr > self.nr_results and self.initnumitems + SizeList.SetNrResults(self, nr) + + actitem = self.guiutility.frame.actlist.GetItem(5) + num_items = getattr(actitem, 'num_items', None) + if num_items: + num_items.SetValue(str(nr)) + actitem.hSizer.Layout() + if highlight: + actitem.Highlight() + self.initnumitems = True + + def OnFilter(self, keyword): + pass + + def MatchFilter(self, item): + if not self.rawfilter: + return False + + source = item[1][5] + match = (hexlify(self.rawfilter) if len(hexlify(self.rawfilter)) == 40 else self.rawfilter) in source + + if self.boosting_manager.get_source_object(self.rawfilter): + match = match and self.boosting_manager.get_source_object(self.rawfilter).enabled + + return match + + def GotFilter(self, keyword=None): + self.rawfilter = keyword + if self.rawfilter == '' and not self.guiutility.getFamilyFilter(): + wx.CallAfter(self.list.SetFilter, None, None, keyword is None) + else: + wx.CallAfter(self.list.SetFilter, self.MatchFilter, self.GetFilterMessage, True) + + self.OnFilter(self.rawfilter) + + def GetFilterMessage(self, empty=False): + if empty: + return 'Empty', 'No credit mining torrent to show' + else: + return None, 'end of list' + + def MatchFFilter(self, item): + return True + + class ChannelList(List): def __init__(self, parent): @@ -2173,14 +2555,19 @@ def __SetData(self): (2, ['Results'], None, ActivityListItem), (3, ['Channels'], None, ActivityListItem), (4, ['Downloads'], None, ActivityListItem)] + + if self.utility.session.get_creditmining_enable(): + data_list.append((5, ['Credit Mining'], None, ActivityListItem)) + if sys.platform != 'darwin': - data_list.append((5, ['Videoplayer'], None, ActivityListItem)) + data_list.append((6, ['Videoplayer'], None, ActivityListItem)) self.list.SetData(data_list) self.ResizeListItems() self.DisableItem(2) + if not self.guiutility.frame.videoparentpanel and sys.platform != 'darwin': - self.DisableItem(5) + self.DisableItem(6) self.DisableCollapse() self.selectTab('home') @@ -2191,7 +2578,7 @@ def __SetData(self): self.expandedPanel_channels.Hide() if sys.platform != 'darwin': - videoplayer_item = self.list.GetItem(5) + videoplayer_item = self.list.GetItem(6) self.expandedPanel_videoplayer = VideoplayerExpandedPanel(videoplayer_item) videoplayer_item.AddEvents(self.expandedPanel_videoplayer) self.expandedPanel_videoplayer.Hide() @@ -2271,6 +2658,8 @@ def OnExpand(self, item): if self.guiutility.guiPage not in ['videoplayer']: self.guiutility.ShowPage('videoplayer') return self.expandedPanel_videoplayer + elif item.data[0] == 'Credit Mining': + self.guiutility.ShowPage('creditmining') return True def OnCollapse(self, item, panel, from_expand): @@ -2335,8 +2724,10 @@ def selectTab(self, tab): itemKey = 3 elif tab == 'my_files': itemKey = 4 - elif tab == 'videoplayer': + elif tab == 'creditmining': itemKey = 5 + elif tab == 'videoplayer': + itemKey = 6 if itemKey: wx.CallAfter(self.Select, itemKey, True) return @@ -2357,7 +2748,7 @@ def _DoPage(self, increment): if curPage < 0: curPage = len(pages) - 1 - pageNames = ['home', 'search_results', 'channels', 'my_files', 'videoplayer'] + pageNames = ['home', 'search_results', 'channels', 'my_files', 'creditmining', 'videoplayer'] for i in self.settings.keys(): pageNames.pop(i - 1) self.guiutility.ShowPage(pageNames[curPage]) diff --git a/Tribler/Main/vwxGUI/list_body.py b/Tribler/Main/vwxGUI/list_body.py index 3c1ae7fed69..533fa03e36b 100644 --- a/Tribler/Main/vwxGUI/list_body.py +++ b/Tribler/Main/vwxGUI/list_body.py @@ -371,11 +371,26 @@ def IsSelected(control): else: self.BackgroundColor(self.list_deselected) + def SetSelectedColour(self, selected): + if selected.Get() != self.list_selected.Get(): + self.list_selected = selected + self.ShowSelected() + def SetDeselectedColour(self, deselected): if deselected.Get() != self.list_deselected.Get(): self.list_deselected = deselected self.ShowSelected() + def SetExpandedColour(self, expanded): + if expanded.Get() != self.list_expanded.Get(): + self.list_expanded = expanded + self.ShowSelected() + + def SetExpandedAndSelectedColour(self, selected_and_expanded): + if selected_and_expanded.Get() != self.list_selected_and_expanded.Get(): + self.list_selected_and_expanded = selected_and_expanded + self.ShowSelected() + @warnWxThread def BackgroundColor(self, color): if self.GetBackgroundColour() != color: diff --git a/Tribler/Main/vwxGUI/list_details.py b/Tribler/Main/vwxGUI/list_details.py index 333df91b974..7b8091f9eae 100644 --- a/Tribler/Main/vwxGUI/list_details.py +++ b/Tribler/Main/vwxGUI/list_details.py @@ -2376,7 +2376,7 @@ def RemoveFileindex(self, fileindex): self.library_manager.last_vod_torrent = None def SetNrFiles(self, nr): - videoplayer_item = self.guiutility.frame.actlist.GetItem(5) + videoplayer_item = self.guiutility.frame.actlist.GetItem(6) num_items = getattr(videoplayer_item, 'num_items', None) if num_items and self.guiutility.frame.videoparentpanel: num_items.SetValue(str(nr)) diff --git a/Tribler/Main/vwxGUI/list_item.py b/Tribler/Main/vwxGUI/list_item.py index 5c2ab7528e1..1fb64b97812 100644 --- a/Tribler/Main/vwxGUI/list_item.py +++ b/Tribler/Main/vwxGUI/list_item.py @@ -1124,6 +1124,13 @@ def GetIcons(self): def SetThumbnailIcon(self): pass +class CreditMiningListItem(ListItem): + + def AddComponents(self, leftSpacer, rightSpacer): + ListItem.AddComponents(self, 5, 5) + + def GetIcons(self): + return [] class ActivityListItem(ListItem): @@ -1132,7 +1139,7 @@ def __init__(self, *args, **kwargs): def AddComponents(self, leftSpacer, rightSpacer): ListItem.AddComponents(self, leftSpacer, rightSpacer) - if self.data[0] in ['Results', 'Channels', 'Downloads', 'Videoplayer']: + if self.data[0] in ['Results', 'Channels', 'Downloads', 'Credit Mining', 'Videoplayer']: self.num_items = TagText(self, -1, label='0', fill_colour=GRADIENT_DGREY, edge_colour=SEPARATOR_GREY) self.hSizer.Add(self.num_items, 0, wx.CENTER | wx.RIGHT, 5) self.hSizer.Layout() diff --git a/Tribler/Main/vwxGUI/settingsDialog.py b/Tribler/Main/vwxGUI/settingsDialog.py index 3870e499fbc..5cf34a6be8e 100644 --- a/Tribler/Main/vwxGUI/settingsDialog.py +++ b/Tribler/Main/vwxGUI/settingsDialog.py @@ -65,7 +65,7 @@ def add_label(parent, sizer, label): class SettingsDialog(wx.Dialog): def __init__(self): - super(SettingsDialog, self).__init__(None, size=(600, 600), + super(SettingsDialog, self).__init__(None, size=(600, 700), title="Settings", name="settingsDialog", style=wx.DEFAULT_DIALOG_STYLE) self.SetExtraStyle(self.GetExtraStyle() | wx.WS_EX_VALIDATE_RECURSIVELY) self._logger = logging.getLogger(self.__class__.__name__) @@ -318,6 +318,12 @@ def saveAll(self, event, skip_restart_dialog=False): scfg.set_enable_multichain(use_multichain) restart = True + # Credit Mining + use_boosting = self._use_boosting.IsChecked() + if use_boosting != self.utility.session.get_creditmining_enable(): + scfg.set_creditmining_enable(use_boosting) + restart = True + scfg.save(cfgfilename) self.utility.flush_config() @@ -737,6 +743,13 @@ def __create_s5(self, tree_root, sizer): exp_panel, label="Tribler connects to Emercoin over its JSON-RPC API.\nThis requires you to enable it by editing the emercoin.conf file and setting\nserver=1, rpcport, rpcuser, rpcpassword, and rpcconnect.") exp_vsizer.Add(exp_s2_faq_text, 0, wx.EXPAND | wx.TOP, 10) + exp_s3_sizer = create_subsection(exp_panel, exp_vsizer, "Credit Mining", 1, 3) + boosting_text = wx.StaticText(exp_panel, -1, 'Credit Mining is a mechanism to boost your ratio by ' + '\nautomatically download and upload data.') + exp_s3_sizer.Add(boosting_text, 0, wx.EXPAND | wx.TOP, 5) + self._use_boosting = wx.CheckBox(exp_panel, label="Enable credit mining") + exp_s3_sizer.Add(self._use_boosting, 0, wx.EXPAND) + # load values self._use_webui.SetValue(self.utility.read_config('use_webui')) self._webui_port.SetValue(str(self.utility.read_config('webui_port'))) @@ -747,6 +760,8 @@ def __create_s5(self, tree_root, sizer): self._emc_username.SetValue(self.utility.read_config('emc_username')) self._emc_password.SetValue(self.utility.read_config('emc_password')) + self._use_boosting.SetValue(self.utility.session.get_creditmining_enable()) + return exp_panel, item_id def __create_s6(self, tree_root, sizer): diff --git a/Tribler/Policies/BoostingManager.py b/Tribler/Policies/BoostingManager.py new file mode 100644 index 00000000000..d69b1fe4f26 --- /dev/null +++ b/Tribler/Policies/BoostingManager.py @@ -0,0 +1,478 @@ +# -*- coding: utf-8 -*- +# Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H +"""Manage boosting of swarms""" +import logging +import os +import shutil +from binascii import hexlify, unhexlify + +import libtorrent as lt +from twisted.internet.task import LoopingCall + +from Tribler.Core.DownloadConfig import DownloadStartupConfig, DefaultDownloadStartupConfig +from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl +from Tribler.Core.Utilities import utilities +from Tribler.Core.exceptions import OperationNotPossibleAtRuntimeException +from Tribler.Core.simpledefs import DLSTATUS_SEEDING, NTFY_TORRENTS, NTFY_UPDATE, NTFY_CHANNELCAST +from Tribler.Policies.BoostingPolicy import SeederRatioPolicy +from Tribler.Policies.BoostingSource import ChannelSource +from Tribler.Policies.BoostingSource import DirectorySource +from Tribler.Policies.BoostingSource import RSSFeedSource +from Tribler.Policies.credit_mining_util import source_to_string, string_to_source, compare_torrents, \ + validate_source_string +from Tribler.Policies.defs import SAVED_ATTR, CREDIT_MINING_FOLDER_DOWNLOAD, CONFIG_KEY_ARCHIVELIST, \ + CONFIG_KEY_SOURCELIST, CONFIG_KEY_ENABLEDLIST, CONFIG_KEY_DISABLEDLIST +from Tribler.dispersy.taskmanager import TaskManager + + +class BoostingSettings(object): + """ + This class contains settings used by the boosting manager + """ + def __init__(self, session, policy=SeederRatioPolicy, load_config=True): + self.session = session + + # Configurable parameter (changeable in runtime -plus sources-) + self.max_torrents_active = 20 + self.max_torrents_per_source = 10 + self.source_interval = 100 + self.swarm_interval = 100 + + # Can't be changed on runtime + self.tracker_interval = 200 + self.logging_interval = 60 + self.share_mode_target = 3 + self.policy = policy(session) + + # Non-Configurable + self.initial_logging_interval = 20 + self.initial_tracker_interval = 25 + self.initial_swarm_interval = 30 + self.min_connection_start = 5 + self.min_channels_start = 100 + self.credit_mining_path = os.path.join(DefaultDownloadStartupConfig.getInstance().get_dest_dir(), + CREDIT_MINING_FOLDER_DOWNLOAD) + self.load_config = load_config + + # whether we want to check dependencies of BoostingManager + self.check_dependencies = True + self.auto_start_source = True + + +class BoostingManager(TaskManager): + """ + Class to manage all the credit mining activities + """ + + def __init__(self, session, settings=None): + super(BoostingManager, self).__init__() + self._logger = logging.getLogger(self.__class__.__name__) + + BoostingManager.__single = self + self.boosting_sources = {} + self.torrents = {} + + self.session = session + + # use provided settings or a default one + self.settings = settings or BoostingSettings(session, load_config=True) + + if self.settings.check_dependencies: + assert self.session.get_libtorrent() + assert self.session.get_torrent_checking() + assert self.session.get_dispersy() + assert self.session.get_torrent_store() + assert self.session.get_enable_torrent_search() + assert self.session.get_enable_channel_search() + assert self.session.get_megacache() + + self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) + self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) + + if self.settings.load_config: + self.load_config() + + if not os.path.exists(self.settings.credit_mining_path): + os.makedirs(self.settings.credit_mining_path) + + self.session.lm.ltmgr.get_session().set_settings( + {'share_mode_target': self.settings.share_mode_target}) + + self.session.add_observer(self.on_torrent_notify, NTFY_TORRENTS, [NTFY_UPDATE]) + + self.register_task("CreditMining_select", LoopingCall(self._select_torrent), + self.settings.initial_swarm_interval, interval=self.settings.swarm_interval) + + self.register_task("CreditMining_scrape", LoopingCall(self.scrape_trackers), + self.settings.initial_tracker_interval, interval=self.settings.tracker_interval) + + self.register_task("CreditMining_log", LoopingCall(self.log_statistics), + self.settings.initial_logging_interval, interval=self.settings.logging_interval) + + def shutdown(self): + """ + Shutting down boosting manager. It also stops and remove all the sources. + """ + self.save_config() + self._logger.info("Shutting down boostingmanager") + + for sourcekey in self.boosting_sources.keys(): + self.remove_source(sourcekey) + + self.cancel_all_pending_tasks() + + # remove credit mining downloaded data + shutil.rmtree(self.settings.credit_mining_path, ignore_errors=True) + + def get_source_object(self, sourcekey): + """ + Get the actual object of the source key + """ + return self.boosting_sources.get(sourcekey, None) + + def set_enable_mining(self, source, mining_bool=True, force_restart=False): + """ + Dynamically enable/disable mining source. + """ + for ihash, tor in self.torrents.iteritems(): + if tor['source'] == source_to_string(source): + self.torrents[ihash]['enabled'] = mining_bool + + # pause torrent download from disabled source + if not mining_bool: + self.stop_download(tor) + + self.boosting_sources[string_to_source(source)].enabled = mining_bool + + self._logger.info("Set mining source %s %s", source, mining_bool) + + if force_restart: + self._select_torrent() + + def add_source(self, source): + """ + add new source into the boosting manager + """ + if source not in self.boosting_sources: + args = (self.session, source, self.settings, self.on_torrent_insert) + + try: + isdir = os.path.isdir(source) + except TypeError: + # this handle binary data that has null bytes '\00' + isdir = False + + if isdir: + self.boosting_sources[source] = DirectorySource(*args) + elif source.startswith('http://') or source.startswith('https://'): + self.boosting_sources[source] = RSSFeedSource(*args) + elif len(source) == 20: + self.boosting_sources[source] = ChannelSource(*args) + else: + self._logger.error("Cannot add unknown source %s", source) + return + + if self.settings.auto_start_source: + self.boosting_sources[source].start() + + self._logger.info("Added source %s", source) + else: + self._logger.info("Already have source %s", source) + + def remove_source(self, source_key): + """ + remove source by stop the downloading and remove its metainfo for all its swarms + """ + if source_key in self.boosting_sources: + source = self.boosting_sources.pop(source_key) + source.kill_tasks() + self._logger.info("Removed source %s", source_key) + + rm_torrents = [torrent for _, torrent in self.torrents.items() + if torrent['source'] == source_to_string(source_key)] + + for torrent in rm_torrents: + self.stop_download(torrent) + self.torrents.pop(torrent["metainfo"].get_infohash(), None) + + self._logger.info("Torrents download stopped and removed") + + def on_torrent_insert(self, source, infohash, torrent): + """ + This function called when a source is finally determined. Fetch some torrents from it, + then insert it into our data + """ + + # Remember where we got this torrent from + self._logger.debug("remember torrent %s from %s", torrent, source_to_string(source)) + + torrent['source'] = source_to_string(source) + + boost_source = self.boosting_sources.get(source, None) + if not boost_source: + self._logger.info("Dropping torrent insert from removed source: %s", repr(torrent)) + return + elif boost_source.archive: + torrent['preload'] = True + torrent['prio'] = 100 + + # If duplicates exist, set is_duplicate to True, except for the one with the most seeders. + duplicates = [other for other in self.torrents.values() if compare_torrents(torrent, other)] + if duplicates: + duplicates += [torrent] + healthiest_torrent = max([(torrent['num_seeders'], torrent) for torrent in duplicates])[1] + for duplicate in duplicates: + is_duplicate = healthiest_torrent != duplicate + duplicate['is_duplicate'] = is_duplicate + if is_duplicate and duplicate.get('download', None): + self.stop_download(duplicate) + + self.torrents[infohash] = torrent + + def on_torrent_notify(self, subject, change_type, infohash): + """ + Notify us when we have new seeder/leecher value in torrent from tracker + """ + if infohash not in self.torrents: + return + + self._logger.debug("infohash %s %s %s updated", subject, change_type, hexlify(infohash)) + + tdict = self.torrent_db.getTorrent(infohash, keys=['C.torrent_id', 'infohash', 'name', + 'length', 'category', 'status', 'num_seeders', + 'num_leechers']) + + if tdict: + infohash_str = hexlify(tdict['infohash']) + + new_seed = tdict['num_seeders'] + new_leecher = tdict['num_leechers'] + + if new_seed - self.torrents[tdict['infohash']]['num_seeders'] \ + or new_leecher - self.torrents[tdict['infohash']]['num_leechers']: + self.torrents[tdict['infohash']]['num_seeders'] = new_seed + self.torrents[tdict['infohash']]['num_leechers'] = new_leecher + self._logger.info("infohash %s : seeder/leecher changed seed:%d leech:%d", + infohash_str, new_seed, new_leecher) + + def scrape_trackers(self): + """ + Manually scrape tracker by requesting to tracker manager + """ + + for infohash in list(self.torrents): + # torrent handle + lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(lt.big_number(infohash)) + + peer_list = [] + for i in lt_torrent.get_peer_info(): + peer = LibtorrentDownloadImpl.create_peerlist_data(i) + peer_list.append(peer) + + num_seed, num_leech = utilities.translate_peers_into_health(peer_list) + + # calculate number of seeder and leecher by looking at the peers + if self.torrents[infohash]['num_seeders'] == 0: + self.torrents[infohash]['num_seeders'] = num_seed + if self.torrents[infohash]['num_leechers'] == 0: + self.torrents[infohash]['num_leechers'] = num_leech + + self._logger.debug("Seeder/leecher data translated from peers : seeder %s, leecher %s", num_seed, num_leech) + + # check health(seeder/leecher) + self.session.lm.torrent_checker.add_gui_request(infohash, True) + + def set_archive(self, source, enable): + """ + setting archive of a particular source. This affects all the torrents in this source + """ + if source in self.boosting_sources: + self.boosting_sources[source].archive = enable + self._logger.info("Set archive mode for %s to %s", source, enable) + else: + self._logger.error("Could not set archive mode for unknown source %s", source) + + def start_download(self, torrent): + """ + Start downloading a particular torrent and add it to download list in Tribler + """ + dscfg = DownloadStartupConfig() + dscfg.set_dest_dir(self.settings.credit_mining_path) + dscfg.set_safe_seeding(False) + + preload = torrent.get('preload', False) + + if self.session.lm.download_exists(torrent["metainfo"].get_infohash()): + self._logger.error("Already downloading %s. Cancel start_download", + hexlify(torrent["metainfo"].get_infohash())) + return + + self._logger.info("Starting %s preload %s", + hexlify(torrent["metainfo"].get_infohash()), preload) + + torrent['download'] = self.session.lm.add(torrent['metainfo'], dscfg, hidden=True, + share_mode=not preload, checkpoint_disabled=True) + torrent['download'].set_priority(torrent.get('prio', 1)) + + def stop_download(self, torrent): + """ + Stopping torrent that currently downloading + """ + ihash = lt.big_number(torrent["metainfo"].get_infohash()) + self._logger.info("Stopping %s", str(ihash)) + download = torrent.pop('download', False) + lt_torrent = self.session.lm.ltmgr.get_session().find_torrent(ihash) + if download and lt_torrent.is_valid(): + self._logger.info("Writing resume data for %s", str(ihash)) + download.save_resume_data() + self.session.remove_download(download, hidden=True) + + def _select_torrent(self): + """ + Function to select which torrent in the torrent list will be downloaded in the + next iteration. It depends on the source and applied policy + """ + torrents = {} + for infohash, torrent in self.torrents.iteritems(): + # we prioritize archive source + if torrent.get('preload', False): + if 'download' not in torrent: + self.start_download(torrent) + elif torrent['download'].get_status() == DLSTATUS_SEEDING: + self.stop_download(torrent) + elif not torrent.get('is_duplicate', False): + if torrent.get('enabled', True): + torrents[infohash] = torrent + + if self.settings.policy is not None and torrents: + # Determine which torrent to start and which to stop. + torrents_start, torrents_stop = self.settings.policy.apply( + torrents, self.settings.max_torrents_active) + for torrent in torrents_stop: + self.stop_download(torrent) + for torrent in torrents_start: + self.start_download(torrent) + + self._logger.info("Selecting from %s torrents %s start download", len(torrents), len(torrents_start)) + + def load_config(self): + """ + load config in file configuration and apply it to manager + """ + self._logger.info("Loading config file from session configuration") + + def _add_sources(values): + """ + adding sources in configuration file + """ + for boosting_source in values: + boosting_source = validate_source_string(boosting_source) + self.add_source(boosting_source) + + def _archive_sources(values): + """ + setting archive to sources + """ + for archive_source in values: + archive_source = validate_source_string(archive_source) + self.set_archive(archive_source, True) + + def _set_enable_boosting(values, enabled): + """ + set disable/enable source + """ + for boosting_source in values: + boosting_source = validate_source_string(boosting_source) + if boosting_source not in self.boosting_sources.keys(): + self.add_source(boosting_source) + self.boosting_sources[boosting_source].enabled = enabled + + # set policy + self.settings.policy = self.session.get_cm_policy(True)(self.session) + + for k in SAVED_ATTR: + # see the session configuration + object.__setattr__(self.settings, k, getattr(self.session, "get_cm_%s" %k)()) + + for k, val in self.session.get_cm_sources().items(): + if k is "boosting_sources": + _add_sources(val) + elif k is "archive_sources": + _archive_sources(val) + elif k is "boosting_enabled": + _set_enable_boosting(val, True) + elif k is "boosting_disabled": + _set_enable_boosting(val, False) + + def save_config(self): + """ + save the environment parameters in config file + """ + for k in SAVED_ATTR: + try: + setattr(self.session, "set_cm_%s" % k, getattr(self.settings, k)) + except OperationNotPossibleAtRuntimeException: + # some of the attribute can't be changed in runtime. See lm.sessconfig_changed_callback + self._logger.debug("Cannot set attribute %s. Not permitted in runtime", k) + + archive_sources = [] + lboosting_sources = [] + flag_enabled_sources = [] + flag_disabled_sources = [] + for boosting_source_name, boosting_source in \ + self.boosting_sources.iteritems(): + + bsname = source_to_string(boosting_source_name) + + lboosting_sources.append(bsname) + if boosting_source.enabled: + flag_enabled_sources.append(bsname) + else: + flag_disabled_sources.append(bsname) + + if boosting_source.archive: + archive_sources.append(bsname) + + self.session.set_cm_sources(lboosting_sources, CONFIG_KEY_SOURCELIST) + self.session.set_cm_sources(flag_enabled_sources, CONFIG_KEY_ENABLEDLIST) + self.session.set_cm_sources(flag_disabled_sources, CONFIG_KEY_DISABLEDLIST) + self.session.set_cm_sources(archive_sources, CONFIG_KEY_ARCHIVELIST) + + self.session.save_pstate_sessconfig() + + def log_statistics(self): + """Log transfer statistics""" + lt_torrents = self.session.lm.ltmgr.get_session().get_torrents() + + for lt_torrent in lt_torrents: + status = lt_torrent.status() + + if unhexlify(str(status.info_hash)) in self.torrents: + self._logger.debug("Status for %s : %s %s | ul_lim : %d, max_ul %d, maxcon %d", status.info_hash, + status.all_time_download, status.all_time_upload, lt_torrent.upload_limit(), + lt_torrent.max_uploads(), lt_torrent.max_connections()) + + # piece_priorities will fail in libtorrent 1.0.9 + if lt.__version__ == '1.0.9.0': + continue + else: + non_zero_values = [] + for piece_priority in lt_torrent.piece_priorities(): + if piece_priority != 0: + non_zero_values.append(piece_priority) + if non_zero_values: + self._logger.debug("Non zero priorities for %s : %s", status.info_hash, non_zero_values) + + def update_torrent_stats(self, torrent_infohash_str, seeding_stats): + """ + function to update swarm statistics. + + This function called when we get new Downloadstate for active torrents. + Updated downloadstate (seeding_stats) for a particular torrent is stored here. + """ + if 'time_seeding' in self.torrents[torrent_infohash_str]['last_seeding_stats']: + if seeding_stats['time_seeding'] >= self.torrents[torrent_infohash_str][ + 'last_seeding_stats']['time_seeding']: + self.torrents[torrent_infohash_str]['last_seeding_stats'] = seeding_stats + else: + self.torrents[torrent_infohash_str]['last_seeding_stats'] = seeding_stats diff --git a/Tribler/Policies/BoostingPolicy.py b/Tribler/Policies/BoostingPolicy.py new file mode 100644 index 00000000000..3def71ea11f --- /dev/null +++ b/Tribler/Policies/BoostingPolicy.py @@ -0,0 +1,110 @@ +# coding=utf-8 +""" +Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H +Supported boosting policy +""" +import logging +import random + + +class BoostingPolicy(object): + """ + Base class for determining what swarm selection policy will be applied + """ + + def __init__(self, session): + self.session = session + # function that checks if key can be applied to torrent + self.reverse = None + + self._logger = logging.getLogger(self.__class__.__name__) + + def apply(self, torrents, max_active, force=False): + """ + apply the policy to the torrents stored + """ + sorted_torrents = sorted([torrent for torrent in torrents.itervalues() + if self.key_check(torrent)], + key=self.key, reverse=self.reverse) + + torrents_start = [] + for torrent in sorted_torrents[:max_active]: + if not self.session.get_download(torrent["metainfo"].get_infohash()): + torrents_start.append(torrent) + torrents_stop = [] + for torrent in sorted_torrents[max_active:]: + if self.session.get_download(torrent["metainfo"].get_infohash()): + torrents_stop.append(torrent) + + if force: + return torrents_start, torrents_stop + + # if both results are empty for some reason (e.g, key_check too restrictive) + # or torrent started less than half available torrent (try to keep boosting alive) + # if it's already random, just let it be + if not isinstance(self, RandomPolicy) and ((not torrents_start and not torrents_stop) or + (len(torrents_start) < len(torrents) / 2 and len( + torrents_start) < max_active / 2)): + self._logger.error("Start and stop torrent list are empty. Fallback to Random") + # fallback to random policy + torrents_start, torrents_stop = RandomPolicy(self.session).apply(torrents, max_active) + + return torrents_start, torrents_stop + + def key(self, key): + """ + function to find a key of an object + """ + return None + + def key_check(self, key): + """ + function to check whether a swarm is included to download + """ + return False + +class RandomPolicy(BoostingPolicy): + """ + A credit mining policy that chooses a swarm randomly + """ + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.reverse = False + + def key_check(self, key): + return True + + def key(self, key): + return random.random() + + +class CreationDatePolicy(BoostingPolicy): + """ + A credit mining policy that chooses swarm by its creation date + + The idea is, older swarms need to be boosted. + """ + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.reverse = True + + def key_check(self, key): + return key['creation_date'] > 0 + + def key(self, key): + return key['creation_date'] + + +class SeederRatioPolicy(BoostingPolicy): + """ + Default policy. Find the most underseeded swarm to boost. + """ + def __init__(self, session): + BoostingPolicy.__init__(self, session) + self.reverse = False + + def key(self, key): + return key['num_seeders'] / float(key['num_seeders'] + key['num_leechers']) + + def key_check(self, key): + return (key['num_seeders'] + key['num_leechers']) > 0 diff --git a/Tribler/Policies/BoostingSource.py b/Tribler/Policies/BoostingSource.py new file mode 100644 index 00000000000..7fe6dc39ec4 --- /dev/null +++ b/Tribler/Policies/BoostingSource.py @@ -0,0 +1,463 @@ +# coding=utf-8 +""" +Written by Egbert Bouman, Mihai Capotă, Elric Milon, and Ardhi Putra Pratama H +Supported boosting sources +""" +import glob +import logging +import os +import re +import urllib +from binascii import hexlify +from hashlib import sha1 +import feedparser + +import libtorrent as lt +from twisted.internet import defer +from twisted.internet import reactor +from twisted.internet.defer import CancelledError +from twisted.internet.task import LoopingCall +from twisted.web.client import Agent, readBody, getPage +from twisted.web.error import Error +from twisted.web.http_headers import Headers + +from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.simpledefs import NTFY_INSERT, NTFY_TORRENTS, NTFY_UPDATE +from Tribler.Core.version import version_id +from Tribler.Main.Utility.GuiDBTuples import Torrent, Channel +from Tribler.Policies.credit_mining_util import TorrentManagerCM, ent2chr +from Tribler.community.allchannel.community import AllChannelCommunity +from Tribler.community.channel.community import ChannelCommunity +from Tribler.dispersy.exception import CommunityNotFoundException +from Tribler.dispersy.taskmanager import TaskManager + + +class BoostingSource(TaskManager): + """ + Base class for boosting source. For now, it can be RSS, directory, and channel + """ + + def __init__(self, session, source, boost_settings, torrent_insert_cb): + super(BoostingSource, self).__init__() + self.session = session + self.channelcast_db = session.lm.channelcast_db + + self.torrents = {} + self.source = source + self.interval = boost_settings.source_interval + self.max_torrents = boost_settings.max_torrents_per_source + self.torrent_insert_callback = torrent_insert_cb + self.archive = False + + self.enabled = True + + self.av_uprate = 0 + self.av_dwnrate = 0 + self.storage_used = 0 + self.ready = False + + self.min_connection = boost_settings.min_connection_start + self.min_channels = boost_settings.min_channels_start + + self.torrent_mgr = TorrentManagerCM(session) + + self._logger = logging.getLogger(BoostingSource.__name__) + + self.boosting_manager = self.session.lm.boosting_manager + + def start(self): + """ + Start operating mining for this source + """ + d = self._load_if_ready(self.source) + self.register_task(str(self.source) + "_load", d, value=self.source) + self._logger.debug("Start mining on %s", self.source) + + def kill_tasks(self): + """ + kill tasks on this source + """ + self.ready = False + self.cancel_all_pending_tasks() + + def _load_if_ready(self, source): + """ + load source if and only if the overall system is ready. + + This is useful so we don't burden the application during the startup + """ + def check_system(defer_param=None): + """ + function that check the system whether it's ready or not + + it depends on #connection and #channel + """ + if defer_param is None: + defer_param = defer.Deferred() + + nr_channels = self.channelcast_db.getNrChannels() + nr_connections = 0 + + for community in self.session.lm.dispersy.get_communities(): + from Tribler.community.search.community import SearchCommunity + if isinstance(community, SearchCommunity): + nr_connections = community.get_nr_connections() + + if nr_channels > self.min_channels and nr_connections > self.min_connection: + defer_param.callback(source) + else: + self.register_task(str(self.source)+"_check_sys", reactor.callLater(10, check_system, defer_param)) + + return defer_param + + defer_check = check_system() + defer_check.addCallbacks(self._load, self._on_err) + return defer_check + + def _load(self, source): + pass + + def _update(self): + pass + + def get_source_text(self): + """ + returning 'raw' source. May be overriden + """ + return self.source + + def _on_err(self, err_msg): + self._logger.error(err_msg) + + def check_and_register_task(self, name, task, delay=None, value=None, interval=None): + """ + Helper function to avoid assertion in register task. + + It will register task if it has not already registered + """ + task_ret = None + if not self.is_pending_task_active(name): + task_ret = self.register_task(name, task, delay, value, interval) + + return task_ret + +class ChannelSource(BoostingSource): + """ + Credit mining source from a channel. + """ + def __init__(self, session, dispersy_cid, boost_settings, torrent_insert_cb): + BoostingSource.__init__(self, session, dispersy_cid, boost_settings, torrent_insert_cb) + + self.channel_id = None + + self.channel = None + self.community = None + self.database_updated = True + + self.check_torrent_interval = 10 + self.dispersy_cid = dispersy_cid + + self.session.add_observer(self._on_database_updated, NTFY_TORRENTS, [NTFY_INSERT, NTFY_UPDATE]) + + self.unavail_torrent = {} + + def kill_tasks(self): + BoostingSource.kill_tasks(self) + + self.session.remove_observer(self._on_database_updated) + + def _load(self, dispersy_cid): + dispersy = self.session.get_dispersy_instance() + + def join_community(): + """ + find the community/channel id, then join + """ + try: + self.community = dispersy.get_community(dispersy_cid, True) + self.register_task(str(self.source) + "_get_id", reactor.callLater(1, get_channel_id)) + + except CommunityNotFoundException: + + allchannelcommunity = None + for community in dispersy.get_communities(): + if isinstance(community, AllChannelCommunity): + allchannelcommunity = community + break + + if allchannelcommunity: + self.community = ChannelCommunity.init_community(dispersy, dispersy.get_member(mid=dispersy_cid), + allchannelcommunity._my_member, self.session) + self._logger.info("Joined channel community %s", dispersy_cid.encode("HEX")) + self.register_task(str(self.source) + "_get_id", reactor.callLater(1, get_channel_id)) + else: + self._logger.error("Could not find AllChannelCommunity") + + def get_channel_id(): + """ + find channel id by looking at the network + """ + if self.community and self.community._channel_id: + self.channel_id = self.community._channel_id + + channel_dict = self.channelcast_db.getChannel(self.channel_id) + self.channel = Channel(*channel_dict) + + task_call = self.check_and_register_task(str(self.source) + "_update", + LoopingCall(self._update)).start(self.interval, now=True) + if task_call: + self._logger.debug("Registering update call") + + self._logger.info("Got channel id %s", self.channel_id) + + self.ready = True + else: + self._logger.warning("Could not get channel id, retrying in 10 s") + self.register_task(str(self.source) + "_get_id", reactor.callLater(10, get_channel_id)) + + self.register_task(str(self.source) + "_join_comm", reactor.callLater(1, join_community)) + + def _check_tor(self): + """ + periodically check torrents in channel. Will return the torrent data if finished. + """ + def showtorrent(torrent): + """ + assembly torrent data, call the callback + """ + if torrent.files: + if len(self.torrents) >= self.max_torrents: + self._logger.debug("Max torrents in source reached. Not adding %s", torrent.infohash) + del self.unavail_torrent[torrent.infohash] + return + + infohash = torrent.infohash + self._logger.debug("[ChannelSource] Got torrent %s", hexlify(infohash)) + self.torrents[infohash] = {} + self.torrents[infohash]['name'] = torrent.name + self.torrents[infohash]['metainfo'] = torrent.tdef + self.torrents[infohash]['creation_date'] = torrent.creation_date + self.torrents[infohash]['length'] = torrent.tdef.get_length() + self.torrents[infohash]['num_files'] = len(torrent.files) + self.torrents[infohash]['num_seeders'] = torrent.swarminfo[0] or 0 + self.torrents[infohash]['num_leechers'] = torrent.swarminfo[1] or 0 + self.torrents[infohash]['enabled'] = self.enabled + + # seeding stats from DownloadState + self.torrents[infohash]['last_seeding_stats'] = {} + + del self.unavail_torrent[infohash] + + if self.torrent_insert_callback: + self.torrent_insert_callback(self.source, infohash, self.torrents[infohash]) + self.database_updated = False + + self._logger.debug("Unavailable #torrents : %d from %s", len(self.unavail_torrent), hexlify(self.source)) + + if len(self.unavail_torrent) and self.enabled: + for torrent in self.unavail_torrent.values(): + self.torrent_mgr.load_torrent(torrent, showtorrent) + + def _update(self): + if len(self.torrents) < self.max_torrents and self.database_updated: + CHANTOR_DB = ['ChannelTorrents.channel_id', 'Torrent.torrent_id', 'infohash', '""', 'length', + 'category', 'status', 'num_seeders', 'num_leechers', 'ChannelTorrents.id', + 'ChannelTorrents.dispersy_id', 'ChannelTorrents.name', 'Torrent.name', + 'ChannelTorrents.description', 'ChannelTorrents.time_stamp', 'ChannelTorrents.inserted'] + + torrent_values = self.channelcast_db.getTorrentsFromChannelId(self.channel_id, True, CHANTOR_DB, + self.max_torrents) + + listtor = self.torrent_mgr.create_torrents(torrent_values, True, + {self.channel_id: self.channelcast_db.getChannel( + self.channel_id)}) + + # dict {key_infohash(binary):Torrent(object-GUIDBTuple)} + self.unavail_torrent.update({t.infohash: t for t in listtor if t.infohash not in self.torrents}) + + # it's highly probable the checktor function is running at this time (if it's already running) + # if not running, start the checker + + task_call = self.check_and_register_task(hexlify(self.source) + "_checktor", LoopingCall(self._check_tor)) + if task_call: + self._logger.debug("Registering check torrent function") + task_call.start(self.check_torrent_interval, now=True) + + def _on_database_updated(self, dummy_subject, dummy_change_type, dummy_infohash): + self.database_updated = True + + def get_source_text(self): + return self.channel.name if self.channel else None + + +class RSSFeedSource(BoostingSource): + """ + Credit mining source from a RSS feed. + """ + + def __init__(self, session, rss_feed, boost_settings, torrent_insert_cb): + BoostingSource.__init__(self, session, rss_feed, boost_settings, torrent_insert_cb) + + self.parsed_rss = None + + self.torrent_store = self.session.lm.torrent_store + + # Not all RSS feeds provide us with the infohash, + # so we use a fake infohash based on the URL (generated by sha1) to identify the torrents. + # keys : fake infohash, value : real infohash. Type : (length 20 string, binary) + self.fake_infohash_id = {} + + self.title = "" + self.description = "" + self.total_torrents = 0 + + self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) + + def _on_success_rss(self, body_rss, rss_feed): + """ + function called when RSS successfully read + """ + self.register_task(str(self.source) + "_update", LoopingCall(self._update), + 10, interval=self.interval) + self.parsed_rss = feedparser.parse(body_rss) + self._logger.info("Got RSS feed %s", rss_feed) + self.ready = True + + def _on_error_rss(self, failure, rss_feed): + """ + function called when RSS failed except from 503 + aborting load the source + """ + failure.trap(CancelledError, Error) + self._logger.error("Aborting load on : %s. Reason : %s.", rss_feed, failure.getErrorMessage()) + + if "503" in failure.getErrorMessage(): + self.register_task(str(self.source)+"_load_delay", reactor.callLater(10, self._load, rss_feed)) + return + + if rss_feed in self.boosting_manager.boosting_sources: + self.boosting_manager.set_enable_mining(rss_feed, False) + + def _load(self, rss_feed): + + defer_feed = getPage(rss_feed) + defer_feed.addCallback(self._on_success_rss, rss_feed) + defer_feed.addErrback(self._on_error_rss, rss_feed) + + self.register_task(str(self.source)+"_wait_feed", defer_feed) + + def _update(self): + if len(self.torrents) >= self.max_torrents: + return + + feed_elem = self.parsed_rss['feed'] + + self.title = feed_elem['title'] + self.description = feed_elem['subtitle'] + + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', + 'enabled', 'last_seeding_stats'] + + def __cb_body(body_bin, item_torrent_entry): + tdef = None + metainfo = None + + # tdef.get_infohash returned binary string by length 20 + try: + metainfo = lt.bdecode(body_bin) + tdef = TorrentDef.load_from_dict(metainfo) + self.session.save_collected_torrent(tdef.get_infohash(), body_bin) + except ValueError, err: + self._logger.error("Could not parse/save torrent, skipping %s. Reason: %s", + item_torrent_entry['link'], err.message + + ", metainfo is " + ("not " if metainfo else "") +"None") + + if tdef and len(self.torrents) < self.max_torrents: + # Create a torrent dict. + real_infohash = tdef.get_infohash() + torrent_values = [item_torrent_entry['title'], tdef, tdef.get_creation_date(), tdef.get_length(), + len(tdef.get_files()), -1, -1, self.enabled, {}] + + # store the real infohash to generated infohash + self.torrents[real_infohash] = dict(zip(torrent_keys, torrent_values)) + self.fake_infohash_id[sha1(item_torrent_entry['id']).digest()] = real_infohash + + # manually generate an ID and put this into DB + self.torrent_db.addOrGetTorrentID(real_infohash) + self.torrent_db.addExternalTorrent(tdef) + + # create Torrent object and store it + self.torrent_mgr.load_torrent(Torrent.fromTorrentDef(tdef)) + + # Notify the BoostingManager and provide the real infohash. + if self.torrent_insert_callback: + self.torrent_insert_callback(self.source, real_infohash, self.torrents[real_infohash]) + elif tdef: + self._logger.debug("Max torrents in source reached. Not adding %s", tdef.get_infohash()) + + def __success_cb(response, item_dict): + return readBody(response).addCallback(__cb_body, item_dict).addErrback(self._on_err) + + regex_unescape_xml = re.compile(r"\&\#(x?[0-9a-fA-F]+);") + + for item in self.parsed_rss['entries']: + f_links = item['links'] + for link in f_links: + if link['type'] == u'application/x-bittorrent': + url = regex_unescape_xml.sub(ent2chr, str(link['href'])) + fake_infohash = sha1(url).digest() + if fake_infohash not in self.fake_infohash_id.keys(): + # create Agent to download torrent file + self.fake_infohash_id[fake_infohash] = None + agent = Agent(reactor) + ses_agent = agent.request( + 'GET', # http://stackoverflow.com/a/845595 + urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]"), + Headers({'User-Agent': ['Tribler ' + version_id]}), + None) + ses_agent.addCallback(__success_cb, item).addErrback(self._on_err) + + +class DirectorySource(BoostingSource): + """ + Credit mining source from a local directory + + The directory must exist. + """ + + def _load(self, directory): + if os.path.isdir(directory): + # Wait for __init__ to finish so the source is registered with the + # BoostinManager, otherwise adding torrents won't work + self.register_task(str(self.source) + "_update", + LoopingCall(self._update), delay=2, interval=self.interval) + self._logger.info("Got directory %s", directory) + self.ready = True + else: + self._logger.error("Could not find directory %s", directory) + + def _update(self): + torrent_keys = ['name', 'metainfo', 'creation_date', 'length', 'num_files', 'num_seeders', 'num_leechers', + 'enabled', 'last_seeding_stats'] + + # Wait for __init__ to finish so the source is registered with the + # BoostingManager, otherwise adding torrents won't work. Although we already include delay when call this + if not self.ready: + return + + for torrent_filename in glob.glob(self.source + '/*.torrent'): + if torrent_filename not in self.torrents and len(self.torrents) < self.max_torrents: + try: + tdef = TorrentDef.load(torrent_filename) + except ValueError, verr: + self._logger.error("Could not load %s. Reason %s", torrent_filename, verr) + continue + + # Create a torrent dict. + infohash = tdef.get_infohash() + torrent_values = [tdef.get_name_as_unicode(), tdef, tdef.get_creation_date(), tdef.get_length(), + len(tdef.get_files()), -1, -1, self.enabled, {}] + self.torrents[infohash] = dict(zip(torrent_keys, torrent_values)) + # Notify the BoostingManager. + if self.torrent_insert_callback: + self.torrent_insert_callback(self.source, tdef.get_infohash(), self.torrents[infohash]) diff --git a/Tribler/Policies/__init__.py b/Tribler/Policies/__init__.py new file mode 100644 index 00000000000..07aca525373 --- /dev/null +++ b/Tribler/Policies/__init__.py @@ -0,0 +1,40 @@ +# __init__.py --- +# +# Filename: __init__.py +# Description: +# Author: Elric Milon +# Maintainer: +# Created: Thu Aug 13 17:17:18 2015 (+0200) + +# Commentary: +# +# +# +# + +# Change Log: +# +# +# +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNU Emacs. If not, see . +# +# + +# Code: + + + +# +# __init__.py ends here diff --git a/Tribler/Policies/credit_mining_util.py b/Tribler/Policies/credit_mining_util.py new file mode 100644 index 00000000000..fa6f3433670 --- /dev/null +++ b/Tribler/Policies/credit_mining_util.py @@ -0,0 +1,199 @@ +""" +File containing function used in credit mining module. +""" + + +import os +from binascii import hexlify, unhexlify + +from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.simpledefs import NTFY_CHANNELCAST +from Tribler.Core.simpledefs import NTFY_TORRENTS +from Tribler.Core.simpledefs import NTFY_VOTECAST +from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent, RemoteTorrent, NotCollectedTorrent, Channel, \ + ChannelTorrent +from Tribler.Policies.defs import SIMILARITY_TRESHOLD +from Tribler.dispersy.taskmanager import TaskManager + + +def validate_source_string(source): + """ + Function to check whether a source string is a valid source or not + """ + return unhexlify(source) if len(source) == 40 and not source.startswith("http") else source + + +def levenshtein_dist(t1_fname, t2_fname): + """ + Calculates the Levenshtein distance between a and b. + + Levenshtein distance (LD) is a measure of the similarity between two strings. + (from http://people.cs.pitt.edu/~kirk/cs1501/Pruhs/Fall2006/Assignments/editdistance/Levenshtein%20Distance.htm) + """ + len_t1_fname, len_t2_fname = len(t1_fname), len(t2_fname) + if len_t1_fname > len_t2_fname: + # Make sure len_t1_fname <= len_t2_fname, to use O(min(len_t1_fname,len_t2_fname)) space + t1_fname, t2_fname = t2_fname, t1_fname + len_t1_fname, len_t2_fname = len_t2_fname, len_t1_fname + + current = range(len_t1_fname + 1) + for i in xrange(1, len_t2_fname + 1): + previous, current = current, [i] + [0] * len_t1_fname + for j in xrange(1, len_t1_fname + 1): + add, delete = previous[j] + 1, current[j - 1] + 1 + change = previous[j - 1] + if t1_fname[j - 1] != t2_fname[i - 1]: + change += 1 + current[j] = min(add, delete, change) + + return current[len_t1_fname] + + +def source_to_string(source_obj): + return hexlify(source_obj) if len(source_obj) == 20 and not (source_obj.startswith('http://') + or source_obj.startswith('https://')) else source_obj + + +def string_to_source(source_str): + # don't need to handle null byte because lazy evaluation + return source_str.decode('hex') \ + if len(source_str) == 40 and not (os.path.isdir(source_str) or source_str.startswith('http://')) else source_str + + +def compare_torrents(torrent_1, torrent_2): + """ + comparing swarms. We don't want to download the same swarm with different infohash + :return: whether those t1 and t2 similar enough + """ + files1 = [files for files in torrent_1['metainfo'].get_files_with_length() if files[1] > 1024 * 1024] + files2 = [files for files in torrent_2['metainfo'].get_files_with_length() if files[1] > 1024 * 1024] + + if len(files1) == len(files2): + for ft1 in files1: + for ft2 in files2: + if ft1[1] != ft2[1] or levenshtein_dist(ft1[0], ft2[0]) > SIMILARITY_TRESHOLD: + return False + return True + return False + + +def ent2chr(input_str): + """ + Function to unescape literal string in XML to symbols + source : http://www.gossamer-threads.com/lists/python/python/177423 + """ + code = input_str.group(1) + code_int = int(code) if code.isdigit() else int(code[1:], 16) + return chr(code_int) if code_int < 256 else '?' + +# TODO(ardhi) : temporary function until GUI and core code are separated +class TorrentManagerCM(TaskManager): + """ + *Temporary* class to handle load torrent. + + Adapted from TorrentManager in SearchGridManager + """ + def __init__(self, session): + super(TorrentManagerCM, self).__init__() + + self.session = session + self.torrent_db = self.session.open_dbhandler(NTFY_TORRENTS) + self.channelcast_db = self.session.open_dbhandler(NTFY_CHANNELCAST) + self.votecastdb = self.session.open_dbhandler(NTFY_VOTECAST) + + self.dslist = [] + + def load_torrent(self, torrent, callback=None): + """ + function to load torrent dictionary to torrent object. + + From TorrentManager.loadTorrent in SearchGridManager + """ + + # session is quitting + if not (self.session and self.session.get_torrent_store() and self.session.lm.torrent_store): + return + + if not isinstance(torrent, CollectedTorrent): + if torrent.torrent_id <= 0: + torrent_id = self.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + if not self.session.has_collected_torrent(torrent.infohash): + files = [] + trackers = [] + + # see if we have most info in our tables + if isinstance(torrent, RemoteTorrent): + torrent_id = self.torrent_db.getTorrentID(torrent.infohash) + else: + torrent_id = torrent.torrent_id + + trackers.extend(self.torrent_db.getTrackerListByTorrentID(torrent_id)) + + if 'DHT' in trackers: + trackers.remove('DHT') + if 'no-DHT' in trackers: + trackers.remove('no-DHT') + + # replacement # self.downloadTorrentfileFromPeers(torrent, None) + if self.session.has_download(torrent.infohash): + return False + + if torrent.query_candidates is None or len(torrent.query_candidates) == 0: + self.session.download_torrentfile(torrent.infohash, None, 0) + else: + for candidate in torrent.query_candidates: + self.session.download_torrentfile_from_peer(candidate, torrent.infohash, None, 0) + + torrent = NotCollectedTorrent(torrent, files, trackers) + + else: + tdef = TorrentDef.load_from_memory(self.session.get_collected_torrent(torrent.infohash)) + + if torrent.torrent_id <= 0: + del torrent.torrent_id + + torrent = CollectedTorrent(torrent, tdef) + + # replacement # self.library_manager.addDownloadState(torrent) + for dl_state in self.dslist: + torrent.addDs(dl_state) + + # return + if callback is not None: + callback(torrent) + else: + return torrent + + def create_torrents(self, tor_values, _, channel_dict): + """ + function to create torrents from channel. Adapted from + ChannelManager in SearchGridManager + """ + + #adding new channel from the one that can't be detected from torrent values + fetch_channels = set(hit[0] for hit in tor_values if hit[0] not in channel_dict) + if len(fetch_channels) > 0: + channels_new_dict = self.channelcast_db.getChannels(fetch_channels) + channels = [] + for hit in channels_new_dict: + channel = Channel(*hit) + channels.append(channel) + + for channel in channels: + channel_dict[channel.id] = channel + + # creating torrents + torrents = [] + for hit in tor_values: + if hit: + chan_torrent = ChannelTorrent(*hit[1:] + [channel_dict.get(hit[0], None), None]) + chan_torrent.torrent_db = self.torrent_db + chan_torrent.channelcast_db = self.channelcast_db + + if chan_torrent.name: + torrents.append(chan_torrent) + + return torrents diff --git a/Tribler/Policies/defs.py b/Tribler/Policies/defs.py new file mode 100644 index 00000000000..d37c83d6ec4 --- /dev/null +++ b/Tribler/Policies/defs.py @@ -0,0 +1,23 @@ +""" +Definition/Constant that used in credit mining +""" + +from Tribler.Core.Utilities.install_dir import determine_install_dir + +SIMILARITY_TRESHOLD = 5 + +TRIBLER_ROOT = determine_install_dir() + +SAVED_ATTR = ["max_torrents_per_source", + "max_torrents_active", "source_interval", + "swarm_interval", "share_mode_target", + "tracker_interval", "logging_interval"] + +CREDIT_MINING_FOLDER_DOWNLOAD = "credit_mining" + +CONFIG_OP_ADD = "add" +CONFIG_OP_RM = "rm" +CONFIG_KEY_SOURCELIST = "boosting_sources" +CONFIG_KEY_ARCHIVELIST = "archive_sources" +CONFIG_KEY_ENABLEDLIST = "boosting_enabled" +CONFIG_KEY_DISABLEDLIST = "boosting_disabled" diff --git a/Tribler/Test/Core/CreditMining/__init__.py b/Tribler/Test/Core/CreditMining/__init__.py new file mode 100644 index 00000000000..3dc228c3f6d --- /dev/null +++ b/Tribler/Test/Core/CreditMining/__init__.py @@ -0,0 +1,3 @@ +""" +This package contains tests for the credit mining-related code of Tribler. +""" diff --git a/Tribler/Test/Core/CreditMining/mock_creditmining.py b/Tribler/Test/Core/CreditMining/mock_creditmining.py new file mode 100644 index 00000000000..525abae90d5 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/mock_creditmining.py @@ -0,0 +1,138 @@ +""" +Module of Credit mining mock classes + +Written by Ardhi Putra Pratama H +""" +from twisted.web.resource import Resource + + +class MockLtTorrent(object): + """ + Class representing libtorrent handle for getting peer info + """ + def __init__(self, infohash="12345"): + self.info_hash = infohash + self.all_time_download = 0 + self.all_time_upload = 0 + + def upload_limit(self): + return 12 + + def max_uploads(self): + return 13 + + def max_connections(self): + return 14 + + def piece_priorities(self): + return [0, 1, 1, 0, 1, 1, 1] + + def get_peer_info(self): + """ + class returning peer info for a particular handle + """ + peer = [None] * 6 + peer[0] = MockLtPeer(1, "ip1") + peer[0].setvalue(True, True, True) + peer[1] = MockLtPeer(2, "ip2") + peer[1].setvalue(False, False, True) + peer[2] = MockLtPeer(3, "ip3") + peer[2].setvalue(True, False, True) + peer[3] = MockLtPeer(4, "ip4") + peer[3].setvalue(False, True, False) + peer[4] = MockLtPeer(5, "ip5") + peer[4].setvalue(False, True, True) + peer[5] = MockLtPeer(6, "ip6") + peer[5].setvalue(False, False, False) + return peer + + def is_valid(self): + """ + check whether the handle is valid or not + """ + return True + + def status(self): + return self + +class MockLtPeer(object): + """ + Dummy peer object returned by libtorrent python binding + """ + def __init__(self, pid, ip): + self.pid = pid + self.client = 2 + self.ip = [ip, "port"] + self.flags = 1 + self.local_connection = True + self.payload_up_speed = 0 + self.remote_interested = 1 + self.remote_choked = 1 + self.upload_queue_length = 0 + self.used_send_buffer = 0 + self.payload_down_speed = 0 + self.interesting = True + self.choked = True + self.total_upload = 0 + self.total_download = 0 + self.progress = 0 + self.pieces = 0 + self.remote_dl_rate = 0 + self.country = "ID" + self.connection_type = 0 + self.seed = 1 + self.upload_only = 1 + self.read_state = False + self.write_state = False + + def setvalue(self, upload_only, uinterested, completed): + self.upload_only = upload_only + self.remote_interested = uinterested + self.progress = 1 if completed else 0 + + +class MockMeta(object): + """ + class for mocking the torrent metainfo + """ + + def __init__(self, id_hash): + self.infohash = id_hash + + def get_infohash(self): + """ + returning infohash of torrents + """ + return self.infohash + + +class MockLtSession(object): + """ + Mock for session and LibTorrentMgr + """ + def __init__(self): + pass + + def get_session(self): + """ + supposed to get libtorrent session + """ + return self + + def set_settings(self, _): + """ + set settings (don't do anything) + """ + pass + + def shutdown(self): + """ + obligatory shutdown function + """ + pass + + +class ResourceFailClass(Resource): + def render_GET(self, request): + request.setResponseCode(503) + return "Error 503." diff --git a/Tribler/Test/Core/CreditMining/test_creditmining.py b/Tribler/Test/Core/CreditMining/test_creditmining.py new file mode 100644 index 00000000000..eeb2d85b195 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/test_creditmining.py @@ -0,0 +1,405 @@ +# coding=utf-8 +""" +Module of Credit mining function testing + +Written by Mihai Capotă and Ardhi Putra Pratama H +""" + +import binascii +import random +import re + +import Tribler.Policies.BoostingManager as bm +from Tribler.Core.DownloadConfig import DefaultDownloadStartupConfig +from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl +from Tribler.Core.SessionConfig import SessionConfigInterface +from Tribler.Core.Utilities import utilities +from Tribler.Core.defaults import sessdefaults +from Tribler.Policies.BoostingPolicy import CreationDatePolicy, SeederRatioPolicy, RandomPolicy +from Tribler.Policies.BoostingSource import ent2chr +from Tribler.Policies.credit_mining_util import levenshtein_dist, source_to_string +from Tribler.Test.Core.CreditMining.mock_creditmining import MockMeta, MockLtPeer, MockLtSession, MockLtTorrent +from Tribler.Test.test_as_server import TestAsServer + + +class TestBoostingManagerPolicies(TestAsServer): + """ + The class to test core function of credit mining policies + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerPolicies, self).__init__(*argv, **kwargs) + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerPolicies, self).setUp() + self.session.get_download = lambda x: x % 2 + random.seed(0) + self.torrents = dict() + for i in xrange(1, 11): + mock_metainfo = MockMeta(i) + + self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": i, + "num_leechers": i-1, "creation_date": i} + + def test_random_policy(self): + """ + testing random policy + """ + policy = RandomPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(3, len(ids_start), "Start failed %s vs %s" % (ids_start, torrents_start)) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(2, len(ids_stop), "Stop failed %s vs %s" % (ids_stop, torrents_stop)) + + def test_seederratio_policy(self): + """ + testing seeder ratio policy + """ + policy = SeederRatioPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(ids_start, [10, 8, 6]) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(ids_stop, [3, 1]) + + def test_fallback_policy(self): + """ + testing policy (seederratio) and then fallback + """ + + for i in xrange(1, 11): + mock_metainfo = MockMeta(i) + self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": -i, + "num_leechers": -i, "creation_date": i} + + policy = SeederRatioPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(3, len(ids_start), "Start failed %s vs %s" % (ids_start, torrents_start)) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(2, len(ids_stop), "Stop failed %s vs %s" % (ids_stop, torrents_stop)) + + def test_creationdate_policy(self): + """ + test policy based on creation date + """ + policy = CreationDatePolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 5, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(ids_start, [10, 8, 6]) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(ids_stop, [5, 3, 1]) + + +class TestBoostingManagerUtilities(TestAsServer): + """ + Test several utilities used in credit mining + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerUtilities, self).__init__(*argv, **kwargs) + + self.peer = [None] * 6 + self.peer[0] = MockLtPeer(1, "ip1") + self.peer[0].setvalue(True, True, True) + self.peer[1] = MockLtPeer(2, "ip2") + self.peer[1].setvalue(False, False, True) + self.peer[2] = MockLtPeer(3, "ip3") + self.peer[2].setvalue(True, False, True) + self.peer[3] = MockLtPeer(4, "ip4") + self.peer[3].setvalue(False, True, False) + self.peer[4] = MockLtPeer(5, "ip5") + self.peer[4].setvalue(False, True, True) + self.peer[5] = MockLtPeer(6, "ip6") + self.peer[5].setvalue(False, False, False) + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerUtilities, self).setUp() + + self.session.get_libtorrent = lambda: True + + self.bsettings = bm.BoostingSettings(self.session) + self.bsettings.credit_mining_path = self.session_base_dir + self.bsettings.load_config = False + self.bsettings.check_dependencies = False + self.bsettings.initial_logging_interval = 900 + + def tearDown(self): + # TODO(ardhi) : remove it when Tribler free of singleton + # and 1 below + DefaultDownloadStartupConfig.delInstance() + + super(TestBoostingManagerUtilities, self).tearDown() + + def test_boosting_dependencies(self): + """ + Test whether boosting manager dependencies works or not. + + In all test, check dependencies always off. In production, it is on by default. + """ + self.bsettings.check_dependencies = True + self.bsettings.initial_swarm_interval = 9000 + self.bsettings.initial_tracker_interval = 9000 + self.bsettings.initial_logging_interval = 9000 + self.session.open_dbhandler = lambda _: None + self.session.lm.ltmgr = MockLtSession() + + self.session.get_torrent_checking = lambda: True + self.session.get_dispersy = lambda: True + self.session.get_torrent_store = lambda: True + self.session.get_enable_torrent_search = lambda: True + self.session.get_enable_channel_search = lambda: True + self.session.get_megacache = lambda: False + + self.assertRaises(AssertionError, bm.BoostingManager, self.session, self.bsettings) + + def test_load_default(self): + """ + Test load default configuration in BoostingManager + """ + self.bsettings.load_config = True + self.bsettings.auto_start_source = False + self.bsettings.initial_swarm_interval = 9000 + self.bsettings.initial_tracker_interval = 9000 + self.bsettings.initial_logging_interval = 9000 + self.session.open_dbhandler = lambda _: None + self.session.lm.ltmgr = MockLtSession() + + # it will automatically load the default configuration + boost_man = bm.BoostingManager(self.session, self.bsettings) + + # def validate(d_defer): + self.assertEqual(sessdefaults['credit_mining']['source_interval'], boost_man.settings.source_interval) + self.assertEqual(sessdefaults['credit_mining']['archive_sources'], + [source_to_string(src.source) for src in boost_man.boosting_sources.values() + if src.archive]) + + boost_man.cancel_all_pending_tasks() + + def test_sessionconfig(self): + """ + test basic credit mining preferences + """ + sci = SessionConfigInterface() + + sci.set_cm_logging_interval(100) + self.assertEqual(sci.get_cm_logging_interval(), 100) + + sci.set_cm_max_torrents_active(20) + self.assertEqual(sci.get_cm_max_torrents_active(), 20) + + sci.set_cm_max_torrents_per_source(10) + self.assertEqual(sci.get_cm_max_torrents_per_source(), 10) + + sci.set_cm_source_interval(100) + self.assertEqual(sci.get_cm_source_interval(), 100) + + sci.set_cm_policy("random") + self.assertIs(sci.get_cm_policy(as_class=True), RandomPolicy) + + sci.set_cm_policy(SeederRatioPolicy(self.session)) + self.assertEqual(sci.get_cm_policy(as_class=False), "seederratio") + + sci.set_cm_share_mode_target(2) + self.assertEqual(sci.get_cm_share_mode_target(), 2) + + sci.set_cm_swarm_interval(200) + self.assertEqual(sci.get_cm_swarm_interval(), 200) + + sci.set_cm_tracker_interval(300) + self.assertEqual(sci.get_cm_tracker_interval(), 300) + + def test_translate_peer_info(self): + """ + test - predict number of seeder and leecher only based on peer discovered and + their activities + """ + peerlist_dict = [] + for peer in self.peer: + peerlist_dict.append(LibtorrentDownloadImpl.create_peerlist_data(peer)) + + num_seed, num_leech = utilities.translate_peers_into_health(peerlist_dict) + self.assertEqual(num_seed, 4, "Seeder number don't match") + self.assertEqual(num_leech, 3, "Leecher number don't match") + + def test_levenshtein(self): + """ + test levenshtein between two string (in this case, file name) + + source : + http://people.cs.pitt.edu/~kirk/cs1501/Pruhs/Fall2006/Assignments/editdistance/Levenshtein%20Distance.htm + """ + string1 = "GUMBO" + string2 = "GAMBOL" + dist = levenshtein_dist(string1, string2) + dist_swap = levenshtein_dist(string2, string1) + + # random string check + self.assertEqual(dist, 2, "Wrong levenshtein distance") + self.assertEqual(dist_swap, 2, "Wrong levenshtein distance") + + string1 = "ubuntu-15.10-desktop-i386.iso" + string2 = "ubuntu-15.10-desktop-amd64.iso" + dist = levenshtein_dist(string1, string2) + + # similar filename check + self.assertEqual(dist, 4, "Wrong levenshtein distance") + + dist = levenshtein_dist(string1, string1) + # equal filename check + self.assertEqual(dist, 0, "Wrong levenshtein distance") + + string2 = "Learning-Ubuntu-Linux-Server.tgz" + dist = levenshtein_dist(string1, string2) + # equal filename check + self.assertEqual(dist, 28, "Wrong levenshtein distance") + + def test_update_statistics(self): + """ + test updating statistics of a torrent (pick a new one) + """ + self.session.open_dbhandler = lambda _: None + + infohash_1 = "a"*20 + infohash_2 = "b"*20 + torrents = { + infohash_1: { + "last_seeding_stats": { + "time_seeding": 100, + "value": 5 + } + }, + infohash_2: { + "last_seeding_stats": {} + } + } + + new_seeding_stats = { + "time_seeding": 110, + "value": 1 + } + new_seeding_stats_unexist = { + "time_seeding": 10, + "value": 8 + } + + self.session.lm.ltmgr = MockLtSession() + + boost_man = bm.BoostingManager(self.session, self.bsettings) + boost_man.torrents = torrents + + boost_man.update_torrent_stats(infohash_1, new_seeding_stats) + self.assertEqual(boost_man.torrents[infohash_1]['last_seeding_stats'] + ['value'], 1) + + boost_man.update_torrent_stats(infohash_2, new_seeding_stats_unexist) + self.assertEqual(boost_man.torrents[infohash_2]['last_seeding_stats'] + ['value'], 8) + + boost_man.cancel_all_pending_tasks() + + def test_escape_xml(self): + """ + testing escape symbols occured in xml/rss document file. + """ + re_symbols = re.compile(r'\&\#(x?[0-9a-fA-F]+);') + + ampersand_str = re_symbols.sub(ent2chr, '&') + self.assertEqual(ampersand_str, "&", "wrong ampersand conversion %s" % ampersand_str) + + str_123 = re_symbols.sub(ent2chr, "123") + self.assertEqual(str_123, "123", "wrong number conversion %s" % str_123) + + def test_logging(self): + self.session.open_dbhandler = lambda _: None + + infohash_1 = "a"*20 + infohash_2 = "b"*20 + torrents = { + infohash_1: { + "last_seeding_stats": { + "time_seeding": 100, + "value": 5 + } + }, + infohash_2: { + "last_seeding_stats": {} + } + } + self.session.lm.ltmgr = MockLtSession() + + boost_man = bm.BoostingManager(self.session, self.bsettings) + boost_man.torrents = torrents + + boost_man.session.lm.ltmgr.get_session().get_torrents = \ + lambda: [MockLtTorrent(binascii.hexlify(infohash_1)), + MockLtTorrent(binascii.hexlify(infohash_2))] + + boost_man.log_statistics() + boost_man.cancel_all_pending_tasks() + + +class TestBoostingManagerError(TestAsServer): + """ + Class to test a bunch of credit mining error handle + """ + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerError, self).setUp() + + self.session.open_dbhandler = lambda _: True + self.session.get_libtorrent = lambda: True + self.session.lm.ltmgr = MockLtSession() + + self.boost_setting = bm.BoostingSettings(self.session) + self.boost_setting.load_config = False + self.boost_setting.initial_logging_interval = 900 + self.boost_setting.check_dependencies = False + self.boosting_manager = bm.BoostingManager(self.session, self.boost_setting) + self.session.lm.boosting_manager = self.boosting_manager + + def tearDown(self): + DefaultDownloadStartupConfig.delInstance() + super(TestBoostingManagerError, self).tearDown() + + def test_insert_torrent_unknown_source(self): + """ + testing insert torrent on unknown source + """ + torrent = { + 'preload': False, + 'metainfo': MockMeta("1234"), + 'infohash': '12345' + } + + self.boosting_manager.on_torrent_insert(binascii.unhexlify("abcd" * 10), '12345', torrent) + self.assertNotIn('12345', self.boosting_manager.torrents) + + def test_unknown_source(self): + """ + testing uknkown source added to boosting source, and try to apply archive + on top of that + """ + unknown_key = "1234567890" + + sources = len(self.boosting_manager.boosting_sources.keys()) + self.boosting_manager.add_source(unknown_key) + self.boosting_manager.set_archive(unknown_key, False) + self.assertEqual(sources, len(self.boosting_manager.boosting_sources.keys()), "unknown source added") + + def test_failed_start_download(self): + """ + test assertion error then not download the actual torrent + """ + torrent = { + 'preload': False, + 'metainfo': MockMeta("1234") + } + self.session.lm.download_exists = lambda _: True + self.boosting_manager.start_download(torrent) + + self.assertNotIn('download', torrent, "%s downloading despite error" % torrent) diff --git a/Tribler/Test/Core/CreditMining/test_creditmining_sys.py b/Tribler/Test/Core/CreditMining/test_creditmining_sys.py new file mode 100644 index 00000000000..d79790f7826 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/test_creditmining_sys.py @@ -0,0 +1,517 @@ +# coding=utf-8 +""" +Module of Credit mining function testing + +Written by Ardhi Putra Pratama H +""" +import binascii +import os +import shutil + +from twisted.internet import defer +from twisted.web.server import Site +from twisted.web.static import File + +from Tribler.Core.DownloadConfig import DefaultDownloadStartupConfig +from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.Utilities.twisted_thread import deferred, reactor +from Tribler.Core.simpledefs import NTFY_TORRENTS, NTFY_UPDATE +from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent +from Tribler.Policies.BoostingManager import BoostingManager, BoostingSettings +from Tribler.Test.Core.CreditMining.mock_creditmining import MockLtTorrent, ResourceFailClass +from Tribler.Test.Core.Modules.RestApi.test_channels_endpoints import AbstractTestChannelsEndpoint +from Tribler.Test.test_as_server import TestAsServer, TESTS_DATA_DIR +from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_FILE_INFOHASH +from Tribler.Test.util import prepare_xml_rss +from Tribler.community.channel.community import ChannelCommunity +from Tribler.dispersy.dispersy import Dispersy +from Tribler.dispersy.endpoint import ManualEnpoint +from Tribler.dispersy.util import blocking_call_on_reactor_thread + + +class TestBoostingManagerSys(TestAsServer): + """ + base class to test base credit mining function + """ + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSys, self).setUp() + + self.set_boosting_settings() + + self.session.lm.ltmgr.get_session().find_torrent = lambda _: MockLtTorrent() + + self.boosting_manager = BoostingManager(self.session, self.bsettings) + + self.session.lm.boosting_manager = self.boosting_manager + + def set_boosting_settings(self): + """ + set settings in credit mining + """ + self.bsettings = BoostingSettings(self.session) + self.bsettings.credit_mining_path = os.path.join(self.session_base_dir, "credit_mining") + self.bsettings.load_config = False + self.bsettings.check_dependencies = False + self.bsettings.min_connection_start = -1 + self.bsettings.min_channels_start = -1 + + self.bsettings.max_torrents_active = 8 + self.bsettings.max_torrents_per_source = 5 + + self.bsettings.tracker_interval = 5 + self.bsettings.initial_tracker_interval = 5 + self.bsettings.logging_interval = 30 + self.bsettings.initial_logging_interval = 3 + + def setUpPreSession(self): + super(TestBoostingManagerSys, self).setUpPreSession() + + self.config.set_torrent_checking(True) + self.config.set_megacache(True) + self.config.set_dispersy(True) + self.config.set_torrent_store(True) + self.config.set_enable_torrent_search(True) + self.config.set_enable_channel_search(True) + self.config.set_libtorrent(True) + + def tearDown(self): + DefaultDownloadStartupConfig.delInstance() + self.boosting_manager.shutdown() + + super(TestBoostingManagerSys, self).tearDown() + + def check_torrents(self, src, defer_param=None, target=1): + """ + function to check if a torrent is already added to the source + + In this function, + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + if len(src_obj.torrents) < target: + reactor.callLater(1, self.check_torrents, src, defer_param, target=target) + else: + # notify torrent (emulate scraping) + self.boosting_manager.scrape_trackers() + + def _get_tor_dummy(_, keys=123, include_mypref=True): + """ + function to emulate get_torrent in torrent_db + """ + return {'C.torrent_id': 93, 'category': u'Compressed', 'torrent_id': 41, + 'infohash': src_obj.torrents.keys()[0], 'length': 1150844928, 'last_tracker_check': 10001, + 'myDownloadHistory': False, 'name': u'ubuntu-15.04-desktop-amd64.iso', + 'num_leechers': 999, 'num_seeders': 123, 'status': u'unknown', 'tracker_check_retries': 0} + self.boosting_manager.torrent_db.getTorrent = _get_tor_dummy + self.session.notifier.notify(NTFY_TORRENTS, NTFY_UPDATE, src_obj.torrents.keys()[0]) + + # log it + self.boosting_manager.log_statistics() + + defer_param.callback(src) + return defer_param + + def check_source(self, src, defer_param=None, ready=True): + """ + function to check if a source is ready initializing + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + + if not ready: + defer_param.callback(src) + elif not src_obj or not src_obj.ready: + reactor.callLater(1, self.check_source, src, defer_param) + else: + defer_param.callback(src) + + return defer_param + + +class TestBoostingManagerSysRSS(TestBoostingManagerSys): + """ + testing class for RSS (dummy) source + """ + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSysRSS, self).setUp() + + files_path, self.file_server_port = prepare_xml_rss(self.session_base_dir, 'test_rss_cm.xml') + + shutil.copyfile(TORRENT_FILE, os.path.join(files_path, 'ubuntu.torrent')) + self.setUpFileServer(self.file_server_port, self.session_base_dir) + + self.rss_error_deferred = defer.Deferred() + # now the rss should be at : + # http://localhost:port/test_rss_cm.xml + # which resides in sessiondir/http_torrent_files + + def set_boosting_settings(self): + super(TestBoostingManagerSysRSS, self).set_boosting_settings() + self.bsettings.auto_start_source = False + + def setUpFileServer(self, port, path): + resource = File(path) + resource.putChild("err503", ResourceFailClass()) + factory = Site(resource) + self._logger.debug("Listen to port %s, factory %s", port, factory) + self.file_server = reactor.listenTCP(port, factory) + + @deferred(timeout=15) + def test_rss(self): + """ + test rss source + """ + url = 'http://localhost:%s/test_rss_cm.xml' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj.start() + + d = self.check_source(url) + d.addCallback(self.check_torrents, target=1) + return d + + def _on_error_rss(self, dummy_1, dummy_2): + """ + dummy errback when RSS source produces an error + """ + self.rss_error_deferred.callback(True) + + @deferred(timeout=8) + def test_rss_unexist(self): + """ + Testing an unexisting RSS feed + """ + url = 'http://localhost:%s/nothingness' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj._on_error_rss = self._on_error_rss + rss_obj.start() + + defer_err_rss = self.check_source(url, ready=False) + defer_err_rss.chainDeferred(self.rss_error_deferred) + return defer_err_rss + + @deferred(timeout=8) + def test_rss_unavailable(self): + """ + Testing an unavailable RSS feed + """ + url = 'http://localhost:%s/err503' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj._on_error_rss = self._on_error_rss + rss_obj.start() + + defer_err_rss = self.check_source(url, ready=False) + defer_err_rss.chainDeferred(self.rss_error_deferred) + return defer_err_rss + + +class TestBoostingManagerSysDir(TestBoostingManagerSys): + """ + testing class for directory source + """ + + @deferred(timeout=10) + def test_dir(self): + """ + test directory filled with .torrents + """ + self.boosting_manager.add_source(TESTS_DATA_DIR) + len_source = len(self.boosting_manager.boosting_sources) + + # deliberately try to add the same source + self.boosting_manager.add_source(TESTS_DATA_DIR) + self.assertEqual(len(self.boosting_manager.boosting_sources), len_source, "identical source added") + + dir_obj = self.boosting_manager.get_source_object(TESTS_DATA_DIR) + self.assertTrue(dir_obj.ready, "Not Ready") + + d = self.check_torrents(TESTS_DATA_DIR, target=2) + d.addCallback(lambda _: True) + return d + + @deferred(timeout=10) + def test_dir_archive_example(self): + """ + test archive mode. Use diretory because easier to fetch torrent + """ + self.boosting_manager.add_source(TESTS_DATA_DIR) + self.boosting_manager.set_archive(TESTS_DATA_DIR, True) + + dir_obj = self.boosting_manager.get_source_object(TESTS_DATA_DIR) + self.assertTrue(dir_obj.ready, "Not Ready") + + d = self.check_torrents(TESTS_DATA_DIR, target=2) + d.addCallback(lambda _: self.boosting_manager._select_torrent()) + return d + + +class TestBoostingManagerSysChannel(AbstractTestChannelsEndpoint, TestBoostingManagerSys): + """ + testing class for channel source + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerSysChannel, self).__init__(*argv, **kwargs) + self.tdef = TorrentDef.load(TORRENT_FILE) + self.channel_id = 0 + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSysChannel, self).setUp() + + def set_boosting_settings(self): + super(TestBoostingManagerSysChannel, self).set_boosting_settings() + self.bsettings.swarm_interval = 1 + self.bsettings.initial_swarm_interval = 1 + self.bsettings.max_torrents_active = 1 + self.bsettings.max_torrents_per_source = 1 + + def setUpPreSession(self): + super(TestBoostingManagerSysChannel, self).setUpPreSession() + + # we use dummy dispersy here + self.config.set_dispersy(False) + + @blocking_call_on_reactor_thread + def create_torrents_in_channel(self, dispersy_cid_hex): + """ + Helper function to insert 10 torrent into designated channel + """ + for i in xrange(0, 10): + self.insert_channel_in_db('rand%d' % i, 42 + i, 'Test channel %d' % i, 'Test description %d' % i) + + self.channel_id = self.insert_channel_in_db(dispersy_cid_hex.decode('hex'), 42, + 'Simple Channel', 'Channel description') + + torrent_list = [[self.channel_id, 1, 1, TORRENT_FILE_INFOHASH, 1460000000, TORRENT_FILE, + self.tdef.get_files_as_unicode_with_length(), self.tdef.get_trackers_as_single_tuple()]] + + self.insert_torrents_into_channel(torrent_list) + + @deferred(timeout=20) + def test_chn_lookup(self): + """ + testing channel source. + + It includes finding and downloading actual torrent + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + torrent = CollectedTorrent(torrent, self.tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + def check_torrents_channel(src, defer_param=None, target=1): + """ + check if a torrent already in channel and ready to download + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + success = True + if not src_obj or len(src_obj.torrents) < target: + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + elif not self.boosting_manager.torrents.get(TORRENT_FILE_INFOHASH, None): + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + elif not self.boosting_manager.torrents[TORRENT_FILE_INFOHASH].get('download', None): + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + + if success: + self.boosting_manager.set_enable_mining(src, False, force_restart=True) + if src_obj.community: + src_obj.community.cancel_all_pending_tasks() + + defer_param.callback(src) + + return defer_param + + chn_obj.torrent_mgr.load_torrent = _load + + d = self.check_source(dispersy_cid) + d.addCallback(check_torrents_channel, target=1) + return d + + @deferred(timeout=20) + def test_chn_exist_lookup(self): + """ + testing existing channel as a source. + + It also tests how boosting manager cope with unknown channel with retrying + the lookup + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + # channel is exist + community = ChannelCommunity.init_community(self.session.lm.dispersy, + self.session.lm.dispersy.get_member(mid=dispersy_cid), + self.session.lm.dispersy._communities['allchannel']._my_member, + self.session) + + # make the id unknown so boosting manager can test repeating search + id_tmp = community._channel_id + community._channel_id = 0 + + def _set_id_channel(channel_id): + """ + set channel id manually (emulate finding) + """ + community._channel_id = channel_id + + reactor.callLater(5, _set_id_channel, id_tmp) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + torrent = CollectedTorrent(torrent, self.tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + chn_obj.torrent_mgr.load_torrent = _load + + def clean_community(_): + """ + cleanly exit the community we are in + """ + if chn_obj.community: + chn_obj.community.cancel_all_pending_tasks() + + chn_obj.kill_tasks() + + + d = self.check_source(dispersy_cid) + d.addCallback(clean_community) + return d + + @deferred(timeout=20) + def test_chn_max_torrents(self): + """ + Test the restriction of max_torrents in a source. + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + pioneer_file = os.path.join(TESTS_DATA_DIR, "Pioneer.One.S01E06.720p.x264-VODO.torrent") + pioneer_tdef = TorrentDef.load(pioneer_file) + pioneer_ihash = binascii.unhexlify("66ED7F30E3B30FA647ABAA19A36E7503AA071535") + + torrent_list = [[self.channel_id, 1, 1, pioneer_ihash, 1460000001, pioneer_file, + pioneer_tdef.get_files_as_unicode_with_length(), pioneer_tdef.get_trackers_as_single_tuple()]] + self.insert_torrents_into_channel(torrent_list) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + chn_obj.max_torrents = 2 + chn_obj.torrent_mgr.load_torrent = lambda dummy_1, dummy_2: None + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + infohash_str = binascii.hexlify(torrent.infohash) + torrent = CollectedTorrent(torrent, self.tdef if infohash_str.startswith("fc") else pioneer_tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + def activate_mgr(): + """ + activate ltmgr and adjust max torrents to emulate overflow torrents + """ + chn_obj.max_torrents = 1 + chn_obj.torrent_mgr.load_torrent = _load + + reactor.callLater(5, activate_mgr) + + def check_torrents_channel(src, defer_param=None): + """ + check if a torrent already in channel and ready to download + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + success = True + if len(src_obj.unavail_torrent) == 0: + self.assertLessEqual(len(src_obj.torrents), src_obj.max_torrents) + else: + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param) + + if success: + src_obj.community.cancel_all_pending_tasks() + src_obj.kill_tasks() + defer_param.callback(src) + + return defer_param + + d = self.check_source(dispersy_cid) + d.addCallback(check_torrents_channel) + return d + + def tearDown(self): + self.session.lm.dispersy._communities['allchannel'].cancel_all_pending_tasks() + self.session.lm.dispersy.cancel_all_pending_tasks() + self.session.lm.dispersy = None + super(TestBoostingManagerSysChannel, self).tearDown() diff --git a/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py b/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py index bd441d4f3b7..3215e85a9b6 100644 --- a/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py +++ b/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py @@ -20,6 +20,8 @@ def setUp(self, autoload_discovery=True): self.votecast_db_handler = self.session.open_dbhandler(NTFY_VOTECAST) self.channel_db_handler._get_my_dispersy_cid = lambda: "myfakedispersyid" + self.create_votecast_called = False + def insert_channel_in_db(self, dispersy_cid, peer_id, name, description): return self.channel_db_handler.on_channel_from_dispersy(dispersy_cid, peer_id, name, description) @@ -29,6 +31,26 @@ def vote_for_channel(self, cid, vote_time): def insert_torrents_into_channel(self, torrent_list): self.channel_db_handler.on_torrents_from_dispersy(torrent_list) + @blocking_call_on_reactor_thread + def create_fake_allchannel_community(self): + """ + This method creates a fake AllChannel community so we can check whether a request is made in the community + when doing stuff with a channel. + """ + self.session.lm.dispersy._database.open() + fake_member = DummyMember(self.session.lm.dispersy, 1, "a" * 20) + member = self.session.lm.dispersy.get_new_member(u"curve25519") + fake_community = AllChannelCommunity.init_community(self.session.lm.dispersy, fake_member, member) + fake_community.disp_create_votecast = self.on_dispersy_create_votecast + self.session.lm.dispersy._communities = {"allchannel": fake_community} + return fake_community + + def on_dispersy_create_votecast(self, cid, vote, _): + """ + Check whether we have the expected parameters when this method is called. + """ + self.create_votecast_called = True + class TestChannelsEndpoint(AbstractTestChannelsEndpoint): @@ -132,39 +154,19 @@ def setUp(self, autoload_discovery=True): super(TestChannelsSubscriptionEndpoint, self).setUp(autoload_discovery) self.expected_votecast_cid = None self.expected_votecast_vote = None - self.create_votecast_called = False self.session.get_dispersy = lambda: True self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + self.create_fake_allchannel_community() for i in xrange(0, 10): self.insert_channel_in_db('rand%d' % i, 42 + i, 'Test channel %d' % i, 'Test description %d' % i) def on_dispersy_create_votecast(self, cid, vote, _): - """ - Check whether we have the expected parameters when this method is called. - """ + super(TestChannelsSubscriptionEndpoint, self).on_dispersy_create_votecast(cid, vote, _) self.assertEqual(cid, self.expected_votecast_cid) self.assertEqual(vote, self.expected_votecast_vote) - self.create_votecast_called = True - - @blocking_call_on_reactor_thread - def create_fake_allchannel_community(self): - """ - This method creates a fake AllChannel community so we can check whether a request is made in the community - when doing stuff with a channel. - """ - self.session.lm.dispersy._database.open() - fake_member = DummyMember(self.session.lm.dispersy, 1, "a" * 20) - member = self.session.lm.dispersy.get_new_member(u"curve25519") - fake_community = AllChannelCommunity(self.session.lm.dispersy, fake_member, member) - fake_community.disp_create_votecast = self.on_dispersy_create_votecast - self.session.lm.dispersy._communities = {"allchannel": fake_community} - - def tearDown(self): - self.session.lm.dispersy = None - super(TestChannelsSubscriptionEndpoint, self).tearDown() @deferred(timeout=10) def test_subscribe_channel_not_exist(self): @@ -229,3 +231,8 @@ def verify_votecast_made(_): self.expected_votecast_vote = VOTE_UNSUBSCRIBE return self.do_request('channels/subscribed/%s' % 'rand1'.encode('hex'), expected_code=200, expected_json=expected_json, request_type='DELETE').addCallback(verify_votecast_made) + + def tearDown(self): + self.session.lm.dispersy._communities['allchannel'].cancel_all_pending_tasks() + self.session.lm.dispersy = None + super(TestChannelsSubscriptionEndpoint, self).tearDown() diff --git a/Tribler/Test/data/test_rss_cm.xml b/Tribler/Test/data/test_rss_cm.xml new file mode 100644 index 00000000000..18044929497 --- /dev/null +++ b/Tribler/Test/data/test_rss_cm.xml @@ -0,0 +1,31 @@ + + + + + Test RSS for Credit Mining + test_rss_cm.xml + Test RSS CM feed. + + en + Tue, 12 May 2015 08:39:23 GMT + Tue, 12 May 2015 08:39:23 GMT + + + + + ubuntu-15.04-desktop-amd64.iso + http://localhost:RANDOMPORT/ubuntu.torrent + + http://localhost:RANDOMPORT.ubuntu.torrent + + + + Pioneer.One.S01E06.720p.x264-VODO.torrent + http://localhost:RANDOMPORT/pioneer.torrent + + http://localhost:RANDOMPORT.pioneer.torrent + + + diff --git a/Tribler/Test/test_as_server.py b/Tribler/Test/test_as_server.py index 982abd49029..4eaf84ac38e 100644 --- a/Tribler/Test/test_as_server.py +++ b/Tribler/Test/test_as_server.py @@ -262,6 +262,7 @@ def setUpPreSession(self): self.config.set_upgrader_enabled(False) self.config.set_http_api_enabled(False) self.config.set_tunnel_community_enabled(False) + self.config.set_creditmining_enable(False) def tearDown(self): self.annotate(self._testMethodName, start=False) diff --git a/Tribler/Test/test_my_channel.py b/Tribler/Test/test_my_channel.py index b0106e95487..1b67946feaa 100644 --- a/Tribler/Test/test_my_channel.py +++ b/Tribler/Test/test_my_channel.py @@ -1,16 +1,15 @@ # Written by Niels Zeilemaker # see LICENSE.txt for license information -from binascii import hexlify import os import shutil -from Tribler.Core.Utilities.network_utils import get_random_port +from binascii import hexlify +from Tribler.Core.TorrentDef import TorrentDef from Tribler.Test.common import UBUNTU_1504_INFOHASH -from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_VIDEO_FILE from Tribler.Test.test_as_server import TestGuiAsServer, TESTS_DATA_DIR - -from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_VIDEO_FILE +from Tribler.Test.util import prepare_xml_rss DEBUG = True @@ -21,15 +20,8 @@ def setUp(self): super(TestMyChannel, self).setUp() # Prepare test_rss.xml file, replace the port with a random one - self.file_server_port = get_random_port() - with open(os.path.join(TESTS_DATA_DIR, 'test_rss.xml'), 'r') as source_xml,\ - open(os.path.join(self.session_base_dir, 'test_rss.xml'), 'w') as destination_xml: - for line in source_xml: - destination_xml.write(line.replace('RANDOMPORT', str(self.file_server_port))) - - # Setup file server to serve torrent file and thumbnails - files_path = os.path.join(self.session_base_dir, 'http_torrent_files') - os.mkdir(files_path) + files_path, self.file_server_port = prepare_xml_rss(self.session_base_dir, 'test_rss.xml') + shutil.copyfile(TORRENT_FILE, os.path.join(files_path, 'ubuntu.torrent')) shutil.copyfile(TORRENT_VIDEO_FILE, os.path.join(files_path, 'video.torrent')) shutil.copyfile(os.path.join(TESTS_DATA_DIR, 'ubuntu-logo14.png'), diff --git a/Tribler/Test/util.py b/Tribler/Test/util.py index 6e3f8e20ad1..e4e629724f6 100644 --- a/Tribler/Test/util.py +++ b/Tribler/Test/util.py @@ -36,11 +36,15 @@ import logging +import os import sys # logging.basicConfig() from twisted.python.log import addObserver +from Tribler.Core.Utilities.network_utils import get_random_port + + __all__ = ["process_unhandled_exceptions"] @@ -130,6 +134,23 @@ def check_exceptions(self): % (num_twisted_exceptions, self._twisted_exceptions[-1]['log_text'])) +def prepare_xml_rss(target_path, filename): + """ + Function to prepare test_rss.xml file, replace the port with a random one + """ + files_path = os.path.join(target_path, 'http_torrent_files') + os.mkdir(files_path) + + port = get_random_port() + + from Tribler.Test.test_as_server import TESTS_DATA_DIR + with open(os.path.join(TESTS_DATA_DIR, filename), 'r') as source_xml,\ + open(os.path.join(target_path, filename), 'w') as destination_xml: + for line in source_xml: + destination_xml.write(line.replace('RANDOMPORT', str(port))) + + return files_path, port + _catcher = UnhandledExceptionCatcher() _twisted_catcher = UnhandledTwistedExceptionCatcher() diff --git a/logger.conf b/logger.conf index af408213581..074db031cd2 100644 --- a/logger.conf +++ b/logger.conf @@ -1,5 +1,5 @@ [loggers] -keys=root,candidates,twisted,MetadataInjector,HiddenTunnelCommunity,TunnelMain,TunnelLogger,BarterCommunity,BarterCommunityCrawler, MultiChainCommunity +keys=root,candidates,twisted,MetadataInjector,HiddenTunnelCommunity,TunnelMain,TunnelLogger,BarterCommunity,BarterCommunityCrawler, BoostingManager, MultiChainCommunity, BoostingSource [handlers] keys=debugging,default @@ -64,6 +64,18 @@ qualname=MultiChainCommunity handlers=default propagate=0 +[logger_BoostingManager] +level=INFO +qualname=BoostingManager +handlers=default +propagate=0 + +[logger_BoostingSource] +level=INFO +qualname=BoostingSource +handlers=default +propagate=0 + [handler_default] class=StreamHandler level=NOTSET