diff --git a/Tribler/Test/Core/CreditMining/__init__.py b/Tribler/Test/Core/CreditMining/__init__.py new file mode 100644 index 00000000000..3dc228c3f6d --- /dev/null +++ b/Tribler/Test/Core/CreditMining/__init__.py @@ -0,0 +1,3 @@ +""" +This package contains tests for the credit mining-related code of Tribler. +""" diff --git a/Tribler/Test/Core/CreditMining/mock_creditmining.py b/Tribler/Test/Core/CreditMining/mock_creditmining.py new file mode 100644 index 00000000000..525abae90d5 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/mock_creditmining.py @@ -0,0 +1,138 @@ +""" +Module of Credit mining mock classes + +Written by Ardhi Putra Pratama H +""" +from twisted.web.resource import Resource + + +class MockLtTorrent(object): + """ + Class representing libtorrent handle for getting peer info + """ + def __init__(self, infohash="12345"): + self.info_hash = infohash + self.all_time_download = 0 + self.all_time_upload = 0 + + def upload_limit(self): + return 12 + + def max_uploads(self): + return 13 + + def max_connections(self): + return 14 + + def piece_priorities(self): + return [0, 1, 1, 0, 1, 1, 1] + + def get_peer_info(self): + """ + class returning peer info for a particular handle + """ + peer = [None] * 6 + peer[0] = MockLtPeer(1, "ip1") + peer[0].setvalue(True, True, True) + peer[1] = MockLtPeer(2, "ip2") + peer[1].setvalue(False, False, True) + peer[2] = MockLtPeer(3, "ip3") + peer[2].setvalue(True, False, True) + peer[3] = MockLtPeer(4, "ip4") + peer[3].setvalue(False, True, False) + peer[4] = MockLtPeer(5, "ip5") + peer[4].setvalue(False, True, True) + peer[5] = MockLtPeer(6, "ip6") + peer[5].setvalue(False, False, False) + return peer + + def is_valid(self): + """ + check whether the handle is valid or not + """ + return True + + def status(self): + return self + +class MockLtPeer(object): + """ + Dummy peer object returned by libtorrent python binding + """ + def __init__(self, pid, ip): + self.pid = pid + self.client = 2 + self.ip = [ip, "port"] + self.flags = 1 + self.local_connection = True + self.payload_up_speed = 0 + self.remote_interested = 1 + self.remote_choked = 1 + self.upload_queue_length = 0 + self.used_send_buffer = 0 + self.payload_down_speed = 0 + self.interesting = True + self.choked = True + self.total_upload = 0 + self.total_download = 0 + self.progress = 0 + self.pieces = 0 + self.remote_dl_rate = 0 + self.country = "ID" + self.connection_type = 0 + self.seed = 1 + self.upload_only = 1 + self.read_state = False + self.write_state = False + + def setvalue(self, upload_only, uinterested, completed): + self.upload_only = upload_only + self.remote_interested = uinterested + self.progress = 1 if completed else 0 + + +class MockMeta(object): + """ + class for mocking the torrent metainfo + """ + + def __init__(self, id_hash): + self.infohash = id_hash + + def get_infohash(self): + """ + returning infohash of torrents + """ + return self.infohash + + +class MockLtSession(object): + """ + Mock for session and LibTorrentMgr + """ + def __init__(self): + pass + + def get_session(self): + """ + supposed to get libtorrent session + """ + return self + + def set_settings(self, _): + """ + set settings (don't do anything) + """ + pass + + def shutdown(self): + """ + obligatory shutdown function + """ + pass + + +class ResourceFailClass(Resource): + def render_GET(self, request): + request.setResponseCode(503) + return "Error 503." diff --git a/Tribler/Test/Core/CreditMining/test_creditmining.py b/Tribler/Test/Core/CreditMining/test_creditmining.py new file mode 100644 index 00000000000..eeb2d85b195 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/test_creditmining.py @@ -0,0 +1,405 @@ +# coding=utf-8 +""" +Module of Credit mining function testing + +Written by Mihai Capotă and Ardhi Putra Pratama H +""" + +import binascii +import random +import re + +import Tribler.Policies.BoostingManager as bm +from Tribler.Core.DownloadConfig import DefaultDownloadStartupConfig +from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentDownloadImpl +from Tribler.Core.SessionConfig import SessionConfigInterface +from Tribler.Core.Utilities import utilities +from Tribler.Core.defaults import sessdefaults +from Tribler.Policies.BoostingPolicy import CreationDatePolicy, SeederRatioPolicy, RandomPolicy +from Tribler.Policies.BoostingSource import ent2chr +from Tribler.Policies.credit_mining_util import levenshtein_dist, source_to_string +from Tribler.Test.Core.CreditMining.mock_creditmining import MockMeta, MockLtPeer, MockLtSession, MockLtTorrent +from Tribler.Test.test_as_server import TestAsServer + + +class TestBoostingManagerPolicies(TestAsServer): + """ + The class to test core function of credit mining policies + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerPolicies, self).__init__(*argv, **kwargs) + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerPolicies, self).setUp() + self.session.get_download = lambda x: x % 2 + random.seed(0) + self.torrents = dict() + for i in xrange(1, 11): + mock_metainfo = MockMeta(i) + + self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": i, + "num_leechers": i-1, "creation_date": i} + + def test_random_policy(self): + """ + testing random policy + """ + policy = RandomPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(3, len(ids_start), "Start failed %s vs %s" % (ids_start, torrents_start)) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(2, len(ids_stop), "Stop failed %s vs %s" % (ids_stop, torrents_stop)) + + def test_seederratio_policy(self): + """ + testing seeder ratio policy + """ + policy = SeederRatioPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(ids_start, [10, 8, 6]) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(ids_stop, [3, 1]) + + def test_fallback_policy(self): + """ + testing policy (seederratio) and then fallback + """ + + for i in xrange(1, 11): + mock_metainfo = MockMeta(i) + self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": -i, + "num_leechers": -i, "creation_date": i} + + policy = SeederRatioPolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 6) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(3, len(ids_start), "Start failed %s vs %s" % (ids_start, torrents_start)) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(2, len(ids_stop), "Stop failed %s vs %s" % (ids_stop, torrents_stop)) + + def test_creationdate_policy(self): + """ + test policy based on creation date + """ + policy = CreationDatePolicy(self.session) + torrents_start, torrents_stop = policy.apply(self.torrents, 5, force=True) + ids_start = [torrent["metainfo"].get_infohash() for torrent in + torrents_start] + self.assertEqual(ids_start, [10, 8, 6]) + ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] + self.assertEqual(ids_stop, [5, 3, 1]) + + +class TestBoostingManagerUtilities(TestAsServer): + """ + Test several utilities used in credit mining + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerUtilities, self).__init__(*argv, **kwargs) + + self.peer = [None] * 6 + self.peer[0] = MockLtPeer(1, "ip1") + self.peer[0].setvalue(True, True, True) + self.peer[1] = MockLtPeer(2, "ip2") + self.peer[1].setvalue(False, False, True) + self.peer[2] = MockLtPeer(3, "ip3") + self.peer[2].setvalue(True, False, True) + self.peer[3] = MockLtPeer(4, "ip4") + self.peer[3].setvalue(False, True, False) + self.peer[4] = MockLtPeer(5, "ip5") + self.peer[4].setvalue(False, True, True) + self.peer[5] = MockLtPeer(6, "ip6") + self.peer[5].setvalue(False, False, False) + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerUtilities, self).setUp() + + self.session.get_libtorrent = lambda: True + + self.bsettings = bm.BoostingSettings(self.session) + self.bsettings.credit_mining_path = self.session_base_dir + self.bsettings.load_config = False + self.bsettings.check_dependencies = False + self.bsettings.initial_logging_interval = 900 + + def tearDown(self): + # TODO(ardhi) : remove it when Tribler free of singleton + # and 1 below + DefaultDownloadStartupConfig.delInstance() + + super(TestBoostingManagerUtilities, self).tearDown() + + def test_boosting_dependencies(self): + """ + Test whether boosting manager dependencies works or not. + + In all test, check dependencies always off. In production, it is on by default. + """ + self.bsettings.check_dependencies = True + self.bsettings.initial_swarm_interval = 9000 + self.bsettings.initial_tracker_interval = 9000 + self.bsettings.initial_logging_interval = 9000 + self.session.open_dbhandler = lambda _: None + self.session.lm.ltmgr = MockLtSession() + + self.session.get_torrent_checking = lambda: True + self.session.get_dispersy = lambda: True + self.session.get_torrent_store = lambda: True + self.session.get_enable_torrent_search = lambda: True + self.session.get_enable_channel_search = lambda: True + self.session.get_megacache = lambda: False + + self.assertRaises(AssertionError, bm.BoostingManager, self.session, self.bsettings) + + def test_load_default(self): + """ + Test load default configuration in BoostingManager + """ + self.bsettings.load_config = True + self.bsettings.auto_start_source = False + self.bsettings.initial_swarm_interval = 9000 + self.bsettings.initial_tracker_interval = 9000 + self.bsettings.initial_logging_interval = 9000 + self.session.open_dbhandler = lambda _: None + self.session.lm.ltmgr = MockLtSession() + + # it will automatically load the default configuration + boost_man = bm.BoostingManager(self.session, self.bsettings) + + # def validate(d_defer): + self.assertEqual(sessdefaults['credit_mining']['source_interval'], boost_man.settings.source_interval) + self.assertEqual(sessdefaults['credit_mining']['archive_sources'], + [source_to_string(src.source) for src in boost_man.boosting_sources.values() + if src.archive]) + + boost_man.cancel_all_pending_tasks() + + def test_sessionconfig(self): + """ + test basic credit mining preferences + """ + sci = SessionConfigInterface() + + sci.set_cm_logging_interval(100) + self.assertEqual(sci.get_cm_logging_interval(), 100) + + sci.set_cm_max_torrents_active(20) + self.assertEqual(sci.get_cm_max_torrents_active(), 20) + + sci.set_cm_max_torrents_per_source(10) + self.assertEqual(sci.get_cm_max_torrents_per_source(), 10) + + sci.set_cm_source_interval(100) + self.assertEqual(sci.get_cm_source_interval(), 100) + + sci.set_cm_policy("random") + self.assertIs(sci.get_cm_policy(as_class=True), RandomPolicy) + + sci.set_cm_policy(SeederRatioPolicy(self.session)) + self.assertEqual(sci.get_cm_policy(as_class=False), "seederratio") + + sci.set_cm_share_mode_target(2) + self.assertEqual(sci.get_cm_share_mode_target(), 2) + + sci.set_cm_swarm_interval(200) + self.assertEqual(sci.get_cm_swarm_interval(), 200) + + sci.set_cm_tracker_interval(300) + self.assertEqual(sci.get_cm_tracker_interval(), 300) + + def test_translate_peer_info(self): + """ + test - predict number of seeder and leecher only based on peer discovered and + their activities + """ + peerlist_dict = [] + for peer in self.peer: + peerlist_dict.append(LibtorrentDownloadImpl.create_peerlist_data(peer)) + + num_seed, num_leech = utilities.translate_peers_into_health(peerlist_dict) + self.assertEqual(num_seed, 4, "Seeder number don't match") + self.assertEqual(num_leech, 3, "Leecher number don't match") + + def test_levenshtein(self): + """ + test levenshtein between two string (in this case, file name) + + source : + http://people.cs.pitt.edu/~kirk/cs1501/Pruhs/Fall2006/Assignments/editdistance/Levenshtein%20Distance.htm + """ + string1 = "GUMBO" + string2 = "GAMBOL" + dist = levenshtein_dist(string1, string2) + dist_swap = levenshtein_dist(string2, string1) + + # random string check + self.assertEqual(dist, 2, "Wrong levenshtein distance") + self.assertEqual(dist_swap, 2, "Wrong levenshtein distance") + + string1 = "ubuntu-15.10-desktop-i386.iso" + string2 = "ubuntu-15.10-desktop-amd64.iso" + dist = levenshtein_dist(string1, string2) + + # similar filename check + self.assertEqual(dist, 4, "Wrong levenshtein distance") + + dist = levenshtein_dist(string1, string1) + # equal filename check + self.assertEqual(dist, 0, "Wrong levenshtein distance") + + string2 = "Learning-Ubuntu-Linux-Server.tgz" + dist = levenshtein_dist(string1, string2) + # equal filename check + self.assertEqual(dist, 28, "Wrong levenshtein distance") + + def test_update_statistics(self): + """ + test updating statistics of a torrent (pick a new one) + """ + self.session.open_dbhandler = lambda _: None + + infohash_1 = "a"*20 + infohash_2 = "b"*20 + torrents = { + infohash_1: { + "last_seeding_stats": { + "time_seeding": 100, + "value": 5 + } + }, + infohash_2: { + "last_seeding_stats": {} + } + } + + new_seeding_stats = { + "time_seeding": 110, + "value": 1 + } + new_seeding_stats_unexist = { + "time_seeding": 10, + "value": 8 + } + + self.session.lm.ltmgr = MockLtSession() + + boost_man = bm.BoostingManager(self.session, self.bsettings) + boost_man.torrents = torrents + + boost_man.update_torrent_stats(infohash_1, new_seeding_stats) + self.assertEqual(boost_man.torrents[infohash_1]['last_seeding_stats'] + ['value'], 1) + + boost_man.update_torrent_stats(infohash_2, new_seeding_stats_unexist) + self.assertEqual(boost_man.torrents[infohash_2]['last_seeding_stats'] + ['value'], 8) + + boost_man.cancel_all_pending_tasks() + + def test_escape_xml(self): + """ + testing escape symbols occured in xml/rss document file. + """ + re_symbols = re.compile(r'\&\#(x?[0-9a-fA-F]+);') + + ampersand_str = re_symbols.sub(ent2chr, '&') + self.assertEqual(ampersand_str, "&", "wrong ampersand conversion %s" % ampersand_str) + + str_123 = re_symbols.sub(ent2chr, "123") + self.assertEqual(str_123, "123", "wrong number conversion %s" % str_123) + + def test_logging(self): + self.session.open_dbhandler = lambda _: None + + infohash_1 = "a"*20 + infohash_2 = "b"*20 + torrents = { + infohash_1: { + "last_seeding_stats": { + "time_seeding": 100, + "value": 5 + } + }, + infohash_2: { + "last_seeding_stats": {} + } + } + self.session.lm.ltmgr = MockLtSession() + + boost_man = bm.BoostingManager(self.session, self.bsettings) + boost_man.torrents = torrents + + boost_man.session.lm.ltmgr.get_session().get_torrents = \ + lambda: [MockLtTorrent(binascii.hexlify(infohash_1)), + MockLtTorrent(binascii.hexlify(infohash_2))] + + boost_man.log_statistics() + boost_man.cancel_all_pending_tasks() + + +class TestBoostingManagerError(TestAsServer): + """ + Class to test a bunch of credit mining error handle + """ + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerError, self).setUp() + + self.session.open_dbhandler = lambda _: True + self.session.get_libtorrent = lambda: True + self.session.lm.ltmgr = MockLtSession() + + self.boost_setting = bm.BoostingSettings(self.session) + self.boost_setting.load_config = False + self.boost_setting.initial_logging_interval = 900 + self.boost_setting.check_dependencies = False + self.boosting_manager = bm.BoostingManager(self.session, self.boost_setting) + self.session.lm.boosting_manager = self.boosting_manager + + def tearDown(self): + DefaultDownloadStartupConfig.delInstance() + super(TestBoostingManagerError, self).tearDown() + + def test_insert_torrent_unknown_source(self): + """ + testing insert torrent on unknown source + """ + torrent = { + 'preload': False, + 'metainfo': MockMeta("1234"), + 'infohash': '12345' + } + + self.boosting_manager.on_torrent_insert(binascii.unhexlify("abcd" * 10), '12345', torrent) + self.assertNotIn('12345', self.boosting_manager.torrents) + + def test_unknown_source(self): + """ + testing uknkown source added to boosting source, and try to apply archive + on top of that + """ + unknown_key = "1234567890" + + sources = len(self.boosting_manager.boosting_sources.keys()) + self.boosting_manager.add_source(unknown_key) + self.boosting_manager.set_archive(unknown_key, False) + self.assertEqual(sources, len(self.boosting_manager.boosting_sources.keys()), "unknown source added") + + def test_failed_start_download(self): + """ + test assertion error then not download the actual torrent + """ + torrent = { + 'preload': False, + 'metainfo': MockMeta("1234") + } + self.session.lm.download_exists = lambda _: True + self.boosting_manager.start_download(torrent) + + self.assertNotIn('download', torrent, "%s downloading despite error" % torrent) diff --git a/Tribler/Test/Core/CreditMining/test_creditmining_sys.py b/Tribler/Test/Core/CreditMining/test_creditmining_sys.py new file mode 100644 index 00000000000..d79790f7826 --- /dev/null +++ b/Tribler/Test/Core/CreditMining/test_creditmining_sys.py @@ -0,0 +1,517 @@ +# coding=utf-8 +""" +Module of Credit mining function testing + +Written by Ardhi Putra Pratama H +""" +import binascii +import os +import shutil + +from twisted.internet import defer +from twisted.web.server import Site +from twisted.web.static import File + +from Tribler.Core.DownloadConfig import DefaultDownloadStartupConfig +from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Core.Utilities.twisted_thread import deferred, reactor +from Tribler.Core.simpledefs import NTFY_TORRENTS, NTFY_UPDATE +from Tribler.Main.Utility.GuiDBTuples import CollectedTorrent +from Tribler.Policies.BoostingManager import BoostingManager, BoostingSettings +from Tribler.Test.Core.CreditMining.mock_creditmining import MockLtTorrent, ResourceFailClass +from Tribler.Test.Core.Modules.RestApi.test_channels_endpoints import AbstractTestChannelsEndpoint +from Tribler.Test.test_as_server import TestAsServer, TESTS_DATA_DIR +from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_FILE_INFOHASH +from Tribler.Test.util import prepare_xml_rss +from Tribler.community.channel.community import ChannelCommunity +from Tribler.dispersy.dispersy import Dispersy +from Tribler.dispersy.endpoint import ManualEnpoint +from Tribler.dispersy.util import blocking_call_on_reactor_thread + + +class TestBoostingManagerSys(TestAsServer): + """ + base class to test base credit mining function + """ + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSys, self).setUp() + + self.set_boosting_settings() + + self.session.lm.ltmgr.get_session().find_torrent = lambda _: MockLtTorrent() + + self.boosting_manager = BoostingManager(self.session, self.bsettings) + + self.session.lm.boosting_manager = self.boosting_manager + + def set_boosting_settings(self): + """ + set settings in credit mining + """ + self.bsettings = BoostingSettings(self.session) + self.bsettings.credit_mining_path = os.path.join(self.session_base_dir, "credit_mining") + self.bsettings.load_config = False + self.bsettings.check_dependencies = False + self.bsettings.min_connection_start = -1 + self.bsettings.min_channels_start = -1 + + self.bsettings.max_torrents_active = 8 + self.bsettings.max_torrents_per_source = 5 + + self.bsettings.tracker_interval = 5 + self.bsettings.initial_tracker_interval = 5 + self.bsettings.logging_interval = 30 + self.bsettings.initial_logging_interval = 3 + + def setUpPreSession(self): + super(TestBoostingManagerSys, self).setUpPreSession() + + self.config.set_torrent_checking(True) + self.config.set_megacache(True) + self.config.set_dispersy(True) + self.config.set_torrent_store(True) + self.config.set_enable_torrent_search(True) + self.config.set_enable_channel_search(True) + self.config.set_libtorrent(True) + + def tearDown(self): + DefaultDownloadStartupConfig.delInstance() + self.boosting_manager.shutdown() + + super(TestBoostingManagerSys, self).tearDown() + + def check_torrents(self, src, defer_param=None, target=1): + """ + function to check if a torrent is already added to the source + + In this function, + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + if len(src_obj.torrents) < target: + reactor.callLater(1, self.check_torrents, src, defer_param, target=target) + else: + # notify torrent (emulate scraping) + self.boosting_manager.scrape_trackers() + + def _get_tor_dummy(_, keys=123, include_mypref=True): + """ + function to emulate get_torrent in torrent_db + """ + return {'C.torrent_id': 93, 'category': u'Compressed', 'torrent_id': 41, + 'infohash': src_obj.torrents.keys()[0], 'length': 1150844928, 'last_tracker_check': 10001, + 'myDownloadHistory': False, 'name': u'ubuntu-15.04-desktop-amd64.iso', + 'num_leechers': 999, 'num_seeders': 123, 'status': u'unknown', 'tracker_check_retries': 0} + self.boosting_manager.torrent_db.getTorrent = _get_tor_dummy + self.session.notifier.notify(NTFY_TORRENTS, NTFY_UPDATE, src_obj.torrents.keys()[0]) + + # log it + self.boosting_manager.log_statistics() + + defer_param.callback(src) + return defer_param + + def check_source(self, src, defer_param=None, ready=True): + """ + function to check if a source is ready initializing + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + + if not ready: + defer_param.callback(src) + elif not src_obj or not src_obj.ready: + reactor.callLater(1, self.check_source, src, defer_param) + else: + defer_param.callback(src) + + return defer_param + + +class TestBoostingManagerSysRSS(TestBoostingManagerSys): + """ + testing class for RSS (dummy) source + """ + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSysRSS, self).setUp() + + files_path, self.file_server_port = prepare_xml_rss(self.session_base_dir, 'test_rss_cm.xml') + + shutil.copyfile(TORRENT_FILE, os.path.join(files_path, 'ubuntu.torrent')) + self.setUpFileServer(self.file_server_port, self.session_base_dir) + + self.rss_error_deferred = defer.Deferred() + # now the rss should be at : + # http://localhost:port/test_rss_cm.xml + # which resides in sessiondir/http_torrent_files + + def set_boosting_settings(self): + super(TestBoostingManagerSysRSS, self).set_boosting_settings() + self.bsettings.auto_start_source = False + + def setUpFileServer(self, port, path): + resource = File(path) + resource.putChild("err503", ResourceFailClass()) + factory = Site(resource) + self._logger.debug("Listen to port %s, factory %s", port, factory) + self.file_server = reactor.listenTCP(port, factory) + + @deferred(timeout=15) + def test_rss(self): + """ + test rss source + """ + url = 'http://localhost:%s/test_rss_cm.xml' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj.start() + + d = self.check_source(url) + d.addCallback(self.check_torrents, target=1) + return d + + def _on_error_rss(self, dummy_1, dummy_2): + """ + dummy errback when RSS source produces an error + """ + self.rss_error_deferred.callback(True) + + @deferred(timeout=8) + def test_rss_unexist(self): + """ + Testing an unexisting RSS feed + """ + url = 'http://localhost:%s/nothingness' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj._on_error_rss = self._on_error_rss + rss_obj.start() + + defer_err_rss = self.check_source(url, ready=False) + defer_err_rss.chainDeferred(self.rss_error_deferred) + return defer_err_rss + + @deferred(timeout=8) + def test_rss_unavailable(self): + """ + Testing an unavailable RSS feed + """ + url = 'http://localhost:%s/err503' % self.file_server_port + self.boosting_manager.add_source(url) + + rss_obj = self.boosting_manager.get_source_object(url) + rss_obj._on_error_rss = self._on_error_rss + rss_obj.start() + + defer_err_rss = self.check_source(url, ready=False) + defer_err_rss.chainDeferred(self.rss_error_deferred) + return defer_err_rss + + +class TestBoostingManagerSysDir(TestBoostingManagerSys): + """ + testing class for directory source + """ + + @deferred(timeout=10) + def test_dir(self): + """ + test directory filled with .torrents + """ + self.boosting_manager.add_source(TESTS_DATA_DIR) + len_source = len(self.boosting_manager.boosting_sources) + + # deliberately try to add the same source + self.boosting_manager.add_source(TESTS_DATA_DIR) + self.assertEqual(len(self.boosting_manager.boosting_sources), len_source, "identical source added") + + dir_obj = self.boosting_manager.get_source_object(TESTS_DATA_DIR) + self.assertTrue(dir_obj.ready, "Not Ready") + + d = self.check_torrents(TESTS_DATA_DIR, target=2) + d.addCallback(lambda _: True) + return d + + @deferred(timeout=10) + def test_dir_archive_example(self): + """ + test archive mode. Use diretory because easier to fetch torrent + """ + self.boosting_manager.add_source(TESTS_DATA_DIR) + self.boosting_manager.set_archive(TESTS_DATA_DIR, True) + + dir_obj = self.boosting_manager.get_source_object(TESTS_DATA_DIR) + self.assertTrue(dir_obj.ready, "Not Ready") + + d = self.check_torrents(TESTS_DATA_DIR, target=2) + d.addCallback(lambda _: self.boosting_manager._select_torrent()) + return d + + +class TestBoostingManagerSysChannel(AbstractTestChannelsEndpoint, TestBoostingManagerSys): + """ + testing class for channel source + """ + + def __init__(self, *argv, **kwargs): + super(TestBoostingManagerSysChannel, self).__init__(*argv, **kwargs) + self.tdef = TorrentDef.load(TORRENT_FILE) + self.channel_id = 0 + + def setUp(self, autoload_discovery=True): + super(TestBoostingManagerSysChannel, self).setUp() + + def set_boosting_settings(self): + super(TestBoostingManagerSysChannel, self).set_boosting_settings() + self.bsettings.swarm_interval = 1 + self.bsettings.initial_swarm_interval = 1 + self.bsettings.max_torrents_active = 1 + self.bsettings.max_torrents_per_source = 1 + + def setUpPreSession(self): + super(TestBoostingManagerSysChannel, self).setUpPreSession() + + # we use dummy dispersy here + self.config.set_dispersy(False) + + @blocking_call_on_reactor_thread + def create_torrents_in_channel(self, dispersy_cid_hex): + """ + Helper function to insert 10 torrent into designated channel + """ + for i in xrange(0, 10): + self.insert_channel_in_db('rand%d' % i, 42 + i, 'Test channel %d' % i, 'Test description %d' % i) + + self.channel_id = self.insert_channel_in_db(dispersy_cid_hex.decode('hex'), 42, + 'Simple Channel', 'Channel description') + + torrent_list = [[self.channel_id, 1, 1, TORRENT_FILE_INFOHASH, 1460000000, TORRENT_FILE, + self.tdef.get_files_as_unicode_with_length(), self.tdef.get_trackers_as_single_tuple()]] + + self.insert_torrents_into_channel(torrent_list) + + @deferred(timeout=20) + def test_chn_lookup(self): + """ + testing channel source. + + It includes finding and downloading actual torrent + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + torrent = CollectedTorrent(torrent, self.tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + def check_torrents_channel(src, defer_param=None, target=1): + """ + check if a torrent already in channel and ready to download + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + success = True + if not src_obj or len(src_obj.torrents) < target: + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + elif not self.boosting_manager.torrents.get(TORRENT_FILE_INFOHASH, None): + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + elif not self.boosting_manager.torrents[TORRENT_FILE_INFOHASH].get('download', None): + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param, target=target) + + if success: + self.boosting_manager.set_enable_mining(src, False, force_restart=True) + if src_obj.community: + src_obj.community.cancel_all_pending_tasks() + + defer_param.callback(src) + + return defer_param + + chn_obj.torrent_mgr.load_torrent = _load + + d = self.check_source(dispersy_cid) + d.addCallback(check_torrents_channel, target=1) + return d + + @deferred(timeout=20) + def test_chn_exist_lookup(self): + """ + testing existing channel as a source. + + It also tests how boosting manager cope with unknown channel with retrying + the lookup + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + # channel is exist + community = ChannelCommunity.init_community(self.session.lm.dispersy, + self.session.lm.dispersy.get_member(mid=dispersy_cid), + self.session.lm.dispersy._communities['allchannel']._my_member, + self.session) + + # make the id unknown so boosting manager can test repeating search + id_tmp = community._channel_id + community._channel_id = 0 + + def _set_id_channel(channel_id): + """ + set channel id manually (emulate finding) + """ + community._channel_id = channel_id + + reactor.callLater(5, _set_id_channel, id_tmp) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + torrent = CollectedTorrent(torrent, self.tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + chn_obj.torrent_mgr.load_torrent = _load + + def clean_community(_): + """ + cleanly exit the community we are in + """ + if chn_obj.community: + chn_obj.community.cancel_all_pending_tasks() + + chn_obj.kill_tasks() + + + d = self.check_source(dispersy_cid) + d.addCallback(clean_community) + return d + + @deferred(timeout=20) + def test_chn_max_torrents(self): + """ + Test the restriction of max_torrents in a source. + """ + self.session.get_dispersy = lambda: True + self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + dispersy_cid_hex = "abcd" * 9 + "0012" + dispersy_cid = binascii.unhexlify(dispersy_cid_hex) + + # create channel and insert torrent + self.create_fake_allchannel_community() + self.create_torrents_in_channel(dispersy_cid_hex) + + pioneer_file = os.path.join(TESTS_DATA_DIR, "Pioneer.One.S01E06.720p.x264-VODO.torrent") + pioneer_tdef = TorrentDef.load(pioneer_file) + pioneer_ihash = binascii.unhexlify("66ED7F30E3B30FA647ABAA19A36E7503AA071535") + + torrent_list = [[self.channel_id, 1, 1, pioneer_ihash, 1460000001, pioneer_file, + pioneer_tdef.get_files_as_unicode_with_length(), pioneer_tdef.get_trackers_as_single_tuple()]] + self.insert_torrents_into_channel(torrent_list) + + self.boosting_manager.add_source(dispersy_cid) + chn_obj = self.boosting_manager.get_source_object(dispersy_cid) + chn_obj.max_torrents = 2 + chn_obj.torrent_mgr.load_torrent = lambda dummy_1, dummy_2: None + + def _load(torrent, callback=None): + if not isinstance(torrent, CollectedTorrent): + torrent_id = 0 + if torrent.torrent_id <= 0: + torrent_id = self.session.lm.torrent_db.getTorrentID(torrent.infohash) + if torrent_id: + torrent.update_torrent_id(torrent_id) + + infohash_str = binascii.hexlify(torrent.infohash) + torrent = CollectedTorrent(torrent, self.tdef if infohash_str.startswith("fc") else pioneer_tdef) + if callback is not None: + callback(torrent) + else: + return torrent + + def activate_mgr(): + """ + activate ltmgr and adjust max torrents to emulate overflow torrents + """ + chn_obj.max_torrents = 1 + chn_obj.torrent_mgr.load_torrent = _load + + reactor.callLater(5, activate_mgr) + + def check_torrents_channel(src, defer_param=None): + """ + check if a torrent already in channel and ready to download + """ + if defer_param is None: + defer_param = defer.Deferred() + + src_obj = self.boosting_manager.get_source_object(src) + success = True + if len(src_obj.unavail_torrent) == 0: + self.assertLessEqual(len(src_obj.torrents), src_obj.max_torrents) + else: + success = False + reactor.callLater(1, check_torrents_channel, src, defer_param) + + if success: + src_obj.community.cancel_all_pending_tasks() + src_obj.kill_tasks() + defer_param.callback(src) + + return defer_param + + d = self.check_source(dispersy_cid) + d.addCallback(check_torrents_channel) + return d + + def tearDown(self): + self.session.lm.dispersy._communities['allchannel'].cancel_all_pending_tasks() + self.session.lm.dispersy.cancel_all_pending_tasks() + self.session.lm.dispersy = None + super(TestBoostingManagerSysChannel, self).tearDown() diff --git a/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py b/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py index bd441d4f3b7..3215e85a9b6 100644 --- a/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py +++ b/Tribler/Test/Core/Modules/RestApi/test_channels_endpoints.py @@ -20,6 +20,8 @@ def setUp(self, autoload_discovery=True): self.votecast_db_handler = self.session.open_dbhandler(NTFY_VOTECAST) self.channel_db_handler._get_my_dispersy_cid = lambda: "myfakedispersyid" + self.create_votecast_called = False + def insert_channel_in_db(self, dispersy_cid, peer_id, name, description): return self.channel_db_handler.on_channel_from_dispersy(dispersy_cid, peer_id, name, description) @@ -29,6 +31,26 @@ def vote_for_channel(self, cid, vote_time): def insert_torrents_into_channel(self, torrent_list): self.channel_db_handler.on_torrents_from_dispersy(torrent_list) + @blocking_call_on_reactor_thread + def create_fake_allchannel_community(self): + """ + This method creates a fake AllChannel community so we can check whether a request is made in the community + when doing stuff with a channel. + """ + self.session.lm.dispersy._database.open() + fake_member = DummyMember(self.session.lm.dispersy, 1, "a" * 20) + member = self.session.lm.dispersy.get_new_member(u"curve25519") + fake_community = AllChannelCommunity.init_community(self.session.lm.dispersy, fake_member, member) + fake_community.disp_create_votecast = self.on_dispersy_create_votecast + self.session.lm.dispersy._communities = {"allchannel": fake_community} + return fake_community + + def on_dispersy_create_votecast(self, cid, vote, _): + """ + Check whether we have the expected parameters when this method is called. + """ + self.create_votecast_called = True + class TestChannelsEndpoint(AbstractTestChannelsEndpoint): @@ -132,39 +154,19 @@ def setUp(self, autoload_discovery=True): super(TestChannelsSubscriptionEndpoint, self).setUp(autoload_discovery) self.expected_votecast_cid = None self.expected_votecast_vote = None - self.create_votecast_called = False self.session.get_dispersy = lambda: True self.session.lm.dispersy = Dispersy(ManualEnpoint(0), self.getStateDir()) + self.create_fake_allchannel_community() for i in xrange(0, 10): self.insert_channel_in_db('rand%d' % i, 42 + i, 'Test channel %d' % i, 'Test description %d' % i) def on_dispersy_create_votecast(self, cid, vote, _): - """ - Check whether we have the expected parameters when this method is called. - """ + super(TestChannelsSubscriptionEndpoint, self).on_dispersy_create_votecast(cid, vote, _) self.assertEqual(cid, self.expected_votecast_cid) self.assertEqual(vote, self.expected_votecast_vote) - self.create_votecast_called = True - - @blocking_call_on_reactor_thread - def create_fake_allchannel_community(self): - """ - This method creates a fake AllChannel community so we can check whether a request is made in the community - when doing stuff with a channel. - """ - self.session.lm.dispersy._database.open() - fake_member = DummyMember(self.session.lm.dispersy, 1, "a" * 20) - member = self.session.lm.dispersy.get_new_member(u"curve25519") - fake_community = AllChannelCommunity(self.session.lm.dispersy, fake_member, member) - fake_community.disp_create_votecast = self.on_dispersy_create_votecast - self.session.lm.dispersy._communities = {"allchannel": fake_community} - - def tearDown(self): - self.session.lm.dispersy = None - super(TestChannelsSubscriptionEndpoint, self).tearDown() @deferred(timeout=10) def test_subscribe_channel_not_exist(self): @@ -229,3 +231,8 @@ def verify_votecast_made(_): self.expected_votecast_vote = VOTE_UNSUBSCRIBE return self.do_request('channels/subscribed/%s' % 'rand1'.encode('hex'), expected_code=200, expected_json=expected_json, request_type='DELETE').addCallback(verify_votecast_made) + + def tearDown(self): + self.session.lm.dispersy._communities['allchannel'].cancel_all_pending_tasks() + self.session.lm.dispersy = None + super(TestChannelsSubscriptionEndpoint, self).tearDown() diff --git a/Tribler/Test/data/test_rss_cm.xml b/Tribler/Test/data/test_rss_cm.xml new file mode 100644 index 00000000000..18044929497 --- /dev/null +++ b/Tribler/Test/data/test_rss_cm.xml @@ -0,0 +1,31 @@ + + + + + Test RSS for Credit Mining + test_rss_cm.xml + Test RSS CM feed. + + en + Tue, 12 May 2015 08:39:23 GMT + Tue, 12 May 2015 08:39:23 GMT + + + + + ubuntu-15.04-desktop-amd64.iso + http://localhost:RANDOMPORT/ubuntu.torrent + + http://localhost:RANDOMPORT.ubuntu.torrent + + + + Pioneer.One.S01E06.720p.x264-VODO.torrent + http://localhost:RANDOMPORT/pioneer.torrent + + http://localhost:RANDOMPORT.pioneer.torrent + + + diff --git a/Tribler/Test/test_BoostingManager.py b/Tribler/Test/test_BoostingManager.py deleted file mode 100644 index 48f47fc41c8..00000000000 --- a/Tribler/Test/test_BoostingManager.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# Written by Mihai Capotă -# pylint: disable=too-many-public-methods -"""Test Tribler.Policies.BoostingManager""" - -import mock -import random -import unittest - -import Tribler.Policies.BoostingManager as bm - -class TestBoostingManagerPolicies(unittest.TestCase): - - def setUp(self): - random.seed(0) - self.session = mock.Mock() - self.session.get_download = lambda i: i % 2 - self.torrents = dict() - for i in range(1, 11): - mock_metainfo = mock.Mock() - mock_metainfo.get_id.return_value = i - self.torrents[i] = {"metainfo": mock_metainfo, "num_seeders": i, - "num_leechers": i-1, "creation_date": i} - - def test_RandomPolicy(self): - policy = bm.RandomPolicy(self.session) - torrents_start, torrents_stop = policy.apply(self.torrents, 2) - ids_start = [torrent["metainfo"].get_infohash() for torrent in - torrents_start] - self.assertEqual(ids_start, [4, 8]) - ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] - self.assertEqual(ids_stop, [3, 9, 5, 7, 1]) - - def test_SeederRatioPolicy(self): - policy = bm.SeederRatioPolicy(self.session) - torrents_start, torrents_stop = policy.apply(self.torrents, 6) - ids_start = [torrent["metainfo"].get_infohash() for torrent in - torrents_start] - self.assertEqual(ids_start, [10, 8, 6]) - ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] - self.assertEqual(ids_stop, [3, 1]) - - def test_CreationDatePolicy(self): - policy = bm.CreationDatePolicy(self.session) - torrents_start, torrents_stop = policy.apply(self.torrents, 5) - ids_start = [torrent["metainfo"].get_infohash() for torrent in - torrents_start] - self.assertEqual(ids_start, [10, 8, 6]) - ids_stop = [torrent["metainfo"].get_infohash() for torrent in torrents_stop] - self.assertEqual(ids_stop, [5, 3, 1]) - -if __name__ == "__main__": - unittest.main() diff --git a/Tribler/Test/test_as_server.py b/Tribler/Test/test_as_server.py index 982abd49029..4eaf84ac38e 100644 --- a/Tribler/Test/test_as_server.py +++ b/Tribler/Test/test_as_server.py @@ -262,6 +262,7 @@ def setUpPreSession(self): self.config.set_upgrader_enabled(False) self.config.set_http_api_enabled(False) self.config.set_tunnel_community_enabled(False) + self.config.set_creditmining_enable(False) def tearDown(self): self.annotate(self._testMethodName, start=False) diff --git a/Tribler/Test/test_my_channel.py b/Tribler/Test/test_my_channel.py index b0106e95487..1b67946feaa 100644 --- a/Tribler/Test/test_my_channel.py +++ b/Tribler/Test/test_my_channel.py @@ -1,16 +1,15 @@ # Written by Niels Zeilemaker # see LICENSE.txt for license information -from binascii import hexlify import os import shutil -from Tribler.Core.Utilities.network_utils import get_random_port +from binascii import hexlify +from Tribler.Core.TorrentDef import TorrentDef from Tribler.Test.common import UBUNTU_1504_INFOHASH -from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_VIDEO_FILE from Tribler.Test.test_as_server import TestGuiAsServer, TESTS_DATA_DIR - -from Tribler.Core.TorrentDef import TorrentDef +from Tribler.Test.test_libtorrent_download import TORRENT_FILE, TORRENT_VIDEO_FILE +from Tribler.Test.util import prepare_xml_rss DEBUG = True @@ -21,15 +20,8 @@ def setUp(self): super(TestMyChannel, self).setUp() # Prepare test_rss.xml file, replace the port with a random one - self.file_server_port = get_random_port() - with open(os.path.join(TESTS_DATA_DIR, 'test_rss.xml'), 'r') as source_xml,\ - open(os.path.join(self.session_base_dir, 'test_rss.xml'), 'w') as destination_xml: - for line in source_xml: - destination_xml.write(line.replace('RANDOMPORT', str(self.file_server_port))) - - # Setup file server to serve torrent file and thumbnails - files_path = os.path.join(self.session_base_dir, 'http_torrent_files') - os.mkdir(files_path) + files_path, self.file_server_port = prepare_xml_rss(self.session_base_dir, 'test_rss.xml') + shutil.copyfile(TORRENT_FILE, os.path.join(files_path, 'ubuntu.torrent')) shutil.copyfile(TORRENT_VIDEO_FILE, os.path.join(files_path, 'video.torrent')) shutil.copyfile(os.path.join(TESTS_DATA_DIR, 'ubuntu-logo14.png'), diff --git a/Tribler/Test/util.py b/Tribler/Test/util.py index 6e3f8e20ad1..e4e629724f6 100644 --- a/Tribler/Test/util.py +++ b/Tribler/Test/util.py @@ -36,11 +36,15 @@ import logging +import os import sys # logging.basicConfig() from twisted.python.log import addObserver +from Tribler.Core.Utilities.network_utils import get_random_port + + __all__ = ["process_unhandled_exceptions"] @@ -130,6 +134,23 @@ def check_exceptions(self): % (num_twisted_exceptions, self._twisted_exceptions[-1]['log_text'])) +def prepare_xml_rss(target_path, filename): + """ + Function to prepare test_rss.xml file, replace the port with a random one + """ + files_path = os.path.join(target_path, 'http_torrent_files') + os.mkdir(files_path) + + port = get_random_port() + + from Tribler.Test.test_as_server import TESTS_DATA_DIR + with open(os.path.join(TESTS_DATA_DIR, filename), 'r') as source_xml,\ + open(os.path.join(target_path, filename), 'w') as destination_xml: + for line in source_xml: + destination_xml.write(line.replace('RANDOMPORT', str(port))) + + return files_path, port + _catcher = UnhandledExceptionCatcher() _twisted_catcher = UnhandledTwistedExceptionCatcher()