From bf20f5752dde8c293b45c3a7e997531e5657b6b4 Mon Sep 17 00:00:00 2001 From: Boudewijn Date: Tue, 7 May 2013 11:33:22 +0200 Subject: [PATCH 1/2] Fixes magnetlink unittest. The response protocol changed, instead of responding with data in a 'data' field inside the bencoded dict, the data is placed behind the bencoded dict. Added a sloppy_bdecode to bencode.py to allow us to get the offset where the bencoded dict ends and the data begins. Also, libtorrent does not reject duplicate requests. Instead it gives a normal response and outputs the following warning: LibtorrentDownloadImpl: alert performance_alert with message file.wmv: performance warning: max outstanding piece requests reached Finally, variables named responce were renamed to response. --- Tribler/Core/Utilities/bencode.py | 13 ++++-- Tribler/Test/test_magnetlink.py | 76 +++++++++++++++++-------------- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/Tribler/Core/Utilities/bencode.py b/Tribler/Core/Utilities/bencode.py index 07093555ba1..5d40d1b7e8a 100644 --- a/Tribler/Core/Utilities/bencode.py +++ b/Tribler/Core/Utilities/bencode.py @@ -83,6 +83,15 @@ def decode_dict(x, f): #decode_func['u'] = decode_unicode def bdecode(x, sloppy = 0): + r, l = sloppy_bdecode(x) + if not sloppy and l != len(x): + raise ValueError, "bad bencoded data" + return r + +def sloppy_bdecode(x): + """ + Same as bdecode, except that it returns the decoded data AND the number of bytes read from X. + """ try: r, l = decode_func[x[0]](x, 0) # except (IndexError, KeyError): @@ -90,9 +99,7 @@ def bdecode(x, sloppy = 0): if DEBUG: print_exc() raise ValueError, "bad bencoded data" - if not sloppy and l != len(x): - raise ValueError, "bad bencoded data" - return r + return r, l def test_bdecode(): try: diff --git a/Tribler/Test/test_magnetlink.py b/Tribler/Test/test_magnetlink.py index ef480efefeb..ca46aeb3df1 100644 --- a/Tribler/Test/test_magnetlink.py +++ b/Tribler/Test/test_magnetlink.py @@ -11,7 +11,7 @@ from btconn import BTConnection from Tribler.Core.TorrentDef import TorrentDef from Tribler.Core.DownloadConfig import DownloadStartupConfig -from Tribler.Core.Utilities.bencode import bencode, bdecode +from Tribler.Core.Utilities.bencode import bencode, bdecode, sloppy_bdecode from Tribler.Core.MessageID import EXTEND from Tribler.Core.simpledefs import dlstatus_strings, DLSTATUS_SEEDING from Tribler.Core.DecentralizedTracking.MagnetLink.MagnetLink import MagnetHandler @@ -54,24 +54,24 @@ def metadata_id_from_extend_handshake(self, data): return val def read_extend_handshake(self, conn): - responce = conn.recv() - self.assert_(len(responce) > 0) - # print >>sys.stderr,"test: Got reply", getMessageName(responce[0]) - self.assert_(responce[0] == EXTEND) - return self.metadata_id_from_extend_handshake(responce[1:]) + response = conn.recv() + self.assert_(len(response) > 0) + # print >>sys.stderr,"test: Got reply", getMessageName(response[0]) + self.assert_(response[0] == EXTEND) + return self.metadata_id_from_extend_handshake(response[1:]) def read_extend_metadata_request(self, conn): while True: - responce = conn.recv() - assert len(responce) > 0 - # print >>sys.stderr,"test: Got data", getMessageName(responce[0]) - if responce[0] == EXTEND: + response = conn.recv() + assert len(response) > 0 + # print >>sys.stderr,"test: Got data", getMessageName(response[0]) + if response[0] == EXTEND: break - assert responce[0] == EXTEND - assert ord(responce[1]) == 42 + assert response[0] == EXTEND + assert ord(response[1]) == 42 - payload = bdecode(responce[2:]) + payload = bdecode(response[2:]) assert "msg_type" in payload assert payload["msg_type"] == 0 assert "piece" in payload @@ -83,36 +83,46 @@ def read_extend_metadata_reply(self, conn, piece): while True: response = conn.recv() assert len(response) > 0 - # print >>sys.stderr,"test: Got data", getMessageName(responce[0]) + # print >>sys.stderr,"test: Got data", getMessageName(response[0]) if response[0] == EXTEND: break assert response[0] == EXTEND assert ord(response[1]) == 42 - try: - payload = bdecode(response[2:]) - assert payload["msg_type"] == 1 - assert payload["piece"] == piece + payload, length = sloppy_bdecode(response[2:]) + assert payload["msg_type"] == 1 + assert payload["piece"] == piece + if "data" in payload: assert payload["data"] == self.metadata_list[piece] - except: - print_exc() - print >> sys.stderr, response[2:] + else: + assert response[2 + length:] == self.metadata_list[piece] def read_extend_metadata_reject(self, conn, piece): while True: - responce = conn.recv() - assert len(responce) > 0 - # print >>sys.stderr,"test: Got reject", getMessageName(responce[0]) - if responce[0] == EXTEND: + response = conn.recv() + assert len(response) > 0 + # print >>sys.stderr,"test: Got reject", getMessageName(response[0]) + if response[0] == EXTEND: break - assert responce[0] == EXTEND - assert ord(responce[1]) == 42 + assert response[0] == EXTEND + assert ord(response[1]) == 42 + + payload, length = sloppy_bdecode(response[2:]) + assert payload["msg_type"] in (1, 2), [payload, response[2:2 + length]] + assert payload["piece"] == piece, [payload, response[2:2 + length]] - payload = bdecode(responce[2:]) - assert payload["msg_type"] == 2 - assert payload["piece"] == piece + # some clients return msg_type 1, unfortunately this is not a reject but a proper response. + # instead libtorrent warns: max outstanding piece requests reached + if payload["msg_type"] == 1: + assert response[2 + length:] == self.metadata_list[piece] + + # some clients return msg_type 2, we must make sure no "data" is given (i.e. the request was + # rejected) + if payload["msg_type"] == 2: + assert payload["piece"] == piece, [payload, response[2:2 + length]] + assert not "data" in payload, [payload, response[2:2 + length]] def read_extend_metadata_close(self, conn): """ @@ -121,10 +131,10 @@ def read_extend_metadata_close(self, conn): """ conn.s.settimeout(10.0) while True: - responce = conn.recv() - if len(responce) == 0: + response = conn.recv() + if len(response) == 0: break - assert not (responce[0] == EXTEND and responce[1] == 42) + assert not (response[0] == EXTEND and response[1] == 42) class TestMagnetMiniBitTorrent(TestAsServer, MagnetHelpers): """ From 6ba52ececdf800d1f3b11c072afa072ca877c269 Mon Sep 17 00:00:00 2001 From: Boudewijn Date: Tue, 7 May 2013 11:41:54 +0200 Subject: [PATCH 2/2] We use nosetests to run all unitests. --- dispersy-test | 90 --------------------------------------------------- 1 file changed, 90 deletions(-) delete mode 100755 dispersy-test diff --git a/dispersy-test b/dispersy-test deleted file mode 100755 index 1555faece94..00000000000 --- a/dispersy-test +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -BRANCH="." -if [ -d "$1" ]; then - BRANCH="$1" -fi - -echo "================================================================================" -echo "Testcases in __debug__ mode" -echo "================================================================================" - -rm -f sqlite/dispersy.db -rm -f sqlite/dispersy.db-journal -rm -f dispersy.log - -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-batch || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-classification || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-crypto || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-destroy-community || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-dynamic-settings || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-identical-payload || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-member-tag || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-missing-message || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-signature || exit 1 -# python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-subjective-set || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-sync || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-timeline || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-undo || exit 1 - -rm -f sqlite/dispersy.db -rm -f sqlite/dispersy.db-journal -rm -f dispersy.log - -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-batch || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-classification || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-crypto || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-destroy-community || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-dynamic-settings || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-identical-payload || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-member-tag || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-missing-message || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-signature || exit 1 -# python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-subjective-set || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-sync || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-timeline || exit 1 -python "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-undo || exit 1 - -echo "================================================================================" -echo "Testcases in optimized mode" -echo "================================================================================" - -rm -f sqlite/dispersy.db -rm -f sqlite/dispersy.db-journal -rm -f dispersy.log - -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-batch || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-classification || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-crypto || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-destroy-community || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-dynamic-settings || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-identical-payload || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-member-tag || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-missing-message || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-signature || exit 1 -# python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-subjective-set || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-sync || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-timeline || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --enable-dispersy-script --script dispersy-undo || exit 1 - -rm -f sqlite/dispersy.db -rm -f sqlite/dispersy.db-journal -rm -f dispersy.log - -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-batch || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-classification || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-crypto || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-destroy-community || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-dynamic-settings || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-identical-payload || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-member-tag || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-missing-message || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-signature || exit 1 -# python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-subjective-set || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-sync || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-timeline || exit 1 -python -O "$BRANCH/Tribler/Main/dispersy.py" --swiftproc --enable-dispersy-script --script dispersy-undo || exit 1 - -echo "================================================================================" -echo "Finished testcases successfully" -echo "================================================================================"