From 917e854ac3610b542954eabaddc9af00ca2df312 Mon Sep 17 00:00:00 2001 From: DanGraur Date: Sun, 28 Oct 2018 16:06:34 +0100 Subject: [PATCH] Added a new endpoint for retrieving the latest TrustChain block via the DHT mechanism. Added a new callback in the TrustChainCommunity class which is called whenever a block is being added in the DB, irrelevant of the block's type. It is now possible to retrieve the latest TrustChain block of some peer via a HTTP request. Created a new test set for the newly added block retrieval mechanism. In addition to this, made a few changes to the REST API test suite in terms of the communication modules, such as moving classes from one module to another and creating new modules which host logic for HTTP requests and communication for specific endpoints. --- ipv8/REST/dht_endpoint.py | 91 +++- ipv8/attestation/trustchain/community.py | 11 + .../test/REST/attestationendpoint/__init__.py | 0 .../peer_communication.py | 7 - .../rest_peer_communication.py | 390 ++++++++++++++++++ .../test_attestation_endpoint.py | 14 +- ipv8/test/REST/dht/__init__.py | 0 ipv8/test/REST/dht/rest_peer_communication.py | 43 ++ ipv8/test/REST/dht/test_dht_endpoint.py | 160 +++++++ ipv8/test/mocking/rest/comunities.py | 27 ++ ipv8/test/mocking/rest/ipv8.py | 6 +- .../mocking/rest/rest_peer_communication.py | 362 +--------------- test_classes_list.txt | 3 +- 13 files changed, 735 insertions(+), 379 deletions(-) create mode 100644 ipv8/test/REST/attestationendpoint/__init__.py rename ipv8/test/{mocking/rest => REST/attestationendpoint}/peer_communication.py (96%) create mode 100644 ipv8/test/REST/attestationendpoint/rest_peer_communication.py rename ipv8/test/REST/{ => attestationendpoint}/test_attestation_endpoint.py (98%) create mode 100644 ipv8/test/REST/dht/__init__.py create mode 100644 ipv8/test/REST/dht/rest_peer_communication.py create mode 100644 ipv8/test/REST/dht/test_dht_endpoint.py diff --git a/ipv8/REST/dht_endpoint.py b/ipv8/REST/dht_endpoint.py index 4a825ab7f..9d34d2151 100644 --- a/ipv8/REST/dht_endpoint.py +++ b/ipv8/REST/dht_endpoint.py @@ -1,13 +1,17 @@ from __future__ import absolute_import -from base64 import b64encode +from base64 import b64encode, b64decode from binascii import hexlify, unhexlify +from hashlib import sha1 import json +import struct +from twisted.internet.defer import inlineCallbacks, returnValue from twisted.web import http, resource from twisted.web.server import NOT_DONE_YET -from ..dht.community import DHTCommunity +from ..dht.community import DHTCommunity, MAX_ENTRY_SIZE +from ..attestation.trustchain.community import TrustChainCommunity from ..dht.discovery import DHTDiscoveryCommunity @@ -20,10 +24,87 @@ def __init__(self, session): resource.Resource.__init__(self) dht_overlays = [overlay for overlay in session.overlays if isinstance(overlay, DHTCommunity)] + tc_overlays = [overlay for overlay in session.overlays if isinstance(overlay, TrustChainCommunity)] if dht_overlays: - self.putChild("statistics", DHTStatisticsEndpoint(dht_overlays[0])) - self.putChild("values", DHTValuesEndpoint(dht_overlays[0])) - self.putChild("peers", DHTPeersEndpoint(dht_overlays[0])) + self.putChild(b"statistics", DHTStatisticsEndpoint(dht_overlays[0])) + self.putChild(b"values", DHTValuesEndpoint(dht_overlays[0])) + self.putChild(b"peers", DHTPeersEndpoint(dht_overlays[0])) + self.putChild(b"block", DHTBlockEndpoint(dht_overlays[0], tc_overlays[0])) + + +class DHTBlockEndpoint(resource.Resource): + """ + This endpoint is responsible for returning the latest Trustchain block of a peer. Additionally, it ensures + this peer's latest TC block is available + """ + + KEY_SUFFIX = b'_BLOCK' + + def __init__(self, dht, trustchain): + resource.Resource.__init__(self) + self.dht = dht + self.trustchain = trustchain + self.block_version = 0 + + self._hashed_dht_key = sha1(self.trustchain.my_peer.mid + self.KEY_SUFFIX).digest() + + trustchain.set_new_block_cb(self.publish_latest_block) + + @inlineCallbacks + def publish_latest_block(self): + """ + Publish the latest block of this node's Trustchain to the DHT + + :return: + """ + # latest_block = self.trustchain.persistence.get_latest(self.trustchain.my_peer.key.pub().key_to_bin()) + latest_block = self.trustchain.persistence.get_latest(self.trustchain.my_peer.public_key.key_to_bin()) + + if latest_block: + latest_block = latest_block.pack() + version = struct.pack("H", self.block_version) + self.block_version += 1 + + for i in range(0, len(latest_block), MAX_ENTRY_SIZE - 3): + blob_chunk = version + latest_block[i:i + MAX_ENTRY_SIZE - 3] + yield self.dht.store_value(self._hashed_dht_key, blob_chunk) + + def render_GET(self, request): + """ + Return the latest TC block of a peer, as identified in the request + + :param request: the request for retrieving the latest TC block of a peer. It must contain the peer's + public key of the peer + :return: the latest block of the peer, if found + """ + if not self.dht: + request.setResponseCode(http.NOT_FOUND) + return json.dumps({"error": "DHT community not found"}).encode('utf-8') + + if not request.args or b'public_key' not in request.args: + request.setResponseCode(http.BAD_REQUEST) + return json.dumps({"error": "Must specify the peer's public key"}).encode('utf-8') + + hash_key = sha1(b64decode(request.args[b'public_key'][0]) + self.KEY_SUFFIX).digest() + block_chunks = self.dht.storage.get(hash_key) + + if not block_chunks: + request.setResponseCode(http.NOT_FOUND) + return json.dumps({"error": "Could not find a block for the specified key."}).encode('utf-8') + + new_blocks = {} + max_version = 0 + + for entry in block_chunks: + this_version = struct.unpack("I", entry[1:3] + b'\x00\x00')[0] + max_version = max_version if max_version > this_version else this_version + + if this_version in new_blocks: + new_blocks[this_version] = entry[3:] + new_blocks[this_version] + else: + new_blocks[this_version] = entry[3:] + + return json.dumps({"block": b64encode(new_blocks[max_version]).decode('utf-8')}).encode('utf-8') class DHTStatisticsEndpoint(resource.Resource): diff --git a/ipv8/attestation/trustchain/community.py b/ipv8/attestation/trustchain/community.py index d9ab51743..157e92bb1 100644 --- a/ipv8/attestation/trustchain/community.py +++ b/ipv8/attestation/trustchain/community.py @@ -73,6 +73,8 @@ def __init__(self, *args, **kwargs): self.db_cleanup_lc = self.register_task("db_cleanup", LoopingCall(self.do_db_cleanup)) self.db_cleanup_lc.start(600) + self.new_block_cb = lambda: None + self.decode_map.update({ chr(1): self.received_half_block, chr(2): self.received_crawl_request, @@ -83,6 +85,13 @@ def __init__(self, *args, **kwargs): chr(7): self.received_empty_crawl_response, }) + def set_new_block_cb(self, f): + """ + Set the callback for when a new block is added to the DB. The callback should take no arguments and should + return nothing: [] -> None + """ + self.new_block_cb = f + def do_db_cleanup(self): """ Cleanup the database if necessary. @@ -251,6 +260,7 @@ def sign_block(self, peer, public_key=EMPTY_PK, block_type=b'unknown', transacti if not self.persistence.contains(block): self.persistence.add_block(block) + self.new_block_cb() self.notify_listeners(block) # This is a source block with no counterparty @@ -342,6 +352,7 @@ def validate_persist_block(self, block): pass elif not self.persistence.contains(block): self.persistence.add_block(block) + self.new_block_cb() self.notify_listeners(block) return validation diff --git a/ipv8/test/REST/attestationendpoint/__init__.py b/ipv8/test/REST/attestationendpoint/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ipv8/test/mocking/rest/peer_communication.py b/ipv8/test/REST/attestationendpoint/peer_communication.py similarity index 96% rename from ipv8/test/mocking/rest/peer_communication.py rename to ipv8/test/REST/attestationendpoint/peer_communication.py index 5b7cbbd54..6a059c92f 100644 --- a/ipv8/test/mocking/rest/peer_communication.py +++ b/ipv8/test/REST/attestationendpoint/peer_communication.py @@ -1,4 +1,3 @@ - from abc import abstractmethod @@ -111,9 +110,3 @@ def make_outstanding_verify(self, param_dict): :return: None """ pass - - -class RequestException(Exception): - """ - Custom exception used to model request errors - """ diff --git a/ipv8/test/REST/attestationendpoint/rest_peer_communication.py b/ipv8/test/REST/attestationendpoint/rest_peer_communication.py new file mode 100644 index 000000000..c54aa7bb5 --- /dev/null +++ b/ipv8/test/REST/attestationendpoint/rest_peer_communication.py @@ -0,0 +1,390 @@ +from twisted.internet.defer import inlineCallbacks, returnValue + +from .peer_communication import IGetStyleRequestsAE, IPostStyleRequestsAE +from ...mocking.rest.rest_peer_communication import HTTPRequester, RequestException, process_json_response + + +class HTTPGetRequesterAE(IGetStyleRequestsAE, HTTPRequester): + """ + Implements the GetStyleRequests abstract methods using the HTTP protocol for the attestation endpoint. + """ + + def __init__(self): + IGetStyleRequestsAE.__init__(self) + HTTPRequester.__init__(self) + + @process_json_response + @inlineCallbacks + def make_outstanding(self, param_dict): + """ + Forward a request for outstanding attestation requests. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + {'type': 'outstanding'}, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_verification_output(self, param_dict): + """ + Forward a request for the verification outputs. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + {'type': 'verification_output'}, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_peers(self, param_dict): + """ + Forward a request for the known peers in the network. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + {'type': 'peers'}, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_attributes(self, param_dict): + """ + Forward a request for the attributes of a peer. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (attributes), and the (optional) b64_mid of the attester + request_parameters = param_dict.get('request_parameters', dict()) + request_parameters.update({'type': 'attributes'}) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_dht_block(self, param_dict): + """ + Forward a request for the latest TC block of a peer + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'public_key': the public key of the peer whose latest TC block is being requested + (optional) 'callback': single parameter callback for the request's response + } + :return: None + :raises RequestException: raised when the method could not find one of the required pieces of information + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + request_parameters = {} + + if 'public_key' in param_dict: + request_parameters['public_key'] = param_dict['public_key'] + else: + raise RequestException("Malformed request: did not specify the public_key") + + response = yield self.make_request("http://{0}:{1}/{2}".format(interface, port, endpoint), + 'GET', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + @inlineCallbacks + def make_drop_identity(self, param_dict): + """ + Forward a request for dropping a peer's identity. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + {'type': 'drop_identity'}, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_outstanding_verify(self, param_dict): + """ + Forward a request which requests information on the outstanding verify requests + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (request), and the rest of the parameters + request_parameters = {'type': 'outstanding_verify'} + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + +class HTTPPostRequesterAE(IPostStyleRequestsAE, HTTPRequester): + """ + Implements the PostStyleRequests abstract methods using the HTTP protocol for the AttestationEndpoint + """ + + def __init__(self): + IPostStyleRequestsAE.__init__(self) + HTTPRequester.__init__(self) + + @inlineCallbacks + def make_attestation_request(self, param_dict): + """ + Forward a request for the attestation of an attribute. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'attribute_name': attribute_name + 'mid': attester b64_mid + (optional) 'metadata': JSON style metadata required for the attestation process + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (request), and the rest of the parameters + request_parameters = {'type': 'request'} + + # Add the request parameters one-by-one; if required parameter is missing, then raise error + if 'attribute_name' in param_dict: + request_parameters['attribute_name'] = param_dict['attribute_name'] + else: + raise RequestException("Malformed request: did not specify the attribute_name") + + if 'mid' in param_dict: + request_parameters['mid'] = param_dict['mid'] + else: + raise RequestException("Malformed request: did not specify the attester's mid") + + if 'metadata' in param_dict: + request_parameters['metadata'] = param_dict['metadata'] + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'POST', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + @inlineCallbacks + def make_attest(self, param_dict): + """ + Forward a request which attests an attestation request. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'attribute_name': attribute_name + 'mid': attestee's b64_mid + 'attribute_value': b64 hash of the attestation blob + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (attest), and the rest of the parameters + request_parameters = {'type': 'attest'} + + # Add the request parameters one-by-one; if required parameter is missing, then raise error + if 'attribute_name' in param_dict: + request_parameters['attribute_name'] = param_dict['attribute_name'] + else: + raise RequestException("Malformed request: did not specify the attribute_name") + + if 'mid' in param_dict: + request_parameters['mid'] = param_dict['mid'] + else: + raise RequestException("Malformed request: did not specify the attestee's mid") + + if 'attribute_value' in param_dict: + request_parameters['attribute_value'] = param_dict['attribute_value'] + else: + raise RequestException("Malformed request: did not specify the attribute_value, i.e. the attestation" + "blob hash") + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'POST', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + @inlineCallbacks + def make_verify(self, param_dict): + """ + Forward a request which demands the verification of an attestation + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'attribute_hash': the b64 hash of the attestation blob which needs to be verified + 'mid': verifier's b64_mid + 'attribute_values': a string of b64 encoded values, which are separated by ',' characters + e.g. "val_1,val_2,val_3, ..., val_N" + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (attest), and the rest of the parameters + request_parameters = {'type': 'verify'} + + # Add the request parameters one-by-one; if required parameter is missing, then raise error + if 'attribute_hash' in param_dict: + request_parameters['attribute_hash'] = param_dict['attribute_hash'] + else: + raise RequestException("Malformed request: did not specify the attribute_hash") + + if 'mid' in param_dict: + request_parameters['mid'] = param_dict['mid'] + else: + raise RequestException("Malformed request: did not specify the verifier's mid") + + if 'attribute_values' in param_dict: + request_parameters['attribute_values'] = param_dict['attribute_values'] + else: + raise RequestException("Malformed request: did not specify the attribute_values") + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'POST', + request_parameters, + param_dict.get('callback', None)) + + returnValue(response) + + @inlineCallbacks + def make_allow_verify(self, param_dict): + """ + Forward a request which requests that verifications be allowed for a particular peer for a particular attribute + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'attribute_name': attribute_name + 'mid': verifier's b64_mid + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (request), and the rest of the parameters + request_parameters = {'type': 'allow_verify'} + + # Add the request parameters one-by-one; if required parameter is missing, then raise error + if 'attribute_name' in param_dict: + request_parameters['attribute_name'] = param_dict['attribute_name'] + else: + raise RequestException("Malformed request: did not specify the attribute_name") + + if 'mid' in param_dict: + request_parameters['mid'] = param_dict['mid'] + else: + raise RequestException("Malformed request: did not specify the attester's mid") + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'POST', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) diff --git a/ipv8/test/REST/test_attestation_endpoint.py b/ipv8/test/REST/attestationendpoint/test_attestation_endpoint.py similarity index 98% rename from ipv8/test/REST/test_attestation_endpoint.py rename to ipv8/test/REST/attestationendpoint/test_attestation_endpoint.py index 2a6fd6404..c3b91f2a4 100644 --- a/ipv8/test/REST/test_attestation_endpoint.py +++ b/ipv8/test/REST/attestationendpoint/test_attestation_endpoint.py @@ -3,13 +3,13 @@ from json import dumps from twisted.internet.defer import returnValue, inlineCallbacks -from ..mocking.rest.base import RESTTestBase -from ..mocking.rest.peer_interactive_behavior import RequesterRestTestPeer -from ..mocking.rest.rest_peer_communication import string_to_url -from ..mocking.rest.rest_api_peer import RestTestPeer -from ..mocking.rest.rest_peer_communication import HTTPGetRequesterAE, HTTPPostRequesterAE -from ..mocking.rest.comunities import TestAttestationCommunity, TestIdentityCommunity -from ...attestation.identity.community import IdentityCommunity +from .rest_peer_communication import HTTPGetRequesterAE, HTTPPostRequesterAE +from ...mocking.rest.base import RESTTestBase +from ...mocking.rest.peer_interactive_behavior import RequesterRestTestPeer +from ...mocking.rest.rest_peer_communication import string_to_url +from ...mocking.rest.rest_api_peer import RestTestPeer +from ...mocking.rest.comunities import TestAttestationCommunity, TestIdentityCommunity +from ....attestation.identity.community import IdentityCommunity class TestAttestationEndpoint(RESTTestBase): diff --git a/ipv8/test/REST/dht/__init__.py b/ipv8/test/REST/dht/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ipv8/test/REST/dht/rest_peer_communication.py b/ipv8/test/REST/dht/rest_peer_communication.py new file mode 100644 index 000000000..22225c515 --- /dev/null +++ b/ipv8/test/REST/dht/rest_peer_communication.py @@ -0,0 +1,43 @@ +from twisted.internet.defer import inlineCallbacks, returnValue + +from ...mocking.rest.rest_peer_communication import HTTPRequester, RequestException, process_json_response + + +class HTTPGetRequesterDHT(HTTPRequester): + """ + Implements the HTTP GET type requests for the DHT endpoint + """ + + def __init__(self): + HTTPRequester.__init__(self) + + @process_json_response + @inlineCallbacks + def make_dht_block(self, param_dict): + """ + Forward a request for the latest TC block of a peer + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'public_key': the public key of the peer whose latest TC block is being requested + (optional) 'callback': single parameter callback for the request's response + } + :return: None + :raises RequestException: raised when the method could not find one of the required pieces of information + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + request_parameters = {} + + if 'public_key' in param_dict: + request_parameters['public_key'] = param_dict['public_key'] + else: + raise RequestException("Malformed request: did not specify the public_key") + + response = yield self.make_request("http://{0}:{1}/{2}".format(interface, port, endpoint), + 'GET', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) diff --git a/ipv8/test/REST/dht/test_dht_endpoint.py b/ipv8/test/REST/dht/test_dht_endpoint.py new file mode 100644 index 000000000..a2d79a54a --- /dev/null +++ b/ipv8/test/REST/dht/test_dht_endpoint.py @@ -0,0 +1,160 @@ +import struct + +from hashlib import sha1 +from base64 import b64encode, b64decode +from twisted.internet.defer import inlineCallbacks + +from .rest_peer_communication import HTTPGetRequesterDHT +from ...attestation.trustchain.test_block import TestBlock +from ...mocking.rest.base import RESTTestBase +from ...mocking.rest.rest_peer_communication import string_to_url +from ...mocking.rest.rest_api_peer import RestTestPeer +from ...mocking.rest.comunities import TestDHTCommunity, TestTrustchainCommunity +from ....attestation.trustchain.payload import HalfBlockPayload +from ....attestation.trustchain.community import TrustChainCommunity +from ....dht.community import DHTCommunity, MAX_ENTRY_SIZE +from ....REST.dht_endpoint import DHTBlockEndpoint +from ....messaging.serialization import Serializer + + +class TestDHTEndpoint(RESTTestBase): + """ + Class for testing the DHT Endpoint in the REST API of the IPv8 object + """ + + def setUp(self): + super(TestDHTEndpoint, self).setUp() + self.initialize([(2, RestTestPeer)], HTTPGetRequesterDHT(), None) + self.serializer = Serializer() + + def create_new_peer(self, peer_cls, port, *args, **kwargs): + self._create_new_peer_inner(peer_cls, port, [TestDHTCommunity, TestTrustchainCommunity], *args, **kwargs) + + @inlineCallbacks + def publish_to_DHT(self, peer, key, data, numeric_version): + """ + Publish data to the DHT via a peer + + :param peer: the peer via which the data is published to the DHT + :param key: the key of the added data + :param data: the data itself; should be a string + :param numeric_version: the version of the data + :return: None + """ + version = struct.pack("H", numeric_version) + + for i in range(0, len(data), MAX_ENTRY_SIZE - 3): + blob_chunk = version + data[i:i + MAX_ENTRY_SIZE - 3] + yield peer.get_overlay_by_class(DHTCommunity).store_value(key, blob_chunk) + + def deserialize_payload(self, serializables, data): + """ + Deserialize data + + :param serializables: the list of serializable formats + :param data: the serialized data + :return: The payload obtained from deserializing the data + """ + payload = self.serializer.unpack_to_serializables(serializables, data) + return payload[:-1][0] + + @inlineCallbacks + def test_added_block_explicit(self): + """ + Test the publication of a block which has been added by hand to the DHT + """ + param_dict = { + 'port': self.nodes[0].port, + 'interface': self.nodes[0].interface, + 'endpoint': 'dht/block', + 'public_key': string_to_url(b64encode(self.nodes[0].get_keys()['my_peer'].mid)) + } + # Introduce the nodes + yield self.introduce_nodes(DHTCommunity) + + # Manually add a block to the Trustchain + original_block = TestBlock() + hash_key = sha1(self.nodes[0].get_keys()['my_peer'].mid + DHTBlockEndpoint.KEY_SUFFIX).digest() + + yield self.publish_to_DHT(self.nodes[0], hash_key, original_block.pack(), 4536) + + # Get the block through the REST API + response = yield self._get_style_requests.make_dht_block(param_dict) + self.assertTrue('block' in response and response['block'], "Response is not as expected: %s" % response) + response = b64decode(response['block']) + + # Reconstruct the block from what was received in the response + payload = self.deserialize_payload((HalfBlockPayload, ), response) + reconstructed_block = self.nodes[0].get_overlay_by_class(TrustChainCommunity).get_block_class(payload.type) \ + .from_payload(payload, self.serializer) + + self.assertEqual(reconstructed_block, original_block, "The received block was not the one which was expected") + + @inlineCallbacks + def test_added_block_implicit(self): + """ + Test the publication of a block which has been added implicitly to the DHT + """ + param_dict = { + 'port': self.nodes[1].port, + 'interface': self.nodes[1].interface, + 'endpoint': 'dht/block', + 'public_key': string_to_url(b64encode(self.nodes[0].get_keys()['my_peer'].mid)) + } + # Introduce the nodes + yield self.introduce_nodes(DHTCommunity) + + publisher_pk = self.nodes[0].get_overlay_by_class(TrustChainCommunity).my_peer.public_key.key_to_bin() + + yield self.nodes[0].get_overlay_by_class(TrustChainCommunity).create_source_block(b'test', {}) + original_block = self.nodes[0].get_overlay_by_class(TrustChainCommunity).persistence.get(publisher_pk, 1) + yield self.deliver_messages() + + # Get the block through the REST API + response = yield self._get_style_requests.make_dht_block(param_dict) + self.assertTrue('block' in response and response['block'], "Response is not as expected: %s" % response) + response = b64decode(response['block']) + + # Reconstruct the block from what was received in the response + payload = self.deserialize_payload((HalfBlockPayload,), response) + reconstructed_block = self.nodes[0].get_overlay_by_class(TrustChainCommunity).get_block_class(payload.type)\ + .from_payload(payload, self.serializer) + + self.assertEqual(reconstructed_block, original_block, "The received block was not the one which was expected") + + @inlineCallbacks + def test_latest_block(self): + """ + Test the retrieval of the latest block via the DHT, when there is + more than one block in the DHT under the same key + """ + param_dict = { + 'port': self.nodes[1].port, + 'interface': self.nodes[1].interface, + 'endpoint': 'dht/block', + 'public_key': string_to_url(b64encode(self.nodes[0].get_keys()['my_peer'].mid)) + } + # Introduce the nodes + yield self.introduce_nodes(DHTCommunity) + + # Manually add a block to the Trustchain + original_block_1 = TestBlock(transaction={1: 'asd'}) + original_block_2 = TestBlock(transaction={1: 'mmm'}) + hash_key = sha1(self.nodes[0].get_keys()['my_peer'].mid + DHTBlockEndpoint.KEY_SUFFIX).digest() + + # Publish the two blocks under the same key in the first peer + yield self.publish_to_DHT(self.nodes[0], hash_key, original_block_1.pack(), 4536) + yield self.publish_to_DHT(self.nodes[0], hash_key, original_block_2.pack(), 7636) + + # Get the block through the REST API from the second peer + response = yield self._get_style_requests.make_dht_block(param_dict) + self.assertTrue('block' in response and response['block'], "Response is not as expected: %s" % response) + response = b64decode(response['block']) + + # Reconstruct the block from what was received in the response + payload = self.deserialize_payload((HalfBlockPayload,), response) + reconstructed_block = self.nodes[0].get_overlay_by_class(TrustChainCommunity).get_block_class( + payload.type).from_payload(payload, self.serializer) + + self.assertEqual(reconstructed_block, original_block_2, "The received block was not equal to the latest block") + self.assertNotEqual(reconstructed_block, original_block_1, "The received block was equal to the older block") diff --git a/ipv8/test/mocking/rest/comunities.py b/ipv8/test/mocking/rest/comunities.py index 89ed1b711..812d1cbf2 100644 --- a/ipv8/test/mocking/rest/comunities.py +++ b/ipv8/test/mocking/rest/comunities.py @@ -1,5 +1,7 @@ from ....attestation.identity.community import IdentityCommunity from ....attestation.wallet.community import AttestationCommunity +from ....attestation.trustchain.community import TrustChainCommunity +from ....dht.community import DHTCommunity from ....keyvault.crypto import ECCrypto from ....peer import Peer @@ -10,3 +12,28 @@ class TestAttestationCommunity(AttestationCommunity): class TestIdentityCommunity(IdentityCommunity): master_peer = Peer(ECCrypto().generate_key(u'high')) + + +class TestDHTCommunity(DHTCommunity): + master_peer = Peer(ECCrypto().generate_key(u'high')) + + +class TestTrustchainCommunity(TrustChainCommunity): + master_peer = Peer(ECCrypto().generate_key(u'high')) + + +def overlay_initializer(overlay_class, my_peer, endpoint, network, working_directory): + """ + Wrapper class, which instantiates new overlay classes. + + :param overlay_class: the overlay's class + :param my_peer: the peer passed to the overlay + :param endpoint: the endpoint passed to the overlay + :param network: the network passer to the overlay + :param working_directory: the overlay's working directory + :return: an initialized object of the overlay_class type + """ + if issubclass(overlay_class, DHTCommunity): + return overlay_class(my_peer, endpoint, network) + + return overlay_class(my_peer, endpoint, network, working_directory=working_directory) diff --git a/ipv8/test/mocking/rest/ipv8.py b/ipv8/test/mocking/rest/ipv8.py index 90cf48e94..f1eb9a87a 100644 --- a/ipv8/test/mocking/rest/ipv8.py +++ b/ipv8/test/mocking/rest/ipv8.py @@ -2,7 +2,7 @@ from twisted.internet.task import LoopingCall -from .comunities import TestIdentityCommunity, TestAttestationCommunity +from .comunities import TestIdentityCommunity, TestAttestationCommunity, overlay_initializer from ....keyvault.crypto import ECCrypto from ....messaging.interfaces.udp.endpoint import UDPEndpoint from ....peer import Peer @@ -26,8 +26,8 @@ def __init__(self, crypto_curve, port, interface, overlay_classes, memory_dbs=Tr self.overlays = [] for overlay_class in overlay_classes: - self.overlays.append(overlay_class(my_peer, self.endpoint, self.network, - working_directory=database_working_dir)) + self.overlays.append(overlay_initializer(overlay_class, my_peer, self.endpoint, self.network, + working_directory=database_working_dir)) self.strategies = [ (RandomWalk(overlay), 20) for overlay in self.overlays diff --git a/ipv8/test/mocking/rest/rest_peer_communication.py b/ipv8/test/mocking/rest/rest_peer_communication.py index 8260f427a..9f49c08e9 100644 --- a/ipv8/test/mocking/rest/rest_peer_communication.py +++ b/ipv8/test/mocking/rest/rest_peer_communication.py @@ -6,7 +6,6 @@ from twisted.web.client import Agent, readBody from twisted.web.http_headers import Headers -from .peer_communication import IGetStyleRequestsAE, RequestException, IPostStyleRequestsAE from ....util import quote @@ -107,361 +106,6 @@ def basic_url_builder(interface, port, endpoint, protocol='http'): return "%s://%s:%d/%s" % (protocol, interface, port, endpoint) -class HTTPGetRequesterAE(IGetStyleRequestsAE, HTTPRequester): - """ - Implements the GetStyleRequests abstract methods using the HTTP protocol for the attestation endpoint - """ - - def __init__(self): - IGetStyleRequestsAE.__init__(self) - HTTPRequester.__init__(self) - - @process_json_response - @inlineCallbacks - def make_outstanding(self, param_dict): - """ - Forward a request for outstanding attestation requests. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - {'type': 'outstanding'}, - param_dict.get('callback', None)) - returnValue(response) - - @process_json_response - @inlineCallbacks - def make_verification_output(self, param_dict): - """ - Forward a request for the verification outputs. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - {'type': 'verification_output'}, - param_dict.get('callback', None)) - returnValue(response) - - @process_json_response - @inlineCallbacks - def make_peers(self, param_dict): - """ - Forward a request for the known peers in the network. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - {'type': 'peers'}, - param_dict.get('callback', None)) - returnValue(response) - - @process_json_response - @inlineCallbacks - def make_attributes(self, param_dict): - """ - Forward a request for the attributes of a peer. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (attributes), and the (optional) b64_mid of the attester - request_parameters = param_dict.get('request_parameters', dict()) - request_parameters.update({'type': 'attributes'}) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - @inlineCallbacks - def make_drop_identity(self, param_dict): - """ - Forward a request for dropping a peer's identity. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - {'type': 'drop_identity'}, - param_dict.get('callback', None)) - returnValue(response) - - @process_json_response - @inlineCallbacks - def make_outstanding_verify(self, param_dict): - """ - Forward a request which requests information on the outstanding verify requests - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (request), and the rest of the parameters - request_parameters = {'type': 'outstanding_verify'} - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - -class HTTPPostRequesterAE(IPostStyleRequestsAE, HTTPRequester): - """ - Implements the PostStyleRequests abstract methods using the HTTP protocol for the AttestationEndpoint - """ - - def __init__(self): - IPostStyleRequestsAE.__init__(self) - HTTPRequester.__init__(self) - - @inlineCallbacks - def make_attestation_request(self, param_dict): - """ - Forward a request for the attestation of an attribute. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - 'attribute_name': attribute_name - 'mid': attester b64_mid - (optional) 'metadata': JSON style metadata required for the attestation process - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (request), and the rest of the parameters - request_parameters = {'type': 'request'} - - # Add the request parameters one-by-one; if required parameter is missing, then raise error - if 'attribute_name' in param_dict: - request_parameters['attribute_name'] = param_dict['attribute_name'] - else: - raise RequestException("Malformed request: did not specify the attribute_name") - - if 'mid' in param_dict: - request_parameters['mid'] = param_dict['mid'] - else: - raise RequestException("Malformed request: did not specify the attester's mid") - - if 'metadata' in param_dict: - request_parameters['metadata'] = param_dict['metadata'] - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'POST', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - @inlineCallbacks - def make_attest(self, param_dict): - """ - Forward a request which attests an attestation request. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - 'attribute_name': attribute_name - 'mid': attestee's b64_mid - 'attribute_value': b64 hash of the attestation blob - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (attest), and the rest of the parameters - request_parameters = {'type': 'attest'} - - # Add the request parameters one-by-one; if required parameter is missing, then raise error - if 'attribute_name' in param_dict: - request_parameters['attribute_name'] = param_dict['attribute_name'] - else: - raise RequestException("Malformed request: did not specify the attribute_name") - - if 'mid' in param_dict: - request_parameters['mid'] = param_dict['mid'] - else: - raise RequestException("Malformed request: did not specify the attestee's mid") - - if 'attribute_value' in param_dict: - request_parameters['attribute_value'] = param_dict['attribute_value'] - else: - raise RequestException("Malformed request: did not specify the attribute_value, i.e. the attestation" - "blob hash") - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'POST', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - @inlineCallbacks - def make_verify(self, param_dict): - """ - Forward a request which demands the verification of an attestation - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - 'attribute_hash': the b64 hash of the attestation blob which needs to be verified - 'mid': verifier's b64_mid - 'attribute_values': a string of b64 encoded values, which are separated by ',' characters - e.g. "val_1,val_2,val_3, ..., val_N" - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (attest), and the rest of the parameters - request_parameters = {'type': 'verify'} - - # Add the request parameters one-by-one; if required parameter is missing, then raise error - if 'attribute_hash' in param_dict: - request_parameters['attribute_hash'] = param_dict['attribute_hash'] - else: - raise RequestException("Malformed request: did not specify the attribute_hash") - - if 'mid' in param_dict: - request_parameters['mid'] = param_dict['mid'] - else: - raise RequestException("Malformed request: did not specify the verifier's mid") - - if 'attribute_values' in param_dict: - request_parameters['attribute_values'] = param_dict['attribute_values'] - else: - raise RequestException("Malformed request: did not specify the attribute_values") - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'POST', - request_parameters, - param_dict.get('callback', None)) - - returnValue(response) - - @inlineCallbacks - def make_allow_verify(self, param_dict): - """ - Forward a request which requests that verifications be allowed for a particular peer for a particular attribute - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - 'attribute_name': attribute_name - 'mid': verifier's b64_mid - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (request), and the rest of the parameters - request_parameters = {'type': 'allow_verify'} - - # Add the request parameters one-by-one; if required parameter is missing, then raise error - if 'attribute_name' in param_dict: - request_parameters['attribute_name'] = param_dict['attribute_name'] - else: - raise RequestException("Malformed request: did not specify the attribute_name") - - if 'mid' in param_dict: - request_parameters['mid'] = param_dict['mid'] - else: - raise RequestException("Malformed request: did not specify the attester's mid") - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'POST', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - def string_to_url(string, quote_string=False, to_utf_8=False): """ Convert a string to a format which is compatible to it being passed via a url @@ -476,3 +120,9 @@ def string_to_url(string, quote_string=False, to_utf_8=False): string = string.replace("+", "%2B") if not quote_string else quote(string.replace("+", "%2B")) return string.encode('utf-8') if to_utf_8 else string + + +class RequestException(Exception): + """ + Custom exception used to model request errors + """ diff --git a/test_classes_list.txt b/test_classes_list.txt index b3261833e..227456931 100644 --- a/test_classes_list.txt +++ b/test_classes_list.txt @@ -41,4 +41,5 @@ ipv8/test/dht/test_routing.py:TestBucket ipv8/test/dht/test_routing.py:TestRoutingTable ipv8/test/dht/test_storage.py:TestStorage -ipv8/test/REST/test_attestation_endpoint.py:TestAttestationEndpoint \ No newline at end of file +ipv8/test/REST/attestationendpoint/test_attestation_endpoint.py:TestAttestationEndpoint +ipv8/test/REST/dht/test_dht_endpoint.py:TestDHTEndpoint \ No newline at end of file