diff --git a/ipv8/REST/dht_endpoint.py b/ipv8/REST/dht_endpoint.py index 4a825ab7f..9d34d2151 100644 --- a/ipv8/REST/dht_endpoint.py +++ b/ipv8/REST/dht_endpoint.py @@ -1,13 +1,17 @@ from __future__ import absolute_import -from base64 import b64encode +from base64 import b64encode, b64decode from binascii import hexlify, unhexlify +from hashlib import sha1 import json +import struct +from twisted.internet.defer import inlineCallbacks, returnValue from twisted.web import http, resource from twisted.web.server import NOT_DONE_YET -from ..dht.community import DHTCommunity +from ..dht.community import DHTCommunity, MAX_ENTRY_SIZE +from ..attestation.trustchain.community import TrustChainCommunity from ..dht.discovery import DHTDiscoveryCommunity @@ -20,10 +24,87 @@ def __init__(self, session): resource.Resource.__init__(self) dht_overlays = [overlay for overlay in session.overlays if isinstance(overlay, DHTCommunity)] + tc_overlays = [overlay for overlay in session.overlays if isinstance(overlay, TrustChainCommunity)] if dht_overlays: - self.putChild("statistics", DHTStatisticsEndpoint(dht_overlays[0])) - self.putChild("values", DHTValuesEndpoint(dht_overlays[0])) - self.putChild("peers", DHTPeersEndpoint(dht_overlays[0])) + self.putChild(b"statistics", DHTStatisticsEndpoint(dht_overlays[0])) + self.putChild(b"values", DHTValuesEndpoint(dht_overlays[0])) + self.putChild(b"peers", DHTPeersEndpoint(dht_overlays[0])) + self.putChild(b"block", DHTBlockEndpoint(dht_overlays[0], tc_overlays[0])) + + +class DHTBlockEndpoint(resource.Resource): + """ + This endpoint is responsible for returning the latest Trustchain block of a peer. Additionally, it ensures + this peer's latest TC block is available + """ + + KEY_SUFFIX = b'_BLOCK' + + def __init__(self, dht, trustchain): + resource.Resource.__init__(self) + self.dht = dht + self.trustchain = trustchain + self.block_version = 0 + + self._hashed_dht_key = sha1(self.trustchain.my_peer.mid + self.KEY_SUFFIX).digest() + + trustchain.set_new_block_cb(self.publish_latest_block) + + @inlineCallbacks + def publish_latest_block(self): + """ + Publish the latest block of this node's Trustchain to the DHT + + :return: + """ + # latest_block = self.trustchain.persistence.get_latest(self.trustchain.my_peer.key.pub().key_to_bin()) + latest_block = self.trustchain.persistence.get_latest(self.trustchain.my_peer.public_key.key_to_bin()) + + if latest_block: + latest_block = latest_block.pack() + version = struct.pack("H", self.block_version) + self.block_version += 1 + + for i in range(0, len(latest_block), MAX_ENTRY_SIZE - 3): + blob_chunk = version + latest_block[i:i + MAX_ENTRY_SIZE - 3] + yield self.dht.store_value(self._hashed_dht_key, blob_chunk) + + def render_GET(self, request): + """ + Return the latest TC block of a peer, as identified in the request + + :param request: the request for retrieving the latest TC block of a peer. It must contain the peer's + public key of the peer + :return: the latest block of the peer, if found + """ + if not self.dht: + request.setResponseCode(http.NOT_FOUND) + return json.dumps({"error": "DHT community not found"}).encode('utf-8') + + if not request.args or b'public_key' not in request.args: + request.setResponseCode(http.BAD_REQUEST) + return json.dumps({"error": "Must specify the peer's public key"}).encode('utf-8') + + hash_key = sha1(b64decode(request.args[b'public_key'][0]) + self.KEY_SUFFIX).digest() + block_chunks = self.dht.storage.get(hash_key) + + if not block_chunks: + request.setResponseCode(http.NOT_FOUND) + return json.dumps({"error": "Could not find a block for the specified key."}).encode('utf-8') + + new_blocks = {} + max_version = 0 + + for entry in block_chunks: + this_version = struct.unpack("I", entry[1:3] + b'\x00\x00')[0] + max_version = max_version if max_version > this_version else this_version + + if this_version in new_blocks: + new_blocks[this_version] = entry[3:] + new_blocks[this_version] + else: + new_blocks[this_version] = entry[3:] + + return json.dumps({"block": b64encode(new_blocks[max_version]).decode('utf-8')}).encode('utf-8') class DHTStatisticsEndpoint(resource.Resource): diff --git a/ipv8/attestation/trustchain/community.py b/ipv8/attestation/trustchain/community.py index d9ab51743..157e92bb1 100644 --- a/ipv8/attestation/trustchain/community.py +++ b/ipv8/attestation/trustchain/community.py @@ -73,6 +73,8 @@ def __init__(self, *args, **kwargs): self.db_cleanup_lc = self.register_task("db_cleanup", LoopingCall(self.do_db_cleanup)) self.db_cleanup_lc.start(600) + self.new_block_cb = lambda: None + self.decode_map.update({ chr(1): self.received_half_block, chr(2): self.received_crawl_request, @@ -83,6 +85,13 @@ def __init__(self, *args, **kwargs): chr(7): self.received_empty_crawl_response, }) + def set_new_block_cb(self, f): + """ + Set the callback for when a new block is added to the DB. The callback should take no arguments and should + return nothing: [] -> None + """ + self.new_block_cb = f + def do_db_cleanup(self): """ Cleanup the database if necessary. @@ -251,6 +260,7 @@ def sign_block(self, peer, public_key=EMPTY_PK, block_type=b'unknown', transacti if not self.persistence.contains(block): self.persistence.add_block(block) + self.new_block_cb() self.notify_listeners(block) # This is a source block with no counterparty @@ -342,6 +352,7 @@ def validate_persist_block(self, block): pass elif not self.persistence.contains(block): self.persistence.add_block(block) + self.new_block_cb() self.notify_listeners(block) return validation diff --git a/ipv8/test/REST/attestationendpoint/__init__.py b/ipv8/test/REST/attestationendpoint/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ipv8/test/mocking/rest/peer_communication.py b/ipv8/test/REST/attestationendpoint/peer_communication.py similarity index 96% rename from ipv8/test/mocking/rest/peer_communication.py rename to ipv8/test/REST/attestationendpoint/peer_communication.py index 5b7cbbd54..6a059c92f 100644 --- a/ipv8/test/mocking/rest/peer_communication.py +++ b/ipv8/test/REST/attestationendpoint/peer_communication.py @@ -1,4 +1,3 @@ - from abc import abstractmethod @@ -111,9 +110,3 @@ def make_outstanding_verify(self, param_dict): :return: None """ pass - - -class RequestException(Exception): - """ - Custom exception used to model request errors - """ diff --git a/ipv8/test/REST/attestationendpoint/rest_peer_communication.py b/ipv8/test/REST/attestationendpoint/rest_peer_communication.py new file mode 100644 index 000000000..c54aa7bb5 --- /dev/null +++ b/ipv8/test/REST/attestationendpoint/rest_peer_communication.py @@ -0,0 +1,390 @@ +from twisted.internet.defer import inlineCallbacks, returnValue + +from .peer_communication import IGetStyleRequestsAE, IPostStyleRequestsAE +from ...mocking.rest.rest_peer_communication import HTTPRequester, RequestException, process_json_response + + +class HTTPGetRequesterAE(IGetStyleRequestsAE, HTTPRequester): + """ + Implements the GetStyleRequests abstract methods using the HTTP protocol for the attestation endpoint. + """ + + def __init__(self): + IGetStyleRequestsAE.__init__(self) + HTTPRequester.__init__(self) + + @process_json_response + @inlineCallbacks + def make_outstanding(self, param_dict): + """ + Forward a request for outstanding attestation requests. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + {'type': 'outstanding'}, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_verification_output(self, param_dict): + """ + Forward a request for the verification outputs. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + {'type': 'verification_output'}, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_peers(self, param_dict): + """ + Forward a request for the known peers in the network. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + {'type': 'peers'}, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_attributes(self, param_dict): + """ + Forward a request for the attributes of a peer. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (attributes), and the (optional) b64_mid of the attester + request_parameters = param_dict.get('request_parameters', dict()) + request_parameters.update({'type': 'attributes'}) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_dht_block(self, param_dict): + """ + Forward a request for the latest TC block of a peer + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'public_key': the public key of the peer whose latest TC block is being requested + (optional) 'callback': single parameter callback for the request's response + } + :return: None + :raises RequestException: raised when the method could not find one of the required pieces of information + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + request_parameters = {} + + if 'public_key' in param_dict: + request_parameters['public_key'] = param_dict['public_key'] + else: + raise RequestException("Malformed request: did not specify the public_key") + + response = yield self.make_request("http://{0}:{1}/{2}".format(interface, port, endpoint), + 'GET', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + @inlineCallbacks + def make_drop_identity(self, param_dict): + """ + Forward a request for dropping a peer's identity. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + {'type': 'drop_identity'}, + param_dict.get('callback', None)) + returnValue(response) + + @process_json_response + @inlineCallbacks + def make_outstanding_verify(self, param_dict): + """ + Forward a request which requests information on the outstanding verify requests + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (request), and the rest of the parameters + request_parameters = {'type': 'outstanding_verify'} + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'GET', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + +class HTTPPostRequesterAE(IPostStyleRequestsAE, HTTPRequester): + """ + Implements the PostStyleRequests abstract methods using the HTTP protocol for the AttestationEndpoint + """ + + def __init__(self): + IPostStyleRequestsAE.__init__(self) + HTTPRequester.__init__(self) + + @inlineCallbacks + def make_attestation_request(self, param_dict): + """ + Forward a request for the attestation of an attribute. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'attribute_name': attribute_name + 'mid': attester b64_mid + (optional) 'metadata': JSON style metadata required for the attestation process + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (request), and the rest of the parameters + request_parameters = {'type': 'request'} + + # Add the request parameters one-by-one; if required parameter is missing, then raise error + if 'attribute_name' in param_dict: + request_parameters['attribute_name'] = param_dict['attribute_name'] + else: + raise RequestException("Malformed request: did not specify the attribute_name") + + if 'mid' in param_dict: + request_parameters['mid'] = param_dict['mid'] + else: + raise RequestException("Malformed request: did not specify the attester's mid") + + if 'metadata' in param_dict: + request_parameters['metadata'] = param_dict['metadata'] + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'POST', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + @inlineCallbacks + def make_attest(self, param_dict): + """ + Forward a request which attests an attestation request. + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'attribute_name': attribute_name + 'mid': attestee's b64_mid + 'attribute_value': b64 hash of the attestation blob + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (attest), and the rest of the parameters + request_parameters = {'type': 'attest'} + + # Add the request parameters one-by-one; if required parameter is missing, then raise error + if 'attribute_name' in param_dict: + request_parameters['attribute_name'] = param_dict['attribute_name'] + else: + raise RequestException("Malformed request: did not specify the attribute_name") + + if 'mid' in param_dict: + request_parameters['mid'] = param_dict['mid'] + else: + raise RequestException("Malformed request: did not specify the attestee's mid") + + if 'attribute_value' in param_dict: + request_parameters['attribute_value'] = param_dict['attribute_value'] + else: + raise RequestException("Malformed request: did not specify the attribute_value, i.e. the attestation" + "blob hash") + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'POST', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) + + @inlineCallbacks + def make_verify(self, param_dict): + """ + Forward a request which demands the verification of an attestation + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'attribute_hash': the b64 hash of the attestation blob which needs to be verified + 'mid': verifier's b64_mid + 'attribute_values': a string of b64 encoded values, which are separated by ',' characters + e.g. "val_1,val_2,val_3, ..., val_N" + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (attest), and the rest of the parameters + request_parameters = {'type': 'verify'} + + # Add the request parameters one-by-one; if required parameter is missing, then raise error + if 'attribute_hash' in param_dict: + request_parameters['attribute_hash'] = param_dict['attribute_hash'] + else: + raise RequestException("Malformed request: did not specify the attribute_hash") + + if 'mid' in param_dict: + request_parameters['mid'] = param_dict['mid'] + else: + raise RequestException("Malformed request: did not specify the verifier's mid") + + if 'attribute_values' in param_dict: + request_parameters['attribute_values'] = param_dict['attribute_values'] + else: + raise RequestException("Malformed request: did not specify the attribute_values") + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'POST', + request_parameters, + param_dict.get('callback', None)) + + returnValue(response) + + @inlineCallbacks + def make_allow_verify(self, param_dict): + """ + Forward a request which requests that verifications be allowed for a particular peer for a particular attribute + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'attribute_name': attribute_name + 'mid': verifier's b64_mid + (optional) 'callback': single parameter callback for the request's response + } + :return: the request's response + :raises RequestException: raised when the method could not find some element required for the construction of + the request + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + + # Add the type of the request (request), and the rest of the parameters + request_parameters = {'type': 'allow_verify'} + + # Add the request parameters one-by-one; if required parameter is missing, then raise error + if 'attribute_name' in param_dict: + request_parameters['attribute_name'] = param_dict['attribute_name'] + else: + raise RequestException("Malformed request: did not specify the attribute_name") + + if 'mid' in param_dict: + request_parameters['mid'] = param_dict['mid'] + else: + raise RequestException("Malformed request: did not specify the attester's mid") + + response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), + 'POST', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) diff --git a/ipv8/test/REST/test_attestation_endpoint.py b/ipv8/test/REST/attestationendpoint/test_attestation_endpoint.py similarity index 98% rename from ipv8/test/REST/test_attestation_endpoint.py rename to ipv8/test/REST/attestationendpoint/test_attestation_endpoint.py index 2a6fd6404..c3b91f2a4 100644 --- a/ipv8/test/REST/test_attestation_endpoint.py +++ b/ipv8/test/REST/attestationendpoint/test_attestation_endpoint.py @@ -3,13 +3,13 @@ from json import dumps from twisted.internet.defer import returnValue, inlineCallbacks -from ..mocking.rest.base import RESTTestBase -from ..mocking.rest.peer_interactive_behavior import RequesterRestTestPeer -from ..mocking.rest.rest_peer_communication import string_to_url -from ..mocking.rest.rest_api_peer import RestTestPeer -from ..mocking.rest.rest_peer_communication import HTTPGetRequesterAE, HTTPPostRequesterAE -from ..mocking.rest.comunities import TestAttestationCommunity, TestIdentityCommunity -from ...attestation.identity.community import IdentityCommunity +from .rest_peer_communication import HTTPGetRequesterAE, HTTPPostRequesterAE +from ...mocking.rest.base import RESTTestBase +from ...mocking.rest.peer_interactive_behavior import RequesterRestTestPeer +from ...mocking.rest.rest_peer_communication import string_to_url +from ...mocking.rest.rest_api_peer import RestTestPeer +from ...mocking.rest.comunities import TestAttestationCommunity, TestIdentityCommunity +from ....attestation.identity.community import IdentityCommunity class TestAttestationEndpoint(RESTTestBase): diff --git a/ipv8/test/REST/dht/__init__.py b/ipv8/test/REST/dht/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ipv8/test/REST/dht/rest_peer_communication.py b/ipv8/test/REST/dht/rest_peer_communication.py new file mode 100644 index 000000000..22225c515 --- /dev/null +++ b/ipv8/test/REST/dht/rest_peer_communication.py @@ -0,0 +1,43 @@ +from twisted.internet.defer import inlineCallbacks, returnValue + +from ...mocking.rest.rest_peer_communication import HTTPRequester, RequestException, process_json_response + + +class HTTPGetRequesterDHT(HTTPRequester): + """ + Implements the HTTP GET type requests for the DHT endpoint + """ + + def __init__(self): + HTTPRequester.__init__(self) + + @process_json_response + @inlineCallbacks + def make_dht_block(self, param_dict): + """ + Forward a request for the latest TC block of a peer + + :param param_dict: Should have at least the following structure: + { + 'interface': target peer IP or alias + 'port': port_number + 'endpoint': endpoint_name + 'public_key': the public key of the peer whose latest TC block is being requested + (optional) 'callback': single parameter callback for the request's response + } + :return: None + :raises RequestException: raised when the method could not find one of the required pieces of information + """ + interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) + request_parameters = {} + + if 'public_key' in param_dict: + request_parameters['public_key'] = param_dict['public_key'] + else: + raise RequestException("Malformed request: did not specify the public_key") + + response = yield self.make_request("http://{0}:{1}/{2}".format(interface, port, endpoint), + 'GET', + request_parameters, + param_dict.get('callback', None)) + returnValue(response) diff --git a/ipv8/test/REST/dht/test_dht_endpoint.py b/ipv8/test/REST/dht/test_dht_endpoint.py new file mode 100644 index 000000000..a2d79a54a --- /dev/null +++ b/ipv8/test/REST/dht/test_dht_endpoint.py @@ -0,0 +1,160 @@ +import struct + +from hashlib import sha1 +from base64 import b64encode, b64decode +from twisted.internet.defer import inlineCallbacks + +from .rest_peer_communication import HTTPGetRequesterDHT +from ...attestation.trustchain.test_block import TestBlock +from ...mocking.rest.base import RESTTestBase +from ...mocking.rest.rest_peer_communication import string_to_url +from ...mocking.rest.rest_api_peer import RestTestPeer +from ...mocking.rest.comunities import TestDHTCommunity, TestTrustchainCommunity +from ....attestation.trustchain.payload import HalfBlockPayload +from ....attestation.trustchain.community import TrustChainCommunity +from ....dht.community import DHTCommunity, MAX_ENTRY_SIZE +from ....REST.dht_endpoint import DHTBlockEndpoint +from ....messaging.serialization import Serializer + + +class TestDHTEndpoint(RESTTestBase): + """ + Class for testing the DHT Endpoint in the REST API of the IPv8 object + """ + + def setUp(self): + super(TestDHTEndpoint, self).setUp() + self.initialize([(2, RestTestPeer)], HTTPGetRequesterDHT(), None) + self.serializer = Serializer() + + def create_new_peer(self, peer_cls, port, *args, **kwargs): + self._create_new_peer_inner(peer_cls, port, [TestDHTCommunity, TestTrustchainCommunity], *args, **kwargs) + + @inlineCallbacks + def publish_to_DHT(self, peer, key, data, numeric_version): + """ + Publish data to the DHT via a peer + + :param peer: the peer via which the data is published to the DHT + :param key: the key of the added data + :param data: the data itself; should be a string + :param numeric_version: the version of the data + :return: None + """ + version = struct.pack("H", numeric_version) + + for i in range(0, len(data), MAX_ENTRY_SIZE - 3): + blob_chunk = version + data[i:i + MAX_ENTRY_SIZE - 3] + yield peer.get_overlay_by_class(DHTCommunity).store_value(key, blob_chunk) + + def deserialize_payload(self, serializables, data): + """ + Deserialize data + + :param serializables: the list of serializable formats + :param data: the serialized data + :return: The payload obtained from deserializing the data + """ + payload = self.serializer.unpack_to_serializables(serializables, data) + return payload[:-1][0] + + @inlineCallbacks + def test_added_block_explicit(self): + """ + Test the publication of a block which has been added by hand to the DHT + """ + param_dict = { + 'port': self.nodes[0].port, + 'interface': self.nodes[0].interface, + 'endpoint': 'dht/block', + 'public_key': string_to_url(b64encode(self.nodes[0].get_keys()['my_peer'].mid)) + } + # Introduce the nodes + yield self.introduce_nodes(DHTCommunity) + + # Manually add a block to the Trustchain + original_block = TestBlock() + hash_key = sha1(self.nodes[0].get_keys()['my_peer'].mid + DHTBlockEndpoint.KEY_SUFFIX).digest() + + yield self.publish_to_DHT(self.nodes[0], hash_key, original_block.pack(), 4536) + + # Get the block through the REST API + response = yield self._get_style_requests.make_dht_block(param_dict) + self.assertTrue('block' in response and response['block'], "Response is not as expected: %s" % response) + response = b64decode(response['block']) + + # Reconstruct the block from what was received in the response + payload = self.deserialize_payload((HalfBlockPayload, ), response) + reconstructed_block = self.nodes[0].get_overlay_by_class(TrustChainCommunity).get_block_class(payload.type) \ + .from_payload(payload, self.serializer) + + self.assertEqual(reconstructed_block, original_block, "The received block was not the one which was expected") + + @inlineCallbacks + def test_added_block_implicit(self): + """ + Test the publication of a block which has been added implicitly to the DHT + """ + param_dict = { + 'port': self.nodes[1].port, + 'interface': self.nodes[1].interface, + 'endpoint': 'dht/block', + 'public_key': string_to_url(b64encode(self.nodes[0].get_keys()['my_peer'].mid)) + } + # Introduce the nodes + yield self.introduce_nodes(DHTCommunity) + + publisher_pk = self.nodes[0].get_overlay_by_class(TrustChainCommunity).my_peer.public_key.key_to_bin() + + yield self.nodes[0].get_overlay_by_class(TrustChainCommunity).create_source_block(b'test', {}) + original_block = self.nodes[0].get_overlay_by_class(TrustChainCommunity).persistence.get(publisher_pk, 1) + yield self.deliver_messages() + + # Get the block through the REST API + response = yield self._get_style_requests.make_dht_block(param_dict) + self.assertTrue('block' in response and response['block'], "Response is not as expected: %s" % response) + response = b64decode(response['block']) + + # Reconstruct the block from what was received in the response + payload = self.deserialize_payload((HalfBlockPayload,), response) + reconstructed_block = self.nodes[0].get_overlay_by_class(TrustChainCommunity).get_block_class(payload.type)\ + .from_payload(payload, self.serializer) + + self.assertEqual(reconstructed_block, original_block, "The received block was not the one which was expected") + + @inlineCallbacks + def test_latest_block(self): + """ + Test the retrieval of the latest block via the DHT, when there is + more than one block in the DHT under the same key + """ + param_dict = { + 'port': self.nodes[1].port, + 'interface': self.nodes[1].interface, + 'endpoint': 'dht/block', + 'public_key': string_to_url(b64encode(self.nodes[0].get_keys()['my_peer'].mid)) + } + # Introduce the nodes + yield self.introduce_nodes(DHTCommunity) + + # Manually add a block to the Trustchain + original_block_1 = TestBlock(transaction={1: 'asd'}) + original_block_2 = TestBlock(transaction={1: 'mmm'}) + hash_key = sha1(self.nodes[0].get_keys()['my_peer'].mid + DHTBlockEndpoint.KEY_SUFFIX).digest() + + # Publish the two blocks under the same key in the first peer + yield self.publish_to_DHT(self.nodes[0], hash_key, original_block_1.pack(), 4536) + yield self.publish_to_DHT(self.nodes[0], hash_key, original_block_2.pack(), 7636) + + # Get the block through the REST API from the second peer + response = yield self._get_style_requests.make_dht_block(param_dict) + self.assertTrue('block' in response and response['block'], "Response is not as expected: %s" % response) + response = b64decode(response['block']) + + # Reconstruct the block from what was received in the response + payload = self.deserialize_payload((HalfBlockPayload,), response) + reconstructed_block = self.nodes[0].get_overlay_by_class(TrustChainCommunity).get_block_class( + payload.type).from_payload(payload, self.serializer) + + self.assertEqual(reconstructed_block, original_block_2, "The received block was not equal to the latest block") + self.assertNotEqual(reconstructed_block, original_block_1, "The received block was equal to the older block") diff --git a/ipv8/test/mocking/rest/comunities.py b/ipv8/test/mocking/rest/comunities.py index 89ed1b711..812d1cbf2 100644 --- a/ipv8/test/mocking/rest/comunities.py +++ b/ipv8/test/mocking/rest/comunities.py @@ -1,5 +1,7 @@ from ....attestation.identity.community import IdentityCommunity from ....attestation.wallet.community import AttestationCommunity +from ....attestation.trustchain.community import TrustChainCommunity +from ....dht.community import DHTCommunity from ....keyvault.crypto import ECCrypto from ....peer import Peer @@ -10,3 +12,28 @@ class TestAttestationCommunity(AttestationCommunity): class TestIdentityCommunity(IdentityCommunity): master_peer = Peer(ECCrypto().generate_key(u'high')) + + +class TestDHTCommunity(DHTCommunity): + master_peer = Peer(ECCrypto().generate_key(u'high')) + + +class TestTrustchainCommunity(TrustChainCommunity): + master_peer = Peer(ECCrypto().generate_key(u'high')) + + +def overlay_initializer(overlay_class, my_peer, endpoint, network, working_directory): + """ + Wrapper class, which instantiates new overlay classes. + + :param overlay_class: the overlay's class + :param my_peer: the peer passed to the overlay + :param endpoint: the endpoint passed to the overlay + :param network: the network passer to the overlay + :param working_directory: the overlay's working directory + :return: an initialized object of the overlay_class type + """ + if issubclass(overlay_class, DHTCommunity): + return overlay_class(my_peer, endpoint, network) + + return overlay_class(my_peer, endpoint, network, working_directory=working_directory) diff --git a/ipv8/test/mocking/rest/ipv8.py b/ipv8/test/mocking/rest/ipv8.py index 90cf48e94..f1eb9a87a 100644 --- a/ipv8/test/mocking/rest/ipv8.py +++ b/ipv8/test/mocking/rest/ipv8.py @@ -2,7 +2,7 @@ from twisted.internet.task import LoopingCall -from .comunities import TestIdentityCommunity, TestAttestationCommunity +from .comunities import TestIdentityCommunity, TestAttestationCommunity, overlay_initializer from ....keyvault.crypto import ECCrypto from ....messaging.interfaces.udp.endpoint import UDPEndpoint from ....peer import Peer @@ -26,8 +26,8 @@ def __init__(self, crypto_curve, port, interface, overlay_classes, memory_dbs=Tr self.overlays = [] for overlay_class in overlay_classes: - self.overlays.append(overlay_class(my_peer, self.endpoint, self.network, - working_directory=database_working_dir)) + self.overlays.append(overlay_initializer(overlay_class, my_peer, self.endpoint, self.network, + working_directory=database_working_dir)) self.strategies = [ (RandomWalk(overlay), 20) for overlay in self.overlays diff --git a/ipv8/test/mocking/rest/rest_peer_communication.py b/ipv8/test/mocking/rest/rest_peer_communication.py index 8260f427a..9f49c08e9 100644 --- a/ipv8/test/mocking/rest/rest_peer_communication.py +++ b/ipv8/test/mocking/rest/rest_peer_communication.py @@ -6,7 +6,6 @@ from twisted.web.client import Agent, readBody from twisted.web.http_headers import Headers -from .peer_communication import IGetStyleRequestsAE, RequestException, IPostStyleRequestsAE from ....util import quote @@ -107,361 +106,6 @@ def basic_url_builder(interface, port, endpoint, protocol='http'): return "%s://%s:%d/%s" % (protocol, interface, port, endpoint) -class HTTPGetRequesterAE(IGetStyleRequestsAE, HTTPRequester): - """ - Implements the GetStyleRequests abstract methods using the HTTP protocol for the attestation endpoint - """ - - def __init__(self): - IGetStyleRequestsAE.__init__(self) - HTTPRequester.__init__(self) - - @process_json_response - @inlineCallbacks - def make_outstanding(self, param_dict): - """ - Forward a request for outstanding attestation requests. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - {'type': 'outstanding'}, - param_dict.get('callback', None)) - returnValue(response) - - @process_json_response - @inlineCallbacks - def make_verification_output(self, param_dict): - """ - Forward a request for the verification outputs. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - {'type': 'verification_output'}, - param_dict.get('callback', None)) - returnValue(response) - - @process_json_response - @inlineCallbacks - def make_peers(self, param_dict): - """ - Forward a request for the known peers in the network. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - {'type': 'peers'}, - param_dict.get('callback', None)) - returnValue(response) - - @process_json_response - @inlineCallbacks - def make_attributes(self, param_dict): - """ - Forward a request for the attributes of a peer. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (attributes), and the (optional) b64_mid of the attester - request_parameters = param_dict.get('request_parameters', dict()) - request_parameters.update({'type': 'attributes'}) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - @inlineCallbacks - def make_drop_identity(self, param_dict): - """ - Forward a request for dropping a peer's identity. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - {'type': 'drop_identity'}, - param_dict.get('callback', None)) - returnValue(response) - - @process_json_response - @inlineCallbacks - def make_outstanding_verify(self, param_dict): - """ - Forward a request which requests information on the outstanding verify requests - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (request), and the rest of the parameters - request_parameters = {'type': 'outstanding_verify'} - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'GET', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - -class HTTPPostRequesterAE(IPostStyleRequestsAE, HTTPRequester): - """ - Implements the PostStyleRequests abstract methods using the HTTP protocol for the AttestationEndpoint - """ - - def __init__(self): - IPostStyleRequestsAE.__init__(self) - HTTPRequester.__init__(self) - - @inlineCallbacks - def make_attestation_request(self, param_dict): - """ - Forward a request for the attestation of an attribute. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - 'attribute_name': attribute_name - 'mid': attester b64_mid - (optional) 'metadata': JSON style metadata required for the attestation process - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (request), and the rest of the parameters - request_parameters = {'type': 'request'} - - # Add the request parameters one-by-one; if required parameter is missing, then raise error - if 'attribute_name' in param_dict: - request_parameters['attribute_name'] = param_dict['attribute_name'] - else: - raise RequestException("Malformed request: did not specify the attribute_name") - - if 'mid' in param_dict: - request_parameters['mid'] = param_dict['mid'] - else: - raise RequestException("Malformed request: did not specify the attester's mid") - - if 'metadata' in param_dict: - request_parameters['metadata'] = param_dict['metadata'] - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'POST', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - @inlineCallbacks - def make_attest(self, param_dict): - """ - Forward a request which attests an attestation request. - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - 'attribute_name': attribute_name - 'mid': attestee's b64_mid - 'attribute_value': b64 hash of the attestation blob - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (attest), and the rest of the parameters - request_parameters = {'type': 'attest'} - - # Add the request parameters one-by-one; if required parameter is missing, then raise error - if 'attribute_name' in param_dict: - request_parameters['attribute_name'] = param_dict['attribute_name'] - else: - raise RequestException("Malformed request: did not specify the attribute_name") - - if 'mid' in param_dict: - request_parameters['mid'] = param_dict['mid'] - else: - raise RequestException("Malformed request: did not specify the attestee's mid") - - if 'attribute_value' in param_dict: - request_parameters['attribute_value'] = param_dict['attribute_value'] - else: - raise RequestException("Malformed request: did not specify the attribute_value, i.e. the attestation" - "blob hash") - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'POST', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - @inlineCallbacks - def make_verify(self, param_dict): - """ - Forward a request which demands the verification of an attestation - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - 'attribute_hash': the b64 hash of the attestation blob which needs to be verified - 'mid': verifier's b64_mid - 'attribute_values': a string of b64 encoded values, which are separated by ',' characters - e.g. "val_1,val_2,val_3, ..., val_N" - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (attest), and the rest of the parameters - request_parameters = {'type': 'verify'} - - # Add the request parameters one-by-one; if required parameter is missing, then raise error - if 'attribute_hash' in param_dict: - request_parameters['attribute_hash'] = param_dict['attribute_hash'] - else: - raise RequestException("Malformed request: did not specify the attribute_hash") - - if 'mid' in param_dict: - request_parameters['mid'] = param_dict['mid'] - else: - raise RequestException("Malformed request: did not specify the verifier's mid") - - if 'attribute_values' in param_dict: - request_parameters['attribute_values'] = param_dict['attribute_values'] - else: - raise RequestException("Malformed request: did not specify the attribute_values") - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'POST', - request_parameters, - param_dict.get('callback', None)) - - returnValue(response) - - @inlineCallbacks - def make_allow_verify(self, param_dict): - """ - Forward a request which requests that verifications be allowed for a particular peer for a particular attribute - - :param param_dict: Should have at least the following structure: - { - 'interface': target peer IP or alias - 'port': port_number - 'endpoint': endpoint_name - 'attribute_name': attribute_name - 'mid': verifier's b64_mid - (optional) 'callback': single parameter callback for the request's response - } - :return: the request's response - :raises RequestException: raised when the method could not find some element required for the construction of - the request - """ - interface, port, endpoint = HTTPRequester.get_access_parameters(param_dict) - - # Add the type of the request (request), and the rest of the parameters - request_parameters = {'type': 'allow_verify'} - - # Add the request parameters one-by-one; if required parameter is missing, then raise error - if 'attribute_name' in param_dict: - request_parameters['attribute_name'] = param_dict['attribute_name'] - else: - raise RequestException("Malformed request: did not specify the attribute_name") - - if 'mid' in param_dict: - request_parameters['mid'] = param_dict['mid'] - else: - raise RequestException("Malformed request: did not specify the attester's mid") - - response = yield self.make_request(HTTPRequester.basic_url_builder(interface, port, endpoint), - 'POST', - request_parameters, - param_dict.get('callback', None)) - returnValue(response) - - def string_to_url(string, quote_string=False, to_utf_8=False): """ Convert a string to a format which is compatible to it being passed via a url @@ -476,3 +120,9 @@ def string_to_url(string, quote_string=False, to_utf_8=False): string = string.replace("+", "%2B") if not quote_string else quote(string.replace("+", "%2B")) return string.encode('utf-8') if to_utf_8 else string + + +class RequestException(Exception): + """ + Custom exception used to model request errors + """ diff --git a/test_classes_list.txt b/test_classes_list.txt index b3261833e..227456931 100644 --- a/test_classes_list.txt +++ b/test_classes_list.txt @@ -41,4 +41,5 @@ ipv8/test/dht/test_routing.py:TestBucket ipv8/test/dht/test_routing.py:TestRoutingTable ipv8/test/dht/test_storage.py:TestStorage -ipv8/test/REST/test_attestation_endpoint.py:TestAttestationEndpoint \ No newline at end of file +ipv8/test/REST/attestationendpoint/test_attestation_endpoint.py:TestAttestationEndpoint +ipv8/test/REST/dht/test_dht_endpoint.py:TestDHTEndpoint \ No newline at end of file