Added a new endpoint for retrieving the latest TrustChain block via t…

…he DHT mechanism. Added a new callback in the TrustChainCommunity class which is called whenever a block is being added in the DB, irrelevant of the block's type. It is now possible to retrieve the latest TrustChain block of some peer via a HTTP request. Created a new test set for the newly added block retrieval mechanism. In addition to this, made a few changes to the REST API test suite in terms of the communication modules, such as moving classes from one module to another and creating new modules which host logic for HTTP requests and communication for specific endpoints.

ipv8/REST/dht_endpoint.py

@@ -1,13 +1,17 @@
 from __future__ import absolute_import
 
-from base64 import b64encode
+from base64 import b64encode, b64decode
 from binascii import hexlify, unhexlify
+from hashlib import sha1
 import json
+import struct
 
+from twisted.internet.defer import inlineCallbacks, returnValue
 from twisted.web import http, resource
 from twisted.web.server import NOT_DONE_YET
 
-from ..dht.community import DHTCommunity
+from ..dht.community import DHTCommunity, MAX_ENTRY_SIZE
+from ..attestation.trustchain.community import TrustChainCommunity
 from ..dht.discovery import DHTDiscoveryCommunity
 
 
@@ -20,10 +24,87 @@ def __init__(self, session):
  resource.Resource.__init__(self)
 
  dht_overlays = [overlay for overlay in session.overlays if isinstance(overlay, DHTCommunity)]
+ tc_overlays = [overlay for overlay in session.overlays if isinstance(overlay, TrustChainCommunity)]
  if dht_overlays:
- self.putChild("statistics", DHTStatisticsEndpoint(dht_overlays[0]))
- self.putChild("values", DHTValuesEndpoint(dht_overlays[0]))
- self.putChild("peers", DHTPeersEndpoint(dht_overlays[0]))
+ self.putChild(b"statistics", DHTStatisticsEndpoint(dht_overlays[0]))
+ self.putChild(b"values", DHTValuesEndpoint(dht_overlays[0]))
+ self.putChild(b"peers", DHTPeersEndpoint(dht_overlays[0]))
+ self.putChild(b"block", DHTBlockEndpoint(dht_overlays[0], tc_overlays[0]))
+
+
+class DHTBlockEndpoint(resource.Resource):
+ """
+ This endpoint is responsible for returning the latest Trustchain block of a peer. Additionally, it ensures
+ this peer's latest TC block is available
+ """
+
+ KEY_SUFFIX = b'_BLOCK'
+
+ def __init__(self, dht, trustchain):
+ resource.Resource.__init__(self)
+ self.dht = dht
+ self.trustchain = trustchain
+ self.block_version = 0
+
+ self._hashed_dht_key = sha1(self.trustchain.my_peer.mid + self.KEY_SUFFIX).digest()
+
+ trustchain.set_new_block_cb(self.publish_latest_block)
+
+ @inlineCallbacks
+ def publish_latest_block(self):
+ """
+ Publish the latest block of this node's Trustchain to the DHT
+
+ :return:
+ """
+ # latest_block = self.trustchain.persistence.get_latest(self.trustchain.my_peer.key.pub().key_to_bin())
+ latest_block = self.trustchain.persistence.get_latest(self.trustchain.my_peer.public_key.key_to_bin())
+
+ if latest_block:
+ latest_block = latest_block.pack()
+ version = struct.pack("H", self.block_version)
+ self.block_version += 1
+
+ for i in range(0, len(latest_block), MAX_ENTRY_SIZE - 3):
+ blob_chunk = version + latest_block[i:i + MAX_ENTRY_SIZE - 3]
+ yield self.dht.store_value(self._hashed_dht_key, blob_chunk)
+
+ def render_GET(self, request):
+ """
+ Return the latest TC block of a peer, as identified in the request
+
+ :param request: the request for retrieving the latest TC block of a peer. It must contain the peer's
+ public key of the peer
+ :return: the latest block of the peer, if found
+ """
+ if not self.dht:
+ request.setResponseCode(http.NOT_FOUND)
+ return json.dumps({"error": "DHT community not found"}).encode('utf-8')
+
+ if not request.args or b'public_key' not in request.args:
+ request.setResponseCode(http.BAD_REQUEST)
+ return json.dumps({"error": "Must specify the peer's public key"}).encode('utf-8')
+
+ hash_key = sha1(b64decode(request.args[b'public_key'][0]) + self.KEY_SUFFIX).digest()
+ block_chunks = self.dht.storage.get(hash_key)
+
+ if not block_chunks:
+ request.setResponseCode(http.NOT_FOUND)
+ return json.dumps({"error": "Could not find a block for the specified key."}).encode('utf-8')
+
+ new_blocks = {}
+ max_version = 0
+
+ for entry in block_chunks:
+ this_version = struct.unpack("I", entry[1:3] + b'\x00\x00')[0]
+ max_version = max_version if max_version > this_version else this_version
+
+ if this_version in new_blocks:
+ new_blocks[this_version] = entry[3:] + new_blocks[this_version]
+ else:
+ new_blocks[this_version] = entry[3:]
+
+ return json.dumps({"block": b64encode(new_blocks[max_version]).decode('utf-8')}).encode('utf-8')
 
 
 class DHTStatisticsEndpoint(resource.Resource):

ipv8/attestation/trustchain/community.py

@@ -73,6 +73,8 @@ def __init__(self, *args, **kwargs):
  self.db_cleanup_lc = self.register_task("db_cleanup", LoopingCall(self.do_db_cleanup))
  self.db_cleanup_lc.start(600)
 
+ self.new_block_cb = lambda: None
+
  self.decode_map.update({
  chr(1): self.received_half_block,
  chr(2): self.received_crawl_request,
@@ -83,6 +85,13 @@ def __init__(self, *args, **kwargs):
  chr(7): self.received_empty_crawl_response,
  })
 
+ def set_new_block_cb(self, f):
+ """
+ Set the callback for when a new block is added to the DB. The callback should take no arguments and should
+ return nothing: [] -> None
+ """
+ self.new_block_cb = f
+
  def do_db_cleanup(self):
  """
  Cleanup the database if necessary.
@@ -251,6 +260,7 @@ def sign_block(self, peer, public_key=EMPTY_PK, block_type=b'unknown', transacti
 
  if not self.persistence.contains(block):
  self.persistence.add_block(block)
+ self.new_block_cb()
  self.notify_listeners(block)
 
  # This is a source block with no counterparty
@@ -342,6 +352,7 @@ def validate_persist_block(self, block):
  pass
  elif not self.persistence.contains(block):
  self.persistence.add_block(block)
+ self.new_block_cb()
  self.notify_listeners(block)
 
  return validation

ipv8/test/mocking/rest/peer_communication.py → ipv8/test/REST/attestationendpoint/peer_communication.py

@@ -1,4 +1,3 @@
-
 from abc import abstractmethod
 
 
@@ -111,9 +110,3 @@ def make_outstanding_verify(self, param_dict):
  :return: None
  """
  pass
-
-
-class RequestException(Exception):
- """
- Custom exception used to model request errors
- """