Fluffy portal testnet support (#2383)

* Bump portal-mainnet repo. * Update command line arguments and parsing on startup. * Read in angelfood bootstrap nodes and update Fluffy guide. * Configure subnetwork protocol ids.
status-im · Jun 18, 2024 · e3d14bd · e3d14bd
1 parent 8926da0
commit e3d14bd
Show file tree

Hide file tree

Showing 16 changed files with 133 additions and 72 deletions.
diff --git a/fluffy/conf.nim b/fluffy/conf.nim
@@ -56,17 +56,6 @@ type
  PortalCmd* = enum
  noCommand
 
- PortalNetwork* = enum
- none
- mainnet
- testnet
-
- # The networks alias Portal sub-protocols
- Network* = enum
- beacon
- history
- state
-
  PortalConf* = object
  logLevel* {.
  desc:
@@ -94,27 +83,37 @@ type
  name: "listen-address"
  .}: IpAddress
 
- portalNetwork* {.
+ portalNetworkDeprecated* {.
+ hidden,
+ desc:
+ "DEPRECATED: The --portal-network flag will be removed in the future, " &
+ "please use the drop in replacement --network flag instead",
+ defaultValue: none(PortalNetwork),
+ name: "portal-network"
+ .}: Option[PortalNetwork]
+
+ network* {.
  desc:
  "Select which Portal network to join. This will set the " &
  "Portal network specific bootstrap nodes automatically",
  defaultValue: PortalNetwork.mainnet,
- name: "portal-network"
+ name: "network"
  .}: PortalNetwork
 
- portalNetworkDeprecated* {.
+ networksDeprecated* {.
  hidden,
  desc:
- "DEPRECATED: The --network flag will be removed in the future, please use the drop in replacement --portal-network flag instead",
- defaultValue: none(PortalNetwork),
- name: "network"
- .}: Option[PortalNetwork.mainnet]
+ "DEPRECATED: The --networks flag will be removed in the future, " &
+ "please use the drop in replacement --portal-subnetworks flag instead",
+ defaultValue: {},
+ name: "networks"
+ .}: set[PortalSubnetwork]
 
- networks* {.
+ portalSubnetworks* {.
  desc: "Select which networks (Portal sub-protocols) to enable",
- defaultValue: {Network.history},
- name: "networks"
- .}: set[Network]
+ defaultValue: {PortalSubnetwork.history},
+ name: "portal-subnetworks"
+ .}: set[PortalSubnetwork]
 
  # Note: This will add bootstrap nodes for both Discovery v5 network and each
  # enabled Portal network. No distinction is made on bootstrap nodes per
@@ -363,21 +362,23 @@ proc parseCmdArg*(T: type ClientConfig, p: string): T {.raises: [ValueError].} =
 proc completeCmdArg*(T: type ClientConfig, val: string): seq[string] =
  return @[]
 
-proc parseCmdArg*(T: type set[Network], p: string): T {.raises: [ValueError].} =
- var res: set[Network] = {}
+proc parseCmdArg*(
+ T: type set[PortalSubnetwork], p: string
+): T {.raises: [ValueError].} =
+ var res: set[PortalSubnetwork] = {}
  let values = p.split({' ', ','})
  for value in values:
  let stripped = value.strip()
  let network =
  try:
- parseEnum[Network](stripped)
+ parseEnum[PortalSubnetwork](stripped)
  except ValueError:
  raise newException(ValueError, "Invalid network: " & stripped)
 
  res.incl(network)
  res
 
-proc completeCmdArg*(T: type set[Network], val: string): seq[string] =
+proc completeCmdArg*(T: type set[PortalSubnetwork], val: string): seq[string] =
  return @[]
 
 chronicles.formatIt(InputDir):

diff --git a/fluffy/docs/the_fluffy_book/docs/connect-to-portal.md b/fluffy/docs/the_fluffy_book/docs/connect-to-portal.md
@@ -6,15 +6,14 @@ Connecting to the current Portal network is as easy as running following command
 ./build/fluffy --rpc
 ```
 
-This will connect to the public [Portal testnet](https://github.com/ethereum/portal-network-specs/blob/master/testnet.md#portal-network-testnet)
-which contains nodes of the different clients.
+This will connect to the public Portal mainnet which contains nodes of the different clients.
 
 !!! note
- Default the Fluffy node will connect to the
- [bootstrap nodes](https://github.com/ethereum/portal-network-specs/blob/master/testnet.md#bootnodes) of the public testnet.
+ By default the Fluffy node will connect to the
+ [bootstrap nodes](https://github.com/ethereum/portal-network-specs/blob/master/bootnodes.md#bootnodes-mainnet) of the public mainnet.
 
- When testing locally the `--portal-network:none` option can be provided to avoid
- connecting to any of the testnet bootstrap nodes.
+ When testing locally the `--network:none` option can be provided to avoid
+ connecting to any of the default bootstrap nodes.
 
 The `--rpc` option will also enable the different JSON-RPC interfaces through
 which you can access the Portal Network.
@@ -32,4 +31,3 @@ Fluffy also supports a small subset of the [Execution JSON-RPC API](https://ethe
 
 <!-- TODO: Explain some of the more important cli options here? Or in a separate
 page? -->
-
diff --git a/fluffy/fluffy.nim b/fluffy/fluffy.nim
@@ -35,7 +35,7 @@ import
  ./network/state/[state_network, state_content],
  ./network/history/[history_network, history_content],
  ./network/beacon/[beacon_init_loader, beacon_light_client],
- ./network/wire/[portal_stream, portal_protocol_config],
+ ./network/wire/[portal_stream, portal_protocol_config, portal_protocol],
  ./eth_data/history_data_ssz_e2s,
  ./database/content_db,
  ./version,
@@ -101,24 +101,35 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
  loadBootstrapFile(string config.bootstrapNodesFile, bootstrapRecords)
  bootstrapRecords.add(config.bootstrapNodes)
 
- var portalNetwork: PortalNetwork
- if config.portalNetworkDeprecated.isSome():
- warn "DEPRECATED: The --network flag will be removed in the future, please use the drop in replacement --portal-network flag instead"
- portalNetwork = config.portalNetworkDeprecated.get()
- else:
- portalNetwork = config.portalNetwork
+ let portalNetwork =
+ if config.portalNetworkDeprecated.isNone():
+ config.network
+ else:
+ warn "DEPRECATED: The --portal-network flag will be removed in the future, " &
+ "please use the drop in replacement --network flag instead"
+ config.portalNetworkDeprecated.get()
+
+ let portalSubnetworks =
+ if config.networksDeprecated == {}:
+ config.portalSubnetworks
+ else:
+ warn "DEPRECATED: The --networks flag will be removed in the future, " &
+ "please use the drop in replacement --portal-subnetworks flag instead"
+ config.networksDeprecated
 
  case portalNetwork
- of mainnet:
+ of PortalNetwork.none:
+ discard # don't connect to any network bootstrap nodes
+ of PortalNetwork.mainnet:
  for enrURI in mainnetBootstrapNodes:
  var record: Record
  if fromURI(record, enrURI):
  bootstrapRecords.add(record)
- of testnet:
- # TODO: add testnet repo with bootstrap file.
- discard
- else:
- discard
+ of PortalNetwork.angelfood:
+ for enrURI in angelfoodBootstrapNodes:
+  var record: Record
+  if fromURI(record, enrURI):
+  bootstrapRecords.add(record)
 
  let
  discoveryConfig =
@@ -210,9 +221,10 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
  loadAccumulator()
 
  historyNetwork =
- if Network.history in config.networks:
+ if PortalSubnetwork.history in portalSubnetworks:
  Opt.some(
  HistoryNetwork.new(
+ portalNetwork,
  d,
  db,
  streamManager,
@@ -225,9 +237,10 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
  Opt.none(HistoryNetwork)
 
  stateNetwork =
- if Network.state in config.networks:
+ if PortalSubnetwork.state in portalSubnetworks:
  Opt.some(
  StateNetwork.new(
+ portalNetwork,
  d,
  db,
  streamManager,
@@ -243,12 +256,15 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
  beaconLightClient =
  # TODO: Currently disabled by default as it is not sufficiently polished.
  # Eventually this should be always-on functionality.
- if Network.beacon in config.networks and config.trustedBlockRoot.isSome():
+ if PortalSubnetwork.beacon in portalSubnetworks and
+ config.trustedBlockRoot.isSome():
  let
  # Portal works only over mainnet data currently
+ # TODO: investigate this load network data function
  networkData = loadNetworkData("mainnet")
  beaconDb = BeaconDb.new(networkData, config.dataDir / "db" / "beacon_db")
  beaconNetwork = BeaconNetwork.new(
+ portalNetwork,
  d,
  beaconDb,
  streamManager,

diff --git a/fluffy/network/beacon/beacon_network.nim b/fluffy/network/beacon/beacon_network.nim
@@ -23,8 +23,6 @@ export beacon_content, beacon_db
 logScope:
  topics = "beacon_network"
 
-const lightClientProtocolId* = [byte 0x50, 0x1A]
-
 type BeaconNetwork* = ref object
  portalProtocol*: PortalProtocol
  beaconDb*: BeaconDb
@@ -183,6 +181,7 @@ proc getHistoricalSummaries*(
 
 proc new*(
  T: type BeaconNetwork,
+ portalNetwork: PortalNetwork,
  baseProtocol: protocol.Protocol,
  beaconDb: BeaconDb,
  streamManager: StreamManager,
@@ -206,7 +205,7 @@ proc new*(
 
  portalProtocol = PortalProtocol.new(
  baseProtocol,
- lightClientProtocolId,
+ getProtocolId(portalNetwork, PortalSubnetwork.beacon),
  toContentIdHandler,
  createGetHandler(beaconDb),
  stream,

diff --git a/fluffy/network/history/history_network.nim b/fluffy/network/history/history_network.nim
@@ -47,8 +47,6 @@ export accumulator
 proc `$`(x: BlockHeader): string =
  $x
 
-const historyProtocolId* = [byte 0x50, 0x0B]
-
 type
  HistoryNetwork* = ref object
  portalProtocol*: PortalProtocol
@@ -703,6 +701,7 @@ proc validateContent(
 
 proc new*(
  T: type HistoryNetwork,
+ portalNetwork: PortalNetwork,
  baseProtocol: protocol.Protocol,
  contentDB: ContentDB,
  streamManager: StreamManager,
@@ -718,7 +717,7 @@ proc new*(
 
  portalProtocol = PortalProtocol.new(
  baseProtocol,
- historyProtocolId,
+ getProtocolId(portalNetwork, PortalSubnetwork.history),
  toContentIdHandler,
  createGetHandler(contentDB),
  stream,

diff --git a/fluffy/network/network_seed.nim b/fluffy/network/network_seed.nim
@@ -33,8 +33,7 @@ const
  #TODO currently we are using value for history network, but this should be
  #caluculated per netowork basis
  maxItemsPerOfferBySize = getMaxOfferedContentKeys(
- uint32(len(history_network.historyProtocolId)),
- uint32(history_content.maxContentKeySize),
+ uint32(len(PortalProtocolId)), uint32(history_content.maxContentKeySize)
  )
 
  # Offering is restricted to max 64 items

diff --git a/fluffy/network/state/state_network.nim b/fluffy/network/state/state_network.nim
@@ -26,8 +26,6 @@ export results, state_content
 logScope:
  topics = "portal_state"
 
-const stateProtocolId* = [byte 0x50, 0x0A]
-
 type StateNetwork* = ref object
  portalProtocol*: PortalProtocol
  contentDB*: ContentDB
@@ -41,6 +39,7 @@ func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] =
 
 proc new*(
  T: type StateNetwork,
+ portalNetwork: PortalNetwork,
  baseProtocol: protocol.Protocol,
  contentDB: ContentDB,
  streamManager: StreamManager,
@@ -55,7 +54,7 @@ proc new*(
 
  let portalProtocol = PortalProtocol.new(
  baseProtocol,
- stateProtocolId,
+ getProtocolId(portalNetwork, PortalSubnetwork.state),
  toContentIdHandler,
  createGetHandler(contentDB),
  s,

diff --git a/fluffy/network/wire/portal_protocol.nim b/fluffy/network/wire/portal_protocol.nim
@@ -254,6 +254,41 @@ func init*(
  nodesInterestedInContent: nodesInterestedInContent,
  )
 
+func getProtocolId*(
+ network: PortalNetwork, subnetwork: PortalSubnetwork
+): PortalProtocolId =
+ const portalPrefix = byte(0x50)
+
+ case network
+ of PortalNetwork.none, PortalNetwork.mainnet:
+ case subnetwork
+ of PortalSubnetwork.state:
+ [portalPrefix, 0x0A]
+ of PortalSubnetwork.history:
+ [portalPrefix, 0x0B]
+ of PortalSubnetwork.beacon:
+ [portalPrefix, 0x0C]
+ of PortalSubnetwork.transactionIndex:
+ [portalPrefix, 0x0D]
+ of PortalSubnetwork.verkleState:
+ [portalPrefix, 0x0E]
+ of PortalSubnetwork.transactionGossip:
+ [portalPrefix, 0x0F]
+ of PortalNetwork.angelfood:
+ case subnetwork
+ of PortalSubnetwork.state:
+ [portalPrefix, 0x4A]
+ of PortalSubnetwork.history:
+ [portalPrefix, 0x4B]
+ of PortalSubnetwork.beacon:
+ [portalPrefix, 0x4C]
+ of PortalSubnetwork.transactionIndex:
+ [portalPrefix, 0x4D]
+ of PortalSubnetwork.verkleState:
+ [portalPrefix, 0x4E]
+ of PortalSubnetwork.transactionGossip:
+ [portalPrefix, 0x4F]
+
 func `$`(id: PortalProtocolId): string =
  id.toHex()
 

diff --git a/fluffy/network/wire/portal_protocol_config.nim b/fluffy/network/wire/portal_protocol_config.nim
@@ -10,6 +10,20 @@
 import std/strutils, confutils, chronos, stint, eth/p2p/discoveryv5/routing_table
 
 type
+ PortalNetwork* = enum
+ none
+ mainnet
+ angelfood
+
+ # The Portal sub-protocols
+ PortalSubnetwork* = enum
+ state
+ history
+ beacon
+ transactionIndex
+ verkleState
+ transactionGossip
+
  RadiusConfigKind* = enum
  Static
  Dynamic

diff --git a/fluffy/network_metadata.nim b/fluffy/network_metadata.nim
@@ -44,6 +44,8 @@ const
  # rlp.rawData() in the enr code.
  mainnetBootstrapNodes* =
  loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes.txt")
+ angelfoodBootstrapNodes* =
+ loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes_angelfood.txt")
 
  finishedAccumulatorSSZ* = slurp(portalConfigDir / "finished_accumulator.ssz")