Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fluffy portal testnet support #2383

Merged
merged 4 commits into from
Jun 18, 2024
Merged

Fluffy portal testnet support #2383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5b9667f
Bump portal-mainnet repo.
web3-developer Jun 17, 2024
97bcd34
Update command line arguments and parsing on startup.
web3-developer Jun 17, 2024
d8eb3a9
Read in angelfood bootstrap nodes and update Fluffy guide.
web3-developer Jun 17, 2024
89a244b
Configure subnetwork protocol ids.
web3-developer Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 27 additions & 26 deletions fluffy/conf.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,17 +56,6 @@ type
PortalCmd* = enum
noCommand

PortalNetwork* = enum
none
mainnet
testnet

# The networks alias Portal sub-protocols
Network* = enum
beacon
history
state
Copy link
Contributor Author

@web3-developer web3-developer Jun 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved into portal_protocol_config.nim because these types are needed by portal_protocol.nim


PortalConf* = object
logLevel* {.
desc:
Expand Down Expand Up @@ -94,27 +83,37 @@ type
name: "listen-address"
.}: IpAddress

portalNetwork* {.
portalNetworkDeprecated* {.
hidden,
desc:
"DEPRECATED: The --portal-network flag will be removed in the future, " &
"please use the drop in replacement --network flag instead",
defaultValue: none(PortalNetwork),
name: "portal-network"
.}: Option[PortalNetwork]

network* {.
desc:
"Select which Portal network to join. This will set the " &
"Portal network specific bootstrap nodes automatically",
defaultValue: PortalNetwork.mainnet,
name: "portal-network"
name: "network"
.}: PortalNetwork

portalNetworkDeprecated* {.
networksDeprecated* {.
hidden,
desc:
"DEPRECATED: The --network flag will be removed in the future, please use the drop in replacement --portal-network flag instead",
defaultValue: none(PortalNetwork),
name: "network"
.}: Option[PortalNetwork.mainnet]
"DEPRECATED: The --networks flag will be removed in the future, " &
"please use the drop in replacement --portal-subnetworks flag instead",
defaultValue: {},
name: "networks"
.}: set[PortalSubnetwork]

networks* {.
portalSubnetworks* {.
desc: "Select which networks (Portal sub-protocols) to enable",
defaultValue: {Network.history},
name: "networks"
.}: set[Network]
defaultValue: {PortalSubnetwork.history},
name: "portal-subnetworks"
.}: set[PortalSubnetwork]

# Note: This will add bootstrap nodes for both Discovery v5 network and each
# enabled Portal network. No distinction is made on bootstrap nodes per
Expand Down Expand Up @@ -363,21 +362,23 @@ proc parseCmdArg*(T: type ClientConfig, p: string): T {.raises: [ValueError].} =
proc completeCmdArg*(T: type ClientConfig, val: string): seq[string] =
return @[]

proc parseCmdArg*(T: type set[Network], p: string): T {.raises: [ValueError].} =
var res: set[Network] = {}
proc parseCmdArg*(
T: type set[PortalSubnetwork], p: string
): T {.raises: [ValueError].} =
var res: set[PortalSubnetwork] = {}
let values = p.split({' ', ','})
for value in values:
let stripped = value.strip()
let network =
try:
parseEnum[Network](stripped)
parseEnum[PortalSubnetwork](stripped)
except ValueError:
raise newException(ValueError, "Invalid network: " & stripped)

res.incl(network)
res

proc completeCmdArg*(T: type set[Network], val: string): seq[string] =
proc completeCmdArg*(T: type set[PortalSubnetwork], val: string): seq[string] =
return @[]

chronicles.formatIt(InputDir):
Expand Down
12 changes: 5 additions & 7 deletions fluffy/docs/the_fluffy_book/docs/connect-to-portal.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@ Connecting to the current Portal network is as easy as running following command
./build/fluffy --rpc
```

This will connect to the public [Portal testnet](https://github.com/ethereum/portal-network-specs/blob/master/testnet.md#portal-network-testnet)
which contains nodes of the different clients.
This will connect to the public Portal mainnet which contains nodes of the different clients.

!!! note
Default the Fluffy node will connect to the
[bootstrap nodes](https://github.com/ethereum/portal-network-specs/blob/master/testnet.md#bootnodes) of the public testnet.
By default the Fluffy node will connect to the
[bootstrap nodes](https://github.com/ethereum/portal-network-specs/blob/master/bootnodes.md#bootnodes-mainnet) of the public mainnet.

When testing locally the `--portal-network:none` option can be provided to avoid
connecting to any of the testnet bootstrap nodes.
When testing locally the `--network:none` option can be provided to avoid
connecting to any of the default bootstrap nodes.

The `--rpc` option will also enable the different JSON-RPC interfaces through
which you can access the Portal Network.
Expand All @@ -32,4 +31,3 @@ Fluffy also supports a small subset of the [Execution JSON-RPC API](https://ethe

<!-- TODO: Explain some of the more important cli options here? Or in a separate
page? -->

48 changes: 32 additions & 16 deletions fluffy/fluffy.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import
./network/state/[state_network, state_content],
./network/history/[history_network, history_content],
./network/beacon/[beacon_init_loader, beacon_light_client],
./network/wire/[portal_stream, portal_protocol_config],
./network/wire/[portal_stream, portal_protocol_config, portal_protocol],
./eth_data/history_data_ssz_e2s,
./database/content_db,
./version,
Expand Down Expand Up @@ -101,24 +101,35 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
loadBootstrapFile(string config.bootstrapNodesFile, bootstrapRecords)
bootstrapRecords.add(config.bootstrapNodes)

var portalNetwork: PortalNetwork
if config.portalNetworkDeprecated.isSome():
warn "DEPRECATED: The --network flag will be removed in the future, please use the drop in replacement --portal-network flag instead"
portalNetwork = config.portalNetworkDeprecated.get()
else:
portalNetwork = config.portalNetwork
let portalNetwork =
if config.portalNetworkDeprecated.isNone():
config.network
else:
warn "DEPRECATED: The --portal-network flag will be removed in the future, " &
"please use the drop in replacement --network flag instead"
config.portalNetworkDeprecated.get()

let portalSubnetworks =
if config.networksDeprecated == {}:
config.portalSubnetworks
else:
warn "DEPRECATED: The --networks flag will be removed in the future, " &
"please use the drop in replacement --portal-subnetworks flag instead"
config.networksDeprecated

case portalNetwork
of mainnet:
of PortalNetwork.none:
discard # don't connect to any network bootstrap nodes
of PortalNetwork.mainnet:
for enrURI in mainnetBootstrapNodes:
var record: Record
if fromURI(record, enrURI):
bootstrapRecords.add(record)
of testnet:
# TODO: add testnet repo with bootstrap file.
discard
else:
discard
of PortalNetwork.angelfood:
for enrURI in angelfoodBootstrapNodes:
var record: Record
if fromURI(record, enrURI):
bootstrapRecords.add(record)

let
discoveryConfig =
Expand Down Expand Up @@ -210,9 +221,10 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
loadAccumulator()

historyNetwork =
if Network.history in config.networks:
if PortalSubnetwork.history in portalSubnetworks:
Opt.some(
HistoryNetwork.new(
portalNetwork,
d,
db,
streamManager,
Expand All @@ -225,9 +237,10 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
Opt.none(HistoryNetwork)

stateNetwork =
if Network.state in config.networks:
if PortalSubnetwork.state in portalSubnetworks:
Opt.some(
StateNetwork.new(
portalNetwork,
d,
db,
streamManager,
Expand All @@ -243,12 +256,15 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
beaconLightClient =
# TODO: Currently disabled by default as it is not sufficiently polished.
# Eventually this should be always-on functionality.
if Network.beacon in config.networks and config.trustedBlockRoot.isSome():
if PortalSubnetwork.beacon in portalSubnetworks and
config.trustedBlockRoot.isSome():
let
# Portal works only over mainnet data currently
# TODO: investigate this load network data function
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is for loading beacon network metadata , so this remains the same for angelfood.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok thanks, I'll remove the comment in my next PR.

networkData = loadNetworkData("mainnet")
beaconDb = BeaconDb.new(networkData, config.dataDir / "db" / "beacon_db")
beaconNetwork = BeaconNetwork.new(
portalNetwork,
d,
beaconDb,
streamManager,
Expand Down
5 changes: 2 additions & 3 deletions fluffy/network/beacon/beacon_network.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ export beacon_content, beacon_db
logScope:
topics = "beacon_network"

const lightClientProtocolId* = [byte 0x50, 0x1A]

type BeaconNetwork* = ref object
portalProtocol*: PortalProtocol
beaconDb*: BeaconDb
Expand Down Expand Up @@ -183,6 +181,7 @@ proc getHistoricalSummaries*(

proc new*(
T: type BeaconNetwork,
portalNetwork: PortalNetwork,
baseProtocol: protocol.Protocol,
beaconDb: BeaconDb,
streamManager: StreamManager,
Expand All @@ -206,7 +205,7 @@ proc new*(

portalProtocol = PortalProtocol.new(
baseProtocol,
lightClientProtocolId,
getProtocolId(portalNetwork, PortalSubnetwork.beacon),
toContentIdHandler,
createGetHandler(beaconDb),
stream,
Expand Down
5 changes: 2 additions & 3 deletions fluffy/network/history/history_network.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ export accumulator
proc `$`(x: BlockHeader): string =
$x

const historyProtocolId* = [byte 0x50, 0x0B]

type
HistoryNetwork* = ref object
portalProtocol*: PortalProtocol
Expand Down Expand Up @@ -703,6 +701,7 @@ proc validateContent(

proc new*(
T: type HistoryNetwork,
portalNetwork: PortalNetwork,
baseProtocol: protocol.Protocol,
contentDB: ContentDB,
streamManager: StreamManager,
Expand All @@ -718,7 +717,7 @@ proc new*(

portalProtocol = PortalProtocol.new(
baseProtocol,
historyProtocolId,
getProtocolId(portalNetwork, PortalSubnetwork.history),
toContentIdHandler,
createGetHandler(contentDB),
stream,
Expand Down
3 changes: 1 addition & 2 deletions fluffy/network/network_seed.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ const
#TODO currently we are using value for history network, but this should be
#caluculated per netowork basis
maxItemsPerOfferBySize = getMaxOfferedContentKeys(
uint32(len(history_network.historyProtocolId)),
uint32(history_content.maxContentKeySize),
uint32(len(PortalProtocolId)), uint32(history_content.maxContentKeySize)
)

# Offering is restricted to max 64 items
Expand Down
5 changes: 2 additions & 3 deletions fluffy/network/state/state_network.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ export results, state_content
logScope:
topics = "portal_state"

const stateProtocolId* = [byte 0x50, 0x0A]

type StateNetwork* = ref object
portalProtocol*: PortalProtocol
contentDB*: ContentDB
Expand All @@ -41,6 +39,7 @@ func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] =

proc new*(
T: type StateNetwork,
portalNetwork: PortalNetwork,
baseProtocol: protocol.Protocol,
contentDB: ContentDB,
streamManager: StreamManager,
Expand All @@ -55,7 +54,7 @@ proc new*(

let portalProtocol = PortalProtocol.new(
baseProtocol,
stateProtocolId,
getProtocolId(portalNetwork, PortalSubnetwork.state),
toContentIdHandler,
createGetHandler(contentDB),
s,
Expand Down
35 changes: 35 additions & 0 deletions fluffy/network/wire/portal_protocol.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,41 @@ func init*(
nodesInterestedInContent: nodesInterestedInContent,
)

func getProtocolId*(
network: PortalNetwork, subnetwork: PortalSubnetwork
): PortalProtocolId =
const portalPrefix = byte(0x50)

case network
of PortalNetwork.none, PortalNetwork.mainnet:
case subnetwork
of PortalSubnetwork.state:
[portalPrefix, 0x0A]
of PortalSubnetwork.history:
[portalPrefix, 0x0B]
of PortalSubnetwork.beacon:
[portalPrefix, 0x0C]
of PortalSubnetwork.transactionIndex:
[portalPrefix, 0x0D]
of PortalSubnetwork.verkleState:
[portalPrefix, 0x0E]
of PortalSubnetwork.transactionGossip:
[portalPrefix, 0x0F]
of PortalNetwork.angelfood:
case subnetwork
of PortalSubnetwork.state:
[portalPrefix, 0x4A]
of PortalSubnetwork.history:
[portalPrefix, 0x4B]
of PortalSubnetwork.beacon:
[portalPrefix, 0x4C]
of PortalSubnetwork.transactionIndex:
[portalPrefix, 0x4D]
of PortalSubnetwork.verkleState:
[portalPrefix, 0x4E]
of PortalSubnetwork.transactionGossip:
[portalPrefix, 0x4F]

func `$`(id: PortalProtocolId): string =
id.toHex()

Expand Down
14 changes: 14 additions & 0 deletions fluffy/network/wire/portal_protocol_config.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,20 @@
import std/strutils, confutils, chronos, stint, eth/p2p/discoveryv5/routing_table

type
PortalNetwork* = enum
none
mainnet
angelfood

# The Portal sub-protocols
PortalSubnetwork* = enum
state
history
beacon
transactionIndex
verkleState
transactionGossip

RadiusConfigKind* = enum
Static
Dynamic
Expand Down
2 changes: 2 additions & 0 deletions fluffy/network_metadata.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ const
# rlp.rawData() in the enr code.
mainnetBootstrapNodes* =
loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes.txt")
angelfoodBootstrapNodes* =
loadCompileTimeBootstrapNodes(portalConfigDir / "bootstrap_nodes_angelfood.txt")

finishedAccumulatorSSZ* = slurp(portalConfigDir / "finished_accumulator.ssz")

Expand Down
Loading
Loading