Skip to content

Commit

Permalink
Remove vid recycling feature (#2294)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mjfh
mjfh authored Jun 4, 2024
1 parent cc909c9 commit 69a1588
Show file tree
Hide file tree
Showing 25 changed files with 209 additions and 455 deletions.
54 changes: 25 additions & 29 deletions nimbus/db/aristo/aristo_blobify.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,26 +135,24 @@ proc blobifyTo*(vtx: VertexRef; data: var Blob): Result[void,AristoError] =
data &= [0xC0u8 or psLen]
ok()


proc blobify*(vtx: VertexRef): Result[Blob, AristoError] =
## Variant of `blobify()`
var data: Blob
? vtx.blobifyTo data
ok(move(data))

proc blobifyTo*(vGen: openArray[VertexID]; data: var Blob) =
## This function serialises a list of vertex IDs.
## ::
## uint64, ... -- list of IDs
## 0x7c -- marker(8)
##
for w in vGen:
data &= w.uint64.toBytesBE
data.add 0x7Cu8

proc blobify*(vGen: openArray[VertexID]): Blob =
## Variant of `blobify()`
vGen.blobifyTo result
proc blobifyTo*(tuv: VertexID; data: var Blob) =
## This function serialises a top used vertex ID.
data.setLen(9)
let w = tuv.uint64.toBytesBE
(addr data[0]).copyMem(unsafeAddr w[0], 8)
data[8] = 0x7Cu8

proc blobify*(tuv: VertexID): Blob =
## Variant of `blobifyTo()`
tuv.blobifyTo result


proc blobifyTo*(lSst: SavedState; data: var Blob) =
## Serialise a last saved state record
Expand Down Expand Up @@ -315,30 +313,28 @@ proc deblobify*(

proc deblobifyTo*(
data: openArray[byte];
vGen: var seq[VertexID];
tuv: var VertexID;
): Result[void,AristoError] =
## De-serialise the data record encoded with `blobify()` into the vertex ID
## generator argument `vGen`.
## De-serialise a top level vertex ID.
if data.len == 0:
vGen = @[]
tuv = VertexID(0)
elif data.len != 9:
return err(DeblobSizeGarbled)
elif data[^1] != 0x7c:
return err(DeblobWrongType)
else:
if (data.len mod 8) != 1:
return err(DeblobSizeGarbled)
if data[^1] != 0x7c:
return err(DeblobWrongType)
for n in 0 ..< (data.len div 8):
let w = n * 8
vGen.add (uint64.fromBytesBE data.toOpenArray(w, w+7)).VertexID
tuv = (uint64.fromBytesBE data.toOpenArray(0, 7)).VertexID
ok()

proc deblobify*(
data: openArray[byte];
T: type seq[VertexID];
T: type VertexID;
): Result[T,AristoError] =
## Variant of `deblobify()` for deserialising the vertex ID generator state
var vGen: T
? data.deblobifyTo vGen
ok move(vGen)
## Variant of `deblobify()` for deserialising a top level vertex ID.
var vTop: T
? data.deblobifyTo vTop
ok move(vTop)


proc deblobifyTo*(
data: openArray[byte];
Expand Down
115 changes: 45 additions & 70 deletions nimbus/db/aristo/aristo_check/check_be.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,32 +11,18 @@
{.push raises: [].}

import
std/[algorithm, sequtils, sets, tables, typetraits],
std/[sets, tables],
eth/[common, trie/nibbles],
results,
stew/interval_set,
../../aristo,
../aristo_walk/persistent,
".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise]

const
Vid2 = @[VertexID(LEAST_FREE_VID)].toHashSet

# ------------------------------------------------------------------------------
# Private helper
# ------------------------------------------------------------------------------

proc to(s: IntervalSetRef[VertexID,uint64]; T: type HashSet[VertexID]): T =
## Convert the argument list `s` to a set of vertex IDs as it would appear
## with a vertex generator state list.
if s.total < high(uint64):
for w in s.increasing:
if w.maxPt == high(VertexID):
result.incl w.minPt # last interval
else:
for pt in w.minPt .. w.maxPt:
if LEAST_FREE_VID <= pt.distinctBase:
result.incl pt

proc toNodeBE(
vtx: VertexRef; # Vertex to convert
db: AristoDbRef; # Database, top layer
Expand Down Expand Up @@ -76,17 +62,6 @@ proc toNodeBE(
return ok node
return err(vid)

proc vidReorgAlways(vGen: seq[VertexID]): seq[VertexID] =
## See `vidReorg()`, this one always sorts and optimises
##
if 1 < vGen.len:
let lst = vGen.mapIt(uint64(it)).sorted(Descending).mapIt(VertexID(it))
for n in 0 .. lst.len-2:
if lst[n].uint64 != lst[n+1].uint64 + 1:
return lst[n+1 .. lst.len-1] & @[lst[n]]
return @[lst[^1]]
vGen

# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
Expand All @@ -100,11 +75,11 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
): Result[void,(VertexID,AristoError)] =
## Make sure that each vertex has a Merkle hash and vice versa. Also check
## the vertex ID generator state.
let vids = IntervalSetRef[VertexID,uint64].init()
discard vids.merge Interval[VertexID,uint64].new(
VertexID(LEAST_FREE_VID),high(VertexID))
var topVidBe = VertexID(0)

for (vid,vtx) in T.walkVtxBe db:
if topVidBe < vid:
topVidBe = vid
if not vtx.isValid:
return err((vid,CheckBeVtxInvalid))
let rc = db.getKeyBE vid
Expand All @@ -127,6 +102,8 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
return err((vid,CheckBeVtxExtPfxMissing))

for (vid,key) in T.walkKeyBe db:
if topVidBe < vid:
topVidBe = vid
if not key.isValid:
return err((vid,CheckBeKeyInvalid))
let vtx = db.getVtxBE(vid).valueOr:
Expand All @@ -137,29 +114,32 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
let expected = node.digestTo(HashKey)
if expected != key:
return err((vid,CheckBeKeyMismatch))
discard vids.reduce Interval[VertexID,uint64].new(vid,vid)

# Compare calculated state against database state
block:
# Extract vertex ID generator state
let vGen = block:
let rc = db.getIdgBE()
# Compare calculated `vTop` against database state
if topVidBe.isValid:
let vidTuvBe = block:
let rc = db.getTuvBE()
if rc.isOk:
rc.value.vidReorgAlways.toHashSet
elif rc.error == GetIdgNotFound:
EmptyVidSeq.toHashSet
rc.value
elif rc.error == GetTuvNotFound:
VertexID(0)
else:
return err((VertexID(0),rc.error))
let
vGenExpected = vids.to(HashSet[VertexID])
delta = vGenExpected -+- vGen # symmetric difference
if 0 < delta.len:
# Exclude fringe case when there is a single root vertex only
if vGenExpected != Vid2 or 0 < vGen.len:
return err((delta.toSeq.sorted[^1],CheckBeGarbledVGen))

# Check top layer cache against backend
if vidTuvBe != topVidBe:
# All vertices and keys between `topVidBe` and `vidTuvBe` must have
# been deleted.
for vid in max(topVidBe + 1, VertexID(LEAST_FREE_VID)) .. vidTuvBe:
if db.getVtxBE(vid).isOk or db.getKeyBE(vid).isOk:
echo ">>>",
" topVidBe=", topVidBe,
" vidTuvBe=", vidTuvBe,
" vid=", vid
return err((vid,CheckBeGarbledVTop))

# Check layer cache against backend
if cache:
var topVidCache = VertexID(0)

let checkKeysOk = block:
if db.dirty.len == 0:
true
Expand All @@ -170,6 +150,8 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](

# Check structural table
for (vid,vtx) in db.layersWalkVtx:
if vtx.isValid and topVidCache < vid:
topVidCache = vid
let key = block:
let rc = db.layersGetKey(vid)
if rc.isOk:
Expand All @@ -179,22 +161,19 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
return err((vid,CheckBeCacheKeyMissing))
else:
VOID_HASH_KEY
if vtx.isValid:
# Register existing vid against backend generator state
discard vids.reduce Interval[VertexID,uint64].new(vid,vid)
else:
if not vtx.isValid:
# Some vertex is to be deleted, the key must be empty
if checkKeysOk and key.isValid:
return err((vid,CheckBeCacheKeyNonEmpty))
# There must be a representation on the backend DB unless in a TX
if db.getVtxBE(vid).isErr and db.stack.len == 0:
return err((vid,CheckBeCacheVidUnsynced))
# Register deleted vid against backend generator state
discard vids.merge Interval[VertexID,uint64].new(vid,vid)

# Check key table
var list: seq[VertexID]
for (vid,key) in db.layersWalkKey:
if key.isValid and topVidCache < vid:
topVidCache = vid
list.add vid
let vtx = db.getVtx vid
if db.layersGetVtx(vid).isErr and not vtx.isValid:
Expand All @@ -209,22 +188,18 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
if expected != key:
return err((vid,CheckBeCacheKeyMismatch))

# Check vGen
let
vGen = db.vGen.vidReorgAlways.toHashSet
vGenExpected = vids.to(HashSet[VertexID])
delta = vGenExpected -+- vGen # symmetric difference
if 0 < delta.len:
if vGen == Vid2 and vGenExpected.len == 0:
# Fringe case when the database is empty
discard
elif vGen.len == 0 and vGenExpected == Vid2:
# Fringe case when there is a single root vertex only
discard
else:
let delta = delta.toSeq
if delta.len != 1 or delta[0] != VertexID(1) or VertexID(1) in vGen:
return err((delta.sorted[^1],CheckBeCacheGarbledVGen))
# Check vTop
if topVidCache.isValid and topVidCache != db.vTop:
# All vertices and keys between `topVidCache` and `db.vTop` must have
# been deleted.
for vid in max(db.vTop + 1, VertexID(LEAST_FREE_VID)) .. topVidCache:
if db.layersGetVtxOrVoid(vid).isValid or
db.layersGetKeyOrVoid(vid).isValid:
echo ">>>",
" topVidCache=", topVidCache,
" vTop=", db.vTop,
" vid=", vid
return err((db.vTop,CheckBeCacheGarbledVTop))

ok()

Expand Down
17 changes: 13 additions & 4 deletions nimbus/db/aristo/aristo_check/check_top.nim
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
{.push raises: [].}

import
std/[sequtils, sets],
std/[sequtils, sets, typetraits],
eth/[common, trie/nibbles],
results,
".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise, aristo_utils]
Expand Down Expand Up @@ -92,23 +92,25 @@ proc checkTopCommon*(
let
kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len
kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount
vGen = db.vGen.toHashSet
vGenMax = if vGen.len == 0: VertexID(0) else: db.vGen[^1]
vTop = db.vTop
var
topVid = VertexID(0)
stoRoots: HashSet[VertexID]

# Collect leafs and check deleted entries
var nNilVtx = 0
for (vid,vtx) in db.layersWalkVtx:
if vtx.isValid:
if topVid < vid:
topVid = vid
case vtx.vType:
of Leaf:
if vtx.lData.pType == AccountData:
let stoVid = vtx.lData.account.storageID
if stoVid.isValid:
if stoVid in stoRoots:
return err((stoVid,CheckAnyVidSharedStorageRoot))
if vGenMax.isValid and (vGenMax < stoVid or stoVid in vGen):
if vTop < stoVid:
return err((stoVid,CheckAnyVidDeadStorageRoot))
stoRoots.incl stoVid
of Branch:
Expand All @@ -131,6 +133,13 @@ proc checkTopCommon*(
if rc.value.isValid:
return err((vid,CheckAnyVtxEmptyKeyExpected))

if vTop.distinctBase < LEAST_FREE_VID:
# Verify that all vids are below `LEAST_FREE_VID`
if topVid.distinctBase < LEAST_FREE_VID:
for (vid,key) in db.layersWalkKey:
if key.isValid and LEAST_FREE_VID <= vid.distinctBase:
return err((topVid,CheckAnyVTopUnset))

# If present, there are at least as many deleted hashes as there are deleted
# vertices.
if kMapNilCount != 0 and kMapNilCount < nNilVtx:
Expand Down
Loading

0 comments on commit 69a1588

Please sign in to comment.