introduce a freelist interface

This introduces an interface for the freelist, splits it into two concrete
implementations.

fixes etcd-io#773

Signed-off-by: Thomas Jungblut <[email protected]>

allocate_test.go

@@ -4,12 +4,13 @@ import (
  "testing"
 
  "go.etcd.io/bbolt/internal/common"
+ "go.etcd.io/bbolt/internal/freelist"
 )
 
 func TestTx_allocatePageStats(t *testing.T) {
- f := newTestFreelist()
+ f := freelist.NewFreelist(freelist.FreelistArrayType)
  ids := []common.Pgid{2, 3}
- f.readIDs(ids)
+ f.Init(ids)
 
  tx := &Tx{
  db: &DB{
@@ -22,7 +23,7 @@ func TestTx_allocatePageStats(t *testing.T) {
 
  txStats := tx.Stats()
  prePageCnt := txStats.GetPageCount()
- allocateCnt := f.free_count()
+ allocateCnt := f.FreeCount()
 
  if _, err := tx.allocate(allocateCnt); err != nil {
  t.Fatal(err)

bucket.go

@@ -903,7 +903,7 @@ func (b *Bucket) free() {
  var tx = b.tx
  b.forEachPageNode(func(p *common.Page, n *node, _ int) {
  if p != nil {
- tx.db.freelist.free(tx.meta.Txid(), p)
+ tx.db.freelist.Free(tx.meta.Txid(), p)
  } else {
  n.free()
  }

bucket_test.go

@@ -430,6 +430,9 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) {
  if reopenFreePages := db.Stats().FreePageN; freePages != reopenFreePages {
  t.Fatalf("expected %d free pages, got %+v", freePages, db.Stats())
  }
+ if reopenPendingPages := db.Stats().PendingPageN; reopenPendingPages != 0 {
+ t.Fatalf("expected no pending pages, got %+v", db.Stats())
+ }
 }
 
 // Ensure that deleting of non-existing key is a no-op.

cmd/bbolt/command_version.go

@@ -5,6 +5,7 @@ import (
  "runtime"
 
  "github.com/spf13/cobra"
+
  "go.etcd.io/bbolt/version"
 )
 

concurrent_test.go

@@ -17,6 +17,8 @@ import (
  "time"
  "unicode/utf8"
 
+ "go.etcd.io/bbolt/internal/freelist"
+
  "github.com/stretchr/testify/require"
  "golang.org/x/sync/errgroup"
 
@@ -237,9 +239,9 @@ func mustOpenDB(t *testing.T, dbPath string, o *bolt.Options) *bolt.DB {
  o = bolt.DefaultOptions
  }
 
- freelistType := bolt.FreelistArrayType
- if env := os.Getenv("TEST_FREELIST_TYPE"); env == string(bolt.FreelistMapType) {
- freelistType = bolt.FreelistMapType
+ freelistType := freelist.FreelistArrayType
+ if env := os.Getenv("TEST_FREELIST_TYPE"); env == string(freelist.FreelistMapType) {
+ freelistType = freelist.FreelistMapType
  }
 
  o.FreelistType = freelistType
@@ -769,29 +771,29 @@ func TestConcurrentRepeatableRead(t *testing.T) {
  testCases := []struct {
  name string
  noFreelistSync bool
- freelistType bolt.FreelistType
+ freelistType freelist.FreelistType
  }{
  // [array] freelist
  {
  name: "sync array freelist",
  noFreelistSync: false,
- freelistType: bolt.FreelistArrayType,
+ freelistType: freelist.FreelistArrayType,
  },
  {
  name: "not sync array freelist",
  noFreelistSync: true,
- freelistType: bolt.FreelistArrayType,
+ freelistType: freelist.FreelistArrayType,
  },
  // [map] freelist
  {
  name: "sync map freelist",
  noFreelistSync: false,
- freelistType: bolt.FreelistMapType,
+ freelistType: freelist.FreelistMapType,
  },
  {
  name: "not sync map freelist",
  noFreelistSync: true,
- freelistType: bolt.FreelistMapType,
+ freelistType: freelist.FreelistMapType,
  },
  }
 

db.go

@@ -13,25 +13,12 @@ import (
 
  berrors "go.etcd.io/bbolt/errors"
  "go.etcd.io/bbolt/internal/common"
+ fl "go.etcd.io/bbolt/internal/freelist"
 )
 
 // The time elapsed between consecutive file locking attempts.
 const flockRetryTimeout = 50 * time.Millisecond
 
-// FreelistType is the type of the freelist backend
-type FreelistType string
-
-// TODO(ahrtr): eventually we should (step by step)
-// 1. default to `FreelistMapType`;
-// 2. remove the `FreelistArrayType`, do not export `FreelistMapType`
-// and remove field `FreelistType' from both `DB` and `Options`;
-const (
- // FreelistArrayType indicates backend freelist type is array
- FreelistArrayType = FreelistType("array")
- // FreelistMapType indicates backend freelist type is hashmap
- FreelistMapType = FreelistType("hashmap")
-)
-
 // DB represents a collection of buckets persisted to a file on disk.
 // All data access is performed through transactions which can be obtained through the DB.
 // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called.
@@ -70,7 +57,7 @@ type DB struct {
  // The alternative one is using hashmap, it is faster in almost all circumstances
  // but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
  // The default type is array
- FreelistType FreelistType
+ FreelistType fl.FreelistType
 
  // When true, skips the truncate call when growing the database.
  // Setting this to true is only safe on non-ext3/ext4 systems.
@@ -134,8 +121,9 @@ type DB struct {
  rwtx *Tx
  txs []*Tx
 
- freelist *freelist
- freelistLoad sync.Once
+ freelist fl.Freelist
+ freelistSerializer fl.Serializable
+ freelistLoad sync.Once
 
  pagePool sync.Pool
 
@@ -190,6 +178,7 @@ func Open(path string, mode os.FileMode, options *Options) (db *DB, err error) {
  db.NoFreelistSync = options.NoFreelistSync
  db.PreLoadFreelist = options.PreLoadFreelist
  db.FreelistType = options.FreelistType
+ db.freelistSerializer = fl.Serializer{}
  db.Mlock = options.Mlock
 
  // Set default values for later DB operations.
@@ -416,15 +405,16 @@ func (db *DB) getPageSizeFromSecondMeta() (int, bool, error) {
 // concurrent accesses being made to the freelist.
 func (db *DB) loadFreelist() {
  db.freelistLoad.Do(func() {
- db.freelist = newFreelist(db.FreelistType)
+ db.freelist = fl.NewFreelist(db.FreelistType)
  if !db.hasSyncedFreelist() {
  // Reconstruct free list by scanning the DB.
- db.freelist.readIDs(db.freepages())
+ db.freelist.Init(db.freepages())
  } else {
  // Read free list from freelist page.
- db.freelist.read(db.page(db.meta().Freelist()))
+ db.freelistSerializer.Read(db.freelist, db.page(db.meta().Freelist()))
  }
- db.stats.FreePageN = db.freelist.free_count()
+ db.stats.FreePageN = db.freelist.FreeCount()
+ db.stats.PendingPageN = db.freelist.PendingCount()
  })
 }
 
@@ -854,14 +844,14 @@ func (db *DB) freePages() {
  minid = db.txs[0].meta.Txid()
  }
  if minid > 0 {
- db.freelist.release(minid - 1)
+ db.freelist.Release(minid - 1)
  }
  // Release unused txid extents.
  for _, t := range db.txs {
- db.freelist.releaseRange(minid, t.meta.Txid()-1)
+ db.freelist.ReleaseRange(minid, t.meta.Txid()-1)
  minid = t.meta.Txid() + 1
  }
- db.freelist.releaseRange(minid, common.Txid(0xFFFFFFFFFFFFFFFF))
+ db.freelist.ReleaseRange(minid, common.Txid(0xFFFFFFFFFFFFFFFF))
  // Any page both allocated and freed in an extent is safe to release.
 }
 
@@ -1176,7 +1166,7 @@ func (db *DB) allocate(txid common.Txid, count int) (*common.Page, error) {
  p.SetOverflow(uint32(count - 1))
 
  // Use pages from the freelist if they are available.
- p.SetId(db.freelist.allocate(txid, count))
+ p.SetId(db.freelist.Allocate(txid, count))
  if p.Id() != 0 {
  return p, nil
  }
@@ -1305,7 +1295,7 @@ type Options struct {
  // The alternative one is using hashmap, it is faster in almost all circumstances
  // but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
  // The default type is array
- FreelistType FreelistType
+ FreelistType fl.FreelistType
 
  // Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to
  // grab a shared lock (UNIX).
@@ -1360,7 +1350,7 @@ func (o *Options) String() string {
 var DefaultOptions = &Options{
  Timeout: 0,
  NoGrowSync: false,
- FreelistType: FreelistArrayType,
+ FreelistType: fl.FreelistArrayType,
 }
 
 // Stats represents statistics about the database.