Merge pull request #83 from simagix/v1.2

supports readPreferenceTagSets

README.md

@@ -10,8 +10,12 @@ Keyhole is a tool to explore MongoDB deployments. Start with [Survey Your Mongo
 
 This software is not supported by MongoDB, Inc. under any of their commercial support subscriptions or otherwise. Any usage of keyhole is at your own risk.
 
-## New in v1.2
-- Print connected client information from all `mongod` processes
-- Print client information from a log file
-- Compare two clusters using internal metadata
-- Perform deep comparison of two clusters
+## Changes
+### v1.2.1
+- Supports ReadPreferenceTagSets
+
+### v1.2
+- Prints connected client information from all `mongod` processes
+- Prints client information from a log file
+- Compares two clusters using internal metadata
+- Performs deep comparison of two clusters

keyhole.go

@@ -192,6 +192,15 @@ func Run(fullVersion string) {
  }
 
  var client *mongo.Client
+ tag := "readPreferenceTags=nodeType:ANALYTICS"
+ var fastMode bool
+ if strings.Contains(uri, ".mongodb.net") && strings.Contains(uri, tag) {
+ fastMode = true
+ if *allinfo != "" {
+ gox.GetLogger(fullVersion).Infof(`remove "%v" from connection string for fast processing`, tag)
+ }
+ uri = strings.Replace(uri, tag, "", -1)
+ }
  // connection string is required from here forward
  var connString connstring.ConnString
  if connString, err = mdb.ParseURI(uri); err != nil {
@@ -209,6 +218,7 @@ func Run(fullVersion string) {
  stats.SetDBNames(dbNames)
  stats.SetRedaction(*redaction)
  stats.SetVerbose(true)
+ stats.SetFastMode(fastMode)
  if err = stats.GetClusterStats(client, connString); err != nil {
  log.Fatalf("a valid user with roles 'clusterMonitor' and 'readAnyDatabase' on all mongo processes are required.\n%v", err)
  }
@@ -238,6 +248,7 @@ func Run(fullVersion string) {
  ix := mdb.NewIndexStats(fullVersion)
  ix.SetNoColor(*nocolor)
  ix.SetVerbose(*verbose)
+ ix.SetFastMode(fastMode)
  if err = DuplicateIndexesFromFile(ix, client, *createIndex, *drop); err != nil {
  log.Fatal(err)
  }
@@ -253,6 +264,7 @@ func Run(fullVersion string) {
  ix := mdb.NewIndexStats(fullVersion)
  ix.SetNoColor(*nocolor)
  ix.SetVerbose(*verbose)
+ ix.SetFastMode(fastMode)
  if err = CollectIndexStats(ix, client, *maobiURL); err != nil {
  log.Fatal(err)
  }

mdb/cluster_stats.go

@@ -44,14 +44,16 @@ type ClusterStats struct {
  Shards []Shard `bson:"shards"`
  Version string `bson:"version"`
 
- dbNames []string
- redact bool
- verbose bool
+ dbNames []string
+ fastMode bool
+ redact bool
+ signature string
+ verbose bool
 }
 
 // NewClusterStats returns *ClusterStats
 func NewClusterStats(signature string) *ClusterStats {
- s := ClusterStats{Logger: gox.GetLogger(signature)}
+ s := ClusterStats{signature: signature}
  return &s
 }
 
@@ -60,6 +62,11 @@ func (p *ClusterStats) SetDBNames(dbNames []string) {
  p.dbNames = dbNames
 }
 
+// SetFastMode sets fastMode mode
+func (p *ClusterStats) SetFastMode(fastMode bool) {
+ p.fastMode = fastMode
+}
+
 // SetRedaction sets redact
 func (p *ClusterStats) SetRedaction(redact bool) {
  p.redact = redact
@@ -73,9 +80,7 @@ func (p *ClusterStats) SetVerbose(verbose bool) {
 // GetClusterStats collects cluster stats
 func (p *ClusterStats) GetClusterStats(client *mongo.Client, connString connstring.ConnString) error {
  var err error
- if p.Logger == nil {
- p.Logger = gox.GetLogger("")
- }
+ p.Logger = gox.GetLogger(p.signature)
  p.Logger.Info("GetClusterStats() begins")
  if err = p.GetClusterStatsSummary(client); err != nil {
  return err
@@ -100,7 +105,11 @@ func (p *ClusterStats) GetClusterStats(client *mongo.Client, connString connstri
  }
 
  setName := p.ServerStatus.Repl.SetName
- s := fmt.Sprintf(`%v/%v`, setName, strings.Join(p.ServerStatus.Repl.Hosts, ","))
+ hosts := []string{}
+ for _, member := range p.ReplSetGetStatus.Members {
+ hosts = append(hosts, member.Name)
+ }
+ s := fmt.Sprintf(`%v/%v`, setName, strings.Join(hosts, ","))
  oneShard := []Shard{{ID: setName, State: 1, Host: s}}
  if p.Shards, err = p.GetServersStatsSummary(oneShard, connString); err != nil {
  p.Logger.Error(err)
@@ -111,6 +120,7 @@ func (p *ClusterStats) GetClusterStats(client *mongo.Client, connString connstri
  db.SetNumberShards(len(p.Shards))
  db.SetRedaction(p.redact)
  db.SetVerbose(p.verbose)
+ db.SetFastMode(p.fastMode)
  if p.Databases, err = db.GetAllDatabasesStats(client, p.dbNames); err != nil {
  p.Logger.Info(fmt.Sprintf(`GetAllDatabasesStats(): %v`, err))
  }
@@ -120,6 +130,7 @@ func (p *ClusterStats) GetClusterStats(client *mongo.Client, connString connstri
 // GetClusterStatsSummary collects cluster stats
 func (p *ClusterStats) GetClusterStatsSummary(client *mongo.Client) error {
  var err error
+ p.Logger = gox.GetLogger(p.signature)
  if p.BuildInfo, err = GetBuildInfo(client); err != nil {
  return err
  }
@@ -153,9 +164,7 @@ func (p *ClusterStats) GetServersStatsSummary(shards []Shard, connString connstr
  var err error
  var uris []string
  var smap = map[string]Shard{}
- if p.Logger == nil {
- p.Logger = gox.GetLogger("")
- }
+ p.Logger = gox.GetLogger(p.signature)
  for _, v := range shards {
  v.Servers = []ClusterStats{}
  smap[v.ID] = v

mdb/databases_stats.go

@@ -28,6 +28,7 @@ const (
 type DatabaseStats struct {
  Logger *gox.Logger
 
+ fastMode bool
  numShards int
  redaction bool
  threads int
@@ -104,6 +105,11 @@ func NewDatabaseStats(version string) *DatabaseStats {
  return &DatabaseStats{Logger: gox.GetLogger(version), threads: 16, version: version}
 }
 
+// SetFastMode sets fastMode mode
+func (p *DatabaseStats) SetFastMode(fastMode bool) {
+ p.fastMode = fastMode
+}
+
 // SetNumberShards set # of threads
 func (p *DatabaseStats) SetNumberShards(n int) {
  p.numShards = n
@@ -174,10 +180,10 @@ func (p *DatabaseStats) GetAllDatabasesStats(client *mongo.Client, dbNames []str
  if cur, err = client.Database(db.Name).ListCollections(ctx, bson.D{{}}); err != nil {
  return listdb.Databases, err
  }
- defer cur.Close(ctx)
  var collections = []Collection{}
  ir := NewIndexStats(p.version)
  ir.SetVerbose(p.verbose)
+ ir.SetFastMode(p.fastMode)
  collectionNames := []string{}
 
  for cur.Next(ctx) {
@@ -193,6 +199,7 @@ func (p *DatabaseStats) GetAllDatabasesStats(client *mongo.Client, dbNames []str
  }
  collectionNames = append(collectionNames, coll)
  }
+ cur.Close(ctx)
 
  sort.Strings(collectionNames)
  var wg = gox.NewWaitGroup(4) // runs in parallel
@@ -242,27 +249,28 @@ func (p *DatabaseStats) GetAllDatabasesStats(client *mongo.Client, dbNames []str
  if err != nil {
  p.Logger.Error(err)
  }
-
- // stats
  var stats bson.M
- client.Database(db.Name).RunCommand(ctx, bson.D{{Key: "collStats", Value: collectionName}}).Decode(&stats)
  chunks := []Chunk{}
- if isGetChunksDistr && stats["shards"] != nil {
- shardNames := []string{}
- for shard := range stats["shards"].(primitive.M) {
- shardNames = append(shardNames, shard)
- }
- sort.Strings(shardNames)
- for _, k := range shardNames {
- m := (stats["shards"].(primitive.M)[k]).(primitive.M)
- delete(m, "$clusterTime")
- delete(m, "$gleStats")
- if chunk, cerr := p.collectChunksDistribution(client, k, ns); cerr != nil {
- // p.Logger.Error(cerr)
- } else {
- chunk.Objects = toInt64(m["count"])
- chunk.Size = toInt64(m["size"])
- chunks = append(chunks, chunk)
+ if !p.fastMode {
+ // stats
+ client.Database(db.Name).RunCommand(ctx, bson.D{{Key: "collStats", Value: collectionName}}).Decode(&stats)
+ if isGetChunksDistr && stats["shards"] != nil {
+ shardNames := []string{}
+ for shard := range stats["shards"].(primitive.M) {
+ shardNames = append(shardNames, shard)
+ }
+ sort.Strings(shardNames)
+ for _, k := range shardNames {
+ m := (stats["shards"].(primitive.M)[k]).(primitive.M)
+ delete(m, "$clusterTime")
+ delete(m, "$gleStats")
+ if chunk, cerr := p.collectChunksDistribution(client, k, ns); cerr != nil {
+ // p.Logger.Error(cerr)
+ } else {
+ chunk.Objects = toInt64(m["count"])
+ chunk.Size = toInt64(m["size"])
+ chunks = append(chunks, chunk)
+ }
  }
  }
  }

mdb/index_stats.go

@@ -27,6 +27,7 @@ type IndexStats struct {
  Databases []Database `bson:"databases"`
  Logger *gox.Logger `bson:"keyhole"`
 
+ fastMode bool
  filename string
  nocolor bool
  verbose bool
@@ -80,6 +81,11 @@ func NewIndexStats(version string) *IndexStats {
  filename: hostname + indexExt, Databases: []Database{}}
 }
 
+// SetFastMode sets fastMode mode
+func (ix *IndexStats) SetFastMode(fastMode bool) {
+ ix.fastMode = fastMode
+}
+
 // SetFilename sets output file name
 func (ix *IndexStats) SetFilename(filename string) {
  ix.filename = strings.Replace(filename, ":", "_", -1)
@@ -243,8 +249,10 @@ func (ix *IndexStats) GetIndexesFromCollection(client *mongo.Client, collection
  var v map[string]interface{}
  ns := collection.Database().Name() + "." + collection.Name()
  ix.Logger.Debug("GetIndexesFromCollection ", ns, o.KeyString)
- if err = client.Database("config").Collection("collections").FindOne(ctx, bson.M{"_id": ns, "key": o.Key}).Decode(&v); err == nil {
- o.IsShardKey = true
+ if !ix.fastMode {
+ if err = client.Database("config").Collection("collections").FindOne(ctx, bson.M{"_id": ns, "key": o.Key}).Decode(&v); err == nil {
+ o.IsShardKey = true
+ }
  }
  o.EffectiveKey = strings.Replace(o.KeyString[2:len(o.KeyString)-2], ": -1", ": 1", -1)
  o.Usage = []IndexUsage{}

mdb/shards.go

@@ -62,7 +62,7 @@ func GetAllShardURIs(shards []Shard, connString connstring.ConnString) ([]string
  } else if isSRV {
  ruri += "&authSource=admin&tls=true"
  }
- ruri += getQueryParams(connString)
+ ruri += GetQueryParams(connString, false)
  list = append(list, ruri)
  }
  return list, nil
@@ -94,14 +94,15 @@ func GetAllServerURIs(shards []Shard, connString connstring.ConnString) ([]strin
  ruri += "authSource=admin"
  }
  }
- ruri += getQueryParams(connString)
+ ruri += GetQueryParams(connString, true)
  list = append(list, ruri)
  }
  }
  return list, nil
 }
 
-func getQueryParams(connString connstring.ConnString) string {
+// GetQueryParams returns partial connection string from ConnString
+func GetQueryParams(connString connstring.ConnString, isConnectDirect bool) string {
  ruri := ""
  if connString.SSLSet {
  ruri += "&tls=true"
@@ -115,9 +116,22 @@ func getQueryParams(connString connstring.ConnString) string {
  if connString.SSLInsecureSet {
  ruri += "&tlsInsecure=true"
  }
- if connString.ReadPreference != "" {
+ if connString.ReadPreference != "" && !isConnectDirect {
  ruri += "&readPreference=" + connString.ReadPreference
  }
+ if len(connString.ReadPreferenceTagSets) > 0 && !isConnectDirect {
+ ruri += "&readPreferenceTags="
+ cnt := 0
+ for _, amap := range connString.ReadPreferenceTagSets {
+ for k, v := range amap {
+ ruri += k + ":" + v
+ if cnt > 0 {
+ ruri += ","
+ }
+ cnt++
+ }
+ }
+ }
  if connString.WNumberSet {
  ruri += fmt.Sprintf("&w=%v", connString.WNumber)
  } else if connString.WString != "" {

mdb/shards_test.go

@@ -86,6 +86,21 @@ func TestGetAllServerURIs(t *testing.T) {
  }
 }
 
+func TestGetQueryParams(t *testing.T) {
+ var err error
+ var cs connstring.ConnString
+ var expected string
+ uri := "mongodb+srv:https://user:[email protected]/keyhole?readPreference=secondary&readPreferenceTags=nodeType:ANALYTICS"
+ if cs, err = connstring.Parse(uri); err != nil {
+ t.Fatal(err)
+ }
+ expected = "&tls=true&readPreference=secondary&readPreferenceTags=nodeType:ANALYTICS"
+ assertEqual(t, expected, GetQueryParams(cs, false))
+
+ expected = "&tls=true"
+ assertEqual(t, expected, GetQueryParams(cs, true))
+}
+
 func assertEqual(t *testing.T, a interface{}, b interface{}) {
  if a != b {
  t.Fatalf("%s != %s", a, b)

version

@@ -1 +1 @@
-1.2
+1.2.1