All json all the time (textileio#277)

* wip moving to only json mode Signed-off-by: Aaron Sutula <[email protected]> * more refactoring Signed-off-by: Aaron Sutula <[email protected]> * all tests passing Signed-off-by: Aaron Sutula <[email protected]> * remove json naming in query related things Signed-off-by: Aaron Sutula <[email protected]> * remove unused method Signed-off-by: Aaron Sutula <[email protected]> * support nil query Signed-off-by: Aaron Sutula <[email protected]> * got go mode query tests ported Signed-off-by: Aaron Sutula <[email protected]> * file renames Signed-off-by: Aaron Sutula <[email protected]>
AbtinSetyani · Mar 23, 2020 · 47c6a7d · 47c6a7d
1 parent d755705
commit 47c6a7d
Show file tree

Hide file tree

Showing 29 changed files with 1,146 additions and 1,408 deletions.
diff --git a/api/client/client.go b/api/client/client.go
@@ -203,7 +203,7 @@ func (c *Client) Has(ctx context.Context, dbID thread.ID, collectionName string,
 }
 
 // Find finds instances by query
-func (c *Client) Find(ctx context.Context, dbID thread.ID, collectionName string, query *db.JSONQuery, dummySlice interface{}) (interface{}, error) {
+func (c *Client) Find(ctx context.Context, dbID thread.ID, collectionName string, query *db.Query, dummySlice interface{}) (interface{}, error) {
  queryBytes, err := json.Marshal(query)
  if err != nil {
  return nil, err

diff --git a/api/client/client_test.go b/api/client/client_test.go
@@ -211,7 +211,7 @@ func TestFind(t *testing.T) {
  err = client.Create(context.Background(), dbID, collectionName, person)
  checkErr(t, err)
 
- q := db.JSONWhere("lastName").Eq(person.LastName)
+ q := db.Where("lastName").Eq(person.LastName)
 
  rawResults, err := client.Find(context.Background(), dbID, collectionName, q, []*Person{})
  if err != nil {
@@ -250,7 +250,7 @@ func TestFindWithIndex(t *testing.T) {
  err = client.Create(context.Background(), dbID, collectionName, person)
  checkErr(t, err)
 
- q := db.JSONWhere("lastName").Eq(person.LastName).UseIndex("lastName")
+ q := db.Where("lastName").Eq(person.LastName).UseIndex("lastName")
 
  rawResults, err := client.Find(context.Background(), dbID, collectionName, q, []*Person{})
  if err != nil {
@@ -343,7 +343,7 @@ func TestReadTransaction(t *testing.T) {
  t.Fatal("txn collection found by id does't equal the original")
  }
 
- q := db.JSONWhere("lastName").Eq(person.LastName)
+ q := db.Where("lastName").Eq(person.LastName)
 
  rawResults, err := txn.Find(q, []*Person{})
  if err != nil {
@@ -419,7 +419,7 @@ func TestWriteTransaction(t *testing.T) {
  t.Fatalf("txn collection found by id does't equal the original")
  }
 
- q := db.JSONWhere("lastName").Eq(person.LastName)
+ q := db.Where("lastName").Eq(person.LastName)
 
  rawResults, err := txn.Find(q, []*Person{})
  if err != nil {

diff --git a/api/client/read.go b/api/client/read.go
@@ -71,7 +71,7 @@ func (t *ReadTransaction) FindByID(instanceID string, instance interface{}) erro
 }
 
 // Find finds instances by query
-func (t *ReadTransaction) Find(query *db.JSONQuery, dummySlice interface{}) (interface{}, error) {
+func (t *ReadTransaction) Find(query *db.Query, dummySlice interface{}) (interface{}, error) {
  queryBytes, err := json.Marshal(query)
  if err != nil {
  return nil, err

diff --git a/api/client/write.go b/api/client/write.go
@@ -68,7 +68,7 @@ func (t *WriteTransaction) FindByID(instanceID string, instance interface{}) err
 }
 
 // Find finds instances by query
-func (t *WriteTransaction) Find(query *db.JSONQuery, dummySlice interface{}) (interface{}, error) {
+func (t *WriteTransaction) Find(query *db.Query, dummySlice interface{}) (interface{}, error) {
  queryBytes, err := json.Marshal(query)
  if err != nil {
  return nil, err

diff --git a/api/service.go b/api/service.go
@@ -48,7 +48,6 @@ func NewService(network net.Net, conf Config) (*Service, error) {
 
  manager, err := db.NewManager(
  network,
- db.WithJsonMode(true),
  db.WithRepoPath(conf.RepoPath),
  db.WithDebug(conf.Debug))
  if err != nil {
@@ -192,7 +191,7 @@ func (s *Service) Find(_ context.Context, req *pb.FindRequest) (*pb.FindReply, e
  if err != nil {
  return nil, err
  }
- return s.processFindRequest(req, collection.FindJSON)
+ return s.processFindRequest(req, collection.Find)
 }
 
 func (s *Service) FindByID(_ context.Context, req *pb.FindByIDRequest) (*pb.FindByIDReply, error) {
@@ -254,7 +253,7 @@ func (s *Service) ReadTransaction(stream pb.API_ReadTransactionServer) error {
  return err
  }
  case *pb.ReadTransactionRequest_FindRequest:
- innerReply, err := s.processFindRequest(x.FindRequest, txn.FindJSON)
+ innerReply, err := s.processFindRequest(x.FindRequest, txn.Find)
  if err != nil {
  return err
  }
@@ -322,7 +321,7 @@ func (s *Service) WriteTransaction(stream pb.API_WriteTransactionServer) error {
  return err
  }
  case *pb.WriteTransactionRequest_FindRequest:
- innerReply, err := s.processFindRequest(x.FindRequest, txn.FindJSON)
+ innerReply, err := s.processFindRequest(x.FindRequest, txn.Find)
  if err != nil {
  return err
  }
@@ -515,8 +514,8 @@ func (s *Service) processFindByIDRequest(req *pb.FindByIDRequest, findFunc func(
  return &pb.FindByIDReply{Instance: result}, nil
 }
 
-func (s *Service) processFindRequest(req *pb.FindRequest, findFunc func(q *db.JSONQuery) (ret []string, err error)) (*pb.FindReply, error) {
- q := &db.JSONQuery{}
+func (s *Service) processFindRequest(req *pb.FindRequest, findFunc func(q *db.Query) (ret []string, err error)) (*pb.FindReply, error) {
+ q := &db.Query{}
  if err := json.Unmarshal(req.QueryJSON, q); err != nil {
  return nil, err
  }

diff --git a/core/db/db.go b/core/db/db.go
@@ -24,8 +24,10 @@ func (e InstanceID) String() string {
 }
 
 func IsValidInstanceID(instanceID string) bool {
- _, err := uuid.Parse(instanceID)
- return err == nil
+ // ToDo: Decide how we want to apply instance id requirements
+ return len(instanceID) > 0
+ // _, err := uuid.Parse(instanceID)
+ // return err == nil
 }
 
 // Event is a local or remote event generated in collection and dispatcher

diff --git a/db/bench_test.go b/db/bench_test.go
@@ -63,7 +63,6 @@ func createBenchDB(b *testing.B, opts ...Option) (*DB, func()) {
  checkBenchErr(b, err)
  id := thread.NewIDV1(thread.Raw, 32)
  opts = append(opts, WithRepoPath(dir))
- opts = append(opts, WithJsonMode(true))
  d, err := NewDB(context.Background(), n, id, opts...)
  checkBenchErr(b, err)
  return d, func() {
@@ -193,7 +192,7 @@ func BenchmarkNoIndexFind(b *testing.B) {
  b.ResetTimer()
 
  for i := 0; i < b.N; i++ {
- result, err := collection.FindJSON(JSONWhere("Name").Eq("Name0").JSONOr(JSONWhere("Name").Eq("Name6")))
+ result, err := collection.Find(Where("Name").Eq("Name0").Or(Where("Name").Eq("Name6")))
  if err != nil {
  b.Fatalf("Error finding data: %s", err)
  }
@@ -233,7 +232,7 @@ func BenchmarkIndexFind(b *testing.B) {
  b.ResetTimer()
 
  for i := 0; i < b.N; i++ {
- result, err := collection.FindJSON(JSONWhere("Name").Eq("Name0").JSONOr(JSONWhere("Name").Eq("Name6")).UseIndex("Name"))
+ result, err := collection.Find(Where("Name").Eq("Name0").Or(Where("Name").Eq("Name6")).UseIndex("Name"))
  if err != nil {
  b.Fatalf("Error finding data: %s", err)
  }

diff --git a/db/collection.go b/db/collection.go
@@ -6,7 +6,6 @@ import (
  "fmt"
  "reflect"
 
- "github.com/alecthomas/jsonschema"
  jsonpatch "github.com/evanphx/json-patch"
  ds "github.com/ipfs/go-datastore"
  core "github.com/textileio/go-threads/core/db"
@@ -42,20 +41,7 @@ type Collection struct {
  indexes map[string]Index
 }
 
-func newCollection(name string, defaultInstance interface{}, d *DB) *Collection {
- schema := jsonschema.Reflect(defaultInstance)
- schemaLoader := gojsonschema.NewGoLoader(schema)
- c := &Collection{
- name: name,
- schemaLoader: schemaLoader,
- valueType: reflect.TypeOf(defaultInstance),
- db: d,
- indexes: make(map[string]Index),
- }
- return c
-}
-
-func newCollectionFromSchema(name string, schema string, d *DB) *Collection {
+func newCollection(name string, schema string, d *DB) *Collection {
  schemaLoader := gojsonschema.NewStringLoader(schema)
  c := &Collection{
  name: name,
@@ -149,35 +135,23 @@ func (c *Collection) Has(ids ...core.InstanceID) (exists bool, err error) {
  return
 }
 
-// Find executes a Query into result.
-func (c *Collection) Find(result interface{}, q *Query) error {
- return c.ReadTxn(func(txn *Txn) error {
- return txn.Find(result, q)
- })
-}
-
-// FindJSON executes a Query in in JSONMode and returns the result.
-func (c *Collection) FindJSON(q *JSONQuery) (ret []string, err error) {
+// Find executes a Query and returns the result.
+func (c *Collection) Find(q *Query) (ret []string, err error) {
  _ = c.ReadTxn(func(txn *Txn) error {
- ret, err = txn.FindJSON(q)
+ ret, err = txn.Find(q)
  return err
  })
  return
 }
 
 func (c *Collection) validInstance(v interface{}) (bool, error) {
  var vLoader gojsonschema.JSONLoader
- if c.db.jsonMode {
- strJSON := v.(*string)
- vLoader = gojsonschema.NewBytesLoader([]byte(*strJSON))
- } else {
- vLoader = gojsonschema.NewGoLoader(v)
- }
+ strJSON := v.(*string)
+ vLoader = gojsonschema.NewBytesLoader([]byte(*strJSON))
  r, err := gojsonschema.Validate(c.schemaLoader, vLoader)
  if err != nil {
  return false, err
  }
-
  return r.Valid(), nil
 }
 
@@ -196,7 +170,7 @@ type Txn struct {
 }
 
 // Create creates new instances in the collection
-// If the ID value on the instance is nil or otherwise a null value (e.g., "" in jsonMode),
+// If the ID value on the instance is nil or otherwise a null value (e.g., ""),
 // the ID is updated in-place to reflect the automatically-genereted UUID.
 func (t *Txn) Create(new ...interface{}) error {
  for i := range new {
@@ -211,10 +185,9 @@ func (t *Txn) Create(new ...interface{}) error {
  return ErrInvalidSchemaInstance
  }
 
- jsonMode := t.collection.db.jsonMode
- id := getInstanceID(new[i], jsonMode)
+ id := getInstanceID(new[i])
  if id == core.EmptyInstanceID {
- id = setNewInstanceID(new[i], jsonMode)
+ id = setNewInstanceID(new[i])
  }
  key := baseKey.ChildString(t.collection.name).ChildString(id.String())
  exists, err := t.collection.db.datastore.Has(key)
@@ -253,7 +226,7 @@ func (t *Txn) Save(updated ...interface{}) error {
  return ErrInvalidSchemaInstance
  }
 
- id := getInstanceID(updated[i], t.collection.db.jsonMode)
+ id := getInstanceID(updated[i])
  key := baseKey.ChildString(t.collection.name).ChildString(id.String())
  beforeBytes, err := t.collection.db.datastore.Get(key)
  if err == ds.ErrNotFound {
@@ -265,13 +238,6 @@ func (t *Txn) Save(updated ...interface{}) error {
 
  var previous interface{}
  previous = beforeBytes
- if !t.collection.db.jsonMode {
- before := reflect.New(t.collection.valueType.Elem()).Interface()
- if err = json.Unmarshal(beforeBytes, before); err != nil {
- return err
- }
- previous = before
- }
  t.actions = append(t.actions, core.Action{
  Type: core.Save,
  InstanceID: id,
@@ -336,15 +302,12 @@ func (t *Txn) FindByID(id core.InstanceID, v interface{}) error {
  if err != nil {
  return err
  }
- if t.collection.db.jsonMode {
- str := string(bytes)
- rflStr := reflect.ValueOf(str)
- reflV := reflect.ValueOf(v)
- reflV.Elem().Set(rflStr)
+ str := string(bytes)
+ rflStr := reflect.ValueOf(str)
+ reflV := reflect.ValueOf(v)
+ reflV.Elem().Set(rflStr)
 
- return nil
- }
- return json.Unmarshal(bytes, v)
+ return nil
 }
 
 // Commit applies all changes done in the current transaction
@@ -376,48 +339,28 @@ func (t *Txn) Discard() {
  t.discarded = true
 }
 
-func getInstanceID(t interface{}, jsonMode bool) core.InstanceID {
- if jsonMode {
- partial := &struct{ ID *string }{}
- if err := json.Unmarshal([]byte(*(t.(*string))), partial); err != nil {
- log.Fatalf("error when unmarshaling json instance: %v", err)
- }
- if partial.ID == nil {
- log.Fatal("invalid instance: doesn't have an ID attribute")
- }
- if *partial.ID != "" && !core.IsValidInstanceID(*partial.ID) {
- log.Fatal("invalid instance: invalid ID value")
- }
- return core.InstanceID(*partial.ID)
- } else {
- v := reflect.ValueOf(t)
- if v.Type().Kind() != reflect.Ptr {
- v = reflect.New(reflect.TypeOf(v))
- }
- v = v.Elem().FieldByName(idFieldName)
- if !v.IsValid() || v.Type() != reflect.TypeOf(core.EmptyInstanceID) {
- log.Fatal("invalid instance: doesn't have InstanceID attribute")
- }
- return core.InstanceID(v.String())
+func getInstanceID(t interface{}) core.InstanceID {
+ partial := &struct{ ID *string }{}
+ if err := json.Unmarshal([]byte(*(t.(*string))), partial); err != nil {
+ log.Fatalf("error when unmarshaling json instance: %v", err)
+ }
+ if partial.ID == nil {
+ log.Fatal("invalid instance: doesn't have an ID attribute")
  }
+ if *partial.ID != "" && !core.IsValidInstanceID(*partial.ID) {
+ log.Fatal("invalid instance: invalid ID value")
+ }
+ return core.InstanceID(*partial.ID)
 }
 
-func setNewInstanceID(t interface{}, jsonMode bool) core.InstanceID {
+func setNewInstanceID(t interface{}) core.InstanceID {
  newID := core.NewInstanceID()
- if jsonMode {
- patchedValue, err := jsonpatch.MergePatch([]byte(*(t.(*string))), []byte(fmt.Sprintf(`{"ID": %q}`, newID.String())))
- if err != nil {
- log.Fatalf("error while automatically patching autogenerated ID: %v", err)
- }
- strPatchedValue := string(patchedValue)
- reflectPatchedValue := reflect.ValueOf(&strPatchedValue)
- reflect.ValueOf(t).Elem().Set(reflectPatchedValue.Elem())
- } else {
- v := reflect.ValueOf(t)
- if v.Type().Kind() != reflect.Ptr {
- v = reflect.New(reflect.TypeOf(v))
- }
- v.Elem().FieldByName(idFieldName).Set(reflect.ValueOf(newID))
+ patchedValue, err := jsonpatch.MergePatch([]byte(*(t.(*string))), []byte(fmt.Sprintf(`{"ID": %q}`, newID.String())))
+ if err != nil {
+ log.Fatalf("error while automatically patching autogenerated ID: %v", err)
  }
+ strPatchedValue := string(patchedValue)
+ reflectPatchedValue := reflect.ValueOf(&strPatchedValue)
+ reflect.ValueOf(t).Elem().Set(reflectPatchedValue.Elem())
  return newID
 }