From d2d632b0f5d58449b354ac3dc597a97184d87098 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Thu, 23 May 2024 14:35:34 +0200 Subject: [PATCH 1/4] Store design_only option for views in db --- x/sqlite/designdocs.go | 14 +++++----- x/sqlite/json.go | 9 ++++-- x/sqlite/put_designdocs_test.go | 49 ++++++++++++++++++++++++++++++++- x/sqlite/schema.go | 2 ++ 4 files changed, 64 insertions(+), 10 deletions(-) diff --git a/x/sqlite/designdocs.go b/x/sqlite/designdocs.go index 123a7b45..90e2314b 100644 --- a/x/sqlite/designdocs.go +++ b/x/sqlite/designdocs.go @@ -22,8 +22,8 @@ func (d *db) updateDesignDoc(ctx context.Context, tx *sql.Tx, rev revision, data return nil } stmt, err := tx.PrepareContext(ctx, d.query(` - INSERT INTO {{ .Design }} (id, rev, rev_id, language, func_type, func_name, func_body, auto_update) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + INSERT INTO {{ .Design }} (id, rev, rev_id, language, func_type, func_name, func_body, auto_update, include_design) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) `)) if err != nil { return err @@ -32,7 +32,7 @@ func (d *db) updateDesignDoc(ctx context.Context, tx *sql.Tx, rev revision, data for name, view := range data.DesignFields.Views { if view.Map != "" { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "map", name, view.Map, data.DesignFields.AutoUpdate); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "map", name, view.Map, data.DesignFields.AutoUpdate, data.DesignFields.Options.IncludeDesign); err != nil { return err } if err := d.createViewMap(ctx, tx, data.ID, name, rev.String()); err != nil { @@ -40,23 +40,23 @@ func (d *db) updateDesignDoc(ctx context.Context, tx *sql.Tx, rev revision, data } } if view.Reduce != "" { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "reduce", name, view.Reduce, data.DesignFields.AutoUpdate); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "reduce", name, view.Reduce, data.DesignFields.AutoUpdate, false); err != nil { return err } } } for name, update := range data.DesignFields.Updates { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "update", name, update, data.DesignFields.AutoUpdate); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "update", name, update, data.DesignFields.AutoUpdate, false); err != nil { return err } } for name, filter := range data.DesignFields.Filters { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "filter", name, filter, data.DesignFields.AutoUpdate); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "filter", name, filter, data.DesignFields.AutoUpdate, false); err != nil { return err } } if data.DesignFields.ValidateDocUpdates != "" { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "validate_doc_update", "validate", data.DesignFields.ValidateDocUpdates, data.DesignFields.AutoUpdate); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "validate_doc_update", "validate", data.DesignFields.ValidateDocUpdates, data.DesignFields.AutoUpdate, false); err != nil { return err } } diff --git a/x/sqlite/json.go b/x/sqlite/json.go index 7ecb8278..541fc595 100644 --- a/x/sqlite/json.go +++ b/x/sqlite/json.go @@ -85,6 +85,11 @@ type views struct { Reduce string `json:"reduce,omitempty"` } +type designDocViewOptions struct { + LocalSeq bool `json:"local_seq,omitempty"` + IncludeDesign bool `json:"include_design,omitempty"` +} + // designDocData represents a design document. See // https://docs.couchdb.org/en/stable/ddocs/ddocs.html#creation-and-structure type designDocData struct { @@ -95,8 +100,8 @@ type designDocData struct { ValidateDocUpdates string `json:"validate_doc_update,omitempty"` // AutoUpdate indicates whether to automatically build indexes defined in // this design document. Default is true. - AutoUpdate *bool `json:"autoupdate,omitempty"` - Options map[string]interface{} `json:"options,omitempty"` + AutoUpdate *bool `json:"autoupdate,omitempty"` + Options designDocViewOptions `json:"options,omitempty"` } // RevID returns calculated revision ID, possibly setting the MD5sum if it is diff --git a/x/sqlite/put_designdocs_test.go b/x/sqlite/put_designdocs_test.go index 18aa0789..6aac0448 100644 --- a/x/sqlite/put_designdocs_test.go +++ b/x/sqlite/put_designdocs_test.go @@ -120,11 +120,58 @@ func TestDBPut_designDocs(t *testing.T) { }, } }) + tests.Add("options.include_design=true", func(t *testing.T) interface{} { + d := newDB(t) + return test{ + db: d, + docID: "_design/foo", + doc: map[string]interface{}{ + "language": "javascript", + "views": map[string]interface{}{ + "bar": map[string]interface{}{ + "map": "function(doc) { emit(doc._id, null); }", + }, + }, + "options": map[string]interface{}{ + "include_design": true, + }, + }, + wantRev: "1-.*", + wantRevs: []leaf{ + {ID: "_design/foo", Rev: 1}, + }, + wantDDocs: []ddoc{ + { + ID: "_design/foo", + Rev: 1, + Lang: "javascript", + FuncType: "map", + FuncName: "bar", + FuncBody: "function(doc) { emit(doc._id, null); }", + AutoUpdate: true, + }, + }, + check: func(t *testing.T) { + var includeDesign bool + err := d.underlying().QueryRow(` + SELECT include_design + FROM test_design + WHERE func_type = 'map' + LIMIT 1 + `).Scan(&includeDesign) + if err != nil { + t.Fatal(err) + } + if !includeDesign { + t.Errorf("include_design was false, expected true") + } + }, + } + }) /* TODO: - unsupported language? -- ignored? - Drop old indexes when a ddoc changes - */ tests.Run(t, func(t *testing.T, tt test) { diff --git a/x/sqlite/schema.go b/x/sqlite/schema.go index 142abe84..849989fe 100644 --- a/x/sqlite/schema.go +++ b/x/sqlite/schema.go @@ -76,7 +76,9 @@ var schema = []string{ func_type TEXT CHECK (func_type IN ('map', 'reduce', 'update', 'filter', 'validate')) NOT NULL, func_name TEXT NOT NULL, func_body TEXT NOT NULL, + -- Options auto_update BOOLEAN NOT NULL DEFAULT TRUE, + include_design BOOLEAN NOT NULL DEFAULT FALSE, last_seq INTEGER, -- the last map-indexed sequence id, NULL for others FOREIGN KEY (id, rev, rev_id) REFERENCES {{ .Docs }} (id, rev, rev_id) ON DELETE CASCADE, UNIQUE (id, rev, rev_id, func_type, func_name) From 35698174bb35e278f80f6482a0b2664f3c4f507e Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Thu, 23 May 2024 14:38:29 +0200 Subject: [PATCH 2/4] Store local_seq option for views in db --- x/sqlite/designdocs.go | 17 +++++++----- x/sqlite/put_designdocs_test.go | 48 +++++++++++++++++++++++++++++++++ x/sqlite/schema.go | 5 ++-- 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/x/sqlite/designdocs.go b/x/sqlite/designdocs.go index 90e2314b..746ca692 100644 --- a/x/sqlite/designdocs.go +++ b/x/sqlite/designdocs.go @@ -22,8 +22,8 @@ func (d *db) updateDesignDoc(ctx context.Context, tx *sql.Tx, rev revision, data return nil } stmt, err := tx.PrepareContext(ctx, d.query(` - INSERT INTO {{ .Design }} (id, rev, rev_id, language, func_type, func_name, func_body, auto_update, include_design) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + INSERT INTO {{ .Design }} (id, rev, rev_id, language, func_type, func_name, func_body, auto_update, include_design, local_seq) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) `)) if err != nil { return err @@ -32,7 +32,10 @@ func (d *db) updateDesignDoc(ctx context.Context, tx *sql.Tx, rev revision, data for name, view := range data.DesignFields.Views { if view.Map != "" { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "map", name, view.Map, data.DesignFields.AutoUpdate, data.DesignFields.Options.IncludeDesign); err != nil { + if _, err := stmt.ExecContext(ctx, + data.ID, rev.rev, rev.id, data.DesignFields.Language, "map", name, view.Map, + data.DesignFields.AutoUpdate, data.DesignFields.Options.IncludeDesign, data.DesignFields.Options.LocalSeq, + ); err != nil { return err } if err := d.createViewMap(ctx, tx, data.ID, name, rev.String()); err != nil { @@ -40,23 +43,23 @@ func (d *db) updateDesignDoc(ctx context.Context, tx *sql.Tx, rev revision, data } } if view.Reduce != "" { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "reduce", name, view.Reduce, data.DesignFields.AutoUpdate, false); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "reduce", name, view.Reduce, data.DesignFields.AutoUpdate, nil, nil); err != nil { return err } } } for name, update := range data.DesignFields.Updates { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "update", name, update, data.DesignFields.AutoUpdate, false); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "update", name, update, data.DesignFields.AutoUpdate, nil, nil); err != nil { return err } } for name, filter := range data.DesignFields.Filters { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "filter", name, filter, data.DesignFields.AutoUpdate, false); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "filter", name, filter, data.DesignFields.AutoUpdate, nil, nil); err != nil { return err } } if data.DesignFields.ValidateDocUpdates != "" { - if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "validate_doc_update", "validate", data.DesignFields.ValidateDocUpdates, data.DesignFields.AutoUpdate, false); err != nil { + if _, err := stmt.ExecContext(ctx, data.ID, rev.rev, rev.id, data.DesignFields.Language, "validate_doc_update", "validate", data.DesignFields.ValidateDocUpdates, data.DesignFields.AutoUpdate, nil, nil); err != nil { return err } } diff --git a/x/sqlite/put_designdocs_test.go b/x/sqlite/put_designdocs_test.go index 6aac0448..f05f213e 100644 --- a/x/sqlite/put_designdocs_test.go +++ b/x/sqlite/put_designdocs_test.go @@ -168,6 +168,54 @@ func TestDBPut_designDocs(t *testing.T) { }, } }) + tests.Add("options.local_seq=true", func(t *testing.T) interface{} { + d := newDB(t) + return test{ + db: d, + docID: "_design/foo", + doc: map[string]interface{}{ + "language": "javascript", + "views": map[string]interface{}{ + "bar": map[string]interface{}{ + "map": "function(doc) { emit(doc._id, null); }", + }, + }, + "options": map[string]interface{}{ + "local_seq": true, + }, + }, + wantRev: "1-.*", + wantRevs: []leaf{ + {ID: "_design/foo", Rev: 1}, + }, + wantDDocs: []ddoc{ + { + ID: "_design/foo", + Rev: 1, + Lang: "javascript", + FuncType: "map", + FuncName: "bar", + FuncBody: "function(doc) { emit(doc._id, null); }", + AutoUpdate: true, + }, + }, + check: func(t *testing.T) { + var includeDesign bool + err := d.underlying().QueryRow(` + SELECT local_seq + FROM test_design + WHERE func_type = 'map' + LIMIT 1 + `).Scan(&includeDesign) + if err != nil { + t.Fatal(err) + } + if !includeDesign { + t.Errorf("include_design was false, expected true") + } + }, + } + }) /* TODO: - unsupported language? -- ignored? diff --git a/x/sqlite/schema.go b/x/sqlite/schema.go index 849989fe..e21a8077 100644 --- a/x/sqlite/schema.go +++ b/x/sqlite/schema.go @@ -76,9 +76,10 @@ var schema = []string{ func_type TEXT CHECK (func_type IN ('map', 'reduce', 'update', 'filter', 'validate')) NOT NULL, func_name TEXT NOT NULL, func_body TEXT NOT NULL, - -- Options auto_update BOOLEAN NOT NULL DEFAULT TRUE, - include_design BOOLEAN NOT NULL DEFAULT FALSE, + -- Options include_design and local_seq are only stored for 'map' type + include_design BOOLEAN, + local_seq BOOLEAN, last_seq INTEGER, -- the last map-indexed sequence id, NULL for others FOREIGN KEY (id, rev, rev_id) REFERENCES {{ .Docs }} (id, rev, rev_id) ON DELETE CASCADE, UNIQUE (id, rev, rev_id, func_type, func_name) From cc75bd8c3ea94c2c71f82a46280a5c81b5a86d5c Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Thu, 23 May 2024 14:45:55 +0200 Subject: [PATCH 3/4] Honor the include_design view option --- x/sqlite/query.go | 13 ++++++++----- x/sqlite/query_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/x/sqlite/query.go b/x/sqlite/query.go index 2d0262b2..7e9f449d 100644 --- a/x/sqlite/query.go +++ b/x/sqlite/query.go @@ -319,22 +319,24 @@ const batchSize = 100 // ddoc revid and last_seq. If mode is "true", it will also update the index. func (d *db) updateIndex(ctx context.Context, ddoc, view, mode string) (revision, error) { var ( - ddocRev revision - mapFuncJS *string - lastSeq int + ddocRev revision + mapFuncJS *string + lastSeq int + includeDesign sql.NullBool ) err := d.db.QueryRowContext(ctx, d.query(` SELECT docs.rev, docs.rev_id, design.func_body, + design.include_design, COALESCE(design.last_seq, 0) AS last_seq FROM {{ .Docs }} AS docs LEFT JOIN {{ .Design }} AS design ON docs.id = design.id AND docs.rev = design.rev AND docs.rev_id = design.rev_id AND design.func_type = 'map' WHERE docs.id = $1 ORDER BY docs.rev DESC, docs.rev_id DESC LIMIT 1 - `), "_design/"+ddoc).Scan(&ddocRev.rev, &ddocRev.id, &mapFuncJS, &lastSeq) + `), "_design/"+ddoc).Scan(&ddocRev.rev, &ddocRev.id, &mapFuncJS, &includeDesign, &lastSeq) switch { case errors.Is(err, sql.ErrNoRows): return revision{}, &internal.Error{Status: http.StatusNotFound, Message: "missing"} @@ -480,7 +482,8 @@ func (d *db) updateIndex(ctx context.Context, ddoc, view, mode string) (revision } // TODO move this to the query - if strings.HasPrefix(full.ID, "_design/") || strings.HasPrefix(full.ID, "_local/") { + if strings.HasPrefix(full.ID, "_local/") || + (!includeDesign.Bool && strings.HasPrefix(full.ID, "_design/")) { continue } diff --git a/x/sqlite/query_test.go b/x/sqlite/query_test.go index d02620fd..77c514a0 100644 --- a/x/sqlite/query_test.go +++ b/x/sqlite/query_test.go @@ -1650,6 +1650,35 @@ func TestDBQuery(t *testing.T) { }, } }) + tests.Add("include design docs in view output", func(t *testing.T) interface{} { + d := newDB(t) + _ = d.tPut("_design/foo", map[string]interface{}{ + "key": "design", + "views": map[string]interface{}{ + "bar": map[string]string{ + "map": `function(doc) { + if (doc.key) { + emit(doc.key, doc.value); + } + }`, + }, + }, + "options": map[string]interface{}{ + "include_design": true, + }, + }) + _ = d.tPut("a", map[string]interface{}{"key": "a"}) + + return test{ + db: d, + ddoc: "_design/foo", + view: "_view/bar", + want: []rowResult{ + {ID: "a", Key: `"a"`, Value: "null"}, + {ID: "_design/foo", Key: `"design"`, Value: "null"}, + }, + } + }) /* TODO: @@ -1687,6 +1716,8 @@ func TestDBQuery(t *testing.T) { - treat map non-exception errors as exceptions - make sure local docs are properly skipped - make sure deleted docs are properly skipped + - view options: https://docs.couchdb.org/en/stable/api/ddoc/views.html#view-options + - local_seq */ From 0264c9745b836b186be7d5fbef4ece5fca28b158 Mon Sep 17 00:00:00 2001 From: Jonathan Hall Date: Thu, 23 May 2024 14:51:10 +0200 Subject: [PATCH 4/4] Support local_seq view option --- x/sqlite/json.go | 4 +++- x/sqlite/query.go | 15 ++++++++++----- x/sqlite/query_test.go | 31 ++++++++++++++++++++++++++++--- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/x/sqlite/json.go b/x/sqlite/json.go index 541fc595..36e98b6b 100644 --- a/x/sqlite/json.go +++ b/x/sqlite/json.go @@ -453,12 +453,14 @@ func (d *fullDoc) toMap() map[string]interface{} { } result["_attachments"] = attachments } + if d.LocalSeq > 0 { + result["_local_seq"] = d.LocalSeq + } /* Conflicts []string `json:"_conflicts,omitempty"` DeletedConflicts []string `json:"_deleted_conflicts,omitempty"` RevsInfo []map[string]string `json:"_revs_info,omitempty"` Revisions *revsInfo `json:"_revisions,omitempty"` - LocalSeq int `json:"_local_seq,omitempty"` */ return result } diff --git a/x/sqlite/query.go b/x/sqlite/query.go index 7e9f449d..4d755f01 100644 --- a/x/sqlite/query.go +++ b/x/sqlite/query.go @@ -319,10 +319,10 @@ const batchSize = 100 // ddoc revid and last_seq. If mode is "true", it will also update the index. func (d *db) updateIndex(ctx context.Context, ddoc, view, mode string) (revision, error) { var ( - ddocRev revision - mapFuncJS *string - lastSeq int - includeDesign sql.NullBool + ddocRev revision + mapFuncJS *string + lastSeq int + includeDesign, localSeq sql.NullBool ) err := d.db.QueryRowContext(ctx, d.query(` SELECT @@ -330,13 +330,14 @@ func (d *db) updateIndex(ctx context.Context, ddoc, view, mode string) (revision docs.rev_id, design.func_body, design.include_design, + design.local_seq, COALESCE(design.last_seq, 0) AS last_seq FROM {{ .Docs }} AS docs LEFT JOIN {{ .Design }} AS design ON docs.id = design.id AND docs.rev = design.rev AND docs.rev_id = design.rev_id AND design.func_type = 'map' WHERE docs.id = $1 ORDER BY docs.rev DESC, docs.rev_id DESC LIMIT 1 - `), "_design/"+ddoc).Scan(&ddocRev.rev, &ddocRev.id, &mapFuncJS, &includeDesign, &lastSeq) + `), "_design/"+ddoc).Scan(&ddocRev.rev, &ddocRev.id, &mapFuncJS, &includeDesign, &localSeq, &lastSeq) switch { case errors.Is(err, sql.ErrNoRows): return revision{}, &internal.Error{Status: http.StatusNotFound, Message: "missing"} @@ -492,6 +493,10 @@ func (d *db) updateIndex(ctx context.Context, ddoc, view, mode string) (revision continue } + if localSeq.Bool { + full.LocalSeq = seq + } + if err := vm.Set("emit", emit(full.ID, rev)); err != nil { return revision{}, err } diff --git a/x/sqlite/query_test.go b/x/sqlite/query_test.go index 77c514a0..08c88521 100644 --- a/x/sqlite/query_test.go +++ b/x/sqlite/query_test.go @@ -1679,6 +1679,34 @@ func TestDBQuery(t *testing.T) { }, } }) + tests.Add("options.local_seq=true", func(t *testing.T) interface{} { + d := newDB(t) + _ = d.tPut("_design/foo", map[string]interface{}{ + "key": "design", + "views": map[string]interface{}{ + "bar": map[string]string{ + "map": `function(doc) { + if (doc.key) { + emit(doc.key, doc._local_seq); + } + }`, + }, + }, + "options": map[string]interface{}{ + "local_seq": true, + }, + }) + _ = d.tPut("a", map[string]interface{}{"key": "a"}) + + return test{ + db: d, + ddoc: "_design/foo", + view: "_view/bar", + want: []rowResult{ + {ID: "a", Key: `"a"`, Value: "2"}, + }, + } + }) /* TODO: @@ -1716,9 +1744,6 @@ func TestDBQuery(t *testing.T) { - treat map non-exception errors as exceptions - make sure local docs are properly skipped - make sure deleted docs are properly skipped - - view options: https://docs.couchdb.org/en/stable/api/ddoc/views.html#view-options - - local_seq - */ tests.Run(t, func(t *testing.T, tt test) {