Skip to content

Commit

Permalink
Merge pull request #980 from go-kivik/sorted
Browse files Browse the repository at this point in the history 
Various view improvements
  • Loading branch information
@flimzy
flimzy committed May 20, 2024
2 parents 729b531 + 8f73491 commit 4d28f9b
Show file tree
Hide file tree
Showing 7 changed files with 506 additions and 136 deletions.
67 changes: 54 additions & 13 deletions x/sqlite/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func (o optsMap) get(key ...string) (string, interface{}, bool) {
return "", nil, false
}

func parseKey(key string, in any) (string, error) {
func parseJSONKey(key string, in any) (string, error) {
switch t := in.(type) {
case json.RawMessage:
var v interface{}
Expand All @@ -65,31 +65,39 @@ func (o optsMap) endKey() (string, error) {
if !ok {
return "", nil
}
return parseKey(key, value)
return parseJSONKey(key, value)
}

func (o optsMap) endkeyDocID() (string, error) {
key, value, ok := o.get("endkey_doc_id", "end_key_doc_id")
key, value, ok := o.get("endkey_docid", "end_key_doc_id")
if !ok {
return "", nil
}
return parseKey(key, value)
v, ok := value.(string)
if !ok {
return "", &internal.Error{Status: http.StatusBadRequest, Message: fmt.Sprintf("invalid value for '%s': %v", key, value)}
}
return v, nil
}

func (o optsMap) startkeyDocID() (string, error) {
key, value, ok := o.get("startkey_doc_id", "start_key_doc_id")
key, value, ok := o.get("startkey_docid", "start_key_doc_id")
if !ok {
return "", nil
}
return parseKey(key, value)
v, ok := value.(string)
if !ok {
return "", &internal.Error{Status: http.StatusBadRequest, Message: fmt.Sprintf("invalid value for '%s': %v", key, value)}
}
return v, nil
}

func (o optsMap) key() (string, error) {
value, ok := o["key"]
if !ok {
return "", nil
}
return parseKey("key", value)
return parseJSONKey("key", value)
}

func (o optsMap) keys() ([]string, error) {
Expand Down Expand Up @@ -136,7 +144,7 @@ func (o optsMap) startKey() (string, error) {
if !ok {
return "", nil
}
return parseKey(key, value)
return parseJSONKey(key, value)
}

func (o optsMap) rev() string {
Expand Down Expand Up @@ -551,16 +559,44 @@ func (v viewOptions) buildWhere(args *[]any) []string {
where = append(where, `view.key LIKE '"_design/%'`)
}
if v.endkey != "" {
where = append(where, fmt.Sprintf("view.key %s $%d", endKeyOp(v.descending, v.inclusiveEnd), len(*args)+1))
op := endKeyOp(v.descending, v.inclusiveEnd)
where = append(where, fmt.Sprintf("view.key %s $%d", op, len(*args)+1))
*args = append(*args, v.endkey)
if v.endkeyDocID != "" {
where = append(where, fmt.Sprintf("view.id %s $%d", op, len(*args)+1))
*args = append(*args, v.endkeyDocID)
}
}
if v.startkey != "" {
where = append(where, fmt.Sprintf("view.key %s $%d", startKeyOp(v.descending), len(*args)+1))
op := startKeyOp(v.descending)
where = append(where, fmt.Sprintf("view.key %s $%d", op, len(*args)+1))
*args = append(*args, v.startkey)
if v.startkeyDocID != "" {
where = append(where, fmt.Sprintf("view.id %s $%d", op, len(*args)+1))
*args = append(*args, v.startkeyDocID)
}
}
if v.key != "" {
where = append(where, "view.key = $"+strconv.Itoa(len(*args)+1))
*args = append(*args, v.key)
}
if len(v.keys) > 0 {
where = append(where, fmt.Sprintf("view.key IN (%s)", placeholders(len(v.keys), len(*args)+1)))
for _, key := range v.keys {
*args = append(*args, key)
}
}
return where
}

func (v viewOptions) buildOrderBy() string {
if v.sorted {
direction := descendingToDirection(v.descending)
return fmt.Sprintf("ORDER BY view.key %s", direction)
}
return ""
}

// viewOptions are all of the options recognized by the view endpoints
// _desgin/<ddoc>/_view/<view>, _all_docs, _design_docs, and _local_docs.
//
Expand Down Expand Up @@ -602,13 +638,18 @@ func (o optsMap) viewOptions(view string) (*viewOptions, error) {
if err != nil {
return nil, err
}
if reduce != nil && *reduce && isBuiltinView(view) {
return nil, &internal.Error{Status: http.StatusBadRequest, Message: "reduce is invalid for map-only views"}
}
group, err := o.group()
if err != nil {
return nil, err
}
if isBuiltinView(view) {
if group {
return nil, &internal.Error{Status: http.StatusBadRequest, Message: "invalid use of grouping on a map view"}
}
if reduce != nil {
return nil, &internal.Error{Status: http.StatusBadRequest, Message: "reduce is invalid for map-only views"}
}
}
groupLevel, err := o.groupLevel()
if err != nil {
return nil, err
Expand Down
85 changes: 23 additions & 62 deletions x/sqlite/options_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -548,54 +548,35 @@ func Test_viewOptions(t *testing.T) {
wantStatus: http.StatusBadRequest,
})

tests.Add("endkey_doc_id: valid string", test{
options: kivik.Param("endkey_doc_id", "oink"),
tests.Add("endkey_docid: string", test{
options: kivik.Param("endkey_docid", "oink"),
want: &viewOptions{
limit: -1,
inclusiveEnd: true,
update: "true",
sorted: true,
endkeyDocID: `"oink"`,
endkeyDocID: `oink`,
},
})
tests.Add("endkey_doc_id: valid json", test{
options: kivik.Param("endkey_doc_id", json.RawMessage(`"oink"`)),
want: &viewOptions{
limit: -1,
inclusiveEnd: true,
update: "true",
sorted: true,
endkeyDocID: `"oink"`,
},
})
tests.Add("endkey_doc_id: invalid json", test{
options: kivik.Param("endkey_doc_id", json.RawMessage(`oink`)),
wantErr: `invalid value for 'endkey_doc_id': invalid character 'o' looking for beginning of value in key`,
tests.Add("endkey_docid: raw json", test{
options: kivik.Param("endkey_docid", json.RawMessage(`"oink"`)),
wantErr: `invalid value for 'endkey_docid': [34 111 105 110 107 34]`,
wantStatus: http.StatusBadRequest,
})
tests.Add("end_key_doc_id: valid string", test{

tests.Add("end_key_doc_id: string", test{
options: kivik.Param("end_key_doc_id", "oink"),
want: &viewOptions{
limit: -1,
inclusiveEnd: true,
update: "true",
sorted: true,
endkeyDocID: `"oink"`,
},
})
tests.Add("end_key_doc_id: valid json", test{
options: kivik.Param("end_key_doc_id", json.RawMessage(`"oink"`)),
want: &viewOptions{
limit: -1,
inclusiveEnd: true,
update: "true",
sorted: true,
endkeyDocID: `"oink"`,
endkeyDocID: `oink`,
},
})
tests.Add("end_key_doc_id: invalid json", test{
options: kivik.Param("end_key_doc_id", json.RawMessage(`oink`)),
wantErr: `invalid value for 'end_key_doc_id': invalid character 'o' looking for beginning of value in key`,
tests.Add("end_key_doc_id: raw json", test{
options: kivik.Param("end_key_doc_id", json.RawMessage(`"oink"`)),
wantErr: `invalid value for 'end_key_doc_id': [34 111 105 110 107 34]`,
wantStatus: http.StatusBadRequest,
})

Expand Down Expand Up @@ -681,54 +662,34 @@ func Test_viewOptions(t *testing.T) {
wantStatus: http.StatusBadRequest,
})

tests.Add("startkey_doc_id: valid string", test{
options: kivik.Param("startkey_doc_id", "oink"),
tests.Add("startkey_docid: string", test{
options: kivik.Param("startkey_docid", "oink"),
want: &viewOptions{
limit: -1,
inclusiveEnd: true,
update: "true",
sorted: true,
startkeyDocID: `"oink"`,
startkeyDocID: `oink`,
},
})
tests.Add("startkey_doc_id: valid json", test{
options: kivik.Param("startkey_doc_id", json.RawMessage(`"oink"`)),
want: &viewOptions{
limit: -1,
inclusiveEnd: true,
update: "true",
sorted: true,
startkeyDocID: `"oink"`,
},
})
tests.Add("startkey_doc_id: invalid json", test{
options: kivik.Param("startkey_doc_id", json.RawMessage(`oink`)),
wantErr: `invalid value for 'startkey_doc_id': invalid character 'o' looking for beginning of value in key`,
tests.Add("startkey_docid: raw json", test{
options: kivik.Param("startkey_docid", json.RawMessage(`"oink"`)),
wantErr: `invalid value for 'startkey_docid': [34 111 105 110 107 34]`,
wantStatus: http.StatusBadRequest,
})
tests.Add("start_key_doc_id: valid string", test{
tests.Add("start_key_doc_id: string", test{
options: kivik.Param("start_key_doc_id", "oink"),
want: &viewOptions{
limit: -1,
inclusiveEnd: true,
update: "true",
sorted: true,
startkeyDocID: `"oink"`,
},
})
tests.Add("start_key_doc_id: valid json", test{
options: kivik.Param("start_key_doc_id", json.RawMessage(`"oink"`)),
want: &viewOptions{
limit: -1,
inclusiveEnd: true,
update: "true",
sorted: true,
startkeyDocID: `"oink"`,
startkeyDocID: `oink`,
},
})
tests.Add("start_key_doc_id: invalid json", test{
options: kivik.Param("start_key_doc_id", json.RawMessage(`oink`)),
wantErr: `invalid value for 'start_key_doc_id': invalid character 'o' looking for beginning of value in key`,
tests.Add("start_key_doc_id: raw json", test{
options: kivik.Param("start_key_doc_id", json.RawMessage(`"oink"`)),
wantErr: `invalid value for 'start_key_doc_id': [34 111 105 110 107 34]`,
wantStatus: http.StatusBadRequest,
})

Expand Down
46 changes: 24 additions & 22 deletions x/sqlite/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,12 +101,13 @@ func (d *db) performQuery(
vopts *viewOptions,
) (driver.Rows, error) {
if vopts.group {
return d.performGroupQuery(ctx, ddoc, view, vopts.update, vopts.groupLevel)
return d.performGroupQuery(ctx, ddoc, view, vopts.update, vopts.groupLevel, vopts)
}
var (
results *sql.Rows
reducible bool
reduceFuncJS *string
updateSeq string
)
for {
rev, err := d.updateIndex(ctx, ddoc, view, vopts.update)
Expand All @@ -115,7 +116,7 @@ func (d *db) performQuery(
}

args := []interface{}{
vopts.includeDocs, vopts.conflicts, vopts.reduce,
vopts.includeDocs, vopts.conflicts, vopts.reduce, vopts.updateSeq,
"_design/" + ddoc, rev.rev, rev.id, view,
}

Expand All @@ -127,27 +128,27 @@ func (d *db) performQuery(
CASE WHEN MAX(id) IS NOT NULL THEN TRUE ELSE FALSE END AS reducible,
func_body AS reduce_func
FROM {{ .Design }}
WHERE id = $4
AND rev = $5
AND rev_id = $6
WHERE id = $5
AND rev = $6
AND rev_id = $7
AND func_type = 'reduce'
AND func_name = $7
AND func_name = $8
)
SELECT
COALESCE(MAX(last_seq), 0) == (SELECT COALESCE(max(seq),0) FROM {{ .Docs }}) AS up_to_date,
reduce.reducible,
reduce.reduce_func,
NULL,
IIF($4, last_seq, "") AS update_seq,
NULL,
NULL
FROM {{ .Design }} AS map
JOIN reduce
WHERE id = $4
AND rev = $5
AND rev_id = $6
WHERE id = $5
AND rev = $6
AND rev_id = $7
AND func_type = 'map'
AND func_name = $7
AND func_name = $8
UNION ALL
Expand All @@ -160,10 +161,10 @@ func (d *db) performQuery(
NULL AS rev,
NULL AS doc,
NULL AS conflicts
FROM {{ .Map }}
FROM {{ .Map }} AS view
JOIN reduce
WHERE reduce.reducible AND ($3 IS NULL OR $3 == TRUE)
ORDER BY id, key
ORDER BY key
)
UNION ALL
Expand All @@ -184,10 +185,10 @@ func (d *db) performQuery(
WHERE $3 == FALSE OR NOT reduce.reducible
%[2]s
GROUP BY view.id, view.key, view.value, view.rev, view.rev_id
ORDER BY view.key %[1]s
%[1]s
LIMIT %[3]d OFFSET %[4]d
)
`), descendingToDirection(vopts.descending), strings.Join(where, " AND "), vopts.limit, vopts.skip)
`), vopts.buildOrderBy(), strings.Join(where, " AND "), vopts.limit, vopts.skip)
results, err = d.db.QueryContext(ctx, query, args...) //nolint:rowserrcheck // Err checked in Next
switch {
case errIsNoSuchTable(err):
Expand All @@ -204,7 +205,7 @@ func (d *db) performQuery(
}

var upToDate bool
if err := results.Scan(&upToDate, &reducible, &reduceFuncJS, discard{}, discard{}, discard{}); err != nil {
if err := results.Scan(&upToDate, &reducible, &reduceFuncJS, &updateSeq, discard{}, discard{}); err != nil {
_ = results.Close() //nolint:sqlclosecheck // Aborting
return nil, err
}
Expand All @@ -222,17 +223,18 @@ func (d *db) performQuery(
}

if reducible && (vopts.reduce == nil || *vopts.reduce) {
return d.reduceRows(results, reduceFuncJS, false, 0)
return d.reduceRows(results, reduceFuncJS, false, 0, vopts)
}

return &rows{
ctx: ctx,
db: d,
rows: results,
ctx: ctx,
db: d,
rows: results,
updateSeq: updateSeq,
}, nil
}

func (d *db) performGroupQuery(ctx context.Context, ddoc, view, update string, groupLevel uint64) (driver.Rows, error) {
func (d *db) performGroupQuery(ctx context.Context, ddoc, view, update string, groupLevel uint64, vopts *viewOptions) (driver.Rows, error) {
var (
results *sql.Rows
reducible bool
Expand Down Expand Up @@ -326,7 +328,7 @@ func (d *db) performGroupQuery(ctx context.Context, ddoc, view, update string, g
}
}

return d.reduceRows(results, reduceFuncJS, true, groupLevel)
return d.reduceRows(results, reduceFuncJS, true, groupLevel, vopts)
}

const batchSize = 100
Expand Down
Loading

0 comments on commit 4d28f9b

Please sign in to comment.