Support a minModTime input on metadata scans. (stashapp#1951)

* Support a maxAge input on metadata scans.

Extend the GraphQL world with a Duration scalar. It is parsed as a
typical Go duration, i.e., "4h" is 4 hours. Alternatively, one can
pass an integer which is interpreted as seconds.

Extend Mutation.metadataScan(input: $input) to support a new optional
value, maxAge. If set, the scanner will exit early if the file it
is looking at has an mtime older than the cutOff point generated by

now() - maxAge

This speeds up scanning in the case where the user knows how old the
changes on disk are, by exiting the scan early if that is the case.

* Change maxAge into minModTime

Introduce a `Timestamp` scalar, so we have a scalar we control. Let
it accept three formats:

* RFC3339Nano
* @unix where UNIX is a unix-timestamp: seconds after 01-01-1970
* '<4h': a timestamp relative to the current server time

This scalar parses to a time.Time.

Use MinModTime in the scanner to filter out a large number of scan
analyzes by exiting the scan operation early.

* Heed the linter, perform errcheck

* Rename test vars for consistency.

* Code review: move minModTime into queuefiles

* Remove the ability to input Unix timestamps

Test failures on the CI-system explains why this is undesirable. It is
not clear what timezone one is operating in when entering a unix
timestamp. We could go with UTC, but it is so much easier to require an
RFC3339 timestamp, which avoids this problem entirely.

* Move the minModTime field into filters

Create a new filter input object for metadata scans, and push the
minModTime field in there. If we come up with new filters, they can
be added to that input object rather than cluttering the main input
object.

* Use utils.ParseDateStringAsTime

Replace time.Parse with utils.ParseDateStringAsTime

While here, add some more test cases for that parser.

gqlgen.yml

@@ -14,6 +14,10 @@ resolver:
 struct_tag: gqlgen
 
 models:
+ # Scalars
+ Timestamp:
+ model: github.com/stashapp/stash/pkg/models.Timestamp
+ # Objects
  Gallery:
  model: github.com/stashapp/stash/pkg/models.Gallery
  Image:

graphql/schema/types/metadata.graphql

@@ -58,8 +58,15 @@ type GeneratePreviewOptions {
  previewPreset: PreviewPreset
 }
 
+"Filter options for meta data scannning"
+input ScanMetaDataFilterInput {
+ "If set, files with a modification time before this time point are ignored by the scan"
+ minModTime: Timestamp
+}
+
 input ScanMetadataInput {
  paths: [String!]
+
  """Set name, date, details from metadata (if present)"""
  useFileMetadata: Boolean
  """Strip file extension from title"""
@@ -74,6 +81,9 @@ input ScanMetadataInput {
  scanGeneratePhashes: Boolean
  """Generate image thumbnails during scan"""
  scanGenerateThumbnails: Boolean
+
+ "Filter options for the scan"
+ filter: ScanMetaDataFilterInput
 }
 
 type ScanMetadataOptions {
@@ -122,11 +132,11 @@ enum IdentifyFieldStrategy {
  """Never sets the field value"""
  IGNORE
  """
- For multi-value fields, merge with existing. 
+ For multi-value fields, merge with existing.
  For single-value fields, ignore if already set
  """
  MERGE
- """Always replaces the value if a value is found. 
+ """Always replaces the value if a value is found.
  For multi-value fields, any existing values are removed and replaced with the
  scraped values.
  """

graphql/schema/types/scalars.graphql

@@ -0,0 +1,7 @@
+
+"""
+Timestamp is a point in time. It is always output as RFC3339-compatible time points.
+It can be input as a RFC3339 string, or as "<4h" for "4 hours in the past" or ">5m"
+for "5 minutes in the future"
+"""
+scalar Timestamp

pkg/manager/task_scan.go

@@ -146,6 +146,11 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) {
 func (j *ScanJob) queueFiles(ctx context.Context, paths []*models.StashConfig, scanQueue chan<- scanFile, parallelTasks int) (total int, newFiles int) {
  defer close(scanQueue)
 
+ var minModTime time.Time
+ if j.input.Filter != nil && j.input.Filter.MinModTime != nil {
+ minModTime = *j.input.Filter.MinModTime
+ }
+
  wg := sizedwaitgroup.New(parallelTasks)
 
  for _, sp := range paths {
@@ -160,6 +165,11 @@ func (j *ScanJob) queueFiles(ctx context.Context, paths []*models.StashConfig, s
  return context.Canceled
  }
 
+ // exit early on cutoff
+ if info.Mode().IsRegular() && info.ModTime().Before(minModTime) {
+ return nil
+ }
+
  wg.Add()
 
  go func() {

pkg/models/timestamp.go

@@ -0,0 +1,57 @@
+package models
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "strconv"
+ "time"
+
+ "github.com/99designs/gqlgen/graphql"
+ "github.com/stashapp/stash/pkg/logger"
+ "github.com/stashapp/stash/pkg/utils"
+)
+
+var ErrTimestamp = errors.New("cannot parse Timestamp")
+
+func MarshalTimestamp(t time.Time) graphql.Marshaler {
+ if t.IsZero() {
+ return graphql.Null
+ }
+
+ return graphql.WriterFunc(func(w io.Writer) {
+ _, err := io.WriteString(w, strconv.Quote(t.Format(time.RFC3339Nano)))
+ if err != nil {
+ logger.Warnf("could not marshal timestamp: %v", err)
+ }
+ })
+}
+
+func UnmarshalTimestamp(v interface{}) (time.Time, error) {
+ if tmpStr, ok := v.(string); ok {
+ if len(tmpStr) == 0 {
+ return time.Time{}, fmt.Errorf("%w: empty string", ErrTimestamp)
+ }
+
+ switch tmpStr[0] {
+ case '>', '<':
+ d, err := time.ParseDuration(tmpStr[1:])
+ if err != nil {
+ return time.Time{}, fmt.Errorf("%w: cannot parse %v-duration: %v", ErrTimestamp, tmpStr[0], err)
+ }
+ t := time.Now()
+ // Compute point in time:
+ if tmpStr[0] == '<' {
+ t = t.Add(-d)
+ } else {
+ t = t.Add(d)
+ }
+
+ return t, nil
+ }
+
+ return utils.ParseDateStringAsTime(tmpStr)
+ }
+
+ return time.Time{}, fmt.Errorf("%w: not a string", ErrTimestamp)
+}

pkg/models/timestamp_test.go

@@ -0,0 +1,90 @@
+package models
+
+import (
+ "bytes"
+ "strconv"
+ "testing"
+ "time"
+)
+
+func TestTimestampSymmetry(t *testing.T) {
+ n := time.Now()
+ buf := bytes.NewBuffer([]byte{})
+ MarshalTimestamp(n).MarshalGQL(buf)
+
+ str, err := strconv.Unquote(buf.String())
+ if err != nil {
+ t.Fatal("could not unquote string")
+ }
+ got, err := UnmarshalTimestamp(str)
+ if err != nil {
+ t.Fatalf("could not unmarshal time: %v", err)
+ }
+
+ if !n.Equal(got) {
+ t.Fatalf("have %v, want %v", got, n)
+ }
+}
+
+func TestTimestamp(t *testing.T) {
+ n := time.Now().In(time.UTC)
+ testCases := []struct {
+ name string
+ have string
+ want string
+ }{
+ {"reflexivity", n.Format(time.RFC3339Nano), n.Format(time.RFC3339Nano)},
+ {"rfc3339", "2021-11-04T01:02:03Z", "2021-11-04T01:02:03Z"},
+ {"date", "2021-04-05", "2021-04-05T00:00:00Z"},
+ {"datetime", "2021-04-05 14:45:36", "2021-04-05T14:45:36Z"},
+ {"datetime-tz", "2021-04-05 14:45:36 PDT", "2021-04-05T14:45:36Z"},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ p, err := UnmarshalTimestamp(tc.have)
+ if err != nil {
+ t.Fatalf("could not unmarshal time: %v", err)
+ }
+
+ buf := bytes.NewBuffer([]byte{})
+ MarshalTimestamp(p).MarshalGQL(buf)
+
+ got, err := strconv.Unquote(buf.String())
+ if err != nil {
+ t.Fatalf("count not unquote string")
+ }
+ if got != tc.want {
+ t.Errorf("got %s; want %s", got, tc.want)
+ }
+ })
+ }
+}
+
+const epsilon = 10 * time.Second
+
+func TestTimestampRelative(t *testing.T) {
+ n := time.Now()
+ testCases := []struct {
+ name string
+ have string
+ want time.Time
+ }{
+ {"past", "<4h", n.Add(-4 * time.Hour)},
+ {"future", ">5m", n.Add(5 * time.Minute)},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := UnmarshalTimestamp(tc.have)
+ if err != nil {
+ t.Fatalf("could not unmarshal time: %v", err)
+ }
+
+ if got.Sub(tc.want) > epsilon {
+ t.Errorf("not within bound of %v; got %s; want %s", epsilon, got, tc.want)
+ }
+ })
+ }
+
+}