Implement scraping movies by URL

gqlgen.yml

@@ -44,3 +44,7 @@ models:
  model: github.com/stashapp/stash/pkg/models.ScrapedSceneTag
  SceneFileType:
  model: github.com/stashapp/stash/pkg/models.SceneFileType
+ ScrapedMovie:
+ model: github.com/stashapp/stash/pkg/models.ScrapedMovie
+ ScrapedMovieStudio:
+ model: github.com/stashapp/stash/pkg/models.ScrapedMovieStudio

graphql/documents/data/scrapers.graphql

@@ -38,6 +38,12 @@ fragment ScrapedScenePerformerData on ScrapedScenePerformer {
  aliases
 }
 
+fragment ScrapedMovieStudioData on ScrapedMovieStudio {
+ id
+ name
+ url
+}
+
 fragment ScrapedMovieData on ScrapedMovie {
  name
  aliases
@@ -47,6 +53,12 @@ fragment ScrapedMovieData on ScrapedMovie {
  director
  url
  synopsis
+ front_image
+ back_image
+
+ studio {
+ ...ScrapedMovieStudioData
+ }
 }
 
 fragment ScrapedSceneMovieData on ScrapedSceneMovie {
@@ -105,4 +117,4 @@ fragment ScrapedSceneData on ScrapedScene {
  movies {
  ...ScrapedSceneMovieData
  }
-}
+}

graphql/documents/queries/scrapers/scrapers.graphql

@@ -20,6 +20,17 @@ query ListSceneScrapers {
  }
 }
 
+query ListMovieScrapers {
+ listMovieScrapers {
+ id
+ name
+ movie {
+ urls
+ supported_scrapes
+ }
+ }
+}
+
 query ScrapePerformerList($scraper_id: ID!, $query: String!) {
  scrapePerformerList(scraper_id: $scraper_id, query: $query) {
  ...ScrapedPerformerData
@@ -48,4 +59,10 @@ query ScrapeSceneURL($url: String!) {
  scrapeSceneURL(url: $url) {
  ...ScrapedSceneData
  }
-}
+}
+
+query ScrapeMovieURL($url: String!) {
+ scrapeMovieURL(url: $url) {
+ ...ScrapedMovieData
+ }
+}

graphql/schema/schema.graphql

@@ -59,6 +59,7 @@ type Query {
  """List available scrapers"""
  listPerformerScrapers: [Scraper!]!
  listSceneScrapers: [Scraper!]!
+ listMovieScrapers: [Scraper!]!
 
  """Scrape a list of performers based on name"""
  scrapePerformerList(scraper_id: ID!, query: String!): [ScrapedPerformer!]!
@@ -70,6 +71,8 @@ type Query {
  scrapeScene(scraper_id: ID!, scene: SceneUpdateInput!): ScrapedScene
  """Scrapes a complete performer record based on a URL"""
  scrapeSceneURL(url: String!): ScrapedScene
+ """Scrapes a complete movie record based on a URL"""
+ scrapeMovieURL(url: String!): ScrapedMovie
 
  """Scrape a performer using Freeones"""
  scrapeFreeones(performer_name: String!): ScrapedPerformer

graphql/schema/types/scraped-movie.graphql

@@ -1,3 +1,10 @@
+type ScrapedMovieStudio {
+ """Set if studio matched"""
+ id: ID
+ name: String!
+ url: String
+}
+
 """A movie from a scraping operation..."""
 type ScrapedMovie {
  name: String
@@ -8,6 +15,11 @@ type ScrapedMovie {
  director: String
  url: String
  synopsis: String
+ studio: ScrapedMovieStudio
+
+ """This should be base64 encoded"""
+ front_image: String
+ back_image: String
 }
 
 input ScrapedMovieInput {
@@ -19,4 +31,4 @@ input ScrapedMovieInput {
  director: String
  url: String
  synopsis: String
-}
+}

graphql/schema/types/scraper.graphql

@@ -20,6 +20,8 @@ type Scraper {
  performer: ScraperSpec
  """Details for scene scraper"""
  scene: ScraperSpec
+ """Details for movie scraper"""
+ movie: ScraperSpec
 }
 
 

pkg/api/resolver_query_scraper.go

@@ -41,6 +41,10 @@ func (r *queryResolver) ListSceneScrapers(ctx context.Context) ([]*models.Scrape
  return manager.GetInstance().ScraperCache.ListSceneScrapers(), nil
 }
 
+func (r *queryResolver) ListMovieScrapers(ctx context.Context) ([]*models.Scraper, error) {
+ return manager.GetInstance().ScraperCache.ListMovieScrapers(), nil
+}
+
 func (r *queryResolver) ScrapePerformerList(ctx context.Context, scraperID string, query string) ([]*models.ScrapedPerformer, error) {
  if query == "" {
  return nil, nil
@@ -64,3 +68,7 @@ func (r *queryResolver) ScrapeScene(ctx context.Context, scraperID string, scene
 func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models.ScrapedScene, error) {
  return manager.GetInstance().ScraperCache.ScrapeSceneURL(url)
 }
+
+func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models.ScrapedMovie, error) {
+ return manager.GetInstance().ScraperCache.ScrapeMovieURL(url)
+}

pkg/api/urlbuilders/movie.go

@@ -3,13 +3,13 @@ package urlbuilders
 import "strconv"
 
 type MovieURLBuilder struct {
- BaseURL  string
+ BaseURL string
  MovieID string
 }
 
 func NewMovieURLBuilder(baseURL string, movieID int) MovieURLBuilder {
  return MovieURLBuilder{
- BaseURL:  baseURL,
+ BaseURL: baseURL,
  MovieID: strconv.Itoa(movieID),
  }
 }
@@ -21,4 +21,3 @@ func (b MovieURLBuilder) GetMovieFrontImageURL() string {
 func (b MovieURLBuilder) GetMovieBackImageURL() string {
  return b.BaseURL + "/movie/" + b.MovieID + "/backimage"
 }
-

pkg/models/model_scraped_item.go

@@ -135,3 +135,24 @@ type ScrapedSceneTag struct {
  ID *string `graphql:"id" json:"id"`
  Name string `graphql:"name" json:"name"`
 }
+
+type ScrapedMovie struct {
+ Name *string `graphql:"name" json:"name"`
+ Aliases *string `graphql:"aliases" json:"aliases"`
+ Duration *string `graphql:"duration" json:"duration"`
+ Date *string `graphql:"date" json:"date"`
+ Rating *string `graphql:"rating" json:"rating"`
+ Director *string `graphql:"director" json:"director"`
+ Studio *ScrapedMovieStudio `graphql:"studio" json:"studio"`
+ Synopsis *string `graphql:"synopsis" json:"synopsis"`
+ URL *string `graphql:"url" json:"url"`
+ FrontImage *string `graphql:"front_image" json:"front_image"`
+ BackImage *string `graphql:"back_image" json:"back_image"`
+}
+
+type ScrapedMovieStudio struct {
+ // Set if studio matched
+ ID *string `graphql:"id" json:"id"`
+ Name string `graphql:"name" json:"name"`
+ URL *string `graphql:"url" json:"url"`
+}

pkg/scraper/action.go

@@ -39,6 +39,8 @@ type scraper interface {
 
  scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error)
  scrapeSceneByURL(url string) (*models.ScrapedScene, error)
+
+ scrapeMovieByURL(url string) (*models.ScrapedMovie, error)
 }
 
 func getScraper(scraper scraperTypeConfig, config config, globalConfig GlobalConfig) scraper {

pkg/scraper/config.go

@@ -35,6 +35,9 @@ type config struct {
  // Configuration for querying a scene by a URL
  SceneByURL []*scrapeByURLConfig `yaml:"sceneByURL"`
 
+ // Configuration for querying a movie by a URL
+ MovieByURL []*scrapeByURLConfig `yaml:"movieByURL"`
+
  // Scraper debugging options
  DebugOptions *scraperDebugOptions `yaml:"debug"`
 
@@ -86,6 +89,12 @@ func (c config) validate() error {
  }
  }
 
+ for _, s := range c.MovieByURL {
+ if err := s.validate(); err != nil {
+ return err
+ }
+ }
+
  return nil
 }
 
@@ -225,6 +234,18 @@ func (c config) toScraper() *models.Scraper {
  ret.Scene = &scene
  }
 
+ movie := models.ScraperSpec{}
+ if len(c.MovieByURL) > 0 {
+ movie.SupportedScrapes = append(movie.SupportedScrapes, models.ScrapeTypeURL)
+ for _, v := range c.MovieByURL {
+ movie.Urls = append(movie.Urls, v.URL...)
+ }
+ }
+
+ if len(movie.SupportedScrapes) > 0 {
+ ret.Movie = &movie
+ }
+
  return &ret
 }
 
@@ -297,6 +318,20 @@ func (c config) matchesSceneURL(url string) bool {
  return false
 }
 
+func (c config) supportsMovies() bool {
+ return len(c.MovieByURL) > 0
+}
+
+func (c config) matchesMovieURL(url string) bool {
+ for _, scraper := range c.MovieByURL {
+ if scraper.matchesURL(url) {
+ return true
+ }
+ }
+
+ return false
+}
+
 func (c config) ScrapeScene(scene models.SceneUpdateInput, globalConfig GlobalConfig) (*models.ScrapedScene, error) {
  if c.SceneByFragment != nil {
  s := getScraper(*c.SceneByFragment, c, globalConfig)
@@ -323,3 +358,21 @@ func (c config) ScrapeSceneURL(url string, globalConfig GlobalConfig) (*models.S
 
  return nil, nil
 }
+
+func (c config) ScrapeMovieURL(url string, globalConfig GlobalConfig) (*models.ScrapedMovie, error) {
+ for _, scraper := range c.MovieByURL {
+ if scraper.matchesURL(url) {
+ s := getScraper(scraper.scraperTypeConfig, c, globalConfig)
+ ret, err := s.scrapeMovieByURL(url)
+ if err != nil {
+ return nil, err
+ }
+
+ if ret != nil {
+ return ret, nil
+ }
+ }
+ }
+
+ return nil, nil
+}

pkg/scraper/image.go

@@ -47,6 +47,40 @@ func setSceneImage(s *models.ScrapedScene, globalConfig GlobalConfig) error {
  return nil
 }
 
+func setMovieFrontImage(m *models.ScrapedMovie, globalConfig GlobalConfig) error {
+ // don't try to get the image if it doesn't appear to be a URL
+ if m == nil || m.FrontImage == nil || !strings.HasPrefix(*m.FrontImage, "http") {
+ // nothing to do
+ return nil
+ }
+
+ img, err := getImage(*m.FrontImage, globalConfig)
+ if err != nil {
+ return err
+ }
+
+ m.FrontImage = img
+
+ return nil
+}
+
+func setMovieBackImage(m *models.ScrapedMovie, globalConfig GlobalConfig) error {
+ // don't try to get the image if it doesn't appear to be a URL
+ if m == nil || m.BackImage == nil || !strings.HasPrefix(*m.BackImage, "http") {
+ // nothing to do
+ return nil
+ }
+
+ img, err := getImage(*m.BackImage, globalConfig)
+ if err != nil {
+ return err
+ }
+
+ m.BackImage = img
+
+ return nil
+}
+
 func getImage(url string, globalConfig GlobalConfig) (*string, error) {
  client := &http.Client{
  Timeout: imageGetTimeout,

pkg/scraper/json.go

@@ -88,6 +88,16 @@ func (s *jsonScraper) scrapeSceneByURL(url string) (*models.ScrapedScene, error)
  return scraper.scrapeScene(q)
 }
 
+func (s *jsonScraper) scrapeMovieByURL(url string) (*models.ScrapedMovie, error) {
+ doc, scraper, err := s.scrapeURL(url)
+ if err != nil {
+ return nil, err
+ }
+
+ q := s.getJsonQuery(doc)
+ return scraper.scrapeMovie(q)
+}
+
 func (s *jsonScraper) scrapePerformersByName(name string) ([]*models.ScrapedPerformer, error) {
  scraper := s.getJsonScraper()