game indexing + fixes

app/controllers/Game.scala

@@ -9,6 +9,23 @@ object Game extends LilaController with BaseGame {
  private def paginator = Env.game.paginator
  private def analysePaginator = Env.analyse.paginator
  private def cached = Env.game.cached
+ private def searchEnv = Env.gameSearch
+ def searchForm = searchEnv.forms.search
+
+ def search(page: Int) = OpenBody { implicit ctx ⇒
+ Reasonable(page, 100) {
+ implicit def req = ctx.body
+ makeListMenu flatMap { listMenu ⇒
+ searchForm.bindFromRequest.fold(
+ failure ⇒ Ok(html.game.search(listMenu, failure)).fuccess,
+ data ⇒ data.nonEmptyQuery zmap { query ⇒
+ searchEnv.paginator(query, page) map (_.some)
+ } map { pager ⇒
+ Ok(html.game.search(listMenu, searchForm fill data, pager))
+ })
+ }
+ }
+ }
 
  def realtime = Open { implicit ctx ⇒
  GameRepo recentGames 9 zip makeListMenu map {

app/controllers/LilaController.scala

@@ -196,7 +196,6 @@ private[controllers] trait LilaController
  user foreach Env.user.setOnline
  }
 
- private val maxPage = 40
- protected def Reasonable(page: Int)(result: ⇒ Fu[Result]): Fu[Result] =
- (page < maxPage).fold(result, BadRequest("resource too old").fuccess)
+ protected def Reasonable(page: Int, max: Int = 40)(result: ⇒ Fu[Result]): Fu[Result] =
+ (page < max).fold(result, BadRequest("resource too old").fuccess)
 }

old/views/search/form.scala.html → app/views/game/search.scala.html

@@ -1,7 +1,6 @@
-@(listMenu: lila.app.game.ListMenu, form: Form[_], paginator: Option[Paginator[DbGame]] = None)(implicit ctx: Context)
+@(listMenu: lila.game.ListMenu, form: Form[_], paginator: Option[Paginator[Game]] = None)(implicit ctx: Context)
 
-@import lila.app.search.Query
-@import lila.app.search.Sorting
+@import lila.gameSearch.{ Query, Sorting }
 
 @moreCss = {
 @cssTag("search.css")
@@ -18,7 +17,7 @@
 moreCss = moreCss) {
 <div class="content_box no_padding">
  <h1 class="title">@trans.advancedSearch()</h1>
- <form class="search" action="@routes.Search.form()" method="get">
+ <form class="search" action="@routes.Game.search()" method="get">
  <table>
  <tr>
  <th>
@@ -141,11 +140,11 @@ <h1 class="title">@trans.advancedSearch()</h1>
  @if(pager.nbResults > 0) {
  <div class="search_status">
  @pager.nbResults.localize games found -
- <a class="permalink" href="@routes.Search.form()">Permalink</a>
+ <a class="permalink" href="@routes.Game.search()">Permalink</a>
  </div>
  <div class="all_games search_infinitescroll">
  @pager.nextPage.map { n =>
- <div class="pager none"><a href="@routes.Search.form(n)">Next</a></div>
+ <div class="pager none"><a href="@routes.Game.search(n)">Next</a></div>
  }.getOrElse {
  <div class="none"></div>
  }
@@ -154,7 +153,7 @@ <h1 class="title">@trans.advancedSearch()</h1>
  } else {
  <div class="search_status">
  No game found -
- <a class="permalink" href="@routes.Search.form()">Permalink</a>
+ <a class="permalink" href="@routes.Game.search()">Permalink</a>
  </div>
  }
  }.getOrElse {

app/views/game/sideMenu.scala.html

@@ -3,7 +3,7 @@
 <a class="@active.active("realtime")" href="@routes.Game.realtime()">
  @trans.gamesBeingPlayedRightNow()
 </a>
-<a class="@active.active("search")" href="@routes.Search.form()">
+<a class="@active.active("search")" href="@routes.Game.search()">
  @trans.advancedSearch()
 </a>
 <a class="@active.active("import")" href="@routes.Importer.importGame()">

conf/routes

@@ -33,7 +33,7 @@ GET /games/analysed controllers.Game.analysed(page: Int ?= 1)
 GET /games/imported controllers.Game.imported(page: Int ?= 1)
 
 # Search
-GET /games/search controllers.Search.form(page: Int ?= 1)
+GET /games/search controllers.Game.search(page: Int ?= 1)
 
 # Round
 GET /$gameId<[\w\-]{8}> controllers.Round.watcher(gameId: String, color: String = "white")

modules/api/src/main/Cli.scala

@@ -24,23 +24,4 @@ private[api] final class Cli(env: Env) {
  lila.teamSearch.Env.current.cli.process orElse
  lila.forum.Env.current.cli.process orElse
  lila.forumSearch.Env.current.cli.process
-
- // case "average-elo" :: Nil ⇒ infos.averageElo
- // case "i18n-js-dump" :: Nil ⇒ i18n.jsDump
- // case "i18n-fix" :: Nil ⇒ i18n.fileFix
- // case "i18n-fetch" :: from :: Nil ⇒ i18n fetch from
- // case "user-rewrite-history" :: Nil ⇒ users.rewriteHistory
- // case "forum-search" :: text :: Nil ⇒ forum.search(text)
- // case "forum-search-reset" :: Nil ⇒ forum.searchReset
- // case "game-cleanup-next" :: Nil ⇒ titivate.cleanupNext inject "Done"
- // case "game-cleanup-unplayed" :: Nil ⇒ titivate.cleanupUnplayed inject "Done"
- // case "game-finish" :: Nil ⇒ titivate.finishByClock inject "Done"
- // case "search-reset" :: Nil ⇒ search.reset
- // case "team-search" :: text :: Nil ⇒ teams.search(text)
- // case "team-search-reset" :: Nil ⇒ teams.searchReset
- // case "team-join" :: team :: users ⇒ teams.join(team, users)
- // case "team-quit" :: team :: users ⇒ teams.quit(team, users)
- // case "team-enable" :: uid :: Nil ⇒ teams enable uid
- // case "team-disable" :: uid :: Nil ⇒ teams disable uid
- // case "team-delete" :: uid :: Nil ⇒ teams delete uid
 }

modules/db/src/main/Tube.scala

@@ -66,7 +66,11 @@ case class Tube[Doc](
 
 object Tube {
 
- val json = Tube[JsObject](__.read[JsObject], __.write[JsObject])
+ val json = Tube[JsObject](
+ __.read[JsObject],
+ __.write[JsObject],
+ Seq(_.NoId) // no need to rename the ID field as we are not mapping
+ )
 
  def toMongoId(js: JsValue): JsResult[JsObject] =
  js transform Helpers.rename('id, '_id)
@@ -87,7 +91,7 @@ object Tube {
  def andThen(transformer: Reads[JsObject]): Writes[A] =
  writes.transform(Writes[JsValue] { origin ⇒
  origin transform transformer match {
- case err: JsError ⇒ throw LilaException("[tube] Cannot transform %s\n%s".format(origin, err))
+ case err: JsError  ⇒ throw LilaException("[tube] Cannot transform %s\n%s".format(origin, err))
  case JsSuccess(js, _) ⇒ js
  }
  })

modules/db/src/main/api/enumerate.scala

@@ -16,8 +16,8 @@ object $enumerate {
  Iteratee.foreach((obj: A) ⇒ op(obj))
  }
 
- def bulk[A: BSONDocumentReader](query: QueryBuilder, size: Int)(op: List[A] ⇒ Funit): Funit =
- query.batch(size).cursor[A].enumerateBulks run {
+ def bulk[A: BSONDocumentReader](query: QueryBuilder, size: Int, limit: Int = Int.MaxValue)(op: List[A] ⇒ Funit): Funit =
+ query.batch(size).cursor[A].enumerateBulks(limit) run {
  Iteratee.foreach((objs: Iterator[A]) ⇒
  op(objs.toList)
  )

modules/gameSearch/src/main/DataForm.scala

@@ -10,7 +10,7 @@ import org.scala_tools.time.Imports._
 
 import chess.{ Mode }
 
-final class DataForm {
+private[gameSearch] final class DataForm {
 
  val search = Form(mapping(
  "players" -> mapping(
@@ -40,7 +40,7 @@ final class DataForm {
  )(SearchData.apply)(SearchData.unapply)) fill SearchData()
 }
 
-case class SearchData(
+private[gameSearch] case class SearchData(
  players: SearchPlayer = SearchPlayer(),
  variant: Option[Int] = None,
  mode: Option[Int] = None,
@@ -65,7 +65,7 @@ case class SearchData(
  winner = players.cleanWinner,
  variant = variant,
  rated = mode flatMap Mode.apply map (_.rated),
- opening = opening map clean,
+ opening = opening map (_.trim.toLowerCase),
  turns = Range(turnsMin, turnsMax),
  averageElo = Range(eloMin, eloMax),
  hasAi = hasAi map (_ == 1),
@@ -76,7 +76,7 @@ case class SearchData(
  sorting = Sorting(sort.field, sort.order)
  )
 
- private def clean(s: String) = s.trim.toLowerCase
+ def nonEmptyQuery = query.nonEmpty option query
 
  private val DateDelta = """^(\d+)(\w)$""".r
  private def toDate(delta: String): Option[DateTime] = delta match {
@@ -89,7 +89,7 @@ case class SearchData(
  }
 }
 
-case class SearchPlayer(
+private[gameSearch] case class SearchPlayer(
  a: Option[String] = None,
  b: Option[String] = None,
  winner: Option[String] = None) {
@@ -104,6 +104,6 @@ case class SearchPlayer(
  s map (_.trim.toLowerCase) filter (_.nonEmpty)
 }
 
-case class SearchSort(
+private[gameSearch] case class SearchSort(
  field: String = Sorting.default.field,
  order: String = Sorting.default.order)

modules/gameSearch/src/main/Env.scala

@@ -25,7 +25,7 @@ final class Env(
  lowLevel = lowLevelIndexer
  )), name = IndexerName)
 
- lazy val paginatorBuilder = new lila.search.PaginatorBuilder(
+ lazy val paginator = new lila.search.PaginatorBuilder(
  indexer = lowLevelIndexer,
  maxPerPage = PaginatorMaxPerPage,
  converter = responseToGames _)
@@ -59,27 +59,35 @@ final class Env(
  import lila.db.api._
  import lila.db.Implicits.LilaPimpedQueryBuilder
  val selector = DbQuery.frozen ++ sel
- val query = $query(selector).sort(DbQuery.sortCreated) //limit 3000
+ val query = $query(selector) sort DbQuery.sortCreated 
  val size = $count(selector).await
+ val batchSize = 1000
  var nb = 0
- $enumerate.bulk[Option[GameModel]](query, 5000) { gameOptions ⇒
+ var nbSkipped = 0
+ var started = nowMillis
+ $enumerate.bulk[Option[GameModel]](query, batchSize) { gameOptions ⇒
  val games = gameOptions.flatten
- nb = nb + games.size
- if (size > 1000) loginfo("Index %d of %d games".format(nb, size))
+ val nbGames = games.size
+ nb = nb + nbGames
  PgnRepo.associate(games.map(_.id).toSeq) map { pgns ⇒
  val pairs = (pgns map {
  case (id, pgn) ⇒ games.find(_.id == id) map (_ -> pgn)
  }).flatten
- esIndexer bulk {
- pairs map {
+ esIndexer bulk_send {
+ (pairs map {
  case (game, pgn) ⇒ esIndexer.index_prepare(
  IndexName,
  TypeName,
  game.id,
  Json stringify Game.from(game, pgn)
  ).request
- } toList
+ })
  }
+ nbSkipped = nbSkipped + nbGames - pairs.size
+ val perMs = batchSize / (nowMillis - started)
+ started = nowMillis
+ loginfo("[game search] Indexed %d of %d, skipped %d, at %d/s".format(
+ nb, size, nbSkipped, math.round(perMs * 1000)))
  }
  }
  }

modules/gameSearch/src/main/Query.scala

@@ -9,7 +9,7 @@ import org.elasticsearch.index.query._, FilterBuilders._, QueryBuilders._
 import org.joda.time.DateTime
 import org.scala_tools.time.Imports._
 
-private[gameSearch] case class Query(
+case class Query(
  user1: Option[String] = None,
  user2: Option[String] = None,
  winner: Option[String] = None,

modules/round/src/main/Hand.scala

@@ -40,7 +40,8 @@ final class Hand(
  newChessGameAndMove ← chessGame(orig, dest, promotion, lag)
  (newChessGame, move) = newChessGameAndMove
  } yield g2.update(newChessGame, move, blur)).prefixFailuresWith(povRef + " - ").fold(fufail(_), {
- case (progress, pgn) ⇒ if (progress.game.finished)
+ case (progress, pgn) ⇒ 
+ if (progress.game.finished)
  (GameRepo save progress) >>
  PgnRepo.save(povRef.gameId, pgn) >>
  finisher.moveFinish(progress.game, color) map { finishEvents ⇒
@@ -50,7 +51,7 @@ final class Hand(
  initialFen ← progress.game.variant.exotic ?? {
  GameRepo initialFen progress.game.id
  }
- aiResult ← ai.play(progress.game.toChess, pgn.pp, initialFen, ~progress.game.aiLevel)
+ aiResult ← ai.play(progress.game.toChess, pgn, initialFen, ~progress.game.aiLevel)
  eventsAndFen ← aiResult.fold(
  err ⇒ fufail("[round hand] ai response fail " + err), {
  case (newChessGame, move) ⇒ {

modules/search/src/main/TypeIndexer.scala

@@ -5,6 +5,7 @@ import actorApi._
 import scalastic.elasticsearch.{ Indexer ⇒ EsIndexer }
 
 import akka.actor._
+import akka.pattern.pipe
 import play.api.libs.json._
 
 final class TypeIndexer(
@@ -28,7 +29,7 @@ final class TypeIndexer(
 
  case RebuildAll ⇒ {
  self ! Clear
- sender ! indexQuery(Json.obj())
+ indexQuery(Json.obj()) pipeTo sender
  }
 
  case Optimize ⇒ es.optimize(Seq(indexName))