game search engine

app/controllers/Game.scala

@@ -18,12 +18,11 @@ object Game extends LilaController with BaseGame {
  makeListMenu flatMap { listMenu =>
  searchForm.bindFromRequest.fold(
  failure => Ok(html.game.search(listMenu, failure)).fuccess,
- data => ???
- // data.nonEmptyQuery ?? { query =>
- // searchEnv.paginator(query, page) map (_.some)
- // } map { pager =>
- // Ok(html.game.search(listMenu, searchForm fill data, pager))
- // }
+ data => searchEnv.nonEmptyQuery(data) ?? { query =>
+ searchEnv.paginator(query, page) map (_.some)
+ } map { pager =>
+ Ok(html.game.search(listMenu, searchForm fill data, pager))
+ }
  )
  }
  }

app/views/game/search.scala.html

@@ -135,7 +135,7 @@ <h1 class="title">@trans.advancedSearch()</h1>
  </tr>
  <tr>
  <th>
- <label>Analyzed</label>
+ <label>Analysed</label>
  </th>
  <td class="single">
  @{ base.checkbox(form("analysed"), Query.analyseds._2, 1) }

modules/db/src/main/api/enumerate.scala

@@ -19,9 +19,9 @@ object $enumerate {
 
  def bulk[A: BSONDocumentReader](query: QueryBuilder, size: Int, limit: Int = Int.MaxValue)(op: List[A] => Funit): Funit =
  query.batch(size).cursor[A].enumerateBulks(limit) run {
- Iteratee.foreach((objs: Iterator[A]) =>
- op(objs.toList)
- )
+ Iteratee.foldM(()) {
+ case (_, objs) => op(objs.toList)
+ }
  }
 
  def fold[A: BSONDocumentReader, B](query: QueryBuilder)(zero: B)(f: (B, A) => B): Fu[B] =

modules/forumSearch/src/main/Indexer.scala

@@ -25,9 +25,7 @@ private[forumSearch] final class Indexer(
 
  case InsertPost(post) => postApi liteView post foreach {
  _ foreach { view =>
- client execute {
- indexInto(indexType, view)
- }
+ client execute store(view)
  }
  }
 
@@ -62,7 +60,7 @@ private[forumSearch] final class Indexer(
  $enumerate.bulk[Option[Post]]($query[Post](Json.obj()), 200) { postOptions =>
  (postApi liteViews postOptions.flatten) flatMap { views =>
  client bulk {
- views.map { indexInto(indexType, _) }: _*
+ (views map store): _*
  } void
  }
  }, 20 minutes)
@@ -75,7 +73,7 @@ private[forumSearch] final class Indexer(
  }
  }
 
- private def indexInto(indexType: String, view: PostLiteView) =
+ private def store(view: PostLiteView) =
  index into indexType fields {
  List(
  Fields.body -> view.post.text.take(10000),

modules/game/src/main/Game.scala

@@ -16,7 +16,7 @@ case class Game(
  binaryPieces: ByteArray,
  binaryPgn: ByteArray,
  status: Status,
- turns: Int,
+ turns: Int, // = ply
  startedAtTurn: Int,
  clock: Option[Clock],
  castleLastMoveTime: CastleLastMoveTime,
@@ -68,6 +68,8 @@ case class Game(
  def turnOf(p: Player) = p == player
  def turnOf(c: Color) = c == turnColor
 
+ def playedTurns = turns - startedAtTurn
+
  def fullIdOf(player: Player): Option[String] =
  (players contains player) option id + player.id
 

modules/gameSearch/src/main/DataForm.scala

@@ -60,7 +60,8 @@ private[gameSearch] case class SearchData(
  analysed: Option[Int] = None,
  sort: SearchSort = SearchSort()) {
 
- lazy val query = Query(
+ def query(indexType: String) = Query(
+ indexType = indexType,
  user1 = players.cleanA,
  user2 = players.cleanB,
  winner = players.cleanWinner,
@@ -77,7 +78,10 @@ private[gameSearch] case class SearchData(
  analysed = analysed map (_ == 1),
  sorting = Sorting(sort.field, sort.order))
 
- def nonEmptyQuery = query.nonEmpty option query
+ def nonEmptyQuery(indexType: String) = {
+ val q = query(indexType)
+ q.nonEmpty option q
+ }
 
  private val DateDelta = """^(\d+)(\w)$""".r
  private def toDate(delta: String): Option[DateTime] = delta match {

modules/gameSearch/src/main/Env.scala

@@ -1,107 +1,54 @@
 package lila.gameSearch
 
 import akka.actor._
+import com.sksamuel.elastic4s.ElasticClient
 import com.typesafe.config.Config
-import org.elasticsearch.action.search.SearchResponse
-import play.api.libs.json.JsObject
-import scalastic.elasticsearch.{ Indexer => EsIndexer }
 
 import lila.db.api.$find
 import lila.game.tube.gameTube
-import lila.game.{ GameRepo, Game => GameModel, Query => DbQuery }
 
 final class Env(
  config: Config,
  system: ActorSystem,
+ client: ElasticClient,
  analyser: lila.analyse.Analyser) {
 
  private val IndexName = config getString "index"
  private val TypeName = config getString "type"
  private val PaginatorMaxPerPage = config getInt "paginator.max_per_page"
  private val IndexerName = config getString "indexer.name"
 
- // private val lowLevelIndexer: ActorRef = system.actorOf(Props(new TypeIndexer(
- // esIndexer = esIndexer,
- // indexName = IndexName,
- // typeName = TypeName,
- // mapping = Game.jsonMapping,
- // indexQuery = indexQuery _
- // )), name = IndexerName + "-low-level")
+ private val indexer: ActorRef = system.actorOf(Props(new Indexer(
+ client = client,
+ indexName = IndexName,
+ typeName = TypeName,
+ analyser = analyser
+ )), name = IndexerName)
 
- // private val indexer: ActorRef = system.actorOf(Props(new Indexer(
- // lowLevel = lowLevelIndexer,
- // isAnalyzed = analyser.has _
- // )), name = IndexerName)
-
- // lazy val paginator = new lila.search.PaginatorBuilder(
- // indexer = lowLevelIndexer,
- // maxPerPage = PaginatorMaxPerPage,
- // converter = responseToGames _)
+ lazy val paginator = new lila.search.PaginatorBuilder(
+ indexer = indexer,
+ maxPerPage = PaginatorMaxPerPage,
+ converter = res => $find.byOrderedIds[lila.game.Game](res.getHits.hits.toList map (_.id)))
 
  lazy val forms = new DataForm
 
+ def nonEmptyQuery(data: SearchData) = data nonEmptyQuery s"$IndexName/$TypeName"
+
  def cli = new lila.common.Cli {
  import akka.pattern.ask
- import lila.search.actorApi.RebuildAll
  private implicit def timeout = makeTimeout minutes 60
  def process = {
- case "game" :: "search" :: "reset" :: Nil => ???
- // (lowLevelIndexer ? RebuildAll) inject "Game search index rebuilt"
+ case "game" :: "search" :: "reset" :: Nil =>
+ (indexer ? lila.search.actorApi.Reset) inject "Game search index rebuilt"
  }
  }
-
- // private def responseToGames(response: SearchResponse): Fu[List[GameModel]] =
- // $find.byOrderedIds[GameModel](response.getHits.hits.toList map (_.id))
-
- // private def indexQuery(sel: JsObject): Funit = {
- // import play.api.libs.json._
- // import play.api.libs.iteratee._
- // import lila.db.api._
- // import lila.db.Implicits.LilaPimpedQueryBuilder
- // val selector = DbQuery.frozen ++ sel
- // val query = $query(selector) sort DbQuery.sortCreated
- // val size = $count(selector).await
- // val batchSize = 1000
- // var nb = 0
- // var nbSkipped = 0
- // var started = nowMillis
- // esIndexer map { es =>
- // $enumerate.bulk[Option[GameModel]](query, batchSize) { gameOptions =>
- // val games = gameOptions.flatten
- // val gameIds = games.map(_.id).toSeq
- // val nbGames = games.size
- // nb = nb + nbGames
- // GameRepo.associatePgn(gameIds) flatMap { pgns =>
- // analyser hasMany gameIds map { analysedIds =>
- // val pairs = (pgns map {
- // case (id, pgn) => games.find(_.id == id) map (_ -> pgn)
- // }).flatten
- // es bulk_send {
- // (pairs map {
- // case (game, moves) => es.index_prepare(
- // IndexName,
- // TypeName,
- // game.id,
- // Json stringify Game.from(game, analysedIds contains game.id)
- // ).request
- // })
- // }
- // nbSkipped = nbSkipped + nbGames - pairs.size
- // val perMs = batchSize / (nowMillis - started)
- // started = nowMillis
- // loginfo("[game search] Indexed %d of %d, skipped %d, at %d/s".format(
- // nb, size, nbSkipped, math.round(perMs * 1000)))
- // }
- // }
- // }
- // }
- // }
 }
 
 object Env {
 
  lazy val current = "[boot] gameSearch" describes new Env(
  config = lila.common.PlayApp loadConfig "gameSearch",
  system = lila.common.PlayApp.system,
+ client = lila.search.Env.current.client,
  analyser = lila.analyse.Env.current.analyser)
 }

modules/gameSearch/src/main/Fields.scala

@@ -0,0 +1,17 @@
+package lila.gameSearch
+
+private[gameSearch] object Fields {
+
+ val status = "s"
+ val turns = "t"
+ val rated = "r"
+ val variant = "v"
+ val uids = "u"
+ val winner = "w"
+ val averageRating = "a"
+ val ai = "i"
+ val opening = "o"
+ val date = "d"
+ val duration = "l"
+ val analysed = "n"
+}