supercharge game export by IDs

See https://lichess.org/api#operation/gamesExportIds

app/controllers/Api.scala

@@ -176,18 +176,6 @@ object Api extends LilaController {
  }
  }
 
- def games = Action.async(parse.tolerantText) { req =>
- val gameIds = req.body.split(',').take(300)
- val ip = HTTPRequest lastRemoteAddress req
- GameRateLimitPerIP(ip, cost = gameIds.size / 4) {
- lila.mon.api.game.cost(1)
- gameApi.many(
- ids = gameIds,
- withMoves = getBool("with_moves", req)
- ) map toApiResult map toHttp
- }(Zero.instance(tooManyRequests.fuccess))
- }
-
  private val CrosstableRateLimitPerIP = new lila.memo.RateLimit[IpAddress](
  credits = 30,
  duration = 10 minutes,

app/controllers/Game.scala

@@ -29,7 +29,7 @@ object Game extends LilaController {
 
  def exportOne(id: String) = Open { implicit ctx =>
  OptionFuResult(GameRepo game id) { game =>
- if (game.playable) BadRequest("Can't export PGN of game in progress").fuccess
+ if (game.playable) BadRequest("Only bots can access their games in progress. See https://lichess.org/api#tag/Chess-Bot").fuccess
  else {
  val config = GameApiV2.OneConfig(
  format = if (HTTPRequest acceptsJson ctx.req) GameApiV2.Format.JSON else GameApiV2.Format.PGN,
@@ -60,7 +60,7 @@ object Game extends LilaController {
  RequireHttp11(req) {
  Api.GlobalLinearLimitPerIP(HTTPRequest lastRemoteAddress req) {
  Api.GlobalLinearLimitPerUserOption(me) {
- val format = if (HTTPRequest acceptsNdJson req) GameApiV2.Format.JSON else GameApiV2.Format.PGN
+ val format = GameApiV2.Format byRequest req
  WithVs(req) { vs =>
  val config = GameApiV2.ByUserConfig(
  user = user,
@@ -92,6 +92,23 @@ object Game extends LilaController {
  }
  }
 
+ def exportByIds = Action.async(parse.tolerantText) { req =>
+ RequireHttp11(req) {
+ Api.GlobalLinearLimitPerIP(HTTPRequest lastRemoteAddress req) {
+ val format = GameApiV2.Format byRequest req
+ val config = GameApiV2.ByIdsConfig(
+ ids = req.body.split(',').take(300),
+ format = GameApiV2.Format byRequest req,
+ flags = requestPgnFlags(req, extended = false),
+ perSecond = MaxPerSecond(20)
+ )
+ Ok.chunked(Env.api.gameApiV2.exportByIds(config)).withHeaders(
+ CONTENT_TYPE -> gameContentType(config)
+ ).fuccess
+ }
+ }
+ }
+
  private def WithVs(req: RequestHeader)(f: Option[lila.user.User] => Fu[Result]): Fu[Result] =
  get("vs", req) match {
  case None => f(none)
@@ -113,8 +130,8 @@ object Game extends LilaController {
  private def gameContentType(config: GameApiV2.Config) = config.format match {
  case GameApiV2.Format.PGN => pgnContentType
  case GameApiV2.Format.JSON => config match {
- case _: GameApiV2.ByUserConfig => ndJsonContentType
  case _: GameApiV2.OneConfig => JSON
+ case _ => ndJsonContentType
  }
  }
 

conf/routes

@@ -14,6 +14,7 @@ POST /timeline/unsub/:channel controllers.Timeline.unsub(channel: Strin
 GET /games/search controllers.Search.index(page: Int ?= 1)
 
 # Game export
+POST /games/export/_ids controllers.Game.exportByIds
 GET /games/export/:username controllers.Game.exportByUser(username: String)
 
 # TV
@@ -493,7 +494,6 @@ GET /api/user/:name controllers.Api.user(name: String)
 GET /api/user/:name/activity controllers.Api.activity(name: String)
 GET /api/game/:id controllers.Api.game(id: String)
 GET /api/games/team/:teamId controllers.Api.gamesVsTeam(teamId: String)
-POST /api/games controllers.Api.games
 GET /api/tournament controllers.Api.currentTournaments
 GET /api/tournament/:id controllers.Api.tournament(id: String)
 GET /api/status controllers.Api.status

doc/old-api.md

@@ -6,19 +6,6 @@
 
 Parameters and result are similar to the users games API.
 
-### `POST /api/games` fetch many games by ID
-
-```
-> curl --data "x2kpaixn,gtSLJGOK" 'https://lichess.org/api/games'
-```
-
-Games are returned in the order same order as the ids.
-All parameters are optional.
-
-name | type | default | description
---- | --- | --- | ---
-**with_moves** | 1 or 0 | 0 | include a list of PGN moves
-
 ### `GET /api/tournament/<tournamentId>` fetch one tournament
 
 Returns tournament info, and a page of the tournament standing

modules/api/src/main/GameApi.scala

@@ -75,11 +75,6 @@ private[api] final class GameApi(
  }
  }
 
- def many(ids: Seq[String], withMoves: Boolean): Fu[Seq[JsObject]] =
- GameRepo gamesFromPrimary ids flatMap {
- gamesJson(WithFlags(moves = withMoves)) _
- }
-
  def byUsersVs(
  users: (User, User),
  rated: Option[Boolean],

modules/api/src/main/GameApiV2.scala

@@ -3,12 +3,14 @@ package lila.api
 import org.joda.time.DateTime
 import play.api.libs.iteratee._
 import play.api.libs.json._
+import reactivemongo.play.iteratees.cursorProducer
 import scala.concurrent.duration._
 
 import chess.format.FEN
 import chess.format.pgn.Tag
 import lila.analyse.{ AnalysisRepo, JsonView => analysisJson, Analysis }
-import lila.common.{ LightUser, MaxPerSecond }
+import lila.common.{ LightUser, MaxPerSecond, HTTPRequest }
+import lila.db.dsl._
 import lila.game.JsonView._
 import lila.game.PgnDump.WithFlags
 import lila.game.{ Game, GameRepo, Query, PerfPicker }
@@ -47,13 +49,10 @@ final class GameApiV2(
 
  def exportByUser(config: ByUserConfig): Enumerator[String] = {
 
- import reactivemongo.play.iteratees.cursorProducer
- import lila.db.dsl._
-
  val infiniteGames = GameRepo.sortedCursor(
  config.vs.fold(Query.user(config.user.id)) { vs =>
  Query.opponents(config.user, vs)
- } ++ Query.createdBetween(config.since, config.until),
+ } ++ Query.createdBetween(config.since, config.until) ++ Query.finished,
  Query.sortCreated,
  batchSize = config.perSecond.value
  ).bulkEnumerator() &>
@@ -70,21 +69,32 @@ final class GameApiV2(
  }
  }
 
- val formatter = config.format match {
- case Format.PGN => pgnDump.formatter(config.flags)
- case Format.JSON => jsonFormatter(config.flags)
- }
-
- games &> Enumeratee.mapM(enrich(config.flags)) &> formatter
+ games &> Enumeratee.mapM(enrich(config.flags)) &> formatterFor(config)
  }
 
+ def exportByIds(config: ByIdsConfig): Enumerator[String] =
+ GameRepo.sortedCursor(
+ $inIds(config.ids) ++ Query.finished,
+ Query.sortCreated,
+ batchSize = config.perSecond.value
+ ).bulkEnumerator() &>
+ lila.common.Iteratee.delay(1 second) &>
+ Enumeratee.mapConcat(_.toSeq) &>
+ Enumeratee.mapM(enrich(config.flags)) &>
+ formatterFor(config)
+
  private def enrich(flags: WithFlags)(game: Game) =
  GameRepo initialFen game flatMap { initialFen =>
  (flags.evals ?? AnalysisRepo.byGame(game)) map { analysis =>
  (game, initialFen, analysis)
  }
  }
 
+ private def formatterFor(config: Config) = config.format match {
+ case Format.PGN => pgnDump.formatter(config.flags)
+ case Format.JSON => jsonFormatter(config.flags)
+ }
+
  private def jsonFormatter(flags: WithFlags) =
  Enumeratee.mapM[(Game, Option[FEN], Option[Analysis])].apply[String] {
  case (game, initialFen, analysis) => toJson(game, initialFen, analysis, flags) map { json =>
@@ -144,10 +154,12 @@ object GameApiV2 {
  object Format {
  case object PGN extends Format
  case object JSON extends Format
+ def byRequest(req: play.api.mvc.RequestHeader) = if (HTTPRequest acceptsNdJson req) JSON else PGN
  }
 
  sealed trait Config {
  val format: Format
+ val flags: WithFlags
  }
 
  case class OneConfig(
@@ -177,4 +189,11 @@ object GameApiV2 {
  g.player(c).userId has user.id
  } && analysed.fold(true)(g.metadata.analysed ==)
  }
+
+ case class ByIdsConfig(
+ ids: Seq[Game.ID],
+ format: Format,
+ flags: WithFlags,
+ perSecond: MaxPerSecond
+ ) extends Config
 }