From 89ee6cbb8f1bb5913576e56955f0b0e6e5e5b607 Mon Sep 17 00:00:00 2001 From: Thibault Duplessis Date: Tue, 8 May 2018 01:11:37 +0200 Subject: [PATCH] supercharge game export by IDs See https://lichess.org/api#operation/gamesExportIds --- app/controllers/Api.scala | 12 -------- app/controllers/Game.scala | 23 ++++++++++++++-- conf/routes | 2 +- doc/old-api.md | 13 --------- modules/api/src/main/GameApi.scala | 5 ---- modules/api/src/main/GameApiV2.scala | 41 ++++++++++++++++++++-------- 6 files changed, 51 insertions(+), 45 deletions(-) diff --git a/app/controllers/Api.scala b/app/controllers/Api.scala index 41b9bf3d3899..472640cada45 100644 --- a/app/controllers/Api.scala +++ b/app/controllers/Api.scala @@ -176,18 +176,6 @@ object Api extends LilaController { } } - def games = Action.async(parse.tolerantText) { req => - val gameIds = req.body.split(',').take(300) - val ip = HTTPRequest lastRemoteAddress req - GameRateLimitPerIP(ip, cost = gameIds.size / 4) { - lila.mon.api.game.cost(1) - gameApi.many( - ids = gameIds, - withMoves = getBool("with_moves", req) - ) map toApiResult map toHttp - }(Zero.instance(tooManyRequests.fuccess)) - } - private val CrosstableRateLimitPerIP = new lila.memo.RateLimit[IpAddress]( credits = 30, duration = 10 minutes, diff --git a/app/controllers/Game.scala b/app/controllers/Game.scala index ca586d246352..84ff5771ff5f 100644 --- a/app/controllers/Game.scala +++ b/app/controllers/Game.scala @@ -29,7 +29,7 @@ object Game extends LilaController { def exportOne(id: String) = Open { implicit ctx => OptionFuResult(GameRepo game id) { game => - if (game.playable) BadRequest("Can't export PGN of game in progress").fuccess + if (game.playable) BadRequest("Only bots can access their games in progress. See https://lichess.org/api#tag/Chess-Bot").fuccess else { val config = GameApiV2.OneConfig( format = if (HTTPRequest acceptsJson ctx.req) GameApiV2.Format.JSON else GameApiV2.Format.PGN, @@ -60,7 +60,7 @@ object Game extends LilaController { RequireHttp11(req) { Api.GlobalLinearLimitPerIP(HTTPRequest lastRemoteAddress req) { Api.GlobalLinearLimitPerUserOption(me) { - val format = if (HTTPRequest acceptsNdJson req) GameApiV2.Format.JSON else GameApiV2.Format.PGN + val format = GameApiV2.Format byRequest req WithVs(req) { vs => val config = GameApiV2.ByUserConfig( user = user, @@ -92,6 +92,23 @@ object Game extends LilaController { } } + def exportByIds = Action.async(parse.tolerantText) { req => + RequireHttp11(req) { + Api.GlobalLinearLimitPerIP(HTTPRequest lastRemoteAddress req) { + val format = GameApiV2.Format byRequest req + val config = GameApiV2.ByIdsConfig( + ids = req.body.split(',').take(300), + format = GameApiV2.Format byRequest req, + flags = requestPgnFlags(req, extended = false), + perSecond = MaxPerSecond(20) + ) + Ok.chunked(Env.api.gameApiV2.exportByIds(config)).withHeaders( + CONTENT_TYPE -> gameContentType(config) + ).fuccess + } + } + } + private def WithVs(req: RequestHeader)(f: Option[lila.user.User] => Fu[Result]): Fu[Result] = get("vs", req) match { case None => f(none) @@ -113,8 +130,8 @@ object Game extends LilaController { private def gameContentType(config: GameApiV2.Config) = config.format match { case GameApiV2.Format.PGN => pgnContentType case GameApiV2.Format.JSON => config match { - case _: GameApiV2.ByUserConfig => ndJsonContentType case _: GameApiV2.OneConfig => JSON + case _ => ndJsonContentType } } diff --git a/conf/routes b/conf/routes index 3603892c7102..084c03037114 100644 --- a/conf/routes +++ b/conf/routes @@ -14,6 +14,7 @@ POST /timeline/unsub/:channel controllers.Timeline.unsub(channel: Strin GET /games/search controllers.Search.index(page: Int ?= 1) # Game export +POST /games/export/_ids controllers.Game.exportByIds GET /games/export/:username controllers.Game.exportByUser(username: String) # TV @@ -493,7 +494,6 @@ GET /api/user/:name controllers.Api.user(name: String) GET /api/user/:name/activity controllers.Api.activity(name: String) GET /api/game/:id controllers.Api.game(id: String) GET /api/games/team/:teamId controllers.Api.gamesVsTeam(teamId: String) -POST /api/games controllers.Api.games GET /api/tournament controllers.Api.currentTournaments GET /api/tournament/:id controllers.Api.tournament(id: String) GET /api/status controllers.Api.status diff --git a/doc/old-api.md b/doc/old-api.md index 78734072a5f3..f4ef68c883d2 100644 --- a/doc/old-api.md +++ b/doc/old-api.md @@ -6,19 +6,6 @@ Parameters and result are similar to the users games API. -### `POST /api/games` fetch many games by ID - -``` -> curl --data "x2kpaixn,gtSLJGOK" 'https://lichess.org/api/games' -``` - -Games are returned in the order same order as the ids. -All parameters are optional. - -name | type | default | description ---- | --- | --- | --- -**with_moves** | 1 or 0 | 0 | include a list of PGN moves - ### `GET /api/tournament/` fetch one tournament Returns tournament info, and a page of the tournament standing diff --git a/modules/api/src/main/GameApi.scala b/modules/api/src/main/GameApi.scala index aca9a88c0b40..62a69109b81c 100644 --- a/modules/api/src/main/GameApi.scala +++ b/modules/api/src/main/GameApi.scala @@ -75,11 +75,6 @@ private[api] final class GameApi( } } - def many(ids: Seq[String], withMoves: Boolean): Fu[Seq[JsObject]] = - GameRepo gamesFromPrimary ids flatMap { - gamesJson(WithFlags(moves = withMoves)) _ - } - def byUsersVs( users: (User, User), rated: Option[Boolean], diff --git a/modules/api/src/main/GameApiV2.scala b/modules/api/src/main/GameApiV2.scala index 5ba4cc654bda..9733ed1a344c 100644 --- a/modules/api/src/main/GameApiV2.scala +++ b/modules/api/src/main/GameApiV2.scala @@ -3,12 +3,14 @@ package lila.api import org.joda.time.DateTime import play.api.libs.iteratee._ import play.api.libs.json._ +import reactivemongo.play.iteratees.cursorProducer import scala.concurrent.duration._ import chess.format.FEN import chess.format.pgn.Tag import lila.analyse.{ AnalysisRepo, JsonView => analysisJson, Analysis } -import lila.common.{ LightUser, MaxPerSecond } +import lila.common.{ LightUser, MaxPerSecond, HTTPRequest } +import lila.db.dsl._ import lila.game.JsonView._ import lila.game.PgnDump.WithFlags import lila.game.{ Game, GameRepo, Query, PerfPicker } @@ -47,13 +49,10 @@ final class GameApiV2( def exportByUser(config: ByUserConfig): Enumerator[String] = { - import reactivemongo.play.iteratees.cursorProducer - import lila.db.dsl._ - val infiniteGames = GameRepo.sortedCursor( config.vs.fold(Query.user(config.user.id)) { vs => Query.opponents(config.user, vs) - } ++ Query.createdBetween(config.since, config.until), + } ++ Query.createdBetween(config.since, config.until) ++ Query.finished, Query.sortCreated, batchSize = config.perSecond.value ).bulkEnumerator() &> @@ -70,14 +69,20 @@ final class GameApiV2( } } - val formatter = config.format match { - case Format.PGN => pgnDump.formatter(config.flags) - case Format.JSON => jsonFormatter(config.flags) - } - - games &> Enumeratee.mapM(enrich(config.flags)) &> formatter + games &> Enumeratee.mapM(enrich(config.flags)) &> formatterFor(config) } + def exportByIds(config: ByIdsConfig): Enumerator[String] = + GameRepo.sortedCursor( + $inIds(config.ids) ++ Query.finished, + Query.sortCreated, + batchSize = config.perSecond.value + ).bulkEnumerator() &> + lila.common.Iteratee.delay(1 second) &> + Enumeratee.mapConcat(_.toSeq) &> + Enumeratee.mapM(enrich(config.flags)) &> + formatterFor(config) + private def enrich(flags: WithFlags)(game: Game) = GameRepo initialFen game flatMap { initialFen => (flags.evals ?? AnalysisRepo.byGame(game)) map { analysis => @@ -85,6 +90,11 @@ final class GameApiV2( } } + private def formatterFor(config: Config) = config.format match { + case Format.PGN => pgnDump.formatter(config.flags) + case Format.JSON => jsonFormatter(config.flags) + } + private def jsonFormatter(flags: WithFlags) = Enumeratee.mapM[(Game, Option[FEN], Option[Analysis])].apply[String] { case (game, initialFen, analysis) => toJson(game, initialFen, analysis, flags) map { json => @@ -144,10 +154,12 @@ object GameApiV2 { object Format { case object PGN extends Format case object JSON extends Format + def byRequest(req: play.api.mvc.RequestHeader) = if (HTTPRequest acceptsNdJson req) JSON else PGN } sealed trait Config { val format: Format + val flags: WithFlags } case class OneConfig( @@ -177,4 +189,11 @@ object GameApiV2 { g.player(c).userId has user.id } && analysed.fold(true)(g.metadata.analysed ==) } + + case class ByIdsConfig( + ids: Seq[Game.ID], + format: Format, + flags: WithFlags, + perSecond: MaxPerSecond + ) extends Config }