Use GraphQL for list tweets

src/api.nim

@@ -31,7 +31,16 @@ proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timel
  of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
  of TimelineKind.media: (graphUserMedia, Api.userMedia)
  js = await fetch(url ? params, apiId)
- result = parseGraphTimeline(js, after)
+ result = parseGraphTimeline(js, "user", after)
+
+proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
+ if id.len == 0: return
+ let
+ cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+ variables = listTweetsVariables % [id, cursor]
+ params = {"variables": variables, "features": gqlFeatures}
+ js = await fetch(graphListTweets ? params, Api.listTweets)
+ result = parseGraphTimeline(js, "list", after)
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
  let
@@ -60,12 +69,27 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
-proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
+proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
  if id.len == 0: return
  let
- ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
- url = listTimeline ? ps
- result = parseTimeline(await fetch(url, Api.timeline), after)
+ cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+ variables = tweetVariables % [id, cursor]
+ params = {"variables": variables, "features": gqlFeatures}
+ js = await fetch(graphTweet ? params, Api.tweetDetail)
+ result = parseGraphConversation(js, id)
+
+proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
+ result = (await getGraphTweet(id, after)).replies
+ result.beginning = after.len == 0
+
+proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
+ result = await getGraphTweet(id)
+ if after.len > 0:
+ result.replies = await getReplies(id, after)
+
+proc getStatus*(id: string): Future[Tweet] {.async.} =
+ let url = status / (id & ".json") ? genParams()
+ result = parseStatus(await fetch(url, Api.status))
 
 proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
  if name.len == 0: return
@@ -98,28 +122,6 @@ proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
  except InternalError:
  return Result[T](beginning: true, query: query)
 
-proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
- if id.len == 0: return
- let
- cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
- variables = tweetVariables % [id, cursor]
- params = {"variables": variables, "features": gqlFeatures}
- js = await fetch(graphTweet ? params, Api.tweetDetail)
- result = parseGraphConversation(js, id)
-
-proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
- result = (await getGraphTweet(id, after)).replies
- result.beginning = after.len == 0
-
-proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
- result = await getGraphTweet(id)
- if after.len > 0:
- result.replies = await getReplies(id, after)
-
-proc getStatus*(id: string): Future[Tweet] {.async.} =
- let url = status / (id & ".json") ? genParams()
- result = parseStatus(await fetch(url, Api.status))
-
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
  let client = newAsyncHttpClient(maxRedirects=0)
  try:

src/consts.nim

@@ -10,7 +10,6 @@ const
  photoRail* = api / "1.1/statuses/media_timeline.json"
  status* = api / "1.1/statuses/show"
  search* = api / "2/search/adaptive.json"
- listTimeline* = api / "2/timeline/list.json"
 
  graphql = api / "graphql"
  graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets"
@@ -22,6 +21,7 @@ const
  graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
  graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
  graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
+ graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
 
  timelineParams* = {
  "include_profile_interstitial_type": "0",
@@ -100,3 +100,13 @@ const
  "withReactionsPerspective": false,
  "withVoice": false
 }"""
+
+ listTweetsVariables* = """{
+ "listId": "$1", $2
+ "count": 20,
+ "includePromotedContent": false,
+ "withDownvotePerspective": false,
+ "withReactionsMetadata": false,
+ "withReactionsPerspective": false,
+ "withVoice": false
+}"""

src/parser.nim

@@ -455,10 +455,13 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
  elif entryId.startsWith("cursor-bottom"):
  result.replies.bottom = e{"content", "itemContent", "value"}.getStr
 
-proc parseGraphTimeline*(js: JsonNode; after=""): Timeline =
+proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
  result = Timeline(beginning: after.len == 0)
 
- let instructions = ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
+ let instructions =
+ if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"}
+ else: ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
+
  if instructions.len == 0:
  return
 

src/routes/list.nim

@@ -6,7 +6,6 @@ import jester
 import router_utils
 import ".."/[types, redis_cache, api]
 import ../views/[general, timeline, list]
-export getListTimeline, getGraphList
 
 template respList*(list, timeline, title, vnode: typed) =
  if list.id.len == 0 or list.name.len == 0:
@@ -39,7 +38,7 @@ proc createListRouter*(cfg: Config) =
  let
  prefs = cookiePrefs()
  list = await getCachedList(id=(@"id"))
- timeline = await getListTimeline(list.id, getCursor())
+ timeline = await getGraphListTweets(list.id, getCursor())
  vnode = renderTimelineTweets(timeline, prefs, request.path)
  respList(list, timeline, list.title, vnode)
 

src/routes/rss.nim

@@ -159,7 +159,7 @@ proc createRssRouter*(cfg: Config) =
 
  let
  list = await getCachedList(id=id)
- timeline = await getListTimeline(list.id, cursor)
+ timeline = await getGraphListTweets(list.id, cursor)
  rss.cursor = timeline.bottom
  rss.feed = renderListRss(timeline.content, list, cfg)
 

src/tokens.nim

@@ -44,9 +44,9 @@ proc getPoolJson*(): JsonNode =
  of Api.status: 180
  of Api.search: 250
  of Api.timeline: 187
- of Api.listMembers, Api.listBySlug, Api.list, Api.tweetDetail,
+ of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
  Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
- Api.userRestId, Api.userScreenName: 500
+ Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
  reqs = maxReqs - token.apis[api].remaining
 
  reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs

src/types.nim

@@ -20,6 +20,7 @@ type
  list
  listBySlug
  listMembers
+ listTweets
  userRestId
  userScreenName
  userTweets