Replace profile timeline with GraphQL endpoint

src/api.nim

@@ -7,12 +7,9 @@ import experimental/parser as newParser
 proc getGraphUser*(username: string): Future[User] {.async.} =
  if username.len == 0: return
  let
- variables = """{
- "screen_name": "$1",
- "withSafetyModeUserFields": false,
- "withSuperFollowsUserFields": false
- }""" % [username]
- js = await fetchRaw(graphUser ? {"variables": variables}, Api.userScreenName)
+ variables = %*{"screen_name": username}
+ params = {"variables": $variables, "features": userFeatures}
+ js = await fetchRaw(graphUser ? params, Api.userScreenName)
  result = parseGraphUser(js)
 
 proc getGraphUserById*(id: string): Future[User] {.async.} =
@@ -22,6 +19,16 @@ proc getGraphUserById*(id: string): Future[User] {.async.} =
  js = await fetchRaw(graphUserById ? {"variables": variables}, Api.userRestId)
  result = parseGraphUser(js)
 
+proc getGraphUserTweets*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
+ if id.len == 0: return
+ let
+ cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+ variables = userTweetsVariables % [id, cursor]
+ params = {"variables": variables, "features": userTweetsFeatures}
+ url = if replies: graphUserTweetsAndReplies else: graphUserTweets
+ js = await fetch(url ? params, Api.tweetDetail)
+ result = parseGraphTimeline(js, after)
+
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
  let
  variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}

src/consts.nim

@@ -19,8 +19,10 @@ const
  tweet* = timelineApi / "conversation"
 
  graphql = api / "graphql"
+ graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets"
+ graphUserTweetsAndReplies* = graphql / "ehMCHF3Mkgjsfz_aImqOsg/UserTweetsAndReplies"
  graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
- graphUser* = graphql / "7mjxD3-C6BxitPMVQ6w0-Q/UserByScreenName"
+ graphUser* = graphql / "nZjSkpOpSL5rWyIVdsKeLA/UserByScreenName"
  graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
@@ -35,6 +37,7 @@ const
  "include_mute_edge": "0",
  "include_can_dm": "0",
  "include_can_media_tag": "1",
+ "include_ext_is_blue_verified": "true",
  "skip_status": "1",
  "cards_platform": "Web-12",
  "include_cards": "1",
@@ -60,6 +63,40 @@ const
  ## photos: "result_filter: photos"
  ## videos: "result_filter: videos"
 
+ userTweetsVariables* = """{
+ "userId": "$1",
+ $2
+ "count": 20,
+ "includePromotedContent": false,
+ "withDownvotePerspective": false,
+ "withReactionsMetadata": false,
+ "withReactionsPerspective": false,
+ "withVoice": false,
+ "withV2Timeline": true
+}"""
+
+ userTweetsFeatures* = """{
+ "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+ "responsive_web_graphql_exclude_directive_enabled": false,
+ "verified_phone_label_enabled": false,
+ "responsive_web_graphql_timeline_navigation_enabled": false,
+ "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+ "tweetypie_unmention_optimization_enabled": false,
+ "vibe_api_enabled": false,
+ "responsive_web_edit_tweet_api_enabled": false,
+ "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+ "view_counts_everywhere_api_enabled": false,
+ "longform_notetweets_consumption_enabled": true,
+ "tweet_awards_web_tipping_enabled": false,
+ "freedom_of_speech_not_reach_fetch_enabled": false,
+ "standardized_nudges_misinfo": false,
+ "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+ "interactive_text_enabled": false,
+ "responsive_web_text_conversations_enabled": false,
+ "longform_notetweets_richtext_consumption_enabled": false,
+ "responsive_web_enhance_cards_enabled": false
+}"""
+
  tweetVariables* = """{
  "focalTweetId": "$1",
  $2
@@ -79,7 +116,7 @@ const
  "responsive_web_graphql_timeline_navigation_enabled": false,
  "standardized_nudges_misinfo": false,
  "verified_phone_label_enabled": false,
- "responsive_web_twitter_blue_verified_badge_is_enabled": false,
+ "responsive_web_twitter_blue_verified_badge_is_enabled": true,
  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
  "view_counts_everywhere_api_enabled": false,
  "responsive_web_edit_tweet_api_enabled": false,
@@ -90,3 +127,11 @@ const
  "responsive_web_enhance_cards_enabled": false,
  "interactive_text_enabled": false
 }"""
+
+ userFeatures* = """{
+ "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+ "verified_phone_label_enabled": false,
+ "responsive_web_graphql_timeline_navigation_enabled": false,
+ "responsive_web_graphql_exclude_directive_enabled": true,
+ "responsive_web_graphql_skip_user_profile_image_extensions_enabled": true
+}"""

src/experimental/parser/graphql.nim

@@ -11,6 +11,7 @@ proc parseGraphUser*(json: string): User =
 
  result = toUser raw.data.user.result.legacy
  result.id = raw.data.user.result.restId
+ result.verified = result.verified or raw.data.user.result.isBlueVerified
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
  result = Result[User](

src/experimental/types/graphuser.nim

@@ -11,4 +11,5 @@ type
  UserResult = object
  legacy*: RawUser
  restId*: string
+ isBlueVerified*: bool
  reason*: Option[string]

src/parser.nim

@@ -4,6 +4,8 @@ import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
+proc parseGraphTweet(js: JsonNode): Tweet
+
 proc parseUser(js: JsonNode; id=""): User =
  if js.isNull: return
  result = User(
@@ -19,13 +21,20 @@ proc parseUser(js: JsonNode; id=""): User =
  tweets: js{"statuses_count"}.getInt,
  likes: js{"favourites_count"}.getInt,
  media: js{"media_count"}.getInt,
- verified: js{"verified"}.getBool,
+ verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
  protected: js{"protected"}.getBool,
  joinDate: js{"created_at"}.getTime
  )
 
  result.expandUserEntities(js)
 
+proc parseGraphUser(js: JsonNode): User =
+ let user = ? js{"user_results", "result"}
+ result = parseUser(user{"legacy"})
+
+ if "is_blue_verified" in user:
+ result.verified = true
+
 proc parseGraphList*(js: JsonNode): List =
  if js.isNull: return
 
@@ -213,10 +222,16 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
  if js{"is_quote_status"}.getBool:
  result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
 
+ # legacy
  with rt, js{"retweeted_status_id_str"}:
  result.retweet = some Tweet(id: rt.getId)
  return
 
+ # graphql
+ with rt, js{"retweeted_status_result", "result"}:
+ result.retweet = some parseGraphTweet(rt)
+ return
+
  if jsCard.kind != JNull:
  let name = jsCard{"name"}.getStr
  if "poll" in name:
@@ -237,7 +252,10 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
  of "video":
  result.video = some(parseVideo(m))
  with user, m{"additional_media_info", "source_user"}:
- result.attribution = some(parseUser(user))
+ if user{"id"}.getInt > 0:
+ result.attribution = some(parseUser(user))
+ else:
+ result.attribution = some(parseGraphUser(user))
  of "animated_gif":
  result.gif = some(parseGif(m))
  else: discard
@@ -384,7 +402,7 @@ proc parseGraphTweet(js: JsonNode): Tweet =
  jsCard["binding_values"] = values
 
  result = parseTweet(js{"legacy"}, jsCard)
- result.user = parseUser(js{"core", "user_results", "result", "legacy"})
+ result.user = parseGraphUser(js{"core"})
 
  with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
  result.expandNoteTweetEntities(noteTweet)
@@ -435,3 +453,22 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
  result.replies.content.add thread
  elif entryId.startsWith("cursor-bottom"):
  result.replies.bottom = e{"content", "itemContent", "value"}.getStr
+
+proc parseGraphTimeline*(js: JsonNode; after=""): Timeline =
+ result = Timeline(beginning: after.len == 0)
+
+ let instructions = ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
+ if instructions.len == 0:
+ return
+
+ for e in instructions[instructions.len - 1]{"entries"}:
+ let entryId = e{"entryId"}.getStr
+ if entryId.startsWith("tweet"):
+ let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
+ if not tweet.available:
+ tweet.id = parseBiggestInt(entryId.getId())
+ result.content.add tweet
+ elif entryId.startsWith("cursor-bottom"):
+ result.bottom = e{"content", "value"}.getStr
+ elif "cursor-top" notin entryId:
+ echo e

src/routes/timeline.nim

@@ -47,8 +47,8 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
  let
  timeline =
  case query.kind
- of posts: getTimeline(userId, after)
- of replies: getTimeline(userId, after, replies=true)
+ of posts: getGraphUserTweets(userId, after)
+ of replies: getGraphUserTweets(userId, after, replies=true)
  of media: getMediaTimeline(userId, after)
  else: getSearch[Tweet](query, after)
 
@@ -64,6 +64,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
  let tweet = await getCachedTweet(user.pinnedTweet)
  if not tweet.isNil:
  tweet.pinned = true
+ tweet.user = user
  pinned = some tweet
 
  result = Profile(

src/tokens.nim

@@ -42,7 +42,7 @@ proc getPoolJson*(): JsonNode =
  maxReqs =
  case api
  of Api.listMembers, Api.listBySlug, Api.list,
- Api.userRestId, Api.userScreenName, Api.tweetDetail: 500
+ Api.userRestId, Api.userScreenName, Api.userTweets, Api.tweetDetail: 500
  of Api.timeline: 187
  else: 180
  reqs = maxReqs - token.apis[api].remaining

src/types.nim

@@ -19,6 +19,7 @@ type
  listMembers
  userRestId
  userScreenName
+ userTweets
  status
 
  RateLimit* = object

tests/test_profile.py

@@ -17,11 +17,6 @@
 
 invalid = [['thisprofiledoesntexist'], ['%']]
 
-banner_color = [
- ['nim_lang', '22, 25, 32'],
- ['rustlang', '35, 31, 32']
-]
-
 banner_image = [
  ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
 ]
@@ -74,12 +69,6 @@ def test_suspended(self):
  self.open_nitter('user')
  self.assert_text('User "user" has been suspended')
 
- @parameterized.expand(banner_color)
- def test_banner_color(self, username, color):
- self.open_nitter(username)
- banner = self.find_element(Profile.banner + ' a')
- self.assertIn(color, banner.value_of_css_property('background-color'))
-
  @parameterized.expand(banner_image)
  def test_banner_image(self, username, url):
  self.open_nitter(username)