Replace search endpoints, switch Bearer token

src/api.nim

@@ -69,6 +69,24 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
+proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} =
+ let q = genQueryParam(query)
+ if q.len == 0 or q == emptyQuery: return
+ var
+ variables = %*{
+ "rawQuery": q,
+ "count": 20,
+ "product": "Latest",
+ "withDownvotePerspective": false,
+ "withReactionsMetadata": false,
+ "withReactionsPerspective": false
+ }
+ if after.len > 0:
+ variables["cursor"] = % after
+ let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+ result = parseGraphSearch(await fetch(url, Api.search), after)
+ result.query = query
+
 proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
  if id.len == 0: return
  let
@@ -99,32 +117,24 @@ proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
  if name.len == 0: return
  let
  ps = genParams({"screen_name": name, "trim_user": "true"},
- count="18", ext=false)
+  count="18", ext=false)
  url = photoRail ? ps
  result = parsePhotoRail(await fetch(url, Api.timeline))
 
-proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
- when T is User:
- const
- searchMode = ("result_filter", "user")
- parse = parseUsers
- fetchFunc = fetchRaw
- else:
- const
- searchMode = ("tweet_search_mode", "live")
- parse = parseTimeline
- fetchFunc = fetch
-
- let q = genQueryParam(query)
- if q.len == 0 or q == emptyQuery:
- return Result[T](beginning: true, query: query)
-
- let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
- try:
- result = parse(await fetchFunc(url, Api.search), after)
- result.query = query
- except InternalError:
- return Result[T](beginning: true, query: query)
+proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
+ var url = userSearch ? {
+ "q": query.text,
+ "skip_status": "1",
+ "count": "20",
+ "page": page
+ }
+
+ result = parseUsers(await fetchRaw(url, Api.userSearch))
+ result.query = query
+ if page.len == 0:
+ result.bottom = "2"
+ elif page.allCharsInSet(Digits):
+ result.bottom = $(parseInt(page) + 1)
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
  let client = newAsyncHttpClient(maxRedirects=0)

src/apiutils.nim

@@ -17,8 +17,8 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
  result &= p
  if ext:
  result &= ("ext", "mediaStats")
- result &= ("include_ext_alt_text", "true")
- result &= ("include_ext_media_availability", "true")
+ result &= ("include_ext_alt_text", "1")
+ result &= ("include_ext_media_availability", "1")
  if count.len > 0:
  result &= ("count", count)
  if cursor.len > 0:

src/consts.nim

@@ -1,15 +1,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils
+import uri, sequtils, strutils
 
 const
- auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+ auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 
  api = parseUri("https://api.twitter.com")
  activate* = $(api / "1.1/guest/activate.json")
 
  photoRail* = api / "1.1/statuses/media_timeline.json"
- status* = api / "1.1/statuses/show"
- search* = api / "2/search/adaptive.json"
+ userSearch* = api / "1.1/users/search.json"
 
  graphql = api / "graphql"
  graphUser* = graphql / "8mPfHBetXOg-EHAyeVxUoA/UserByScreenName"
@@ -19,6 +18,7 @@ const
  graphUserMedia* = graphql / "MA_EP2a21zpzNWKRkaPBMg/UserMedia"
  graphTweet* = graphql / "6I7Hm635Q6ftv69L8VrSeQ/TweetDetail"
  graphTweetResult* = graphql / "rt-rHeSJ-2H9O9gxWQcPcg/TweetResultByRestId"
+ graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
  graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
  graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
  graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
@@ -33,53 +33,46 @@ const
  "include_mute_edge": "0",
  "include_can_dm": "0",
  "include_can_media_tag": "1",
- "include_ext_is_blue_verified": "true",
+ "include_ext_is_blue_verified": "1",
  "skip_status": "1",
  "cards_platform": "Web-12",
  "include_cards": "1",
- "include_composer_source": "false",
+ "include_composer_source": "0",
  "include_reply_count": "1",
  "tweet_mode": "extended",
- "include_entities": "true",
- "include_user_entities": "true",
- "include_ext_media_color": "false",
- "send_error_codes": "true",
- "simple_quoted_tweet": "true",
- "include_quote_count": "true"
+ "include_entities": "1",
+ "include_user_entities": "1",
+ "include_ext_media_color": "0",
+ "send_error_codes": "1",
+ "simple_quoted_tweet": "1",
+ "include_quote_count": "1"
  }.toSeq
 
- searchParams* = {
- "query_source": "typed_query",
- "pc": "1",
- "spelling_corrections": "1"
- }.toSeq
- ## top: nothing
- ## latest: "tweet_search_mode: live"
- ## user: "result_filter: user"
- ## photos: "result_filter: photos"
- ## videos: "result_filter: videos"
-
  gqlFeatures* = """{
- "longform_notetweets_consumption_enabled": true,
- "longform_notetweets_richtext_consumption_enabled": true,
- "responsive_web_twitter_blue_verified_badge_is_enabled": true,
- "responsive_web_graphql_exclude_directive_enabled": true,
+ "blue_business_profile_image_shape_enabled": false,
  "freedom_of_speech_not_reach_fetch_enabled": false,
  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
  "interactive_text_enabled": false,
+ "longform_notetweets_consumption_enabled": true,
+ "longform_notetweets_richtext_consumption_enabled": true,
+ "longform_notetweets_rich_text_read_enabled": false,
  "responsive_web_edit_tweet_api_enabled": false,
  "responsive_web_enhance_cards_enabled": false,
+ "responsive_web_graphql_exclude_directive_enabled": true,
  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
  "responsive_web_graphql_timeline_navigation_enabled": false,
  "responsive_web_text_conversations_enabled": false,
+ "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+ "spaces_2022_h2_clipping": true,
+ "spaces_2022_h2_spaces_communities": true,
  "standardized_nudges_misinfo": false,
  "tweet_awards_web_tipping_enabled": false,
  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
  "tweetypie_unmention_optimization_enabled": false,
- "view_counts_everywhere_api_enabled": false,
+ "verified_phone_label_enabled": false,
  "vibe_api_enabled": false,
- "verified_phone_label_enabled": false
-}"""
+ "view_counts_everywhere_api_enabled": false
+}""".replace(" ", "").replace("\n", "")
 
  tweetVariables* = """{
  "focalTweetId": "$1",
@@ -109,7 +102,8 @@ const
  "withDownvotePerspective": false,
  "withReactionsMetadata": false,
  "withReactionsPerspective": false,
- "withVoice": false
+ "withVoice": false,
+ "withV2Timeline": true
 }"""
 
  listTweetsVariables* = """{

src/experimental/parser.nim

@@ -1,2 +1,2 @@
-import parser/[user, graphql, timeline]
-export user, graphql, timeline
+import parser/[user, graphql]
+export user, graphql

src/experimental/parser/user.nim

@@ -2,7 +2,7 @@ import std/[algorithm, unicode, re, strutils, strformat, options, nre]
 import jsony
 import utils, slices
 import ../types/user as userType
-from ../../types import User, Error
+from ../../types import Result, User, Error
 
 let
  unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
@@ -76,3 +76,10 @@ proc parseUser*(json: string; username=""): User =
  else: echo "[error - parseUser]: ", error
 
  result = toUser json.fromJson(RawUser)
+
+proc parseUsers*(json: string; after=""): Result[User] =
+ result = Result[User](beginning: after.len == 0)
+
+ let raw = json.fromJson(seq[RawUser])
+ for user in raw:
+ result.content.add user.toUser

src/parser.nim

@@ -359,7 +359,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
  result.top = e.getCursor
  elif "cursor-bottom" in entry:
  result.bottom = e.getCursor
- elif entry.startsWith("sq-C"):
+ elif entry.startsWith("sq-cursor"):
  with cursor, e{"content", "operation", "cursor"}:
  if cursor{"cursorType"}.getStr == "Bottom":
  result.bottom = cursor{"value"}.getStr
@@ -383,6 +383,12 @@ proc parseGraphTweet(js: JsonNode): Tweet =
  if js.kind == JNull or js{"__typename"}.getStr == "TweetUnavailable":
  return Tweet(available: false)
 
+ if js{"__typename"}.getStr == "TweetTombstone":
+ return Tweet(
+ available: false,
+ text: js{"tombstone", "text"}.getTombstone
+ )
+
  var jsCard = copy(js{"card", "legacy"})
  if jsCard.kind != JNull:
  var values = newJObject()
@@ -444,7 +450,7 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
  let tweet = Tweet(
  id: parseBiggestInt(id),
  available: false,
- text: e{"content", "itemContent"}.getTombstone
+ text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
  )
 
  if id == tweetId:
@@ -465,18 +471,44 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
 
  let instructions =
  if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"}
- else: ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
+ else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
 
  if instructions.len == 0:
  return
 
- for e in instructions[instructions.len - 1]{"entries"}:
- let entryId = e{"entryId"}.getStr
- if entryId.startsWith("tweet"):
- with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
- let tweet = parseGraphTweet(tweetResult)
- if not tweet.available:
- tweet.id = parseBiggestInt(entryId.getId())
- result.content.add tweet
- elif entryId.startsWith("cursor-bottom"):
- result.bottom = e{"content", "value"}.getStr
+ for i in instructions:
+ if i{"type"}.getStr == "TimelineAddEntries":
+ for e in i{"entries"}:
+ let entryId = e{"entryId"}.getStr
+ if entryId.startsWith("tweet"):
+ with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+ let tweet = parseGraphTweet(tweetResult)
+ if not tweet.available:
+ tweet.id = parseBiggestInt(entryId.getId())
+ result.content.add tweet
+ elif entryId.startsWith("cursor-bottom"):
+ result.bottom = e{"content", "value"}.getStr
+
+proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
+ result = Timeline(beginning: after.len == 0)
+
+ let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+ if instructions.len == 0:
+ return
+
+ for instruction in instructions:
+ let typ = instruction{"type"}.getStr
+ if typ == "TimelineAddEntries":
+ for e in instructions[0]{"entries"}:
+ let entryId = e{"entryId"}.getStr
+ if entryId.startsWith("tweet"):
+ with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+ let tweet = parseGraphTweet(tweetResult)
+ if not tweet.available:
+ tweet.id = parseBiggestInt(entryId.getId())
+ result.content.add tweet
+ elif entryId.startsWith("cursor-bottom"):
+ result.bottom = e{"content", "value"}.getStr
+ elif typ == "TimelineReplaceEntry":
+ if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
+ result.bottom = instruction{"entry", "content", "value"}.getStr

src/parserutils.nim

@@ -130,12 +130,9 @@ proc getBanner*(js: JsonNode): string =
  return
 
 proc getTombstone*(js: JsonNode): string =
- result = js{"tombstoneInfo", "richText", "text"}.getStr
+ result = js{"text"}.getStr
  result.removeSuffix(" Learn more")
 
- if result.len == 0:
- result = js{"tombstoneInfo", "text"}.getStr
-
 proc getMp4Resolution*(url: string): int =
  # parses the height out of a URL like this one:
  # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4

src/routes/rss.nim

@@ -28,7 +28,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
  var q = query
  q.fromUser = names
  profile = Profile(
- tweets: await getSearch[Tweet](q, after),
+ tweets: await getGraphSearch(q, after),
  # this is kinda dumb
  user: User(
  username: name,
@@ -78,7 +78,7 @@ proc createRssRouter*(cfg: Config) =
  if rss.cursor.len > 0:
  respRss(rss, "Search")
 
- let tweets = await getSearch[Tweet](query, cursor)
+ let tweets = await getGraphSearch(query, cursor)
  rss.cursor = tweets.bottom
  rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
 

src/routes/search.nim

@@ -27,11 +27,11 @@ proc createSearchRouter*(cfg: Config) =
  of users:
  if "," in q:
  redirect("/" & q)
- let users = await getSearch[User](query, getCursor())
+ let users = await getUserSearch(query, getCursor())
  resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
  of tweets:
  let
- tweets = await getSearch[Tweet](query, getCursor())
+ tweets = await getGraphSearch(query, getCursor())
  rss = "/search/rss?" & genQueryUrl(query)
  resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
  request, cfg, prefs, title, rss=rss)