Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GraphQL timeline #812

Merged
merged 31 commits into from
Apr 21, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
ae03170
Update deps
zedeus Mar 19, 2023
8fc3c3d
Replace profile timeline with GraphQL endpoint
zedeus Mar 21, 2023
061694a
Update GraphQL endpoint versions
zedeus Mar 22, 2023
a5826a3
Use GraphQL for profile media tab
zedeus Mar 22, 2023
482b2da
Fix UserByRestId request
zedeus Mar 22, 2023
91c6d59
Improve routing, fixes #814
zedeus Mar 22, 2023
a496616
Fix token pool JSON
zedeus Mar 22, 2023
1f10c1f
Deduplicate GraphQL timeline endpoints
zedeus Mar 22, 2023
56f1ad4
Update list endpoints
zedeus Mar 22, 2023
4332fae
Use GraphQL for list tweets
zedeus Mar 22, 2023
bed060f
Remove debug leftover
zedeus Mar 23, 2023
5676ecc
Replace old pinned tweet endpoint with GraphQL
zedeus Mar 24, 2023
61d65dc
Validate tweet ID
zedeus Mar 25, 2023
1f9d500
Minor token handling fix
zedeus Mar 25, 2023
ec59942
Hide US-only commerce cards
zedeus Mar 26, 2023
2ed1f63
Update config example
zedeus Mar 27, 2023
b2580ed
Remove http pool and gzip from token pool
zedeus Mar 27, 2023
892356e
Support tombstoned tweets in threads
zedeus Mar 27, 2023
bb6f8a2
Retry GraphQL timeout errors
zedeus Mar 27, 2023
f0e1cb5
Remove unnecessary 401 retry
zedeus Mar 27, 2023
bb221fb
Remove broken timeout retry
zedeus Mar 27, 2023
c8bc02c
Update karax, use new bool attribute feature
zedeus Mar 28, 2023
ef2ecb4
Merge branch 'master' into graphql-timeline
zedeus Mar 28, 2023
9cdd419
Merge branch 'master' into graphql-timeline
zedeus Mar 28, 2023
64741de
Update card test
zedeus Mar 28, 2023
34363a2
Fix odd edgecase with broken retweets
zedeus Apr 1, 2023
5dd85c6
Replace search endpoints, switch Bearer token
zedeus Apr 21, 2023
c8e8ea3
Only parse user search if it's a list
zedeus Apr 21, 2023
10a912d
Fix quoted tweet crash
zedeus Apr 21, 2023
376b444
Fix empty search query handling
zedeus Apr 21, 2023
2f3ee72
Fix invalid user search errors again
zedeus Apr 21, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
Update GraphQL endpoint versions
  • Loading branch information
zedeus committed Mar 22, 2023
commit 061694a571223eee95913d08e5fa963be2ee5a8b
19 changes: 4 additions & 15 deletions src/api.nim
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@ proc getGraphUserTweets*(id: string; after=""; replies=false): Future[Timeline]
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = userTweetsVariables % [id, cursor]
params = {"variables": variables, "features": userTweetsFeatures}
url = if replies: graphUserTweetsAndReplies else: graphUserTweets
js = await fetch(url ? params, Api.tweetDetail)
params = {"variables": variables, "features": tweetFeatures}
(url, apiId) = if replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
else: (graphUserTweets, Api.userTweets)
js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, after)

proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
Expand Down Expand Up @@ -65,18 +66,6 @@ proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
url = listTimeline ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)

proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
if id.len == 0: return
let
ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
url = timeline / (id & ".json") ? ps
result = parseTimeline(await fetch(url, Api.timeline), after)

proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
result = parseTimeline(await fetch(url, Api.timeline), after)

proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
if name.len == 0: return
let
Expand Down
93 changes: 34 additions & 59 deletions src/consts.nim
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,20 @@ const
api = parseUri("https://api.twitter.com")
activate* = $(api / "1.1/guest/activate.json")

userShow* = api / "1.1/users/show.json"
photoRail* = api / "1.1/statuses/media_timeline.json"
status* = api / "1.1/statuses/show"
search* = api / "2/search/adaptive.json"

timelineApi = api / "2/timeline"
timeline* = timelineApi / "profile"
mediaTimeline* = timelineApi / "media"
listTimeline* = timelineApi / "list.json"
tweet* = timelineApi / "conversation"

graphql = api / "graphql"
graphUserTweets* = graphql / "9rys0A7w1EyqVd2ME0QCJg/UserTweets"
graphUserTweetsAndReplies* = graphql / "ehMCHF3Mkgjsfz_aImqOsg/UserTweetsAndReplies"
graphTweet* = graphql / "6lWNh96EXDJCXl05SAtn_g/TweetDetail"
graphUser* = graphql / "nZjSkpOpSL5rWyIVdsKeLA/UserByScreenName"
graphUserById* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
graphTweet* = graphql / "6I7Hm635Q6ftv69L8VrSeQ/TweetDetail"
graphUser* = graphql / "8mPfHBetXOg-EHAyeVxUoA/UserByScreenName"
graphUserById* = graphql / "nI8WydSd-X-lQIVo6bdktQ/UserByRestId"
graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
Expand Down Expand Up @@ -63,75 +60,53 @@ const
## photos: "result_filter: photos"
## videos: "result_filter: videos"

userTweetsVariables* = """{
"userId": "$1",
$2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false,
"withV2Timeline": true
userFeatures* = """{
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"responsive_web_graphql_exclude_directive_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": false,
"verified_phone_label_enabled": false
}"""

userTweetsFeatures* = """{
tweetFeatures* = """{
"longform_notetweets_consumption_enabled": true,
"longform_notetweets_richtext_consumption_enabled": true,
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"freedom_of_speech_not_reach_fetch_enabled": false,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
"interactive_text_enabled": false,
"responsive_web_edit_tweet_api_enabled": false,
"responsive_web_enhance_cards_enabled": false,
"responsive_web_graphql_exclude_directive_enabled": false,
"verified_phone_label_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
"tweetypie_unmention_optimization_enabled": false,
"vibe_api_enabled": false,
"responsive_web_edit_tweet_api_enabled": false,
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
"view_counts_everywhere_api_enabled": false,
"longform_notetweets_consumption_enabled": true,
"tweet_awards_web_tipping_enabled": false,
"freedom_of_speech_not_reach_fetch_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"responsive_web_text_conversations_enabled": false,
"standardized_nudges_misinfo": false,
"tweet_awards_web_tipping_enabled": false,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"interactive_text_enabled": false,
"responsive_web_text_conversations_enabled": false,
"longform_notetweets_richtext_consumption_enabled": false,
"responsive_web_enhance_cards_enabled": false
"tweetypie_unmention_optimization_enabled": false,
"view_counts_everywhere_api_enabled": false,
"vibe_api_enabled": false,
"verified_phone_label_enabled": false
}"""

tweetVariables* = """{
"focalTweetId": "$1",
$2
"includePromotedContent": false,
"withBirdwatchNotes": false,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withSuperFollowsTweetFields": false,
"withSuperFollowsUserFields": false,
"withVoice": false,
"withV2Timeline": true
"withVoice": false
}"""

tweetFeatures* = """{
"graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"standardized_nudges_misinfo": false,
"verified_phone_label_enabled": false,
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
"view_counts_everywhere_api_enabled": false,
"responsive_web_edit_tweet_api_enabled": false,
"tweetypie_unmention_optimization_enabled": false,
"vibe_api_enabled": false,
"longform_notetweets_consumption_enabled": true,
"responsive_web_text_conversations_enabled": false,
"responsive_web_enhance_cards_enabled": false,
"interactive_text_enabled": false
}"""

userFeatures* = """{
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"verified_phone_label_enabled": false,
"responsive_web_graphql_timeline_navigation_enabled": false,
"responsive_web_graphql_exclude_directive_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": true
userTweetsVariables* = """{
"userId": "$1", $2
"count": 20,
"includePromotedContent": false,
"withDownvotePerspective": false,
"withReactionsMetadata": false,
"withReactionsPerspective": false,
"withVoice": false
}"""
28 changes: 15 additions & 13 deletions src/parser.nim
Original file line number Diff line number Diff line change
Expand Up @@ -428,23 +428,24 @@ proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
result = Conversation(replies: Result[Chain](beginning: true))

let instructions = ? js{"data", "threaded_conversation_with_injections_v2", "instructions"}
let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"}
if instructions.len == 0:
return

for e in instructions[0]{"entries"}:
let entryId = e{"entryId"}.getStr
# echo entryId
if entryId.startsWith("tweet"):
let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)

if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())

if $tweet.id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
if $tweet.id == tweetId:
result.tweet = tweet
else:
result.before.content.add tweet
elif entryId.startsWith("conversationthread"):
let (thread, self) = parseGraphThread(e)
if self:
Expand All @@ -457,17 +458,18 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
proc parseGraphTimeline*(js: JsonNode; after=""): Timeline =
result = Timeline(beginning: after.len == 0)

let instructions = ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
let instructions = ? js{"data", "user", "result", "timeline", "timeline", "instructions"}
if instructions.len == 0:
return

for e in instructions[instructions.len - 1]{"entries"}:
let entryId = e{"entryId"}.getStr
if entryId.startsWith("tweet"):
let tweet = parseGraphTweet(e{"content", "itemContent", "tweet_results", "result"})
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
let tweet = parseGraphTweet(tweetResult)
if not tweet.available:
tweet.id = parseBiggestInt(entryId.getId())
result.content.add tweet
elif entryId.startsWith("cursor-bottom"):
result.bottom = e{"content", "value"}.getStr
elif "cursor-top" notin entryId:
Expand Down
4 changes: 2 additions & 2 deletions src/types.nim
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@ type

Api* {.pure.} = enum
tweetDetail
userShow
timeline
search
tweet
list
listBySlug
listMembers
userRestId
userScreenName
userTweets
userTweetsAndReplies
userMedia
status

RateLimit* = object
Expand Down