Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GraphQL timeline #812

Merged
merged 31 commits into from
Apr 21, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
ae03170
Update deps
zedeus Mar 19, 2023
8fc3c3d
Replace profile timeline with GraphQL endpoint
zedeus Mar 21, 2023
061694a
Update GraphQL endpoint versions
zedeus Mar 22, 2023
a5826a3
Use GraphQL for profile media tab
zedeus Mar 22, 2023
482b2da
Fix UserByRestId request
zedeus Mar 22, 2023
91c6d59
Improve routing, fixes #814
zedeus Mar 22, 2023
a496616
Fix token pool JSON
zedeus Mar 22, 2023
1f10c1f
Deduplicate GraphQL timeline endpoints
zedeus Mar 22, 2023
56f1ad4
Update list endpoints
zedeus Mar 22, 2023
4332fae
Use GraphQL for list tweets
zedeus Mar 22, 2023
bed060f
Remove debug leftover
zedeus Mar 23, 2023
5676ecc
Replace old pinned tweet endpoint with GraphQL
zedeus Mar 24, 2023
61d65dc
Validate tweet ID
zedeus Mar 25, 2023
1f9d500
Minor token handling fix
zedeus Mar 25, 2023
ec59942
Hide US-only commerce cards
zedeus Mar 26, 2023
2ed1f63
Update config example
zedeus Mar 27, 2023
b2580ed
Remove http pool and gzip from token pool
zedeus Mar 27, 2023
892356e
Support tombstoned tweets in threads
zedeus Mar 27, 2023
bb6f8a2
Retry GraphQL timeout errors
zedeus Mar 27, 2023
f0e1cb5
Remove unnecessary 401 retry
zedeus Mar 27, 2023
bb221fb
Remove broken timeout retry
zedeus Mar 27, 2023
c8bc02c
Update karax, use new bool attribute feature
zedeus Mar 28, 2023
ef2ecb4
Merge branch 'master' into graphql-timeline
zedeus Mar 28, 2023
9cdd419
Merge branch 'master' into graphql-timeline
zedeus Mar 28, 2023
64741de
Update card test
zedeus Mar 28, 2023
34363a2
Fix odd edgecase with broken retweets
zedeus Apr 1, 2023
5dd85c6
Replace search endpoints, switch Bearer token
zedeus Apr 21, 2023
c8e8ea3
Only parse user search if it's a list
zedeus Apr 21, 2023
10a912d
Fix quoted tweet crash
zedeus Apr 21, 2023
376b444
Fix empty search query handling
zedeus Apr 21, 2023
2f3ee72
Fix invalid user search errors again
zedeus Apr 21, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
Deduplicate GraphQL timeline endpoints
  • Loading branch information
zedeus committed Mar 22, 2023
commit 1f10c1fdffd82eff2a31d4d4dec260504ca75b89
27 changes: 10 additions & 17 deletions src/api.nim
Original file line number Diff line number Diff line change
Expand Up @@ -8,38 +8,31 @@ proc getGraphUser*(username: string): Future[User] {.async.} =
if username.len == 0: return
let
variables = %*{"screen_name": username}
params = {"variables": $variables, "features": userFeatures}
params = {"variables": $variables, "features": gqlFeatures}
js = await fetchRaw(graphUser ? params, Api.userScreenName)
result = parseGraphUser(js)

proc getGraphUserById*(id: string): Future[User] {.async.} =
if id.len == 0 or id.any(c => not c.isDigit): return
let
variables = %*{"userId": id, "withSuperFollowsUserFields": true}
params = {"variables": $variables, "features": tweetFeatures}
variables = %*{"userId": id}
params = {"variables": $variables, "features": gqlFeatures}
js = await fetchRaw(graphUserById ? params, Api.userRestId)
result = parseGraphUser(js)

proc getGraphUserTweets*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = userTweetsVariables % [id, cursor]
params = {"variables": variables, "features": tweetFeatures}
(url, apiId) = if replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
else: (graphUserTweets, Api.userTweets)
params = {"variables": variables, "features": gqlFeatures}
(url, apiId) = case kind
of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
of TimelineKind.media: (graphUserMedia, Api.userMedia)
js = await fetch(url ? params, apiId)
result = parseGraphTimeline(js, after)

proc getGraphUserMedia*(id: string; after=""): Future[Timeline] {.async.} =
if id.len == 0: return
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = userTweetsVariables % [id, cursor]
params = {"variables": variables, "features": tweetFeatures}
js = await fetch(graphUserMedia ? params, Api.userMedia)
result = parseGraphTimeline(js, after)

proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
let
variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
Expand Down Expand Up @@ -112,7 +105,7 @@ proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
let
cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
variables = tweetVariables % [id, cursor]
params = {"variables": variables, "features": tweetFeatures}
params = {"variables": variables, "features": gqlFeatures}
js = await fetch(graphTweet ? params, Api.tweetDetail)
result = parseGraphConversation(js, id)

Expand Down
10 changes: 1 addition & 9 deletions src/consts.nim
Original file line number Diff line number Diff line change
Expand Up @@ -58,15 +58,7 @@ const
## photos: "result_filter: photos"
## videos: "result_filter: videos"

userFeatures* = """{
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
"responsive_web_graphql_exclude_directive_enabled": true,
"responsive_web_graphql_skip_user_profile_image_extensions_enabled": true,
"responsive_web_graphql_timeline_navigation_enabled": false,
"verified_phone_label_enabled": false
}"""

tweetFeatures* = """{
gqlFeatures* = """{
"longform_notetweets_consumption_enabled": true,
"longform_notetweets_richtext_consumption_enabled": true,
"responsive_web_twitter_blue_verified_badge_is_enabled": true,
Expand Down
6 changes: 3 additions & 3 deletions src/routes/timeline.nim
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
let
timeline =
case query.kind
of posts: getGraphUserTweets(userId, after)
of replies: getGraphUserTweets(userId, after, replies=true)
of media: getGraphUserMedia(userId, after)
of posts: getGraphUserTweets(userId, TimelineKind.tweets, after)
of replies: getGraphUserTweets(userId, TimelineKind.replies, after)
of media: getGraphUserTweets(userId, TimelineKind.media, after)
else: getSearch[Tweet](query, after)

rail =
Expand Down
5 changes: 5 additions & 0 deletions src/types.nim
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ type
RateLimitError* = object of CatchableError
InternalError* = object of CatchableError

TimelineKind* {.pure.} = enum
tweets
replies
media

Api* {.pure.} = enum
tweetDetail
timeline
Expand Down