From 612637fdc7f667d2e491117ea84cd2e60b5c85e6 Mon Sep 17 00:00:00 2001 From: Vincent Cloutier Date: Sat, 1 Jul 2023 11:27:30 -0400 Subject: [PATCH] token refresh tweaks --- .../Tools/TwitterAuthenticationInitializer.cs | 26 ++++++++++++++----- .../TwitterTweetsService.cs | 23 +++++++++++----- .../TwitterUserService.cs | 5 ++-- .../BirdsiteLive.Twitter.Tests/TweetTests.cs | 4 ++- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs index 56b3d94..247640d 100644 --- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs +++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs @@ -52,6 +52,10 @@ namespace BirdsiteLive.Twitter.Tools _logger = logger; _instanceSettings = settings; _httpClientFactory = httpClientFactory; + + var concuOpt = new ConcurrencyLimiterOptions(); + concuOpt.PermitLimit = 1; + _rateLimiter = new ConcurrencyLimiter(concuOpt); } #endregion @@ -99,17 +103,26 @@ namespace BirdsiteLive.Twitter.Tools string token; var httpClient = _httpClientFactory.CreateClient(); string bearer = await GenerateBearerToken(); - using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json")) - { - request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); + using RateLimitLease lease = await _rateLimiter.AcquireAsync(permitCount: 1); + using var request = new HttpRequestMessage(new HttpMethod("POST"), + "https://api.twitter.com/1.1/guest/activate.json"); + request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); + //request.Headers.Add("User-Agent", + // "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36"); - var httpResponse = await httpClient.SendAsync(request); + HttpResponseMessage httpResponse; + do + { + httpResponse = await httpClient.SendAsync(request); var c = await httpResponse.Content.ReadAsStringAsync(); + if (httpResponse.StatusCode == HttpStatusCode.TooManyRequests) + await Task.Delay(1000); httpResponse.EnsureSuccessStatusCode(); var doc = JsonDocument.Parse(c); token = doc.RootElement.GetProperty("guest_token").GetString(); - } + + } while (httpResponse.StatusCode != HttpStatusCode.OK); return (bearer, token); @@ -124,11 +137,12 @@ namespace BirdsiteLive.Twitter.Tools public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken) { var request = new HttpRequestMessage(m, endpoint); - //(string bearer, string token) = _tokens[r]; (string token, string bearer) = _token2.MaxBy(x => rnd.Next()); request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/"); request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes"); + //request.Headers.Add("User-Agent", + // "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36"); if (addToken) request.Headers.TryAddWithoutValidation("x-guest-token", token); //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/"); diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs index c67151e..da1932d 100644 --- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs +++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.IO; +using System.Net; using System.Net.Http; using System.Text.Json; using System.Text.Json.Nodes; @@ -60,6 +61,11 @@ namespace BirdsiteLive.Twitter { JsonDocument tweet; var httpResponse = await client.SendAsync(request); + if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) + { + _logger.LogError("Error retrieving tweet {statusId}; refreshing client", statusId); + await _twitterAuthenticationInitializer.RefreshClient(request); + } httpResponse.EnsureSuccessStatusCode(); var c = await httpResponse.Content.ReadAsStringAsync(); tweet = JsonDocument.Parse(c); @@ -102,6 +108,9 @@ namespace BirdsiteLive.Twitter var reqURL = "https://api.twitter.com/graphql/pNl8WjKAvaegIoVH--FuoQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D"; + //reqURL = + // """https://twitter.com/i/api/graphql/rIIwMe1ObkGh_ByBtTCtRQ/UserTweets?variables={"userId":"44196397","count":20,"includePromotedContent":true,"withQuickPromoteEligibilityTweetFields":true,"withVoice":true,"withV2Timeline":true}&features={"rweb_lists_timeline_redesign_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":false,"tweet_awards_web_tipping_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_media_download_video_enabled":false,"responsive_web_enhance_cards_enabled":false}&fieldToggles={"withArticleRichContentState":false}"""; + //reqURL = reqURL.Replace("44196397", userId.ToString()); JsonDocument results; List extractedTweets = new List(); using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true); @@ -109,18 +118,18 @@ namespace BirdsiteLive.Twitter { var httpResponse = await client.SendAsync(request); - httpResponse.EnsureSuccessStatusCode(); var c = await httpResponse.Content.ReadAsStringAsync(); + if (httpResponse.StatusCode == HttpStatusCode.Unauthorized) + { + _logger.LogError("Error retrieving timeline of {Username}; refreshing client", username); + await _twitterAuthenticationInitializer.RefreshClient(request); + return null; + } + httpResponse.EnsureSuccessStatusCode(); results = JsonDocument.Parse(c); _statisticsHandler.CalledTweetApi(); } - catch (HttpRequestException e) - { - _logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username); - await _twitterAuthenticationInitializer.RefreshClient(request); - return null; - } catch (Exception e) { _logger.LogError(e, "Error retrieving timeline ", username); diff --git a/src/BirdsiteLive.Twitter/TwitterUserService.cs b/src/BirdsiteLive.Twitter/TwitterUserService.cs index ea1c3b5..bd13edf 100644 --- a/src/BirdsiteLive.Twitter/TwitterUserService.cs +++ b/src/BirdsiteLive.Twitter/TwitterUserService.cs @@ -23,8 +23,9 @@ namespace BirdsiteLive.Twitter private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer; private readonly ITwitterStatisticsHandler _statisticsHandler; private readonly ILogger _logger; - - private readonly string endpoint = "https://twitter.com/i/api/graphql/4LB4fkCe3RDLDmOEEYtueg/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_twitter_blue_new_verification_copy_is_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D"; + + private readonly string endpoint = + "https://api.twitter.com/graphql/oUZZZ8Oddwxs8Cd3iW3UEA/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D"; #region Ctor public TwitterUserService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ILogger logger) diff --git a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs index e3fb441..64acc8d 100644 --- a/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs +++ b/src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs @@ -16,11 +16,13 @@ namespace BirdsiteLive.ActivityPub.Tests [TestClass] public class TweetTests { - private ITwitterTweetsService _tweetService; + private ITwitterTweetsService _tweetService = null; [TestInitialize] public async Task TestInit() { + if (_tweetService != null) + return; var logger1 = new Mock>(MockBehavior.Strict); var logger2 = new Mock>(MockBehavior.Strict);