token refresh tweaks

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -52,6 +52,10 @@ namespace BirdsiteLive.Twitter.Tools
             _logger = logger;
             _instanceSettings = settings;
             _httpClientFactory = httpClientFactory;
+
+            var concuOpt = new ConcurrencyLimiterOptions();
+            concuOpt.PermitLimit = 1;
+            _rateLimiter = new ConcurrencyLimiter(concuOpt);
         }
         #endregion
 
@@ -99,17 +103,26 @@ namespace BirdsiteLive.Twitter.Tools
             string token;
             var httpClient = _httpClientFactory.CreateClient();
             string bearer = await GenerateBearerToken();
-            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json"))
+            using RateLimitLease lease = await _rateLimiter.AcquireAsync(permitCount: 1);
-            {
+            using var request = new HttpRequestMessage(new HttpMethod("POST"),
-                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
+                "https://api.twitter.com/1.1/guest/activate.json");
+            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
+            //request.Headers.Add("User-Agent",
+            //    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36");
 
-                var httpResponse = await httpClient.SendAsync(request);
+            HttpResponseMessage httpResponse;
+            do
+            {
+                httpResponse = await httpClient.SendAsync(request);
 
                 var c = await httpResponse.Content.ReadAsStringAsync();
+                if (httpResponse.StatusCode == HttpStatusCode.TooManyRequests)
+                    await Task.Delay(1000);
                 httpResponse.EnsureSuccessStatusCode();
                 var doc = JsonDocument.Parse(c);
                 token = doc.RootElement.GetProperty("guest_token").GetString();
-            }
+
+            } while (httpResponse.StatusCode != HttpStatusCode.OK);
 
             return (bearer, token);
 
@@ -124,11 +137,12 @@ namespace BirdsiteLive.Twitter.Tools
         public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
         {
             var request = new HttpRequestMessage(m, endpoint);
-            //(string bearer, string token) = _tokens[r];
             (string token, string bearer) = _token2.MaxBy(x => rnd.Next());
             request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
             request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
             request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
+            //request.Headers.Add("User-Agent",
+            //    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36");
             if (addToken)
                 request.Headers.TryAddWithoutValidation("x-guest-token", token);
             //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");

src/BirdsiteLive.Twitter/TwitterTweetsService.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.IO;
+using System.Net;
 using System.Net.Http;
 using System.Text.Json;
 using System.Text.Json.Nodes;
@@ -60,6 +61,11 @@ namespace BirdsiteLive.Twitter
             {
                 JsonDocument tweet;
                 var httpResponse = await client.SendAsync(request);
+                if (httpResponse.StatusCode == HttpStatusCode.Unauthorized)
+                {
+                    _logger.LogError("Error retrieving tweet {statusId}; refreshing client", statusId);
+                    await _twitterAuthenticationInitializer.RefreshClient(request);
+                }
                 httpResponse.EnsureSuccessStatusCode();
                 var c = await httpResponse.Content.ReadAsStringAsync();
                 tweet = JsonDocument.Parse(c);
@@ -102,6 +108,9 @@ namespace BirdsiteLive.Twitter
             var reqURL =
                 "https://api.twitter.com/graphql/pNl8WjKAvaegIoVH--FuoQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" +
                 userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
+            //reqURL =
+            //    """https://twitter.com/i/api/graphql/rIIwMe1ObkGh_ByBtTCtRQ/UserTweets?variables={"userId":"44196397","count":20,"includePromotedContent":true,"withQuickPromoteEligibilityTweetFields":true,"withVoice":true,"withV2Timeline":true}&features={"rweb_lists_timeline_redesign_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":false,"tweet_awards_web_tipping_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_media_download_video_enabled":false,"responsive_web_enhance_cards_enabled":false}&fieldToggles={"withArticleRichContentState":false}""";
+            //reqURL = reqURL.Replace("44196397", userId.ToString());
             JsonDocument results;
             List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
             using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
@@ -109,18 +118,18 @@ namespace BirdsiteLive.Twitter
             {
 
                 var httpResponse = await client.SendAsync(request);
-                httpResponse.EnsureSuccessStatusCode();
                 var c = await httpResponse.Content.ReadAsStringAsync();
+                if (httpResponse.StatusCode == HttpStatusCode.Unauthorized)
+                {
+                    _logger.LogError("Error retrieving timeline of {Username}; refreshing client", username);
+                    await _twitterAuthenticationInitializer.RefreshClient(request);
+                    return null;
+                }
+                httpResponse.EnsureSuccessStatusCode();
                 results = JsonDocument.Parse(c);
 
                 _statisticsHandler.CalledTweetApi();
             }
-            catch (HttpRequestException e)
-            {
-                _logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username);
-                await _twitterAuthenticationInitializer.RefreshClient(request);
-                return null;
-            }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving timeline ", username);

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -23,8 +23,9 @@ namespace BirdsiteLive.Twitter
         private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
         private readonly ITwitterStatisticsHandler _statisticsHandler;
         private readonly ILogger<TwitterUserService> _logger;
-        
+
-        private readonly string endpoint = "https://twitter.com/i/api/graphql/4LB4fkCe3RDLDmOEEYtueg/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_twitter_blue_new_verification_copy_is_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D";
+        private readonly string endpoint =
+            "https://api.twitter.com/graphql/oUZZZ8Oddwxs8Cd3iW3UEA/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=%7B%22hidden_profile_likes_enabled%22%3Afalse%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D";
 
         #region Ctor
         public TwitterUserService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ILogger<TwitterUserService> logger)

src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs

@@ -16,11 +16,13 @@ namespace BirdsiteLive.ActivityPub.Tests
     [TestClass]
     public class TweetTests
     {
-        private ITwitterTweetsService _tweetService;
+        private ITwitterTweetsService _tweetService = null;
         
         [TestInitialize]
         public async Task TestInit()
         {
+            if (_tweetService != null)
+                return;
             
             var logger1 = new Mock<ILogger<TwitterAuthenticationInitializer>>(MockBehavior.Strict);
             var logger2 = new Mock<ILogger<TwitterUserService>>(MockBehavior.Strict);