twitter cache and auth tweaks

.builds/arch.yml

@@ -13,8 +13,12 @@ tasks:
       sudo docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=birdsitelive -e POSTGRES_USER=birdsitelive -e POSTGRES_DB=birdsitelive postgres:15
       cd bird.makeup/src
       dotnet test
-
+  - publish-arm: |
-  - publish: |
+      cd bird.makeup/src/BirdsiteLive
+      dotnet publish --os linux --arch arm64 /t:PublishContainer -c Release
+      docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest-arm
+      docker push cloutier/bird.makeup:latest-arm
+  - publish-x64: |
       cd bird.makeup/src/BirdsiteLive
       dotnet publish --os linux --arch x64 /t:PublishContainer -c Release
       docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -14,7 +14,7 @@
         public int FailingTwitterUserCleanUpThreshold { get; set; }
         public int FailingFollowerCleanUpThreshold { get; set; } = -1;
 
-        public int UserCacheCapacity { get; set; } = 20_000;
+        public int UserCacheCapacity { get; set; } = 40_000;
         public int TweetCacheCapacity { get; set; } = 20_000;
         public int ParallelTwitterRequests { get; set; } = 10;
         public int ParallelFediversePosts { get; set; } = 10;

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -44,7 +44,7 @@ namespace BirdsiteLive.Pipeline
             var twitterUserToRefreshBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions
                 { BoundedCapacity = 1, CancellationToken = ct });
             var retrieveTweetsBlock = new TransformBlock<UserWithDataToSync[], UserWithDataToSync[]>(async x =>  await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions );
-            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct });
+            var retrieveTweetsBufferBlock = new BufferBlock<UserWithDataToSync[]>(new DataflowBlockOptions { BoundedCapacity = 2, CancellationToken = ct });
            // var retrieveFollowersBlock = new TransformManyBlock<UserWithDataToSync[], UserWithDataToSync>(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } );
            // var retrieveFollowersBufferBlock = new BufferBlock<UserWithDataToSync>(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct });
             var sendTweetsToFollowersBlock = new ActionBlock<UserWithDataToSync[]>(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions);

src/BirdsiteLive.Twitter/CachedTwitterService.cs

@@ -11,6 +11,7 @@ namespace BirdsiteLive.Twitter
     {
         void PurgeUser(string username);
         void AddUser(TwitterUser user);
+        bool UserIsCached(string username);
     }
 
     public class CachedTwitterUserService : ICachedTwitterUserService
@@ -39,6 +40,10 @@ namespace BirdsiteLive.Twitter
         }
         #endregion
 
+        public bool UserIsCached(string username)
+        {
+            return _userCache.TryGetValue(username, out _);
+        }
         public async Task<TwitterUser> GetUserAsync(string username)
         {
             if (!_userCache.TryGetValue(username, out Task<TwitterUser> user))

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -23,7 +23,6 @@ namespace BirdsiteLive.Twitter.Tools
     {
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
-        private static System.Timers.Timer aTimer;
         private readonly IHttpClientFactory _httpClientFactory;
         private List<HttpClient> _twitterClients = new List<HttpClient>();
         private List<String> _tokens = new List<string>();
@@ -40,12 +39,6 @@ namespace BirdsiteLive.Twitter.Tools
         {
             _logger = logger;
             _httpClientFactory = httpClientFactory;
-
-            aTimer = new System.Timers.Timer();
-            aTimer.Interval = 20 * 1000; 
-            aTimer.Elapsed += async (sender, e) => await RefreshCred();
-
-            aTimer.Start();
         }
         #endregion
 

src/BirdsiteLive/Controllers/UsersController.cs

@@ -25,7 +25,7 @@ namespace BirdsiteLive.Controllers
 {
     public class UsersController : Controller
     {
-        private readonly ITwitterUserService _twitterUserService;
+        private readonly ICachedTwitterUserService _twitterUserService;
         private readonly ICachedTwitterTweetsService _twitterTweetService;
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
@@ -33,7 +33,7 @@ namespace BirdsiteLive.Controllers
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
+        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
@@ -75,6 +75,14 @@ namespace BirdsiteLive.Controllers
             {
                 try
                 {
+                    if (!_twitterUserService.UserIsCached(id))
+                    {
+                        try
+                        {
+                            await _twitterTweetService.GetTimelineAsync(id);
+                        }
+                        catch (Exception e) { }
+                    }
                     user = await _twitterUserService.GetUserAsync(id);
                 }
                 catch (UserNotFoundException)