From 2a15a3cae660ebbb5d2fc678a57a7fa4790b2cdf Mon Sep 17 00:00:00 2001 From: Vincent Cloutier Date: Wed, 29 Mar 2023 19:03:22 -0400 Subject: [PATCH] twitter cache and auth tweaks --- .builds/arch.yml | 8 ++++++-- src/BirdsiteLive.Common/Settings/InstanceSettings.cs | 2 +- .../StatusPublicationPipeline.cs | 2 +- src/BirdsiteLive.Twitter/CachedTwitterService.cs | 5 +++++ .../Tools/TwitterAuthenticationInitializer.cs | 7 ------- src/BirdsiteLive/Controllers/UsersController.cs | 12 ++++++++++-- 6 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.builds/arch.yml b/.builds/arch.yml index fd6962c..817d9d3 100644 --- a/.builds/arch.yml +++ b/.builds/arch.yml @@ -13,8 +13,12 @@ tasks: sudo docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=birdsitelive -e POSTGRES_USER=birdsitelive -e POSTGRES_DB=birdsitelive postgres:15 cd bird.makeup/src dotnet test - - - publish: | + - publish-arm: | + cd bird.makeup/src/BirdsiteLive + dotnet publish --os linux --arch arm64 /t:PublishContainer -c Release + docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest-arm + docker push cloutier/bird.makeup:latest-arm + - publish-x64: | cd bird.makeup/src/BirdsiteLive dotnet publish --os linux --arch x64 /t:PublishContainer -c Release docker tag cloutier/bird.makeup:1.0 cloutier/bird.makeup:latest diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index 4cec952..93c25a2 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -14,7 +14,7 @@ public int FailingTwitterUserCleanUpThreshold { get; set; } public int FailingFollowerCleanUpThreshold { get; set; } = -1; - public int UserCacheCapacity { get; set; } = 20_000; + public int UserCacheCapacity { get; set; } = 40_000; public int TweetCacheCapacity { get; set; } = 20_000; public int ParallelTwitterRequests { get; set; } = 10; public int ParallelFediversePosts { get; set; } = 10; diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index 47ee70d..cbdd169 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -44,7 +44,7 @@ namespace BirdsiteLive.Pipeline var twitterUserToRefreshBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }); var retrieveTweetsBlock = new TransformBlock(async x => await _retrieveTweetsProcessor.ProcessAsync(x, ct), standardBlockOptions ); - var retrieveTweetsBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }); + var retrieveTweetsBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 2, CancellationToken = ct }); // var retrieveFollowersBlock = new TransformManyBlock(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 } ); // var retrieveFollowersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 500, CancellationToken = ct }); var sendTweetsToFollowersBlock = new ActionBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), standardBlockOptions); diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs index 92ccf52..91f46c9 100644 --- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs +++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs @@ -11,6 +11,7 @@ namespace BirdsiteLive.Twitter { void PurgeUser(string username); void AddUser(TwitterUser user); + bool UserIsCached(string username); } public class CachedTwitterUserService : ICachedTwitterUserService @@ -39,6 +40,10 @@ namespace BirdsiteLive.Twitter } #endregion + public bool UserIsCached(string username) + { + return _userCache.TryGetValue(username, out _); + } public async Task GetUserAsync(string username) { if (!_userCache.TryGetValue(username, out Task user)) diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs index 99cd4f7..6602213 100644 --- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs +++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs @@ -23,7 +23,6 @@ namespace BirdsiteLive.Twitter.Tools { private readonly ILogger _logger; private static bool _initialized; - private static System.Timers.Timer aTimer; private readonly IHttpClientFactory _httpClientFactory; private List _twitterClients = new List(); private List _tokens = new List(); @@ -40,12 +39,6 @@ namespace BirdsiteLive.Twitter.Tools { _logger = logger; _httpClientFactory = httpClientFactory; - - aTimer = new System.Timers.Timer(); - aTimer.Interval = 20 * 1000; - aTimer.Elapsed += async (sender, e) => await RefreshCred(); - - aTimer.Start(); } #endregion diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index d6b0bff..9e3e714 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -25,7 +25,7 @@ namespace BirdsiteLive.Controllers { public class UsersController : Controller { - private readonly ITwitterUserService _twitterUserService; + private readonly ICachedTwitterUserService _twitterUserService; private readonly ICachedTwitterTweetsService _twitterTweetService; private readonly IUserService _userService; private readonly IStatusService _statusService; @@ -33,7 +33,7 @@ namespace BirdsiteLive.Controllers private readonly ILogger _logger; #region Ctor - public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger logger) + public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger logger) { _twitterUserService = twitterUserService; _userService = userService; @@ -75,6 +75,14 @@ namespace BirdsiteLive.Controllers { try { + if (!_twitterUserService.UserIsCached(id)) + { + try + { + await _twitterTweetService.GetTimelineAsync(id); + } + catch (Exception e) { } + } user = await _twitterUserService.GetUserAsync(id); } catch (UserNotFoundException)