diff --git a/sql.md b/sql.md index b5965e5..352a078 100644 --- a/sql.md +++ b/sql.md @@ -29,3 +29,8 @@ SELECT COUNT(*), date_trunc('day', lastsync) FROM (SELECT unnest(followings) as SELECT COUNT(*), date_trunc('hour', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc ORDER BY date_trunc; ``` +# Connections + +```SQL +SELECT SUM(cardinality(followings)) FROM followers; +``` diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index a4f23cb..3b78954 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -38,7 +38,7 @@ namespace BirdsiteLive.Pipeline.Processors { var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000); - var userCount = users.Any() ? Math.Min(users.Length, 50) : 1; + var userCount = users.Any() ? Math.Min(users.Length, 100) : 1; var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList(); foreach (var u in splitUsers) diff --git a/src/BirdsiteLive.Twitter/CachedTwitterService.cs b/src/BirdsiteLive.Twitter/CachedTwitterService.cs index 65f1b71..7d7ce37 100644 --- a/src/BirdsiteLive.Twitter/CachedTwitterService.cs +++ b/src/BirdsiteLive.Twitter/CachedTwitterService.cs @@ -19,7 +19,7 @@ namespace BirdsiteLive.Twitter private readonly MemoryCache _userCache; private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() - .SetSize(1)//Size amount + .SetSize(1000)//Size amount //Priority on removing when reaching size limit (memory pressure) .SetPriority(CacheItemPriority.Low) // Keep in cache for this time, reset time if accessed. diff --git a/src/BirdsiteLive.Twitter/CachedTwitterTweetsService.cs b/src/BirdsiteLive.Twitter/CachedTwitterTweetsService.cs index 89e3f8e..5eb01a3 100644 --- a/src/BirdsiteLive.Twitter/CachedTwitterTweetsService.cs +++ b/src/BirdsiteLive.Twitter/CachedTwitterTweetsService.cs @@ -18,7 +18,7 @@ namespace BirdsiteLive.Twitter private readonly MemoryCache _tweetCache; private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions() - .SetSize(1)//Size amount + .SetSize(1000)//Size amount //Priority on removing when reaching size limit (memory pressure) .SetPriority(CacheItemPriority.Low) // Keep in cache for this time, reset time if accessed. diff --git a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs index f65c708..8f1db5b 100644 --- a/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs +++ b/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs @@ -12,7 +12,6 @@ namespace BirdsiteLive.Twitter.Tools { public interface ITwitterAuthenticationInitializer { - Task EnsureAuthenticationIsInitialized(); Task MakeHttpClient(); HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint); } @@ -26,6 +25,7 @@ namespace BirdsiteLive.Twitter.Tools private List _twitterClients = new List(); private List _tokens = new List(); static Random rnd = new Random(); + private const int _targetClients = 10; public String BearerToken { get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; } } @@ -44,12 +44,7 @@ namespace BirdsiteLive.Twitter.Tools } #endregion - public async Task EnsureAuthenticationIsInitialized() - { - if (_initialized) return; - - await InitTwitterCredentials(); - } + private async Task RefreshCred() { @@ -65,7 +60,7 @@ namespace BirdsiteLive.Twitter.Tools _twitterClients.Add(client); _tokens.Add(guest); - if (_twitterClients.Count > 10) + if (_twitterClients.Count > _targetClients) { _twitterClients.RemoveAt(0); _tokens.RemoveAt(0); @@ -111,7 +106,7 @@ namespace BirdsiteLive.Twitter.Tools public async Task MakeHttpClient() { - if (_twitterClients.Count < 3) + if (_twitterClients.Count < _targetClients) await RefreshCred(); int r = rnd.Next(_twitterClients.Count); return _twitterClients[r]; @@ -123,8 +118,8 @@ namespace BirdsiteLive.Twitter.Tools int r = rnd.Next(_twitterClients.Count); request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken); request.Headers.TryAddWithoutValidation("x-guest-token", _tokens[r]); - request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/"); - request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes"); + //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/"); + //request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes"); return request; } } diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 3fb2e68..3614a51 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -5,7 +5,6 @@ d21486de-a812-47eb-a419-05682bb68856 Linux 0.20.0 - 12 diff --git a/src/BirdsiteLive/Controllers/AboutController.cs b/src/BirdsiteLive/Controllers/AboutController.cs index e64a147..f8ed9b8 100644 --- a/src/BirdsiteLive/Controllers/AboutController.cs +++ b/src/BirdsiteLive/Controllers/AboutController.cs @@ -27,18 +27,6 @@ namespace BirdsiteLive.Controllers return View(stats); } - public IActionResult Blacklisting() - { - var status = GetModerationStatus(); - return View("Blacklisting", status); - } - - public IActionResult Whitelisting() - { - var status = GetModerationStatus(); - return View("Whitelisting", status); - } - private ModerationStatus GetModerationStatus() { var status = new ModerationStatus