magic numbers update & some cleanups

sql.md

@@ -29,3 +29,8 @@ SELECT COUNT(*), date_trunc('day', lastsync) FROM (SELECT unnest(followings) as
 SELECT COUNT(*), date_trunc('hour', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc ORDER BY date_trunc;
 ```
 
+# Connections 
+
+```SQL
+SELECT SUM(cardinality(followings)) FROM followers;
+```

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -38,7 +38,7 @@ namespace BirdsiteLive.Pipeline.Processors
                 {
                     var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000);
 
-                    var userCount = users.Any() ? Math.Min(users.Length, 50) : 1;
+                    var userCount = users.Any() ? Math.Min(users.Length, 100) : 1;
                     var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
 
                     foreach (var u in splitUsers)

src/BirdsiteLive.Twitter/CachedTwitterService.cs

@@ -19,7 +19,7 @@ namespace BirdsiteLive.Twitter
 
         private readonly MemoryCache _userCache;
         private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
-            .SetSize(1)//Size amount
+            .SetSize(1000)//Size amount
             //Priority on removing when reaching size limit (memory pressure)
             .SetPriority(CacheItemPriority.Low)
             // Keep in cache for this time, reset time if accessed.

src/BirdsiteLive.Twitter/CachedTwitterTweetsService.cs

@@ -18,7 +18,7 @@ namespace BirdsiteLive.Twitter
 
         private readonly MemoryCache _tweetCache;
         private readonly MemoryCacheEntryOptions _cacheEntryOptions = new MemoryCacheEntryOptions()
-            .SetSize(1)//Size amount
+            .SetSize(1000)//Size amount
             //Priority on removing when reaching size limit (memory pressure)
             .SetPriority(CacheItemPriority.Low)
             // Keep in cache for this time, reset time if accessed.

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -12,7 +12,6 @@ namespace BirdsiteLive.Twitter.Tools
 {
     public interface ITwitterAuthenticationInitializer
     {
-        Task EnsureAuthenticationIsInitialized();
         Task<HttpClient> MakeHttpClient();
         HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint);
     }
@@ -26,6 +25,7 @@ namespace BirdsiteLive.Twitter.Tools
         private List<HttpClient> _twitterClients = new List<HttpClient>();
         private List<String> _tokens = new List<string>();
         static Random rnd = new Random();
+        private const int _targetClients = 10;
         public String BearerToken { 
             get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
         }
@@ -44,12 +44,7 @@ namespace BirdsiteLive.Twitter.Tools
         }
         #endregion
 
-        public async Task EnsureAuthenticationIsInitialized()
+
-        {
-            if (_initialized) return;
-           
-            await InitTwitterCredentials();
-        }
 
         private async Task RefreshCred()
         {
@@ -65,7 +60,7 @@ namespace BirdsiteLive.Twitter.Tools
             _twitterClients.Add(client);
             _tokens.Add(guest);
 
-            if (_twitterClients.Count > 10)
+            if (_twitterClients.Count > _targetClients)
             {
                 _twitterClients.RemoveAt(0);
                 _tokens.RemoveAt(0);
@@ -111,7 +106,7 @@ namespace BirdsiteLive.Twitter.Tools
 
         public async Task<HttpClient> MakeHttpClient()
         {
-            if (_twitterClients.Count < 3)
+            if (_twitterClients.Count < _targetClients)
                 await RefreshCred();
             int r = rnd.Next(_twitterClients.Count);
             return _twitterClients[r];
@@ -123,8 +118,8 @@ namespace BirdsiteLive.Twitter.Tools
             int r = rnd.Next(_twitterClients.Count);
             request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + BearerToken); 
             request.Headers.TryAddWithoutValidation("x-guest-token", _tokens[r]);
-            request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
+            //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
-            request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
+            //request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
             return request;
         }
     }

src/BirdsiteLive/BirdsiteLive.csproj

@@ -5,7 +5,6 @@
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
     <Version>0.20.0</Version>
-    <ThreadPoolMinThreads>12</ThreadPoolMinThreads>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive/Controllers/AboutController.cs

@@ -27,18 +27,6 @@ namespace BirdsiteLive.Controllers
             return View(stats);
         }
 
-        public IActionResult Blacklisting()
-        {
-            var status = GetModerationStatus();
-            return View("Blacklisting", status);
-        }
-
-        public IActionResult Whitelisting()
-        {
-            var status = GetModerationStatus();
-            return View("Whitelisting", status);
-        }
-
         private ModerationStatus GetModerationStatus()
         {
             var status = new ModerationStatus