UsersController optimisations

README.md

@@ -8,25 +8,26 @@ Bird.makeup is a way to follow Twitter users from any ActivityPub service. The a
 
 Compared to BirdsiteLive, bird.makeup is:
 
+More scalable:
+ - Twitter API calls are not rate-limited
+ - It is possible to split the Twitter crawling to multiple servers
+ - There are now integration tests for the non-official api
+ - The core pipeline has been tweaked to remove bottlenecks. As of writing this, bird.makeup supports without problems more than 20k users. 
+ - Twitter users with no followers on the fediverse will stop being fetched
+
+More native to the fediverse:
+ - Retweets are propagated as boosts
+ - Activities are now "unlisted" which means that they won't polute the public timeline, but they can still be boosted
+ - WIP support for QT
+
 More modern:
  - Moved from .net core 3.1 to .net 6 which is still supported
  - Moved from postgres 9 to 15
  - Moved from Newtonsoft.Json to System.Text.Json
 
-More scalable:
- - Twitter API calls are not rate-limited
- - There are now integration tests for the non-official api
- - The core pipeline has been tweaked to remove bottlenecks. As of writing this, bird.makeup supports without problems more than 10k users. 
- - Twitter users with no followers on the fediverse will stop being fetched
-
-More native to the fediverse:
- - Retweets are propagated as boosts
- - Activities are now "unlisted" which means that they won't polute the public timeline
- - WIP support for QT
-
 ## Official instance 
 
-You can find an official instance here: [bird.makeup](https://bird.makeup). If you are an instance admin that prefers to not have tweets federated to you, please block the entire instance. 
+You can find the official instance here: [bird.makeup](https://bird.makeup). If you are an instance admin that prefers to not have tweets federated to you, please block the entire instance. 
 
 Please consider if you really need another instance before spinning up a new one, as having multiple domain makes it harder for moderators to block twitter content. 
 

sql.md

@@ -29,6 +29,15 @@ SELECT COUNT(*), date_trunc('day', lastsync) FROM (SELECT unnest(followings) as
 SELECT COUNT(*), date_trunc('hour', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc ORDER BY date_trunc;
 ```
 
+Lag by shards: 
+
+```SQL
+SELECT min(lastsync), mod(id, 100) FROM
+    (SELECT acct, id, lastsync FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id) AS f
+GROUP BY mod
+ORDER BY min;
+```
+
 # Connections 
 
 ```SQL

src/BSLManager/App.cs

@@ -1,252 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Moderation.Actions;
-using BSLManager.Domain;
-using BSLManager.Tools;
-using Terminal.Gui;
-
-namespace BSLManager
-{
-    public class App
-    {
-        private readonly IFollowersDal _followersDal;
-        private readonly IRemoveFollowerAction _removeFollowerAction;
-
-        private readonly FollowersListState _state = new FollowersListState();
-
-        #region Ctor
-        public App(IFollowersDal followersDal, IRemoveFollowerAction removeFollowerAction)
-        {
-            _followersDal = followersDal;
-            _removeFollowerAction = removeFollowerAction;
-        }
-        #endregion
-
-        public void Run()
-        {
-            Application.Init();
-            var top = Application.Top;
-
-            // Creates the top-level window to show
-            var win = new Window("BSL Manager")
-            {
-                X = 0,
-                Y = 1, // Leave one row for the toplevel menu
-
-                // By using Dim.Fill(), it will automatically resize without manual intervention
-                Width = Dim.Fill(),
-                Height = Dim.Fill()
-            };
-
-            top.Add(win);
-
-            // Creates a menubar, the item "New" has a help menu.
-            var menu = new MenuBar(new MenuBarItem[]
-            {
-                new MenuBarItem("_File", new MenuItem[]
-                {
-                    new MenuItem("_Quit", "", () =>
-                    {
-                        if (Quit()) top.Running = false;
-                    })
-                }),
-                //new MenuBarItem ("_Edit", new MenuItem [] {
-                //    new MenuItem ("_Copy", "", null),
-                //    new MenuItem ("C_ut", "", null),
-                //    new MenuItem ("_Paste", "", null)
-                //})
-            });
-            top.Add(menu);
-
-            static bool Quit()
-            {
-                var n = MessageBox.Query(50, 7, "Quit BSL Manager", "Are you sure you want to quit?", "Yes", "No");
-                return n == 0;
-            }
-            
-            RetrieveUserList();
-
-            var list = new ListView(_state.GetDisplayableList())
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = Dim.Fill()
-            };
-
-            list.KeyDown += _ =>
-            {
-                if (_.KeyEvent.Key == Key.Enter)
-                {
-                    OpenFollowerDialog(list.SelectedItem);
-                } 
-                else if (_.KeyEvent.Key == Key.Delete
-                           || _.KeyEvent.Key == Key.DeleteChar
-                           || _.KeyEvent.Key == Key.Backspace
-                           || _.KeyEvent.Key == Key.D)
-                {
-                    OpenDeleteDialog(list.SelectedItem);
-                }
-            };
-
-            var listingFollowersLabel = new Label(1, 0, "Listing followers");
-            var filterLabel = new Label("Filter: ") { X = 1, Y = 1 };
-            var filterText = new TextField("")
-            {
-                X = Pos.Right(filterLabel),
-                Y = 1,
-                Width = 40
-            };
-
-            filterText.KeyDown += _ =>
-            {
-                var text = filterText.Text.ToString();
-                if (_.KeyEvent.Key == Key.Enter && !string.IsNullOrWhiteSpace(text))
-                {
-                    _state.FilterBy(text);
-                    ConsoleGui.RefreshUI();
-                }
-            };
-
-            win.Add(
-                listingFollowersLabel,
-                filterLabel,
-                filterText,
-                list
-            );
-
-            Application.Run();
-        }
-
-        private void OpenFollowerDialog(int selectedIndex)
-        {
-            var close = new Button(3, 14, "Close");
-            close.Clicked += () => Application.RequestStop();
-
-            var dialog = new Dialog("Info", 60, 18, close);
-
-            var follower = _state.GetElementAt(selectedIndex);
-
-            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
-            {
-                X = 1,
-                Y = 1,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var following = new Label($"Following Count: {follower.Followings.Count}")
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var errors = new Label($"Posting Errors: {follower.PostingErrorCount}")
-            {
-                X = 1,
-                Y = 4,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var inbox = new Label($"Inbox: {follower.InboxRoute}")
-            {
-                X = 1,
-                Y = 5,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var sharedInbox = new Label($"Shared Inbox: {follower.SharedInboxRoute}")
-            {
-                X = 1,
-                Y = 6,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-
-            dialog.Add(name);
-            dialog.Add(following);
-            dialog.Add(errors);
-            dialog.Add(inbox);
-            dialog.Add(sharedInbox);
-            dialog.Add(close);
-            Application.Run(dialog);
-        }
-
-        private void OpenDeleteDialog(int selectedIndex)
-        {
-            bool okpressed = false;
-            var ok = new Button(10, 14, "Yes");
-            ok.Clicked += () =>
-            {
-                Application.RequestStop();
-                okpressed = true;
-            };
-
-            var cancel = new Button(3, 14, "No");
-            cancel.Clicked += () => Application.RequestStop();
-
-            var dialog = new Dialog("Delete", 60, 18, cancel, ok);
-
-            var follower = _state.GetElementAt(selectedIndex);
-            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
-            {
-                X = 1,
-                Y = 1,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            var entry = new Label("Delete user and remove all their followings?")
-            {
-                X = 1,
-                Y = 3,
-                Width = Dim.Fill(),
-                Height = 1
-            };
-            dialog.Add(name);
-            dialog.Add(entry);
-            Application.Run(dialog);
-
-            if (okpressed)
-            {
-                DeleteAndRemoveUser(selectedIndex);
-            }
-        }
-
-        private void DeleteAndRemoveUser(int el)
-        {
-            Application.MainLoop.Invoke(async () =>
-            {
-                try
-                {
-                    var userToDelete = _state.GetElementAt(el);
-                    
-                    BasicLogger.Log($"Delete {userToDelete.Acct}@{userToDelete.Host}");
-                    await _removeFollowerAction.ProcessAsync(userToDelete);
-                    BasicLogger.Log($"Remove user from list");
-                    _state.RemoveAt(el);
-                }
-                catch (Exception e)
-                {
-                    BasicLogger.Log(e.Message);
-                }
-
-                ConsoleGui.RefreshUI();
-            });
-        }
-
-        private void RetrieveUserList()
-        {
-            Application.MainLoop.Invoke(async () =>
-            {
-                var followers = await _followersDal.GetAllFollowersAsync();
-                _state.Load(followers.ToList());
-                ConsoleGui.RefreshUI();
-            });
-        }
-    }
-}

src/BSLManager/BSLManager.csproj

@@ -1,28 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net6</TargetFramework>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Lamar" Version="5.0.3" />
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
-    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
-    <ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
-    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <None Update="key.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
-</Project>

src/BSLManager/Bootstrapper.cs

@@ -1,94 +0,0 @@
-using System;
-using System.Net.Http;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.Common.Structs;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Postgres.DataAccessLayers;
-using BirdsiteLive.DAL.Postgres.Settings;
-using Lamar;
-using Lamar.Scanning.Conventions;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-
-namespace BSLManager
-{
-    public class Bootstrapper
-    {
-        private readonly DbSettings _dbSettings;
-        private readonly InstanceSettings _instanceSettings;
-
-        #region Ctor
-        public Bootstrapper(DbSettings dbSettings, InstanceSettings instanceSettings)
-        {
-            _dbSettings = dbSettings;
-            _instanceSettings = instanceSettings;
-        }
-        #endregion
-
-        public Container Init()
-        {
-            var container = new Container(x =>
-            {
-                x.For<DbSettings>().Use(x => _dbSettings);
-
-                x.For<InstanceSettings>().Use(x => _instanceSettings);
-
-                if (string.Equals(_dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
-                {
-                    var connString = $"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
-                    var postgresSettings = new PostgresSettings
-                    {
-                        ConnString = connString
-                    };
-                    x.For<PostgresSettings>().Use(x => postgresSettings);
-
-                    x.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
-                    x.For<IFollowersDal>().Use<FollowersPostgresDal>().Singleton();
-                    x.For<IDbInitializerDal>().Use<DbInitializerPostgresDal>().Singleton();
-                }
-                else
-                {
-                    throw new NotImplementedException($"{_dbSettings.Type} is not supported");
-                }
-
-                var serviceProvider = new ServiceCollection().AddHttpClient().BuildServiceProvider();
-                x.For<IHttpClientFactory>().Use(_ => serviceProvider.GetService<IHttpClientFactory>());
-
-                x.For(typeof(ILogger<>)).Use(typeof(DummyLogger<>));
-
-                x.Scan(_ =>
-                {
-                    _.Assembly("BirdsiteLive.Twitter");
-                    _.Assembly("BirdsiteLive.Domain");
-                    _.Assembly("BirdsiteLive.DAL");
-                    _.Assembly("BirdsiteLive.DAL.Postgres");
-                    _.Assembly("BirdsiteLive.Moderation");
-
-                    _.TheCallingAssembly();
-
-                    _.WithDefaultConventions();
-
-                    _.LookForRegistries();
-                });
-            });
-            return container;
-        }
-
-        public class DummyLogger<T> : ILogger<T>
-        {
-            public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
-            {
-            }
-
-            public bool IsEnabled(LogLevel logLevel)
-            {
-                return false;
-            }
-
-            public IDisposable BeginScope<TState>(TState state)
-            {
-                return null;
-            }
-        }
-    }
-}

src/BSLManager/Domain/FollowersListState.cs

@@ -1,81 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-using BirdsiteLive.DAL.Models;
-
-namespace BSLManager.Domain
-{
-    public class FollowersListState
-    {
-        private readonly List<string> _filteredDisplayableUserList = new List<string>();
-
-        private List<Follower> _sourceUserList = new List<Follower>();
-        private List<Follower> _filteredSourceUserList = new List<Follower>();
-        
-        public void Load(List<Follower> followers)
-        {
-            _sourceUserList = followers.OrderByDescending(x => x.Followings.Count).ToList();
-           
-            ResetLists();
-        }
-
-        private void ResetLists()
-        {
-            _filteredSourceUserList = _sourceUserList.ToList();
-
-            _filteredDisplayableUserList.Clear();
-
-            foreach (var follower in _sourceUserList)
-            {
-                var displayedUser = $"{GetFullHandle(follower)}   ({follower.Followings.Count})  (err:{follower.PostingErrorCount})";
-                _filteredDisplayableUserList.Add(displayedUser);
-            }
-        }
-
-        public List<string> GetDisplayableList()
-        {
-            return _filteredDisplayableUserList;
-        }
-
-        public void FilterBy(string pattern)
-        {
-            ResetLists();
-
-            if (!string.IsNullOrWhiteSpace(pattern))
-            {
-                var elToRemove = _filteredSourceUserList
-                    .Where(x => !GetFullHandle(x).Contains(pattern))
-                    .Select(x => x)
-                    .ToList();
-
-                foreach (var el in elToRemove)
-                {
-                    _filteredSourceUserList.Remove(el);
-                    
-                    var dElToRemove = _filteredDisplayableUserList.First(x => x.Contains(GetFullHandle(el)));
-                    _filteredDisplayableUserList.Remove(dElToRemove);
-                }
-            }
-        }
-
-        private string GetFullHandle(Follower follower)
-        {
-            return $"@{follower.Acct}@{follower.Host}";
-        }
-
-        public void RemoveAt(int index)
-        {
-            var displayableUser = _filteredDisplayableUserList[index];
-            var sourceUser = _filteredSourceUserList[index];
-
-            _filteredDisplayableUserList.Remove(displayableUser);
-
-            _filteredSourceUserList.Remove(sourceUser);
-            _sourceUserList.Remove(sourceUser);
-        }
-
-        public Follower GetElementAt(int index)
-        {
-            return _filteredSourceUserList[index];
-        }
-    }
-}

src/BSLManager/Program.cs

@@ -1,39 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection;
-using System.Text;
-using System.Threading.Tasks;
-using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-using BSLManager.Tools;
-using Microsoft.Extensions.Configuration;
-using NStack;
-using Terminal.Gui;
-
-namespace BSLManager
-{
-    class Program
-    {
-        static async Task Main(string[] args)
-        {
-            Console.OutputEncoding = Encoding.Default;
-
-            var settingsManager = new SettingsManager();
-            var settings = settingsManager.GetSettings();
-
-            //var builder = new ConfigurationBuilder()
-            //    .AddEnvironmentVariables();
-            //var configuration = builder.Build();
-
-            //var dbSettings = configuration.GetSection("Db").Get<DbSettings>();
-            //var instanceSettings = configuration.GetSection("Instance").Get<InstanceSettings>();
-
-            var bootstrapper = new Bootstrapper(settings.dbSettings, settings.instanceSettings);
-            var container = bootstrapper.Init();
-
-            var app = container.GetInstance<App>();
-            app.Run();
-        }
-    }
-}

src/BSLManager/Tools/BasicLogger.cs

@@ -1,13 +0,0 @@
-using System;
-using System.IO;
-
-namespace BSLManager.Tools
-{
-    public static class BasicLogger
-    {
-        public static void Log(string log)
-        {
-            File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new []{ log });
-        }
-    }
-}

src/BSLManager/Tools/ConsoleGui.cs

@@ -1,15 +0,0 @@
-using System.Reflection;
-using Terminal.Gui;
-
-namespace BSLManager.Tools
-{
-    public static class ConsoleGui
-    {
-        public static void RefreshUI()
-        {
-            typeof(Application)
-                .GetMethod("TerminalResized", BindingFlags.Static | BindingFlags.NonPublic)
-                .Invoke(null, null);
-        }
-    }
-}

src/BSLManager/Tools/SettingsManager.cs

@@ -1,123 +0,0 @@
-using System;
-using System.IO;
-using System.Text.Json;
-using System.Runtime.CompilerServices;
-using BirdsiteLive.Common.Settings;
-
-namespace BSLManager.Tools
-{
-    public class SettingsManager
-    {
-        private const string LocalFileName = "ManagerSettings.json";
-
-        public (DbSettings dbSettings, InstanceSettings instanceSettings) GetSettings()
-        {
-            var localSettingsData = GetLocalSettingsFile();
-            if (localSettingsData != null) return Convert(localSettingsData);
-
-            Console.WriteLine("We need to set up the manager");
-            Console.WriteLine("Please provide the following information as provided in the docker-compose file");
-
-            LocalSettingsData data;
-            do
-            {
-                data = GetDataFromUser();
-                Console.WriteLine();
-                Console.WriteLine("Please check if all is ok:");
-                Console.WriteLine();
-                Console.WriteLine($"Db Host: {data.DbHost}");
-                Console.WriteLine($"Db Name: {data.DbName}");
-                Console.WriteLine($"Db User: {data.DbUser}");
-                Console.WriteLine($"Db Password: {data.DbPassword}");
-                Console.WriteLine($"Instance Domain: {data.InstanceDomain}");
-                Console.WriteLine();
-
-                string resp;
-                do
-                {
-                    Console.WriteLine("Is it valid? (yes, no)");
-                    resp = Console.ReadLine()?.Trim().ToLowerInvariant();
-
-                    if (resp == "n" || resp == "no") data = null;
-
-                } while (resp != "y" && resp != "yes" && resp != "n" && resp != "no");
-
-            } while (data == null);
-            
-            SaveLocalSettings(data);
-            return Convert(data);
-        }
-
-        private LocalSettingsData GetDataFromUser()
-        {
-            var data = new LocalSettingsData();
-
-            Console.WriteLine("Db Host:");
-            data.DbHost = Console.ReadLine();
-
-            Console.WriteLine("Db Name:");
-            data.DbName = Console.ReadLine();
-
-            Console.WriteLine("Db User:");
-            data.DbUser = Console.ReadLine();
-
-            Console.WriteLine("Db Password:");
-            data.DbPassword = Console.ReadLine();
-
-            Console.WriteLine("Instance Domain:");
-            data.InstanceDomain = Console.ReadLine();
-
-            return data;
-        }
-
-        private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(LocalSettingsData data)
-        {
-            var dbSettings = new DbSettings
-            {
-                Type = data.DbType,
-                Host = data.DbHost,
-                Name = data.DbName,
-                User = data.DbUser,
-                Password = data.DbPassword
-            };
-            var instancesSettings = new InstanceSettings
-            {
-                Domain = data.InstanceDomain
-            };
-            return (dbSettings, instancesSettings);
-        }
-
-        private LocalSettingsData GetLocalSettingsFile()
-        {
-            try
-            {
-                if (!File.Exists(LocalFileName)) return null;
-
-                var jsonContent = File.ReadAllText(LocalFileName);
-                var content = JsonSerializer.Deserialize<LocalSettingsData>(jsonContent);
-                return content;
-            }
-            catch (Exception)
-            {
-                return null;
-            }
-        }
-
-        private void SaveLocalSettings(LocalSettingsData data)
-        {
-            var jsonContent = JsonSerializer.Serialize(data);
-            File.WriteAllText(LocalFileName, jsonContent);
-        }
-    }
-
-    internal class LocalSettingsData
-    {
-        public string DbType { get; set; } = "postgres";
-        public string DbHost { get; set; }
-        public string DbName { get; set; }
-        public string DbUser { get; set; }
-        public string DbPassword { get; set; }
-
-        public string InstanceDomain { get; set; }
-    }
-}

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.ActivityPub/Models/Attachment.cs

@@ -1,9 +1,14 @@
-namespace BirdsiteLive.ActivityPub
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace BirdsiteLive.ActivityPub
 {
     public class Attachment
     {
         public string type { get; set; }
         public string mediaType { get; set; }
         public string url { get; set; }
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public string name { get; set; }
     }
 }

src/BirdsiteLive.Common/BirdsiteLive.Common.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
 </Project>

src/BirdsiteLive.Common/Regexes/HashtagRegexes.cs

@@ -5,6 +5,6 @@ namespace BirdsiteLive.Common.Regexes
     public class HashtagRegexes
     {
         public static readonly Regex HashtagName = new Regex(@"^[a-zA-Z0-9_]+$");
-        public static readonly Regex Hashtag = new Regex(@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])");
+        public static readonly Regex Hashtag = new Regex(@"(^|.?[ \n]+)#([a-zA-Z0-9_]+)(?=\s|$|[\[\]<>.,;:!?/|-])");
     }
 }

src/BirdsiteLive.Common/Regexes/UserRegexes.cs

@@ -5,6 +5,6 @@ namespace BirdsiteLive.Common.Regexes
     public class UserRegexes
     {
         public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
-        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:'\.’!?/—\|-]|(. ))");
+        public static readonly Regex Mention = new Regex(@"(^|.?[ \n\.]+)@([a-zA-Z0-9_]+)(?=\s|$|[\[\]<>,;:'\.’!?/—\|-]|(. ))");
     }
 }

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -18,6 +18,9 @@
         public int TweetCacheCapacity { get; set; } = 20_000;
         // "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
         public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
+        public int m { get; set; } = 1;
+        public int n_start { get; set; } = 0;
+        public int n_end { get; set; } = 1;
         public int ParallelTwitterRequests { get; set; } = 10;
         public int ParallelFediversePosts { get; set; } = 10;
     }

src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs

@@ -42,9 +42,6 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             var twitterUserId = twitterUser.Id;
             if(!follower.Followings.Contains(twitterUserId))
                 follower.Followings.Add(twitterUserId);
-
-            if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                follower.FollowingsSyncStatus.Add(twitterUserId, -1);
             
             // Save Follower
             await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs

@@ -36,9 +36,6 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             if (follower.Followings.Contains(twitterUserId))
                 follower.Followings.Remove(twitterUserId);
 
-            if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                follower.FollowingsSyncStatus.Remove(twitterUserId);
-
             // Save or delete Follower
             if (follower.Followings.Any())
                 await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/StatusService.cs

@@ -137,7 +137,8 @@ namespace BirdsiteLive.Domain
                 {
                     type = "Document",
                     url = x.Url,
-                    mediaType = x.MediaType
+                    mediaType = x.MediaType,
+                    name = x.AltText
                 };
             }).ToArray();
         }

src/BirdsiteLive.Domain/Tools/StatusExtractor.cs

@@ -32,9 +32,6 @@ namespace BirdsiteLive.Domain.Tools
         {
             var tags = new List<Tag>();
 
-            // Replace return lines
-            messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p>");
-            messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/>");
 
 
             // Extract Urls
@@ -124,6 +121,10 @@ namespace BirdsiteLive.Domain.Tools
                         $@"{m.Groups[1]}<span class=""h-card""><a href=""{url}"" class=""u-url mention"">@<span>{mention.ToLower()}</span></a></span>{m.Groups[3]}");
                 }
             }
+            
+            // Replace return lines
+            messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p>");
+            messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/>");
 
             return (messageContent.Trim(), tags.ToArray());
         }

src/BirdsiteLive.Moderation/Actions/RemoveTwitterAccountAction.cs

@@ -41,9 +41,6 @@ namespace BirdsiteLive.Moderation.Actions
                 if (follower.Followings.Contains(twitterUserId))
                     follower.Followings.Remove(twitterUserId);
 
-                if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
-                    follower.FollowingsSyncStatus.Remove(twitterUserId);
-
                 if (follower.Followings.Any())
                     await _followersDal.UpdateFollowerAsync(follower);
                 else

src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 

src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionTask.cs

@@ -1,11 +0,0 @@
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.Pipeline.Models;
-
-namespace BirdsiteLive.Pipeline.Contracts
-{
-    public interface ISaveProgressionTask
-    {
-        Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
-    }
-}

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -61,25 +61,25 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
                         {
                             // skip the first time to avoid sending backlog of tweet
                             var tweetId = tweets.Last().Id;
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
                         }
                         else if (tweets.Length > 0 && user.LastTweetPostedId != -1)
                         {
                             userWtData.Tweets = tweets;
                             usersWtTweets.Add(userWtData);
                             var tweetId = tweets.Last().Id;
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
                         }
                         else
                         {
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
                         }
 
                     } 
                     catch(Exception e)
                     {
                         _logger.LogError(e.Message);
-                        await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
+                        await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
                     }
                 });
                 todo.Add(t);
@@ -104,7 +104,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
                 if (user.LastTweetPostedId == -1)
                     tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct);
                 else
-                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetSynchronizedForAllFollowersId);
+                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetPostedId);
             }
             catch (Exception e)
             {

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -16,16 +17,18 @@ namespace BirdsiteLive.Pipeline.Processors
     {
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
+        private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
         private static Random rng = new Random();
         
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ILogger<RetrieveTwitterUsersProcessor> logger)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
             _followersDal = followersDal;
+            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
@@ -36,31 +39,37 @@ namespace BirdsiteLive.Pipeline.Processors
             {
                 ct.ThrowIfCancellationRequested();
 
-                try
+                if (_instanceSettings.ParallelTwitterRequests == 0)
                 {
-                    var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000);
+                    while (true)
+                        await Task.Delay(10000);
+                }
+                
+                var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
 
-                    var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
-                    var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
+                var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
+                var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
 
-                    foreach (var u in splitUsers)
+                foreach (var users in splitUsers)
+                {
+                    ct.ThrowIfCancellationRequested();
+                    List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
+                    foreach (var u in users)
                     {
-                        ct.ThrowIfCancellationRequested();
-                        UserWithDataToSync[] toSync = await Task.WhenAll(
-                            u.Select(async x => new UserWithDataToSync
-                                { User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) } 
-                            )
-                        );
-
-                        await twitterUsersBufferBlock.SendAsync(toSync, ct);
+                        var followers = await _followersDal.GetFollowersAsync(u.Id);
+                        toSync.Add( new UserWithDataToSync()
+                        {
+                            User = u,
+                            Followers = followers
+                        });
+                        
                     }
 
-                    await Task.Delay(10, ct); // this is somehow necessary
-                }
-                catch (Exception e)
-                {
-                    _logger.LogError(e, "Failing retrieving Twitter Users.");
+                    await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
+
                 }
+                
+                await Task.Delay(10, ct); // this is somehow necessary
             }
         }
     }

src/BirdsiteLive.Pipeline/Processors/SubTasks/SaveProgressionTask.cs

@@ -1,61 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.Pipeline.Contracts;
-using BirdsiteLive.Pipeline.Models;
-using Microsoft.Extensions.Logging;
-
-namespace BirdsiteLive.Pipeline.Processors.SubTasks
-{
-    public class SaveProgressionTask : ISaveProgressionTask
-    {
-        private readonly ITwitterUserDal _twitterUserDal;
-        private readonly ILogger<SaveProgressionTask> _logger;
-
-        #region Ctor
-        public SaveProgressionTask(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionTask> logger)
-        {
-            _twitterUserDal = twitterUserDal;
-            _logger = logger;
-        }
-        #endregion
-
-        public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
-        {
-            try
-            {
-                if (userWithTweetsToSync.Tweets.Length == 0)
-                {
-                    _logger.LogWarning("No tweets synchronized");
-                    return;
-                }
-                if(userWithTweetsToSync.Followers.Length == 0)
-                {
-                    _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct);
-                    return;
-                }
-            
-                var userId = userWithTweetsToSync.User.Id;
-                var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
-            
-                if (followingSyncStatuses.Count == 0)
-                {
-                    _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId);
-                    return;
-                }
-
-                var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
-                var minimumSync = followingSyncStatuses.Min();
-                var now = DateTime.UtcNow;
-                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
-                throw;
-            }
-        }
-    }
-}

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -21,16 +21,14 @@ namespace BirdsiteLive.Pipeline
         private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
         private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
         private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
-        private readonly ISaveProgressionTask _saveProgressionTask;
         private readonly ILogger<StatusPublicationPipeline> _logger;
 
         #region Ctor
-        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionTask saveProgressionTask, ILogger<StatusPublicationPipeline> logger)
+        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger<StatusPublicationPipeline> logger)
         {
             _retrieveTweetsProcessor = retrieveTweetsProcessor;
             _retrieveFollowersProcessor = retrieveFollowersProcessor;
             _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
-            _saveProgressionTask = saveProgressionTask;
             _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
 
             _logger = logger;

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs

@@ -4,5 +4,6 @@
     {
         public string MediaType { get; set; }
         public string Url { get; set; }
+        public string AltText { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs

@@ -17,5 +17,6 @@ namespace BirdsiteLive.Twitter.Models
         public string RetweetUrl { get; set; }
         public long RetweetId { get; set; }
         public TwitterUser OriginalAuthor { get; set; }
+        public TwitterUser Author { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Threading;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
 using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
@@ -26,11 +27,9 @@ namespace BirdsiteLive.Twitter.Tools
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
         private readonly IHttpClientFactory _httpClientFactory;
-        private List<HttpClient> _twitterClients = new List<HttpClient>();
-        private List<(String, String)> _tokens = new List<(string,string)>();
+        private ConcurrentDictionary<String, String> _token2 = new ConcurrentDictionary<string, string>();
         static Random rnd = new Random();
         private RateLimiter _rateLimiter;
-        static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1);
         private const int _targetClients = 3;
         private InstanceSettings _instanceSettings;
         private readonly (string, string)[] _apiKeys = new[]
@@ -40,8 +39,17 @@ namespace BirdsiteLive.Twitter.Tools
             ("CjulERsDeqhhjSme66ECg", "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck"), // iPad
             ("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
         };
+        
+        private readonly string[] _bTokens =  new[] 
+                {
+            // developer.twitter.com
+                    "AAAAAAAAAAAAAAAAAAAAACHguwAAAAAAaSlT0G31NDEyg%2BSnBN5JuyKjMCU%3Dlhg0gv0nE7KKyiJNEAojQbn8Y3wJm1xidDK7VnKGBP4ByJwHPb",
+            // tweetdeck new
+                    "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF",
+            // ipad -- TimelineSearch returns data in a different format, making nitter return empty results. on the other hand, it has high rate limits. build separate token pools per endpoint?
+                    "AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR",
+        };
         public String BearerToken { 
-            //get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
             get
             {
                 return _instanceSettings.TwitterBearerToken;
@@ -54,6 +62,10 @@ namespace BirdsiteLive.Twitter.Tools
             _logger = logger;
             _instanceSettings = settings;
             _httpClientFactory = httpClientFactory;
+
+            var concuOpt = new ConcurrencyLimiterOptions();
+            concuOpt.PermitLimit = 1;
+            _rateLimiter = new ConcurrencyLimiter(concuOpt);
         }
         #endregion
 
@@ -62,6 +74,9 @@ namespace BirdsiteLive.Twitter.Tools
             var httpClient = _httpClientFactory.CreateClient();
             using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
             {
+                int r1 = rnd.Next(_bTokens.Length);
+                return _bTokens[r1];
+                
                 int r = rnd.Next(_apiKeys.Length);
                 var (login, password) = _apiKeys[r];
                 var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
@@ -82,20 +97,8 @@ namespace BirdsiteLive.Twitter.Tools
         public async Task RefreshClient(HttpRequestMessage req)
         {
             string token = req.Headers.GetValues("x-guest-token").First();
-            string bearer = req.Headers.GetValues("Authorization").First().Replace("Bearer ", "");
 
-            var i = _tokens.IndexOf((bearer, token));
-
-            // this is prabably not thread safe but yolo
-            try
-            {
-                _twitterClients.RemoveAt(i);
-                _tokens.RemoveAt(i);
-            }
-            catch (IndexOutOfRangeException _)
-            {
-                _logger.LogError("Error refreshing twitter token");
-            }
+            _token2.TryRemove(token, out _);
 
             await RefreshCred();
             await Task.Delay(1000);
@@ -104,21 +107,8 @@ namespace BirdsiteLive.Twitter.Tools
 
         private async Task RefreshCred()
         {
-            
             (string bearer, string guest) = await GetCred();
-
-            HttpClient client = _httpClientFactory.CreateClient();
-            //HttpClient client = new HttpClient();
-
-            _twitterClients.Add(client);
-            _tokens.Add((bearer,guest));
-
-            if (_twitterClients.Count > _targetClients)
-            {
-                _twitterClients.RemoveAt(0);
-                _tokens.RemoveAt(0);
-            }
-
+            _token2.TryAdd(guest, bearer);
         }
 
         private async Task<(string, string)> GetCred()
@@ -126,17 +116,26 @@ namespace BirdsiteLive.Twitter.Tools
             string token;
             var httpClient = _httpClientFactory.CreateClient();
             string bearer = await GenerateBearerToken();
-            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json"))
-            {
-                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
+            using RateLimitLease lease = await _rateLimiter.AcquireAsync(permitCount: 1);
+            using var request = new HttpRequestMessage(new HttpMethod("POST"),
+                "https://api.twitter.com/1.1/guest/activate.json");
+            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
+            //request.Headers.Add("User-Agent",
+            //    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36");
 
-                var httpResponse = await httpClient.SendAsync(request);
+            HttpResponseMessage httpResponse;
+            do
+            {
+                httpResponse = await httpClient.SendAsync(request);
 
                 var c = await httpResponse.Content.ReadAsStringAsync();
+                if (httpResponse.StatusCode == HttpStatusCode.TooManyRequests)
+                    await Task.Delay(1000);
                 httpResponse.EnsureSuccessStatusCode();
                 var doc = JsonDocument.Parse(c);
                 token = doc.RootElement.GetProperty("guest_token").GetString();
-            }
+
+            } while (httpResponse.StatusCode != HttpStatusCode.OK);
 
             return (bearer, token);
 
@@ -144,19 +143,19 @@ namespace BirdsiteLive.Twitter.Tools
 
         public async Task<HttpClient> MakeHttpClient()
         {
-            if (_twitterClients.Count < 2)
+            if (_token2.Count < _targetClients)
                 await RefreshCred();
-            int r = rnd.Next(_twitterClients.Count);
-            return _twitterClients[r];
+            return _httpClientFactory.CreateClient();
         }
         public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
         {
             var request = new HttpRequestMessage(m, endpoint);
-            int r = rnd.Next(_twitterClients.Count);
-            (string bearer, string token) = _tokens[r];
+            (string token, string bearer) = _token2.MaxBy(x => rnd.Next());
             request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
             request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
             request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
+            //request.Headers.Add("User-Agent",
+            //    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36");
             if (addToken)
                 request.Headers.TryAddWithoutValidation("x-guest-token", token);
             //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");

src/BirdsiteLive.Twitter/TwitterTweetsService.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.IO;
+using System.Net;
 using System.Net.Http;
 using System.Text.Json;
 using System.Text.Json.Nodes;
@@ -31,6 +32,50 @@ namespace BirdsiteLive.Twitter
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<TwitterTweetsService> _logger;
         private readonly InstanceSettings _instanceSettings;
+        private static string gqlFeatures = """
+        { 
+          "android_graphql_skip_api_media_color_palette": false,
+          "blue_business_profile_image_shape_enabled": false,
+          "creator_subscriptions_subscription_count_enabled": false,
+          "creator_subscriptions_tweet_preview_api_enabled": true,
+          "freedom_of_speech_not_reach_fetch_enabled": false,
+          "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+          "hidden_profile_likes_enabled": false,
+          "highlights_tweets_tab_ui_enabled": false,
+          "interactive_text_enabled": false,
+          "longform_notetweets_consumption_enabled": true,
+          "longform_notetweets_inline_media_enabled": false,
+          "longform_notetweets_richtext_consumption_enabled": true,
+          "longform_notetweets_rich_text_read_enabled": false,
+          "responsive_web_edit_tweet_api_enabled": false,
+          "responsive_web_enhance_cards_enabled": false,
+          "responsive_web_graphql_exclude_directive_enabled": true,
+          "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+          "responsive_web_graphql_timeline_navigation_enabled": false,
+          "responsive_web_media_download_video_enabled": false,
+          "responsive_web_text_conversations_enabled": false,
+          "responsive_web_twitter_article_tweet_consumption_enabled": false,
+          "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+          "rweb_lists_timeline_redesign_enabled": true,
+          "spaces_2022_h2_clipping": true,
+          "spaces_2022_h2_spaces_communities": true,
+          "standardized_nudges_misinfo": false,
+          "subscriptions_verification_info_enabled": true,
+          "subscriptions_verification_info_reason_enabled": true,
+          "subscriptions_verification_info_verified_since_enabled": true,
+          "super_follow_badge_privacy_enabled": false,
+          "super_follow_exclusive_tweet_notifications_enabled": false,
+          "super_follow_tweet_api_enabled": false,
+          "super_follow_user_api_enabled": false,
+          "tweet_awards_web_tipping_enabled": false,
+          "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+          "tweetypie_unmention_optimization_enabled": false,
+          "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
+          "verified_phone_label_enabled": false,
+          "vibe_api_enabled": false,
+          "view_counts_everywhere_api_enabled": false
+        }
+        """.Replace(" ", "").Replace("\n", "");
 
         #region Ctor
         public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<TwitterTweetsService> logger)
@@ -51,21 +96,27 @@ namespace BirdsiteLive.Twitter
             var client = await _twitterAuthenticationInitializer.MakeHttpClient();
 
 
+            // https://platform.twitter.com/embed/Tweet.html?id=1633788842770825216
             string reqURL =
-                "https://api.twitter.com/graphql/XjlydVWHFIDaAUny86oh2g/TweetDetail?variables=%7B%22focalTweetId%22%3A%22"
+                "https://api.twitter.com/graphql/83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2?variables={%22focalTweetId%22%3A%22"
                 + statusId +
-                "%22,%22with_rux_injections%22%3Atrue,%22includePromotedContent%22%3Afalse,%22withCommunity%22%3Afalse,%22withQuickPromoteEligibilityTweetFields%22%3Afalse,%22withBirdwatchNotes%22%3Afalse,%22withSuperFollowsUserFields%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Afalse,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
+                "%22,%22count%22:20,%22includeHasBirdwatchNotes%22:false}&features="+ gqlFeatures;
             using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
                 JsonDocument tweet;
                 var httpResponse = await client.SendAsync(request);
+                if (httpResponse.StatusCode == HttpStatusCode.Unauthorized)
+                {
+                    _logger.LogError("Error retrieving tweet {statusId}; refreshing client", statusId);
+                    await _twitterAuthenticationInitializer.RefreshClient(request);
+                }
                 httpResponse.EnsureSuccessStatusCode();
                 var c = await httpResponse.Content.ReadAsStringAsync();
                 tweet = JsonDocument.Parse(c);
 
 
-                var timeline = tweet.RootElement.GetProperty("data").GetProperty("threaded_conversation_with_injections_v2")
+                var timeline = tweet.RootElement.GetProperty("data").GetProperty("timeline_response")
                     .GetProperty("instructions").EnumerateArray().First().GetProperty("entries").EnumerateArray();
 
                 var tweetInDoc = timeline.Where(x => x.GetProperty("entryId").GetString() == "tweet-" + statusId)
@@ -100,8 +151,13 @@ namespace BirdsiteLive.Twitter
 
 
             var reqURL =
-                "https://api.twitter.com/graphql/pNl8WjKAvaegIoVH--FuoQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" +
-                userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
+                "https://api.twitter.com/graphql/8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2?variables=%7B%22rest_id%22%3A%22" +
+                userId +
+                "%22,%22count%22%3A40,%22includeHasBirdwatchNotes%22%3Atrue}&features=" +
+                gqlFeatures;
+            //reqURL =
+            //    """https://twitter.com/i/api/graphql/rIIwMe1ObkGh_ByBtTCtRQ/UserTweets?variables={"userId":"44196397","count":20,"includePromotedContent":true,"withQuickPromoteEligibilityTweetFields":true,"withVoice":true,"withV2Timeline":true}&features={"rweb_lists_timeline_redesign_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":false,"tweet_awards_web_tipping_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_media_download_video_enabled":false,"responsive_web_enhance_cards_enabled":false}&fieldToggles={"withArticleRichContentState":false}""";
+            //reqURL = reqURL.Replace("44196397", userId.ToString());
             JsonDocument results;
             List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
             using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
@@ -109,48 +165,38 @@ namespace BirdsiteLive.Twitter
             {
 
                 var httpResponse = await client.SendAsync(request);
-                httpResponse.EnsureSuccessStatusCode();
                 var c = await httpResponse.Content.ReadAsStringAsync();
+                if (httpResponse.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden)
+                {
+                    _logger.LogError("Error retrieving timeline of {Username}; refreshing client", username);
+                    await _twitterAuthenticationInitializer.RefreshClient(request);
+                    return null;
+                }
+                httpResponse.EnsureSuccessStatusCode();
                 results = JsonDocument.Parse(c);
 
                 _statisticsHandler.CalledTweetApi();
             }
-            catch (HttpRequestException e)
-            {
-                _logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username);
-                //await _twitterAuthenticationInitializer.RefreshClient(request);
-                return null;
-            }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving timeline ", username);
                 return null;
             }
 
-            var timeline = results.RootElement.GetProperty("data").GetProperty("user").GetProperty("result")
-                .GetProperty("timeline_v2").GetProperty("timeline").GetProperty("instructions").EnumerateArray();
+            var timeline = results.RootElement.GetProperty("data").GetProperty("user_result").GetProperty("result")
+                .GetProperty("timeline_response").GetProperty("timeline").GetProperty("instructions").EnumerateArray();
 
             foreach (JsonElement timelineElement in timeline) 
             {
-                if (timelineElement.GetProperty("type").GetString() != "TimelineAddEntries")
+                if (timelineElement.GetProperty("__typename").GetString() != "TimelineAddEntries")
                     continue;
 
                 
                 foreach (JsonElement tweet in timelineElement.GetProperty("entries").EnumerateArray())
                 {
-                    if (tweet.GetProperty("content").GetProperty("entryType").GetString() != "TimelineTimelineItem")
+                    if (tweet.GetProperty("content").GetProperty("__typename").GetString() != "TimelineTimelineItem")
                         continue;
                     
-                    try 
-                    {
-                        JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent")
-                                .GetProperty("tweet_results").GetProperty("core").GetProperty("user_results");
-
-                        TwitterUser tweetUser = _twitterUserService.Extract(userDoc);
-                        _twitterUserService.AddUser(tweetUser);
-                    }
-                    catch (Exception _)
-                    {}
 
                     try 
                     {   
@@ -179,38 +225,45 @@ namespace BirdsiteLive.Twitter
 
             JsonElement retweet;
             TwitterUser OriginalAuthor;
+            TwitterUser author = null;
             JsonElement inReplyToPostIdElement;
             JsonElement inReplyToUserElement;
             string inReplyToUser = null;
             long? inReplyToPostId = null;
             long retweetId = default;
 
-            string userName = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("core").GetProperty("user_results")
+            string userName = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("core").GetProperty("user_result")
                     .GetProperty("result").GetProperty("legacy").GetProperty("screen_name").GetString();
 
-            bool isReply = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+            JsonElement userDoc = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("core")
+                    .GetProperty("user_result").GetProperty("result");
+
+            author = _twitterUserService.Extract(userDoc); 
+            
+            bool isReply = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .TryGetProperty("in_reply_to_status_id_str", out inReplyToPostIdElement);
-            tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+            tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .TryGetProperty("in_reply_to_screen_name", out inReplyToUserElement);
             if (isReply) 
             {
                 inReplyToPostId = Int64.Parse(inReplyToPostIdElement.GetString());
                 inReplyToUser = inReplyToUserElement.GetString();
             }
-            bool isRetweet = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+            bool isRetweet = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .TryGetProperty("retweeted_status_result", out retweet);
             string MessageContent;
             if (!isRetweet)
             {
-                MessageContent = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                MessageContent = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .GetProperty("full_text").GetString();
-                bool isNote = tweet.GetProperty("content").GetProperty("itemContent")
-                        .GetProperty("tweet_results").GetProperty("result")
+                bool isNote = tweet.GetProperty("content").GetProperty("content")
+                        .GetProperty("tweetResult").GetProperty("result")
                         .TryGetProperty("note_tweet", out var note);
                 if (isNote)
                 {
@@ -218,15 +271,16 @@ namespace BirdsiteLive.Twitter
                         .GetProperty("text").GetString();
                 }
                 OriginalAuthor = null;
+                
             }
             else 
             {
-                MessageContent = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                MessageContent = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .GetProperty("retweeted_status_result").GetProperty("result")
                     .GetProperty("legacy").GetProperty("full_text").GetString();
-                bool isNote = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                bool isNote = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .GetProperty("retweeted_status_result").GetProperty("result")
                     .TryGetProperty("note_tweet", out var note);
                 if (isNote)
@@ -234,29 +288,34 @@ namespace BirdsiteLive.Twitter
                     MessageContent = note.GetProperty("note_tweet_results").GetProperty("result")
                         .GetProperty("text").GetString();
                 }
-                string OriginalAuthorUsername = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                string OriginalAuthorUsername = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .GetProperty("retweeted_status_result").GetProperty("result")
-                    .GetProperty("core").GetProperty("user_results").GetProperty("result")
+                    .GetProperty("core").GetProperty("user_result").GetProperty("result")
                     .GetProperty("legacy").GetProperty("screen_name").GetString();
-                OriginalAuthor = await _twitterUserService.GetUserAsync(OriginalAuthorUsername);
-                retweetId = Int64.Parse(tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                JsonElement OriginalAuthorDoc = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+                    .GetProperty("retweeted_status_result").GetProperty("result")
+                    .GetProperty("core").GetProperty("user_result").GetProperty("result");
+                OriginalAuthor = _twitterUserService.Extract(OriginalAuthorDoc); 
+                //OriginalAuthor = await _twitterUserService.GetUserAsync(OriginalAuthorUsername);
+                retweetId = Int64.Parse(tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .GetProperty("retweeted_status_result").GetProperty("result")
                     .GetProperty("rest_id").GetString());
             }
 
-            string creationTime = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+            string creationTime = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .GetProperty("created_at").GetString().Replace(" +0000", "");
 
             JsonElement extendedEntities;
-            bool hasMedia = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+            bool hasMedia = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .TryGetProperty("extended_entities", out extendedEntities);
 
-            JsonElement.ArrayEnumerator urls = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+            JsonElement.ArrayEnumerator urls = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .GetProperty("entities").GetProperty("urls").EnumerateArray();
             foreach (JsonElement url in urls)
             {
@@ -272,7 +331,8 @@ namespace BirdsiteLive.Twitter
                 {
                     var type = media.GetProperty("type").GetString();
                     string url = "";
-                    if (type == "video" || type == "animated_gif")
+                    string altText = null;
+                    if (media.TryGetProperty("video_info", out _))
                     {
                         var bitrate = -1;
                         foreach (JsonElement v in media.GetProperty("video_info").GetProperty("variants").EnumerateArray())
@@ -291,10 +351,16 @@ namespace BirdsiteLive.Twitter
                     {
                         url = media.GetProperty("media_url_https").GetString();
                     }
+
+                    if (media.TryGetProperty("ext_alt_text", out JsonElement altNode))
+                    {
+                        altText = altNode.GetString();
+                    }
                     var m = new ExtractedMedia
                     {
-                        MediaType = GetMediaType(type, media.GetProperty("media_url_https").GetString()),
+                        MediaType = GetMediaType(type, url),
                         Url = url,
+                        AltText = altText
                     };
                     Media.Add(m);
 
@@ -302,20 +368,33 @@ namespace BirdsiteLive.Twitter
                 }
             }
 
-            bool isQuoteTweet = tweet.GetProperty("content").GetProperty("itemContent")
-                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+            bool isQuoteTweet = tweet.GetProperty("content").GetProperty("content")
+                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
                     .GetProperty("is_quote_status").GetBoolean();
 
             if (isQuoteTweet) 
             {
 
-                string quoteTweetLink = tweet.GetProperty("content").GetProperty("itemContent")
-                        .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
-                        .GetProperty("quoted_status_permalink").GetProperty("expanded").GetString();
-                quoteTweetLink = quoteTweetLink.Replace("https://twitter.com/", $"https://{_instanceSettings.Domain}/users/");
-                quoteTweetLink = quoteTweetLink.Replace("/status/", "/statuses/");
+                string quoteTweetId = tweet.GetProperty("content").GetProperty("content")
+                        .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+                        .GetProperty("quoted_status_id_str").GetString();
+                string quoteTweetAcct = tweet.GetProperty("content").GetProperty("content")
+                        .GetProperty("tweetResult").GetProperty("result")
+                        .GetProperty("quoted_status_result").GetProperty("result")
+                        .GetProperty("core").GetProperty("user_result").GetProperty("result")
+                        .GetProperty("legacy").GetProperty("screen_name").GetString();
+                //Uri test = new Uri(quoteTweetLink);
+                //string quoteTweetAcct = test.Segments[1].Replace("/", "");
+                //string quoteTweetId = test.Segments[3];
+                
+                string quoteTweetLink = $"https://{_instanceSettings.Domain}/@{quoteTweetAcct}/{quoteTweetId}";
+
+                //MessageContent.Replace($"https://twitter.com/i/web/status/{}", "");
+                MessageContent = MessageContent.Replace($"https://twitter.com/{quoteTweetAcct}/status/{quoteTweetId}", "");
+                
                 MessageContent = MessageContent + "\n\n" + quoteTweetLink;
             }
+            
             var extractedTweet = new ExtractedTweet
             {
                 Id = Int64.Parse(tweet.GetProperty("entryId").GetString().Replace("tweet-", "")),
@@ -330,6 +409,7 @@ namespace BirdsiteLive.Twitter
                 RetweetUrl = "https://t.co/123",
                 RetweetId = retweetId,
                 OriginalAuthor = OriginalAuthor,
+                Author = author,
             };
        
             return extractedTweet;

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Net;
 using System.Net.Http;
 using System.Text.Json;
 using System.Threading.Tasks;
@@ -23,8 +24,54 @@ namespace BirdsiteLive.Twitter
         private readonly ITwitterAuthenticationInitializer _twitterAuthenticationInitializer;
         private readonly ITwitterStatisticsHandler _statisticsHandler;
         private readonly ILogger<TwitterUserService> _logger;
-        
-        private readonly string endpoint = "https://twitter.com/i/api/graphql/4LB4fkCe3RDLDmOEEYtueg/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_twitter_blue_new_verification_copy_is_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D";
+
+        private readonly string endpoint =
+            "https://api.twitter.com/graphql/pVrmNaXcxPjisIvKtLDMEA/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=" + gqlFeatures;
+
+        private static string gqlFeatures = """
+        { 
+          "android_graphql_skip_api_media_color_palette": false,
+          "blue_business_profile_image_shape_enabled": false,
+          "creator_subscriptions_subscription_count_enabled": false,
+          "creator_subscriptions_tweet_preview_api_enabled": true,
+          "freedom_of_speech_not_reach_fetch_enabled": false,
+          "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+          "hidden_profile_likes_enabled": false,
+          "highlights_tweets_tab_ui_enabled": false,
+          "interactive_text_enabled": false,
+          "longform_notetweets_consumption_enabled": true,
+          "longform_notetweets_inline_media_enabled": false,
+          "longform_notetweets_richtext_consumption_enabled": true,
+          "longform_notetweets_rich_text_read_enabled": false,
+          "responsive_web_edit_tweet_api_enabled": false,
+          "responsive_web_enhance_cards_enabled": false,
+          "responsive_web_graphql_exclude_directive_enabled": true,
+          "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+          "responsive_web_graphql_timeline_navigation_enabled": false,
+          "responsive_web_media_download_video_enabled": false,
+          "responsive_web_text_conversations_enabled": false,
+          "responsive_web_twitter_article_tweet_consumption_enabled": false,
+          "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+          "rweb_lists_timeline_redesign_enabled": true,
+          "spaces_2022_h2_clipping": true,
+          "spaces_2022_h2_spaces_communities": true,
+          "standardized_nudges_misinfo": false,
+          "subscriptions_verification_info_enabled": true,
+          "subscriptions_verification_info_reason_enabled": true,
+          "subscriptions_verification_info_verified_since_enabled": true,
+          "super_follow_badge_privacy_enabled": false,
+          "super_follow_exclusive_tweet_notifications_enabled": false,
+          "super_follow_tweet_api_enabled": false,
+          "super_follow_user_api_enabled": false,
+          "tweet_awards_web_tipping_enabled": false,
+          "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+          "tweetypie_unmention_optimization_enabled": false,
+          "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
+          "verified_phone_label_enabled": false,
+          "vibe_api_enabled": false,
+          "view_counts_everywhere_api_enabled": false
+        }
+        """.Replace(" ", "").Replace("\n", "");
 
         #region Ctor
         public TwitterUserService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ILogger<TwitterUserService> logger)
@@ -45,6 +92,12 @@ namespace BirdsiteLive.Twitter
             {
 
                 var httpResponse = await client.SendAsync(request);
+                if (httpResponse.StatusCode == HttpStatusCode.Unauthorized)
+                {
+                    _logger.LogError("Error retrieving user {Username}, Refreshing client", username);
+                    await _twitterAuthenticationInitializer.RefreshClient(request);
+                    return null;
+                }
                 httpResponse.EnsureSuccessStatusCode();
 
                 var c = await httpResponse.Content.ReadAsStringAsync();
@@ -68,12 +121,6 @@ namespace BirdsiteLive.Twitter
                 //    throw;
                 //}
             }
-            catch (HttpRequestException e)
-            {
-                _logger.LogError(e, "Error retrieving user {Username}, Refreshing client", username);
-                await _twitterAuthenticationInitializer.RefreshClient(request);
-                return null;
-            }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving user {Username}", username);

src/BirdsiteLive.Wikidata/BirdsiteLive.Wikidata.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
+      <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
+      <Content Include="query.sparql">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </Content>
+    </ItemGroup>
+
+</Project>

src/BirdsiteLive.Wikidata/Program.cs

@@ -0,0 +1,48 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading.Tasks;
+using System.Xml;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.DAL.Postgres.DataAccessLayers;
+using BirdsiteLive.DAL.Postgres.Settings;
+
+var settings = new PostgresSettings()
+{
+    ConnString = System.Environment.GetEnvironmentVariable("ConnString"),
+};
+var dal = new TwitterUserPostgresDal(settings);
+
+var twitterUser = new HashSet<string>();
+var twitterUserQuery = await dal.GetAllTwitterUsersAsync();
+Console.WriteLine("Loading twitter users");
+foreach (SyncTwitterUser user in twitterUserQuery)
+{
+    twitterUser.Add(user.Acct);
+}
+Console.WriteLine("Done loading twitter users");
+
+Console.WriteLine("Hello, World!");
+var client = new HttpClient();
+string query = new StreamReader("query.sparql").ReadToEnd();
+
+client.DefaultRequestHeaders.Add("Accept", "text/csv");
+client.DefaultRequestHeaders.Add("User-Agent", "BirdMakeup/1.0 (https://bird.makeup; coolbot@example.org) BirdMakeup/1.0");
+var response = await client.GetAsync($"https://query.wikidata.org/sparql?query={Uri.EscapeDataString(query)}");
+var content = await response.Content.ReadAsStringAsync();
+
+// Console.WriteLine(content);
+
+foreach (string n in content.Split("\n"))
+{
+    var s = n.Split(",");
+    if (n.Length < 2)
+        continue;
+    
+    var acct = s[1].ToLower();
+    var fedi = "@" + s[2];
+    await dal.UpdateTwitterUserFediAcctAsync(acct, fedi);
+    if (twitterUser.Contains(acct))
+        Console.WriteLine(fedi);
+}
+

src/BirdsiteLive.Wikidata/README.md

@@ -0,0 +1,33 @@
+# Wikidata service
+
+Wikidata is the metadata community behind Wikipedia. See for example
+[Hank Green](https://www.wikidata.org/wiki/Q550996). In his page, there are 
+all the links to his wikipedia pages, and many facts about him. What is 
+particularly useful to us is the twitter username (P2002) and mastodon 
+username (P4033). 
+
+From this information, we can build a feature that suggests to follow the 
+native fediverse account of someone you are trying to follow from Twitter.
+
+The main downside is that those redirect are only for somewhat famous
+people/organisations.
+
+## Goals
+### Being reusable by others
+All this data can be useful to many other fediverse projects: tools 
+for finding interesting accounts to follow, "verified" badge powered by 
+Wikipedia, etc. I hope that by working on improving this dataset, we can
+help other projects thrive. 
+### Being independent of Twitter
+Bird.makeup has to build features in a way that can't be suddenly cut off.
+Building this feature with a "Log in with Twitter" is not viable.
+Wikipedia is independent and outside of Elon's reach.
+
+Also this system supports many other services: TikTok, Reddit, YouTube, etc. 
+Which is really useful to expend the scope of this project while reusing as 
+much work as possible
+### Having great moderation
+
+Wikipedia has many tools to help curate data and remove troll's submissions,
+far better than anything I can build. I much prefer contribute to what
+they are doing than try to compete

src/BirdsiteLive.Wikidata/query.sparql

@@ -0,0 +1,9 @@
+#Cats
+SELECT ?item ?username ?username2 ?linkcount ?itemLabel
+WHERE 
+{
+  ?item wdt:P2002 ?username. 
+  ?item wdt:P4033 ?username2. 
+        ?item wikibase:sitelinks ?linkcount .
+  SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } # Helps get the label in your language, if not, then en language
+} ORDER BY DESC(?linkcount) LIMIT 5000

src/BirdsiteLive.sln

@@ -47,12 +47,16 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Moderation.Tes
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Common.Tests", "Tests\BirdsiteLive.Common.Tests\BirdsiteLive.Common.Tests.csproj", "{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BSLManager", "BSLManager\BSLManager.csproj", "{4A84D351-E91B-4E58-8E20-211F0F4991D7}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BSLManager.Tests", "Tests\BSLManager.Tests\BSLManager.Tests.csproj", "{D4457271-620E-465A-B08E-7FC63C99A2F6}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Twitter.Tests", "Tests\BirdsiteLive.Twitter.Tests\BirdsiteLive.Twitter.Tests.csproj", "{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Wikidata", "BirdsiteLive.Wikidata\BirdsiteLive.Wikidata.csproj", "{EAB43087-359C-46BD-8796-5F7D9B473B39}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SocialNetworks", "SocialNetworks", "{7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dotMakeup.HackerNews", "dotMakeup.HackerNews\dotMakeup.HackerNews.csproj", "{060DE3F7-DB7E-45FD-B233-104C3C464F57}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dotMakeup.HackerNews.Tests", "Tests\dotMakeup.HackerNews.Tests\dotMakeup.HackerNews.Tests.csproj", "{6D650384-7BDD-4628-A46C-2FE4A688DBA4}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -131,25 +135,28 @@ Global
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.Build.0 = Release|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.Build.0 = Release|Any CPU
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Release|Any CPU.Build.0 = Release|Any CPU
+		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{160AD138-4E29-4706-8546-9826B529E9B2} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
-		{77C559D1-80A2-4B1C-A566-AE2D156944A4} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
 		{E64E7501-5DB8-4620-BA35-BA59FD746ABA} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
 		{155D46A4-2D05-47F2-8FFC-0B7C412A7652} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{D48450EE-D8BD-4228-9864-043AC88F7EE0} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
@@ -165,8 +172,11 @@ Global
 		{4BE541AC-8A93-4FA3-98AC-956CC2D5B748} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
 		{0A311BF3-4FD9-4303-940A-A3778890561C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
-		{D4457271-620E-465A-B08E-7FC63C99A2F6} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
+		{EAB43087-359C-46BD-8796-5F7D9B473B39} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
+		{060DE3F7-DB7E-45FD-B233-104C3C464F57} = {7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}
+		{77C559D1-80A2-4B1C-A566-AE2D156944A4} = {7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}
+		{6D650384-7BDD-4628-A46C-2FE4A688DBA4} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE}

src/BirdsiteLive/BirdsiteLive.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
     <Version>1.0</Version>

src/BirdsiteLive/Controllers/UsersController.cs

@@ -5,12 +5,13 @@ using System.Linq;
 using System.Text.Json;
 using System.Net.Mime;
 using System.Text.RegularExpressions;
-using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Models;
 using BirdsiteLive.Tools;
@@ -30,16 +31,20 @@ namespace BirdsiteLive.Controllers
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
         private readonly InstanceSettings _instanceSettings;
+        private readonly IFollowersDal _followersDal;
+        private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
+        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, IFollowersDal followersDal, ITwitterUserDal twitterUserDal, ILogger<UsersController> logger)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
             _statusService = statusService;
             _instanceSettings = instanceSettings;
             _twitterTweetService = twitterTweetService;
+            _followersDal = followersDal;
+            _twitterUserDal = twitterUserDal;
             _logger = logger;
         }
         #endregion
@@ -119,6 +124,12 @@ namespace BirdsiteLive.Controllers
             if (isSaturated) return View("ApiSaturated");
             if (notFound) return View("UserNotFound");
 
+            Follower[] followers = new Follower[] { };
+            
+            var userDal = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
+            if (userDal != null)
+                followers = await _followersDal.GetFollowersAsync(userDal.Id);
+
             var displayableUser = new DisplayTwitterUser
             {
                 Name = user.Name,
@@ -127,7 +138,9 @@ namespace BirdsiteLive.Controllers
                 Url = user.Url,
                 ProfileImageUrl = user.ProfileImageUrl,
                 Protected = user.Protected,
-
+                FollowerCount = followers.Length,
+                MostPopularServer = followers.GroupBy(x => x.Host).OrderByDescending(x => x.Count()).Select(x => x.Key).FirstOrDefault("N/A"),
+                FediverseAccount = userDal.FediAcct,
                 InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}"
             };
             return View(displayableUser);
@@ -144,9 +157,10 @@ namespace BirdsiteLive.Controllers
             var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
             if (tweet == null)
                 return NotFound();
-            
-            var user = await _twitterUserService.GetUserAsync(id);
 
+            if (tweet.Author.Acct != id)
+                return NotFound();
+            
             var status = _statusService.GetStatus(id, tweet);
 
             if (acceptHeaders.Any())
@@ -165,12 +179,51 @@ namespace BirdsiteLive.Controllers
             {
                 Text = tweet.MessageContent,
                 OgUrl = $"https://twitter.com/{id}/status/{statusId}",
-                UserProfileImage = user.ProfileImageUrl,
-                UserName = user.Name,
+                UserProfileImage = tweet.Author.ProfileImageUrl,
+                UserName = tweet.Author.Name,
             };
             return View(displayTweet);
         }
 
+        // Mastodon API for QT in some apps
+        [Route("/api/v1/statuses/{statusId}")]
+        public async Task<IActionResult> mastoApi(string id, string statusId)
+        {
+            if (!long.TryParse(statusId, out var parsedStatusId))
+                return NotFound();
+
+            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
+            if (tweet == null)
+                return NotFound();
+            
+            var user = await _twitterUserService.GetUserAsync(tweet.Author.Acct);
+            var status = _statusService.GetActivity(tweet.Author.Acct, tweet);
+            var res = new MastodonPostApi()
+            {
+                id = parsedStatusId,
+                content = status.apObject.content,
+                created_at = status.published,
+                uri = $"https://{_instanceSettings.Domain}/users/{tweet.Author.Acct.ToLower()}/statuses/{tweet.Id}",
+                url = $"https://{_instanceSettings.Domain}/@{tweet.Author.Acct.ToLower()}/{tweet.Id}",
+                account = new MastodonUserApi()
+                {
+                    id = user.Id,
+                    username = user.Acct,
+                    acct = user.Acct,
+                    display_name = user.Name,
+                    note = user.Description,
+                    url = $"https://{_instanceSettings.Domain}/@{tweet.Author.Acct.ToLower()}",
+                    avatar = user.ProfileImageUrl,
+                    avatar_static = user.ProfileImageUrl,
+                    header = user.ProfileBannerURL,
+                    header_static = user.ProfileBannerURL,
+                }
+            };
+
+
+            var jsonApUser = JsonSerializer.Serialize(res);
+            return Content(jsonApUser, "application/activity+json; charset=utf-8");
+        }
         [Route("/users/{id}/statuses/{statusId}/activity")]
         public async Task<IActionResult> Activity(string id, string statusId)
         {
@@ -181,8 +234,6 @@ namespace BirdsiteLive.Controllers
             if (tweet == null)
                 return NotFound();
             
-            var user = await _twitterUserService.GetUserAsync(id);
-
             var status = _statusService.GetActivity(id, tweet);
 
             var jsonApUser = JsonSerializer.Serialize(status);

src/BirdsiteLive/Controllers/WellKnownController.cs

@@ -201,26 +201,30 @@ namespace BirdsiteLive.Controllers
             if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain)
                 return NotFound();
 
-            try
+            var user = await _twitterUserDal.GetTwitterUserAsync(name);
+            if (user is null)
             {
-                await _twitterUserService.GetUserAsync(name);
-            }
-            catch (UserNotFoundException)
-            {
-                return NotFound();
-            }
-            catch (UserHasBeenSuspendedException)
-            {
-                return NotFound();
-            }
-            catch (RateLimitExceededException)
-            {
-                return new ObjectResult("Too Many Requests") { StatusCode = 429 };
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, "Exception getting {Name}", name);
-                throw;
+                try
+                {
+                    await _twitterUserService.GetUserAsync(name);
+                }
+                catch (UserNotFoundException)
+                {
+                    return NotFound();
+                }
+                catch (UserHasBeenSuspendedException)
+                {
+                    return NotFound();
+                }
+                catch (RateLimitExceededException)
+                {
+                    return new ObjectResult("Too Many Requests") { StatusCode = 429 };
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(e, "Exception getting {Name}", name);
+                    throw;
+                }
             }
 
             var actorUrl = UrlFactory.GetActorUrl(_settings.Domain, name);

src/BirdsiteLive/Models/DisplayTwitterUser.cs

@@ -8,7 +8,10 @@
         public string Url { get; set; }
         public string ProfileImageUrl { get; set; }
         public bool Protected { get; set; }
+        public int FollowerCount { get; set; }
+        public string MostPopularServer { get; set; }
 
         public string InstanceHandle { get; set; }
+        public string FediverseAccount { get; set; }
     }
 }

src/BirdsiteLive/Models/MastodonPostApi.cs

@@ -0,0 +1,60 @@
+using System;
+using System.Collections.Generic;
+
+namespace BirdsiteLive.Models;
+
+public class MastodonPostApi
+{
+    public long id { get; set; }
+    public string created_at { get; set; }
+    public long? in_reply_to_id { get; set; } = null;
+    public long? in_reply_to_account_id { get; set; } = null;
+    public bool sensitive { get; set; } = false;
+    public string spoiler_text { get; set; } = "";
+    public string visibility { get; set; } = "public";
+    public string language { get; set; } = "en";
+    public string uri { get; set; } 
+    public string url { get; set; } 
+    public int replies_count { get; set; } = 0;
+    public int reblogs_count { get; set; } = 0;
+    public int favorite_count { get; set; } = 0;
+    public string content { get; set; }
+    public MastodonUserApi account { get; set; }
+    public MastodonAppApi application { get; } = new MastodonAppApi();
+    
+    public List<MastodonAppApi> media_attachments { get; set; } = new List<MastodonAppApi>();
+    public List<MastodonAppApi> mentions { get; set; } = new List<MastodonAppApi>();
+    public List<MastodonAppApi> tags { get; set; } = new List<MastodonAppApi>();
+    public List<MastodonAppApi> emojis { get; set; } = new List<MastodonAppApi>();
+    public string card { get; set; }
+    public string poll { get; set; }
+    public string reblog { get; set; }
+}
+public class MastodonUserApi
+{
+    public long id { get; set; }
+    public string username { get; set; }
+    public string acct { get; set; }
+    public string display_name { get; set; }
+    public bool locked { get; set; } = false;
+    public bool bot { get; set; } = true;
+    public bool group { get; set; } = false;
+    public string note { get; set; }
+    public string url { get; set; }
+    public string avatar { get; set; }
+    public string avatar_static { get; set; }
+    public string header { get; set; }
+    public string header_static { get; set; }
+    public int followers_count { get; set; } = 0;
+    public int following_count { get; set; } = 0;
+    public int statuses_count { get; set; } = 0;
+
+    public List<MastodonAppApi> fields { get; set; } = new List<MastodonAppApi>();
+    public List<MastodonAppApi> emojis { get; set; } = new List<MastodonAppApi>();
+}
+
+public class MastodonAppApi
+{
+    public string name { get; set; } = "bird.makeup";
+    public string url { get; set; } = "https://bird.makeup/";
+}

src/BirdsiteLive/Services/CachedStatisticsService.cs

@@ -15,7 +15,7 @@ namespace BirdsiteLive.Services
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
 
-        private static CachedStatistics _cachedStatistics;
+        private static Task<CachedStatistics> _cachedStatistics;
         private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
@@ -24,28 +24,36 @@ namespace BirdsiteLive.Services
             _twitterUserDal = twitterUserDal;
             _instanceSettings = instanceSettings;
             _followersDal = followersDal;
+            _cachedStatistics = CreateStats();
         }
         #endregion
 
         public async Task<CachedStatistics> GetStatisticsAsync()
         {
-            if (_cachedStatistics == null ||
-                (DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
+            var stats = await _cachedStatistics;
+            if ((DateTime.UtcNow - stats.RefreshedTime).TotalMinutes > 5)
             {
-                var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
-                var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
-                var fediverseUsers = await _followersDal.GetFollowersCountAsync();
-
-                _cachedStatistics = new CachedStatistics
-                {
-                    RefreshedTime = DateTime.UtcNow,
-                    SyncLag = twitterSyncLag,
-                    TwitterUsers = twitterUserCount,
-                    FediverseUsers = fediverseUsers
-                };
+                _cachedStatistics = CreateStats();
             }
 
-            return _cachedStatistics;
+            return stats;
+        }
+
+        private async Task<CachedStatistics> CreateStats()
+        {
+            var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
+            var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
+            var fediverseUsers = await _followersDal.GetFollowersCountAsync();
+
+            var stats = new CachedStatistics
+            {
+                RefreshedTime = DateTime.UtcNow,
+                SyncLag = twitterSyncLag,
+                TwitterUsers = twitterUserCount,
+                FediverseUsers = fediverseUsers
+            };
+        
+            return stats;
         }
     }
 

src/BirdsiteLive/Startup.cs

@@ -9,6 +9,7 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
 using BirdsiteLive.DAL.Postgres.Settings;
 using BirdsiteLive.Models;
+using BirdsiteLive.Services;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Tools;
 using Lamar;
@@ -89,6 +90,8 @@ namespace BirdsiteLive
             services.For<ITwitterUserService>().Use<TwitterUserService>().Singleton();
 
             services.For<ITwitterAuthenticationInitializer>().Use<TwitterAuthenticationInitializer>().Singleton();
+            
+            services.For<ICachedStatisticsService>().Use<CachedStatisticsService>().Singleton();
 
             services.Scan(_ =>
             {

src/BirdsiteLive/Views/Home/Index.cshtml

@@ -12,18 +12,21 @@
     </p>
     
     <form method="POST">
-        @*<div class="form-group">
-            <label for="exampleInputEmail1">Email address</label>
-            <input type="email" class="form-control" id="exampleInputEmail1" aria-describedby="emailHelp" placeholder="Enter email">
-            <small id="emailHelp" class="form-text text-muted">We'll never share your email with anyone else.</small>
-        </div>*@
         <div class="form-group">
-            @*<label for="exampleInputPassword1">Password</label>*@
             <input type="text" class="form-control col-8 col-sm-8 col-md-6 col-lg-4 mx-auto" id="handle" name="handle" autocomplete="off" placeholder="Twitter Handle">
         </div>
         <button type="submit" class="btn btn-primary">Show</button>
     </form>
 
+    <p style = "padding-top: 100px;">
+        <br />
+        bird.makeup is made with ❤️ by <a href="https://social.librem.one/@@vincent"> Vincent Cloutier</a> in 🇨🇦
+        <br />
+        <br />
+        Many thanks to our top Patreon supporters: <br/>
+        <a href="https://mstdn-social.com/@@fishcharlie">Charlie Fish</a>
+        
+    </p>
 
     @*@if (HtmlHelperExtensions.IsDebug())
     {

src/BirdsiteLive/Views/Users/Index.cshtml

@@ -28,6 +28,9 @@
         </div>
     </a>
     <br />
+    <div>
+        This account has @ViewData.Model.FollowerCount followers on the fediverse. The server with the most followers for this account is: @ViewData.Model.MostPopularServer
+    </div>
     <br />
 
     @if (ViewData.Model.Protected)
@@ -44,4 +47,13 @@
             <input type="text" name="textbox" value="@ViewData.Model.InstanceHandle" onclick="this.select()" class="form-control" readonly />
         </div>
     }
+    @if (ViewData.Model.FediverseAccount != null)
+    {
+        <br/>
+        <br/>
+        <div>
+            <p>There is a native fediverse account associated with this Twitter account:</p>
+            <input type="text" name="textbox" value="@ViewData.Model.FediverseAccount" onclick="this.select()" class="form-control" readonly />
+        </div>
+    }
 </div>

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs

@@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
     public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
     {
         private readonly PostgresTools _tools;
-        private readonly Version _currentVersion = new Version(2, 5);
+        private readonly Version _currentVersion = new Version(3, 0);
         private const string DbVersionType = "db-version";
 
         #region Ctor
@@ -136,7 +136,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 new Tuple<Version, Version>(new Version(2,1), new Version(2,2)),
                 new Tuple<Version, Version>(new Version(2,2), new Version(2,3)),
                 new Tuple<Version, Version>(new Version(2,3), new Version(2,4)),
-                new Tuple<Version, Version>(new Version(2,4), new Version(2,5))
+                new Tuple<Version, Version>(new Version(2,4), new Version(2,5)),
+                new Tuple<Version, Version>(new Version(2,5), new Version(3,0))
             };
         }
 
@@ -179,6 +180,48 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 await _tools.ExecuteRequestAsync(alterTwitterUserId);
 
             }
+            else if (from == new Version(2, 5) && to == new Version(3, 0))
+            {
+                var dropFollowingSyncStatus = $@"ALTER TABLE {_settings.FollowersTableName} DROP COLUMN followingssyncstatus";
+                await _tools.ExecuteRequestAsync(dropFollowingSyncStatus);
+
+                var dropLastTweet = $@"ALTER TABLE {_settings.TwitterUserTableName} DROP COLUMN lasttweetsynchronizedforallfollowersid";
+                await _tools.ExecuteRequestAsync(dropLastTweet);
+                
+                var addFediverseEquivalent = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD fediverseaccount text";
+                await _tools.ExecuteRequestAsync(addFediverseEquivalent);
+                
+                var createWorkers = $@"CREATE TABLE {_settings.WorkersTableName}
+                (
+                    id BIGINT PRIMARY KEY,
+                    rangeStart INTEGER,
+                    rangeEnd INTEGER,
+                    lastSeen TIMESTAMP (2) WITHOUT TIME ZONE,
+                    name text
+                );";
+                await _tools.ExecuteRequestAsync(createWorkers);
+                
+                var createWorkerInstance = $@" INSERT INTO {_settings.WorkersTableName} (id,rangeStart,rangeEnd) VALUES(0,0, 100) ";
+                await _tools.ExecuteRequestAsync(createWorkerInstance);
+                
+                var createInstagram = $@"CREATE TABLE {_settings.InstagramUserTableName}
+                (
+                    id SERIAL PRIMARY KEY,
+                    acct VARCHAR(20) UNIQUE, 
+                    
+                    data JSONB
+                );";
+                await _tools.ExecuteRequestAsync(createInstagram);
+                
+                var createInstagramPost = $@"CREATE TABLE {_settings.CachedInstaPostsTableName}
+                (
+                    id VARCHAR(30) PRIMARY KEY,
+                    acct VARCHAR(20) UNIQUE, 
+                    
+                    data JSONB
+                );";
+                await _tools.ExecuteRequestAsync(createInstagramPost);
+            }
             else
             {
                 throw new NotImplementedException();
@@ -207,7 +250,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 $@"DROP TABLE {_settings.DbVersionTableName};",
                 $@"DROP TABLE {_settings.TwitterUserTableName};",
                 $@"DROP TABLE {_settings.FollowersTableName};",
-                $@"DROP TABLE {_settings.CachedTweetsTableName};"
+                $@"DROP TABLE {_settings.CachedTweetsTableName};",
+                $@"DROP TABLE {_settings.InstagramUserTableName};",
+                $@"DROP TABLE {_settings.CachedInstaPostsTableName};",
+                $@"DROP TABLE {_settings.WorkersTableName};"
             };
 
             foreach (var r in dropsRequests)

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs

@@ -21,12 +21,9 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
         }
         #endregion
 
-        public async Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null, Dictionary<int, long> followingSyncStatus = null)
+        public async Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null)
         {
             if(followings == null) followings = new int[0];
-            if(followingSyncStatus == null) followingSyncStatus = new Dictionary<int, long>();
-
-            var serializedDic = JsonSerializer.Serialize(followingSyncStatus);
 
             acct = acct.ToLowerInvariant();
             host = host.ToLowerInvariant();
@@ -34,8 +31,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             using (var dbConnection = Connection)
             {
                 await dbConnection.ExecuteAsync(
-                    $"INSERT INTO {_settings.FollowersTableName} (acct,host,inboxRoute,sharedInboxRoute,followings,followingsSyncStatus,actorId) VALUES(@acct,@host,@inboxRoute,@sharedInboxRoute,@followings,CAST(@followingsSyncStatus as json),@actorId)",
-                    new { acct, host, inboxRoute, sharedInboxRoute, followings, followingsSyncStatus = serializedDic, actorId });
+                    $"INSERT INTO {_settings.FollowersTableName} (acct,host,inboxRoute,sharedInboxRoute,followings,actorId) VALUES(@acct,@host,@inboxRoute,@sharedInboxRoute,@followings,@actorId)",
+                    new { acct, host, inboxRoute, sharedInboxRoute, followings, actorId });
             }
         }
 
@@ -78,13 +75,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             if (!await reader.ReadAsync())
                 return null;
 
-            string syncStatusString = reader["followingsSyncStatus"] as string;
-            var syncStatus = System.Text.Json.JsonSerializer.Deserialize<Dictionary<int, long>>(syncStatusString);
             return new Follower
             {
                 Id = reader["id"] as int? ?? default,
                 Followings = (reader["followings"] as int[] ?? new int[0]).ToList(),
-                FollowingsSyncStatus = syncStatus,
                 ActorId = reader["actorId"] as string,
                 Acct = reader["acct"] as string,
                 Host = reader["host"] as string,
@@ -111,13 +105,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             var followers = new List<Follower>();
             while (await reader.ReadAsync())
             {
-                string syncStatusString = reader["followingsSyncStatus"] as string;
-                var syncStatus = System.Text.Json.JsonSerializer.Deserialize<Dictionary<int, long>>(syncStatusString);
                 followers.Add(new Follower
                 {
                     Id = reader["id"] as int? ?? default,
                     Followings = (reader["followings"] as int[] ?? new int[0]).ToList(),
-                    FollowingsSyncStatus = syncStatus,
                     ActorId = reader["actorId"] as string,
                     Acct = reader["acct"] as string,
                     Host = reader["host"] as string,
@@ -147,14 +138,12 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             if (follower == default) throw new ArgumentException("follower");
             if (follower.Id == default) throw new ArgumentException("id");
 
-            var serializedDic = System.Text.Json.JsonSerializer.Serialize(follower.FollowingsSyncStatus);
-            var query = $"UPDATE {_settings.FollowersTableName} SET followings = $1, followingsSyncStatus = CAST($2 as json), postingErrorCount = $3 WHERE id = $4";
+            var query = $"UPDATE {_settings.FollowersTableName} SET followings = $1, postingErrorCount = $2 WHERE id = $3";
             await using var connection = DataSource.CreateConnection();
             await connection.OpenAsync();
             await using var command = new NpgsqlCommand(query, connection) {
                 Parameters = { 
                     new() { Value = follower.Followings}, 
-                    new() { Value = serializedDic}, 
                     new() { Value = follower.PostingErrorCount}, 
                     new() { Value = follower.Id}
                 }
@@ -204,7 +193,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 ActorId = follower.ActorId,
                 SharedInboxRoute = follower.SharedInboxRoute,
                 Followings = follower.Followings.ToList(),
-                FollowingsSyncStatus = JsonSerializer.Deserialize<Dictionary<int,long>>(follower.FollowingsSyncStatus),
                 PostingErrorCount = follower.PostingErrorCount
             };
         }
@@ -212,10 +200,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
     internal class SerializedFollower {
         public int Id { get; set; }
-
         public int[] Followings { get; set; }
-        public string FollowingsSyncStatus { get; set; }
-
         public string Acct { get; set; }
         public string Host { get; set; }
         public string InboxRoute { get; set; }

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs

@@ -27,8 +27,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             using (var dbConnection = Connection)
             {
                 await dbConnection.ExecuteAsync(
-                    $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)",
-                    new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId });
+                    $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId) VALUES(@acct,@lastTweetPostedId)",
+                    new { acct, lastTweetPostedId });
             }
         }
 
@@ -53,9 +53,9 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 Acct = reader["acct"] as string,
                 TwitterUserId = reader["twitterUserId"] as long? ?? default,
                 LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
-                LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
                 LastSync = reader["lastSync"] as DateTime? ?? default,
                 FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
+                FediAcct = reader["fediverseaccount"] as string,
             };
 
         }
@@ -79,7 +79,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 Acct = reader["acct"] as string,
                 TwitterUserId = reader["twitterUserId"] as long? ?? default,
                 LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
-                LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
                 LastSync = reader["lastSync"] as DateTime? ?? default,
                 FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
             };
@@ -91,7 +90,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
             using (var dbConnection = Connection)
             {
-                var result = (await dbConnection.QueryAsync<TimeSpan>(query)).FirstOrDefault();
+                var result = (await dbConnection.QueryAsync<TimeSpan?>(query)).FirstOrDefault() ?? TimeSpan.Zero;
                 return result;
             }
         }
@@ -118,14 +117,20 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber)
+        public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m)
         {
-            var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id  ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
+            var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id WHERE mod(id, $2) >= $3 AND mod(id, $2) <= $4 ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
 
             await using var connection = DataSource.CreateConnection();
             await connection.OpenAsync();
             await using var command = new NpgsqlCommand(query, connection) {
-                Parameters = { new() { Value = maxNumber}}
+                Parameters =
+                {
+                    new() { Value = maxNumber},
+                    new() { Value = m},
+                    new() { Value = nStart},
+                    new() { Value = nEnd}
+                }
             };
             var reader = await command.ExecuteReaderAsync();
             var results = new List<SyncTwitterUser>();
@@ -137,7 +142,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                         Acct = reader["acct"] as string,
                         TwitterUserId = reader["twitterUserId"] as long? ?? default,
                         LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
-                        LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
                         LastSync = reader["lastSync"] as DateTime? ?? default,
                         FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
                     }
@@ -169,6 +173,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
+        public async Task UpdateTwitterUserFediAcctAsync(string twitterUsername, string fediUsername)
+        {
+            if(twitterUsername == default) throw new ArgumentException("id");
+
+            var query = $"UPDATE {_settings.TwitterUserTableName} SET fediverseaccount = $1 WHERE acct = $2";
+            await using var connection = DataSource.CreateConnection();
+            await connection.OpenAsync();
+            await using var command = new NpgsqlCommand(query, connection) {
+                Parameters = { new() { Value = fediUsername}, new() { Value = twitterUsername}}
+            };
+
+            await command.ExecuteNonQueryAsync();
+        }
         public async Task UpdateTwitterUserIdAsync(string username, long twitterUserId)
         {
             if(username == default) throw new ArgumentException("id");
@@ -183,21 +200,19 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
             await command.ExecuteNonQueryAsync();
         }
-        public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
+        public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, int fetchingErrorCount, DateTime lastSync)
         {
             if(id == default) throw new ArgumentException("id");
             if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId");
-            if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId");
             if(lastSync == default) throw new ArgumentException("lastSync");
 
-            var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = $1, lastTweetSynchronizedForAllFollowersId = $2, fetchingErrorCount = $3, lastSync = $4 WHERE id = $5";
+            var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = $1, fetchingErrorCount = $2, lastSync = $3 WHERE id = $4";
 
             await using var connection = DataSource.CreateConnection();
             await connection.OpenAsync();
             await using var command = new NpgsqlCommand(query, connection) {
                 Parameters = { 
                     new() { Value = lastTweetPostedId}, 
-                    new() { Value = lastTweetSynchronizedForAllFollowersId},
                     new() { Value = fetchingErrorCount},
                     new() { Value = lastSync.ToUniversalTime()},
                     new() { Value = id},
@@ -209,7 +224,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
         public async Task UpdateTwitterUserAsync(SyncTwitterUser user)
         {
-            await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync);
+            await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, user.LastSync);
         }
 
         public async Task DeleteTwitterUserAsync(string acct)

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs

@@ -6,6 +6,9 @@
 
         public string DbVersionTableName { get; set; } = "db_version";
         public string TwitterUserTableName { get; set; } = "twitter_users";
+        public string InstagramUserTableName { get; set; } = "instagram_users";
+        public string WorkersTableName { get; set; } = "workers";
+        public string CachedInstaPostsTableName { get; set; } = "cached_insta_posts";
         public string FollowersTableName { get; set; } = "followers";
         public string CachedTweetsTableName { get; set; } = "cached_tweets";
     }

src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs

@@ -7,8 +7,7 @@ namespace BirdsiteLive.DAL.Contracts
     public interface IFollowersDal
     {
         Task<Follower> GetFollowerAsync(string acct, string host);
-        Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null,
-            Dictionary<int, long> followingSyncStatus = null);
+        Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null);
         Task<Follower[]> GetFollowersAsync(int followedUserId);
         Task<Follower[]> GetAllFollowersAsync();
         Task UpdateFollowerAsync(Follower follower);

src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs

@@ -9,10 +9,10 @@ namespace BirdsiteLive.DAL.Contracts
         Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
         Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
         Task<SyncTwitterUser> GetTwitterUserAsync(int id);
-        Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber);
+        Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
-        Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
+        Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, int fetchingErrorCount, DateTime lastSync);
         Task UpdateTwitterUserIdAsync(string username, long twitterUserId);
         Task UpdateTwitterUserAsync(SyncTwitterUser user);
         Task DeleteTwitterUserAsync(string acct);

src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs

@@ -7,7 +7,6 @@ namespace BirdsiteLive.DAL.Models
         public int Id { get; set; }
         
         public List<int> Followings { get; set; }
-        public Dictionary<int, long> FollowingsSyncStatus { get; set; }
 
         public string ActorId { get; set; }
         public string Acct { get; set; }

src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs

@@ -7,9 +7,9 @@ namespace BirdsiteLive.DAL.Models
         public int Id { get; set; }
         public long TwitterUserId { get; set; }
         public string Acct { get; set; }
+        public string FediAcct { get; set; }
 
         public long LastTweetPostedId { get; set; }
-        public long LastTweetSynchronizedForAllFollowersId { get; set; }
 
         public DateTime LastSync { get; set; }
 

src/Tests/BSLManager.Tests/BSLManager.Tests.csproj

@@ -1,20 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
-
-    <IsPackable>false</IsPackable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
-    <PackageReference Include="coverlet.collector" Version="1.3.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\BSLManager\BSLManager.csproj" />
-  </ItemGroup>
-
-</Project>

src/Tests/BSLManager.Tests/Domain/FollowersListStateTests.cs

@@ -1,307 +0,0 @@
-using System.Collections.Generic;
-using System.Linq;
-using BirdsiteLive.DAL.Models;
-using BSLManager.Domain;
-using Microsoft.VisualStudio.TestTools.UnitTesting;
-
-namespace BSLManager.Tests
-{
-    [TestClass]
-    public class FollowersListStateTests
-    {
-        [TestMethod]
-        public void FilterBy()
-        {
-            #region Stub
-            var followers = new List<Follower>
-            {
-                new Follower
-                {
-                    Id = 0,
-                    Acct = "test",
-                    Host = "host1",
-                    Followings = new List<int>()
-                }, 
-                new Follower
-                {
-                    Id = 1,
-                    Acct = "test",
-                    Host = "host2",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 2,
-                    Acct = "user1",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 3,
-                    Acct = "user2",
-                    Host = "host1",
-                    Followings = new List<int>()
-                }
-            };
-            #endregion
-
-            var state = new FollowersListState();
-            state.Load(followers);
-
-            state.FilterBy("test");
-            
-            #region Validate
-            Assert.AreEqual(2, state.GetDisplayableList().Count);
-            #endregion
-        }
-
-        [TestMethod]
-        public void FilterBy_GetElement()
-        {
-            #region Stub
-            var followers = new List<Follower>
-            {
-                new Follower
-                {
-                    Id = 0,
-                    Acct = "test",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 1,
-                    Acct = "test",
-                    Host = "host2",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 2,
-                    Acct = "user1",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 3,
-                    Acct = "user2",
-                    Host = "host1",
-                    Followings = new List<int>()
-                }
-            };
-            #endregion
-
-            var state = new FollowersListState();
-            state.Load(followers);
-
-            state.FilterBy("test");
-            var el = state.GetElementAt(1);
-
-            #region Validate
-            Assert.AreEqual(followers[1].Id, el.Id);
-            #endregion
-        }
-
-        [TestMethod]
-        public void GetElement()
-        {
-            #region Stub
-            var followers = new List<Follower>
-            {
-                new Follower
-                {
-                    Id = 0,
-                    Acct = "test",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 1,
-                    Acct = "test",
-                    Host = "host2",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 2,
-                    Acct = "user1",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 3,
-                    Acct = "user2",
-                    Host = "host1",
-                    Followings = new List<int>()
-                }
-            };
-            #endregion
-
-            var state = new FollowersListState();
-            state.Load(followers);
-
-            var el = state.GetElementAt(2);
-
-            #region Validate
-            Assert.AreEqual(followers[2].Id, el.Id);
-            #endregion
-        }
-
-        [TestMethod]
-        public void FilterBy_RemoveAt()
-        {
-            #region Stub
-            var followers = new List<Follower>
-            {
-                new Follower
-                {
-                    Id = 0,
-                    Acct = "test",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 1,
-                    Acct = "test",
-                    Host = "host2",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 2,
-                    Acct = "user1",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 3,
-                    Acct = "user2",
-                    Host = "host1",
-                    Followings = new List<int>()
-                }
-            };
-            #endregion
-
-            var state = new FollowersListState();
-            state.Load(followers.ToList());
-
-            state.FilterBy("test");
-            state.RemoveAt(1);
-
-            var list = state.GetDisplayableList();
-
-            #region Validate
-            Assert.AreEqual(1, list.Count);
-            Assert.IsTrue(list[0].Contains("@test@host1"));
-            #endregion
-        }
-
-        [TestMethod]
-        public void RemoveAt()
-        {
-            #region Stub
-            var followers = new List<Follower>
-            {
-                new Follower
-                {
-                    Id = 0,
-                    Acct = "test",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 1,
-                    Acct = "test",
-                    Host = "host2",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 2,
-                    Acct = "user1",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 3,
-                    Acct = "user2",
-                    Host = "host1",
-                    Followings = new List<int>()
-                }
-            };
-            #endregion
-
-            var state = new FollowersListState();
-            state.Load(followers.ToList());
-
-            state.RemoveAt(1);
-
-            var list = state.GetDisplayableList();
-
-            #region Validate
-            Assert.AreEqual(3, list.Count);
-            Assert.IsTrue(list[0].Contains("@test@host1"));
-            Assert.IsFalse(list[1].Contains("@test@host2"));
-            #endregion
-        }
-
-        [TestMethod]
-        public void FilterBy_ResetFilter()
-        {
-            #region Stub
-            var followers = new List<Follower>
-            {
-                new Follower
-                {
-                    Id = 0,
-                    Acct = "test",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 1,
-                    Acct = "test",
-                    Host = "host2",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 2,
-                    Acct = "user1",
-                    Host = "host1",
-                    Followings = new List<int>()
-                },
-                new Follower
-                {
-                    Id = 3,
-                    Acct = "user2",
-                    Host = "host1",
-                    Followings = new List<int>()
-                }
-            };
-            #endregion
-
-            var state = new FollowersListState();
-            state.Load(followers.ToList());
-            
-            #region Validate
-            state.FilterBy("data");
-            var list = state.GetDisplayableList();
-            Assert.AreEqual(0, list.Count);
-
-            state.FilterBy(string.Empty);
-            list = state.GetDisplayableList();
-            Assert.AreEqual(4, list.Count);
-            #endregion
-        }
-    }
-}

src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
 

src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs

@@ -19,6 +19,9 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base
                 CachedTweetsTableName = "CachedTweetsTableName" + RandomGenerator.GetString(4),
                 FollowersTableName = "FollowersTableName" + RandomGenerator.GetString(4),
                 TwitterUserTableName = "TwitterUserTableName" + RandomGenerator.GetString(4),
+                InstagramUserTableName = "InstagramUserTableName" + RandomGenerator.GetString(4),
+                CachedInstaPostsTableName = "CachedInstaPosts" + RandomGenerator.GetString(4),
+                WorkersTableName = "workers" + RandomGenerator.GetString(4),
             };
             _tools = new PostgresTools(_settings);
         }

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs

@@ -44,7 +44,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             var result = await dal.GetFollowerAsync(acct, host);
 
@@ -57,9 +57,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(0, result.PostingErrorCount);
             Assert.AreEqual(following.Length, result.Followings.Count);
             Assert.AreEqual(following[0], result.Followings[0]);
-            Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count);
-            Assert.AreEqual(followingSync.First().Key, result.FollowingsSyncStatus.First().Key);
-            Assert.AreEqual(followingSync.First().Value, result.FollowingsSyncStatus.First().Value);
         }
 
         [TestMethod]
@@ -72,7 +69,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, null, null);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, null);
 
             var result = await dal.GetFollowerAsync(acct, host);
 
@@ -83,7 +80,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(inboxRoute, result.InboxRoute);
             Assert.AreEqual(sharedInboxRoute, result.SharedInboxRoute);
             Assert.AreEqual(0, result.Followings.Count);
-            Assert.AreEqual(0, result.FollowingsSyncStatus.Count);
             Assert.AreEqual(0, result.PostingErrorCount);
         }
 
@@ -112,7 +108,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             var result = await dal.GetFollowerAsync(acct, host);
 
@@ -124,9 +120,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(sharedInboxRoute, result.SharedInboxRoute);
             Assert.AreEqual(following.Length, result.Followings.Count);
             Assert.AreEqual(following[0], result.Followings[0]);
-            Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count);
-            Assert.AreEqual(followingSync.First().Key, result.FollowingsSyncStatus.First().Key);
-            Assert.AreEqual(followingSync.First().Value, result.FollowingsSyncStatus.First().Value);
             Assert.AreEqual(0, result.PostingErrorCount);
         }
 
@@ -143,7 +136,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var inboxRoute = "/myhandle1/inbox";
             var sharedInboxRoute = "/inbox";
             var actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             //User 2 
             acct = "myhandle2";
@@ -152,7 +145,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle2/inbox";
             sharedInboxRoute = "/inbox2";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             //User 2 
             acct = "myhandle3";
@@ -161,7 +154,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle3/inbox";
             sharedInboxRoute = "/inbox3";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             var result = await dal.GetFollowersAsync(2);
             Assert.AreEqual(2, result.Length);
@@ -186,7 +179,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var inboxRoute = "/myhandle1/inbox";
             var sharedInboxRoute = "/inbox";
             var actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             //User 2 
             acct = "myhandle2";
@@ -195,7 +188,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle2/inbox";
             sharedInboxRoute = "/inbox2";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             //User 2 
             acct = "myhandle3";
@@ -204,7 +197,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle3/inbox";
             sharedInboxRoute = "/inbox3";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             var result = await dal.GetAllFollowersAsync();
             Assert.AreEqual(3, result.Length);
@@ -226,7 +219,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var inboxRoute = "/myhandle1/inbox";
             var sharedInboxRoute = "/inbox";
             var actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             //User 2 
             acct = "myhandle2";
@@ -235,7 +228,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle2/inbox";
             sharedInboxRoute = "/inbox2";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             //User 3
             acct = "myhandle3";
@@ -244,7 +237,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle3/inbox";
             sharedInboxRoute = "/inbox3";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             result = await dal.GetFollowersCountAsync();
             Assert.AreEqual(3, result);
@@ -266,7 +259,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var inboxRoute = "/myhandle1/inbox";
             var sharedInboxRoute = "/inbox";
             var actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             //User 2 
             acct = "myhandle2";
@@ -275,7 +268,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle2/inbox";
             sharedInboxRoute = "/inbox2";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             var follower = await dal.GetFollowerAsync(acct, host);
             follower.PostingErrorCount = 1;
@@ -288,7 +281,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle3/inbox";
             sharedInboxRoute = "/inbox3";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
 
             follower = await dal.GetFollowerAsync(acct, host);
             follower.PostingErrorCount = 50;
@@ -315,7 +308,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
             var result = await dal.GetFollowerAsync(acct, host);
 
             var updatedFollowing = new List<int> { 12, 19, 23, 24 };
@@ -326,7 +319,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 {24, 173L}
             };
             result.Followings = updatedFollowing.ToList();
-            result.FollowingsSyncStatus = updatedFollowingSync;
             result.PostingErrorCount = 10;
             
             await dal.UpdateFollowerAsync(result);
@@ -334,9 +326,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(updatedFollowing.Count, result.Followings.Count);
             Assert.AreEqual(updatedFollowing[0], result.Followings[0]);
-            Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count);
-            Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key);
-            Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value);
             Assert.AreEqual(10, result.PostingErrorCount);
         }
 
@@ -357,7 +346,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
             var result = await dal.GetFollowerAsync(acct, host);
 
             var updatedFollowing = new List<int> { 12, 19, 23, 24 };
@@ -368,7 +357,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 {24, 173L}
             };
             result.Followings = updatedFollowing.ToList();
-            result.FollowingsSyncStatus = updatedFollowingSync;
             result.PostingErrorCount = 32768;
 
             await dal.UpdateFollowerAsync(result);
@@ -376,9 +364,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(updatedFollowing.Count, result.Followings.Count);
             Assert.AreEqual(updatedFollowing[0], result.Followings[0]);
-            Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count);
-            Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key);
-            Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value);
             Assert.AreEqual(32768, result.PostingErrorCount);
         }
 
@@ -399,7 +384,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
             var result = await dal.GetFollowerAsync(acct, host);
 
             var updatedFollowing = new[] { 12, 19 };
@@ -409,7 +394,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 {19, 171L}
             };
             result.Followings = updatedFollowing.ToList();
-            result.FollowingsSyncStatus = updatedFollowingSync;
             result.PostingErrorCount = 5;
 
             await dal.UpdateFollowerAsync(result);
@@ -417,9 +401,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(updatedFollowing.Length, result.Followings.Count);
             Assert.AreEqual(updatedFollowing[0], result.Followings[0]);
-            Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count);
-            Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key);
-            Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value);
             Assert.AreEqual(5, result.PostingErrorCount);
         }
 
@@ -440,7 +421,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
             var result = await dal.GetFollowerAsync(acct, host);
             Assert.AreEqual(0, result.PostingErrorCount);
 
@@ -495,7 +476,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
             var result = await dal.GetFollowerAsync(acct, host);
             Assert.IsNotNull(result);
 
@@ -522,7 +503,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
             var result = await dal.GetFollowerAsync(acct, host);
             Assert.IsNotNull(result);
 

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs

@@ -48,7 +48,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(lastTweetId, result.LastTweetPostedId);
-            Assert.AreEqual(lastTweetId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(0, result.FetchingErrorCount);
             Assert.IsTrue(result.Id > 0);
         }
@@ -67,7 +66,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(acct, resultById.Acct);
             Assert.AreEqual(lastTweetId, resultById.LastTweetPostedId);
-            Assert.AreEqual(lastTweetId, resultById.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(result.Id, resultById.Id);
         }
 
@@ -87,13 +85,12 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var updatedLastSyncId = 1550L;
             var now = DateTime.Now;
             var errors = 15;
-            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now);
+            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, errors, now);
 
             result = await dal.GetTwitterUserAsync(acct);
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
-            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(errors, result.FetchingErrorCount);
             Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
         }
@@ -116,7 +113,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var errors = 15;
 
             result.LastTweetPostedId = updatedLastTweetId;
-            result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId;
             result.FetchingErrorCount = errors;
             result.LastSync = now;
             await dal.UpdateTwitterUserAsync(result);
@@ -125,7 +121,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
-            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(errors, result.FetchingErrorCount);
             Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
         }
@@ -148,7 +143,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var errors = 32768;
 
             result.LastTweetPostedId = updatedLastTweetId;
-            result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId;
             result.FetchingErrorCount = errors;
             result.LastSync = now;
             await dal.UpdateTwitterUserAsync(result);
@@ -157,7 +151,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
-            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(errors, result.FetchingErrorCount);
             Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
         }
@@ -167,7 +160,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(default, default, default, DateTime.UtcNow);
         }
 
         [TestMethod]
@@ -175,23 +168,16 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoLastTweetPostedId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(12, default,  default, DateTime.UtcNow);
         }
 
-        [TestMethod]
-        [ExpectedException(typeof(ArgumentException))]
-        public async Task Update_NoLastTweetSynchronizedForAllFollowersId()
-        {
-            var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow);
-        }
 
         [TestMethod]
         [ExpectedException(typeof(ArgumentException))]
         public async Task Update_NoLastSync()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default);
+            await dal.UpdateTwitterUserAsync(12, 9556, 65,  default);
         }
 
         [TestMethod]
@@ -261,7 +247,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
             Assert.IsFalse(result[0].LastTweetPostedId == default);
-            Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
         }
 
         [TestMethod]
@@ -318,7 +303,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             {
                 var user = allUsers[i];
                 var date = i % 2 == 0 ? oldest : newest;
-                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date);
+                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, 0, date);
             }
 
             var result = await dal.GetAllTwitterUsersAsync(10);
@@ -326,7 +311,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
             Assert.IsFalse(result[0].LastTweetPostedId == default);
-            Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
 
             foreach (var acc in result)
                 Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000);
@@ -349,7 +333,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
             Assert.IsFalse(result[0].LastTweetPostedId == default);
-            Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
         }
 
         [TestMethod]
@@ -382,7 +365,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 if (i == 0 || i == 2 || i == 3)
                 {
                     var t = await dal.GetTwitterUserAsync(acct);
-                    await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now);
+                    await dal.UpdateTwitterUserAsync(t.Id ,1L, 50+i*2, DateTime.Now);
                 }
             }
 

src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <LangVersion>11</LangVersion>
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs

@@ -30,7 +30,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 SharedInboxRoute = followerInbox,
                 InboxRoute = inbox,
                 Followings = new List<int>(),
-                FollowingsSyncStatus = new Dictionary<int, long>()
             };
 
             var twitterUser = new SyncTwitterUser
@@ -38,7 +37,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Id = 2,
                 Acct = twitterName,
                 LastTweetPostedId = -1,
-                LastTweetSynchronizedForAllFollowersId = -1
             };
             #endregion
 
@@ -56,14 +54,12 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                     It.Is<string>(y => y == followerInbox),
                     It.Is<string>(y => y == inbox),
                     It.Is<string>(y => y == actorId),
-                    null,
-                    null))
+                    null ))
                 .Returns(Task.CompletedTask);
 
             followersDalMock
                 .Setup(x => x.UpdateFollowerAsync(
-                    It.Is<Follower>(y => y.Followings.Contains(twitterUser.Id)
-                                         && y.FollowingsSyncStatus[twitterUser.Id] == -1)
+                    It.Is<Follower>(y => y.Followings.Contains(twitterUser.Id))
                 ))
                 .Returns(Task.CompletedTask);
 
@@ -108,7 +104,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 SharedInboxRoute = followerInbox,
                 InboxRoute = inbox,
                 Followings = new List<int>(),
-                FollowingsSyncStatus = new Dictionary<int, long>()
             };
 
             var twitterUser = new SyncTwitterUser
@@ -116,7 +111,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Id = 2,
                 Acct = twitterName,
                 LastTweetPostedId = -1,
-                LastTweetSynchronizedForAllFollowersId = -1
             };
             #endregion
 
@@ -128,8 +122,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
             
             followersDalMock
                 .Setup(x => x.UpdateFollowerAsync(
-                    It.Is<Follower>(y => y.Followings.Contains(twitterUser.Id)
-                                         && y.FollowingsSyncStatus[twitterUser.Id] == -1)
+                    It.Is<Follower>(y => y.Followings.Contains(twitterUser.Id) )
                 ))
                 .Returns(Task.CompletedTask);
 

src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessUnfollowUserTests.cs

@@ -52,7 +52,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Acct = username,
                 Host = domain,
                 Followings = new List<int>(),
-                FollowingsSyncStatus = new Dictionary<int, long>()
             };
             #endregion
 
@@ -91,7 +90,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Acct = username,
                 Host = domain,
                 Followings = new List<int> { 2, 3 },
-                FollowingsSyncStatus = new Dictionary<int, long> { { 2, 460 }, { 3, 563} }
             };
 
             var twitterUser = new SyncTwitterUser
@@ -99,7 +97,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Id = 2,
                 Acct = twitterName,
                 LastTweetPostedId = 460,
-                LastTweetSynchronizedForAllFollowersId = 460
             };
 
             var followerList = new List<Follower>
@@ -117,8 +114,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
 
             followersDalMock
                 .Setup(x => x.UpdateFollowerAsync(
-                    It.Is<Follower>(y => !y.Followings.Contains(twitterUser.Id)
-                                         && !y.FollowingsSyncStatus.ContainsKey(twitterUser.Id))
+                    It.Is<Follower>(y => !y.Followings.Contains(twitterUser.Id) )
                 ))
                 .Returns(Task.CompletedTask);
 
@@ -155,7 +151,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Acct = username,
                 Host = domain,
                 Followings = new List<int> { 2 },
-                FollowingsSyncStatus = new Dictionary<int, long> { { 2, 460 } }
             };
 
             var twitterUser = new SyncTwitterUser
@@ -163,7 +158,6 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Id = 2,
                 Acct = twitterName,
                 LastTweetPostedId = 460,
-                LastTweetSynchronizedForAllFollowersId = 460
             };
 
             var followerList = new List<Follower>();

src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs

@@ -111,7 +111,6 @@ namespace BirdsiteLive.Domain.Tests.Tools
             Assert.IsTrue(result.content.Contains(@"<a href=""https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">https://www.</span><span class=""ellipsis"">eff.org/deeplinks/2020/07/pact</span><span class=""invisible"">-act-not-solution-problem-harmful-online-content</span></a>"));
             #endregion
         }
-        [Ignore]
         [TestMethod]
         public void Extract_FormatUrl_Long2_Test()
         {
@@ -128,7 +127,29 @@ namespace BirdsiteLive.Domain.Tests.Tools
 
             #region Validations
             logger.VerifyAll();
-            Assert.AreEqual(result.content, @"<a href=""https://twitterisgoinggreat.com/#twitters-first-dollar15bn-interest-payment-could-be-due-in-two-weeks"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">https://www.</span><span class=""ellipsis"">eff.org/deeplinks/2020/07/pact</span><span class=""invisible"">-act-not-solution-problem-harmful-online-content</span></a>");
+            Assert.AreEqual(result.content, @"<a href=""https://twitterisgoinggreat.com/#twitters-first-dollar15bn-interest-payment-could-be-due-in-two-weeks"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">https://</span><span class=""ellipsis"">twitterisgoinggreat.com/#twitt</span><span class=""invisible"">ers-first-dollar15bn-interest-payment-could-be-due-in-two-weeks</span></a>");
+            Assert.AreEqual(0, result.tags.Length);
+
+            #endregion
+        }
+        
+        [TestMethod]
+        public void Extract_FormatUrl_Long3_Test()
+        {
+            #region Stubs
+            var message = $"https://domain.name/@WeekInEthNews/1668684659855880193";
+            #endregion
+
+            #region Mocks
+            var logger = new Mock<ILogger<StatusExtractor>>();
+            #endregion
+
+            var service = new StatusExtractor(_settings, logger.Object);
+            var result = service.Extract(message);
+
+            #region Validations
+            logger.VerifyAll();
+            Assert.AreEqual(result.content, @"<a href=""https://domain.name/@WeekInEthNews/1668684659855880193"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">https://</span><span class=""ellipsis"">domain.name/@WeekInEthNews/166</span><span class=""invisible"">8684659855880193</span></a>");
             Assert.AreEqual(0, result.tags.Length);
 
             #endregion
@@ -633,7 +654,7 @@ namespace BirdsiteLive.Domain.Tests.Tools
         public void Extract_Emoji_Test()
         {
             #region Stubs
-            var message = $"😤@mynickname 😎😍🤗🤩😘";
+            var message = $"😤 @mynickname 😎😍🤗🤩😘";
             //var message = $"tests@mynickname";
             #endregion
 
@@ -648,12 +669,13 @@ namespace BirdsiteLive.Domain.Tests.Tools
             logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
             Assert.IsTrue(result.content.Contains(
-                @"😤<span class=""h-card""><a href=""https://domain.name/users/mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
+                @"😤 <span class=""h-card""><a href=""https://domain.name/users/mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
 
             Assert.IsTrue(result.content.Contains(@"😎😍🤗🤩😘"));
             #endregion
         }
 
+        [Ignore]
         [TestMethod]
         public void Extract_Parenthesis_Test()
         {

src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveTwitterAccountActionTests.cs

@@ -27,7 +27,6 @@ namespace BirdsiteLive.Moderation.Tests.Actions
                 {
                     Id = 48,
                     Followings = new List<int>{ 24 },
-                    FollowingsSyncStatus = new Dictionary<int, long> { { 24, 1024 } }
                 }
             };
             #endregion
@@ -84,7 +83,6 @@ namespace BirdsiteLive.Moderation.Tests.Actions
                 {
                     Id = 48,
                     Followings = new List<int>{ 24, 36 },
-                    FollowingsSyncStatus = new Dictionary<int, long> { { 24, 1024 }, { 36, 24 } }
                 }
             };
             #endregion
@@ -100,7 +98,6 @@ namespace BirdsiteLive.Moderation.Tests.Actions
                 .Setup(x => x.UpdateFollowerAsync(
                     It.Is<Follower>(y => y.Id == 48
                         && y.Followings.Count == 1
-                        && y.FollowingsSyncStatus.Count == 1
                     )))
                 .Returns(Task.CompletedTask);
 

src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs

@@ -29,12 +29,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 new SyncTwitterUser(),
             };
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users);
             
@@ -47,7 +54,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -72,12 +79,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 users.Add(new SyncTwitterUser());
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -93,7 +107,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -118,12 +132,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 users.Add(new SyncTwitterUser());
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -139,7 +160,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -160,6 +181,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var buffer = new BufferBlock<UserWithDataToSync[]>();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
@@ -167,6 +192,9 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
@@ -178,7 +206,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 1;
             var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -197,12 +225,19 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var buffer = new BufferBlock<UserWithDataToSync[]>();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
+                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
 
@@ -214,7 +249,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -236,6 +271,10 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             canTokenS.Cancel();
 
             var maxUsers = 1000;
+            var instanceSettings = new InstanceSettings()
+            {
+                n_start = 1,
+            };
             #endregion
 
             #region Mocks
@@ -249,7 +288,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
             processor.WaitFactor = 1;
             await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
         }

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs

@@ -1,227 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Threading;
-using System.Threading.Tasks;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.Pipeline.Models;
-using BirdsiteLive.Pipeline.Processors.SubTasks;
-using BirdsiteLive.Twitter.Models;
-using Castle.DynamicProxy.Contributors;
-using Microsoft.Extensions.Logging;
-using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Moq;
-
-namespace BirdsiteLive.Pipeline.Tests.Processors
-{
-    [TestClass]
-    public class SaveProgressionProcessorTests
-    {
-        [TestMethod]
-        public async Task ProcessAsync_Test()
-        {
-            #region Stubs
-            var user = new SyncTwitterUser
-            {
-                Id = 1
-            };
-            var tweet1 = new ExtractedTweet
-            {
-                Id = 36
-            };
-            var tweet2 = new ExtractedTweet
-            {
-                Id = 37
-            };
-            var follower1 = new Follower
-            {
-                FollowingsSyncStatus = new Dictionary<int, long>
-                {
-                    {1, 37}
-                }
-            };
-
-            var usersWithTweets = new UserWithDataToSync
-            {
-                Tweets = new []
-                {
-                    tweet1,
-                    tweet2
-                },
-                Followers = new []
-                {
-                    follower1
-                },
-                User = user
-            };
-
-            var loggerMock = new Mock<ILogger<SaveProgressionTask>>();
-            #endregion
-
-            #region Mocks
-            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
-            twitterUserDalMock
-                .Setup(x => x.UpdateTwitterUserAsync(
-                    It.Is<int>(y => y == user.Id),
-                    It.Is<long>(y => y == tweet2.Id),
-                    It.Is<long>(y => y == tweet2.Id),
-                    It.Is<int>(y => y == 0),
-                    It.IsAny<DateTime>()
-                ))
-                .Returns(Task.CompletedTask);
-            #endregion
-
-            var processor = new SaveProgressionTask(twitterUserDalMock.Object, loggerMock.Object);
-            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
-
-            #region Validations
-            twitterUserDalMock.VerifyAll();
-            loggerMock.VerifyAll();
-            #endregion
-        }
-
-        [TestMethod]
-        public async Task ProcessAsync_PartiallySynchronized_Test()
-        {
-            #region Stubs
-            var user = new SyncTwitterUser
-            {
-                Id = 1
-            };
-            var tweet1 = new ExtractedTweet
-            {
-                Id = 36
-            };
-            var tweet2 = new ExtractedTweet
-            {
-                Id = 37
-            };
-            var tweet3 = new ExtractedTweet
-            {
-                Id = 38
-            };
-            var follower1 = new Follower
-            {
-                FollowingsSyncStatus = new Dictionary<int, long>
-                {
-                    {1, 37}
-                }
-            };
-
-            var usersWithTweets = new UserWithDataToSync
-            {
-                Tweets = new[]
-                {
-                    tweet1,
-                    tweet2,
-                    tweet3
-                },
-                Followers = new[]
-                {
-                    follower1
-                },
-                User = user
-            };
-            #endregion
-
-            #region Mocks
-            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
-            twitterUserDalMock
-                .Setup(x => x.UpdateTwitterUserAsync(
-                    It.Is<int>(y => y == user.Id),
-                    It.Is<long>(y => y == tweet3.Id),
-                    It.Is<long>(y => y == tweet2.Id),
-                    It.Is<int>(y => y == 0),
-                    It.IsAny<DateTime>()
-                ))
-                .Returns(Task.CompletedTask);
-
-            var loggerMock = new Mock<ILogger<SaveProgressionTask>>();
-            #endregion
-
-            var processor = new SaveProgressionTask(twitterUserDalMock.Object, loggerMock.Object);
-            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
-
-            #region Validations
-            twitterUserDalMock.VerifyAll();
-            loggerMock.VerifyAll();
-            #endregion
-        }
-
-        [TestMethod]
-        public async Task ProcessAsync_PartiallySynchronized_MultiUsers_Test()
-        {
-            #region Stubs
-            var user = new SyncTwitterUser
-            {
-                Id = 1
-            };
-            var tweet1 = new ExtractedTweet
-            {
-                Id = 36
-            };
-            var tweet2 = new ExtractedTweet
-            {
-                Id = 37
-            };
-            var tweet3 = new ExtractedTweet
-            {
-                Id = 38
-            };
-            var follower1 = new Follower
-            {
-                FollowingsSyncStatus = new Dictionary<int, long>
-                {
-                    {1, 37}
-                }
-            };
-            var follower2 = new Follower
-            {
-                FollowingsSyncStatus = new Dictionary<int, long>
-                {
-                    {1, 38}
-                }
-            };
-
-            var usersWithTweets = new UserWithDataToSync
-            {
-                Tweets = new[]
-                {
-                    tweet1,
-                    tweet2,
-                    tweet3
-                },
-                Followers = new[]
-                {
-                    follower1,
-                    follower2
-                },
-                User = user
-            };
-            #endregion
-
-            #region Mocks
-            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
-            twitterUserDalMock
-                .Setup(x => x.UpdateTwitterUserAsync(
-                    It.Is<int>(y => y == user.Id),
-                    It.Is<long>(y => y == tweet3.Id),
-                    It.Is<long>(y => y == tweet2.Id),
-                    It.Is<int>(y => y == 0),
-                    It.IsAny<DateTime>()
-                ))
-                .Returns(Task.CompletedTask);
-
-            var loggerMock = new Mock<ILogger<SaveProgressionTask>>();
-            #endregion
-
-            var processor = new SaveProgressionTask(twitterUserDalMock.Object, loggerMock.Object);
-            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
-
-            #region Validations
-            twitterUserDalMock.VerifyAll();
-            loggerMock.VerifyAll();
-            #endregion
-        }
-    }
-}

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs

@@ -77,7 +77,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
             
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -165,7 +164,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 ParallelFediversePosts = 1
             };
 
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
             var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
             #endregion
 
@@ -250,7 +248,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -343,7 +340,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
 
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
             var settings = new InstanceSettings
             {
                 ParallelFediversePosts = 1
@@ -440,7 +436,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
 
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
             var settings = new InstanceSettings
             {
                 ParallelFediversePosts = 1
@@ -519,7 +514,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -600,7 +594,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -689,7 +682,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -775,7 +767,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -865,7 +856,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -959,7 +949,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -1054,7 +1043,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
-            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs

@@ -57,7 +57,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
-                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings
@@ -139,7 +138,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
-                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings { };
@@ -218,7 +216,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
-                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings
@@ -301,7 +298,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
-                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings
@@ -375,7 +371,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
-                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 10 } }
             };
 
             var settings = new InstanceSettings
@@ -456,7 +451,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
-                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 10 } }
             };
 
             var settings = new InstanceSettings
@@ -560,7 +554,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
-                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs

@@ -61,21 +61,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 
@@ -161,21 +158,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 
@@ -262,21 +256,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 
@@ -350,21 +341,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 10}}
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 8}}
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 7}}
                 }
             };
 
@@ -447,21 +435,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 10}}
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 8}}
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 7}}
                 }
             };
 
@@ -568,21 +553,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 
@@ -648,21 +630,18 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
-                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 

src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs

@@ -30,18 +30,16 @@ namespace BirdsiteLive.Pipeline.Tests
             var retrieveTweetsProcessor = new Mock<IRetrieveTweetsProcessor>(MockBehavior.Strict);
             var retrieveFollowersProcessor = new Mock<IRetrieveFollowersProcessor>(MockBehavior.Strict);
             var sendTweetsToFollowersProcessor = new Mock<ISendTweetsToFollowersProcessor>(MockBehavior.Strict);
-            var saveProgressionProcessor = new Mock<ISaveProgressionTask>(MockBehavior.Strict);
             var logger = new Mock<ILogger<StatusPublicationPipeline>>();
             #endregion
 
-            var pipeline = new StatusPublicationPipeline(retrieveTweetsProcessor.Object, retrieveTwitterUserProcessor.Object, retrieveFollowersProcessor.Object, sendTweetsToFollowersProcessor.Object, saveProgressionProcessor.Object, logger.Object);
+            var pipeline = new StatusPublicationPipeline(retrieveTweetsProcessor.Object, retrieveTwitterUserProcessor.Object, retrieveFollowersProcessor.Object, sendTweetsToFollowersProcessor.Object, logger.Object);
             await pipeline.ExecuteAsync(ct.Token);
 
             #region Validations
             retrieveTweetsProcessor.VerifyAll();
             retrieveFollowersProcessor.VerifyAll();
             sendTweetsToFollowersProcessor.VerifyAll();
-            saveProgressionProcessor.VerifyAll();
             logger.VerifyAll();
             #endregion
         }

src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs

@@ -57,12 +57,13 @@ namespace BirdsiteLive.ActivityPub.Tests
         {
             var tweets = await _tweetService.GetTimelineAsync("grantimahara", default);
             Assert.IsTrue(tweets[0].IsReply);
-            Assert.IsTrue(tweets.Length > 30);
+            Assert.IsTrue(tweets.Length > 10);
 
             Assert.AreEqual(tweets[2].MessageContent, "Liftoff!");
             Assert.AreEqual(tweets[2].RetweetId, 1266812530833240064);
             Assert.AreEqual(tweets[2].Id, 1266813644626489345);
             Assert.AreEqual(tweets[2].OriginalAuthor.Acct, "SpaceX");
+            Assert.AreEqual(tweets[2].Author.Acct, "grantimahara");
             Assert.IsTrue(tweets[2].IsRetweet);
         }
 

src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs

@@ -16,10 +16,13 @@ namespace BirdsiteLive.ActivityPub.Tests
     [TestClass]
     public class TweetTests
     {
-        private ITwitterTweetsService _tweetService;
+        private ITwitterTweetsService _tweetService = null;
+        
         [TestInitialize]
         public async Task TestInit()
         {
+            if (_tweetService != null)
+                return;
             
             var logger1 = new Mock<ILogger<TwitterAuthenticationInitializer>>(MockBehavior.Strict);
             var logger2 = new Mock<ILogger<TwitterUserService>>(MockBehavior.Strict);
@@ -39,6 +42,7 @@ namespace BirdsiteLive.ActivityPub.Tests
 
         }
 
+
         [TestMethod]
         public async Task SimpleTextTweet()
         {
@@ -57,7 +61,7 @@ namespace BirdsiteLive.ActivityPub.Tests
 
             Assert.AreEqual(tweet.Media[0].MediaType, "image/jpeg");
             Assert.AreEqual(tweet.Media.Length, 1);
-            // TODO test alt-text of images
+            Assert.AreEqual(tweet.Media[0].AltText, "President Obama with Speaker Nancy Pelosi in DC.");
         }
 
         [TestMethod]
@@ -75,7 +79,18 @@ namespace BirdsiteLive.ActivityPub.Tests
 
             Assert.AreEqual(tweet.Media.Length, 1);
             Assert.AreEqual(tweet.Media[0].MediaType, "video/mp4");
+            Assert.IsNull(tweet.Media[0].AltText);
             Assert.IsTrue(tweet.Media[0].Url.StartsWith("https://video.twimg.com/"));
+            
+            
+            var tweet2 = await _tweetService.GetTweetAsync(1657913781006258178);
+            Assert.AreEqual(tweet2.MessageContent,
+                "Coinbase has big international expansion plans\n\nTom Duff Gordon (@tomduffgordon), VP of International Policy @coinbase has the deets");
+
+            Assert.AreEqual(tweet2.Media.Length, 1);
+            Assert.AreEqual(tweet2.Media[0].MediaType, "video/mp4");
+            Assert.IsNull(tweet2.Media[0].AltText);
+            Assert.IsTrue(tweet2.Media[0].Url.StartsWith("https://video.twimg.com/"));
         }
 
         [Ignore]
@@ -95,7 +110,30 @@ namespace BirdsiteLive.ActivityPub.Tests
         {
             var tweet = await _tweetService.GetTweetAsync(1610807139089383427);
 
-            Assert.AreEqual(tweet.MessageContent, "When you gave them your keys you gave them your coins.\n\nhttps://domain.name/users/kadhim/statuses/1610706613207285773");
+            Assert.AreEqual(tweet.MessageContent, "When you gave them your keys you gave them your coins.\n\nhttps://domain.name/@kadhim/1610706613207285773");
+            Assert.AreEqual(tweet.Author.Acct, "RyanSAdams");
+        }
+        
+        [TestMethod]
+        public async Task QTandTextContainsLink()
+        {
+            var tweet = await _tweetService.GetTweetAsync(1668932525522305026);
+
+            Assert.AreEqual(tweet.MessageContent, @"https://domain.name/@WeekInEthNews/1668684659855880193");
+            Assert.AreEqual(tweet.Author.Acct, "WeekInEthNews");
+        }
+        
+        [Ignore]
+        [TestMethod]
+        public async Task QTandTextContainsWebLink()
+        {
+            var tweet = await _tweetService.GetTweetAsync(1668969663340871682);
+
+            Assert.AreEqual(tweet.MessageContent, @"Friends, our Real World Risk Workshop (now transformed into summer school) #RWRI (18th ed.) takes place July 10-21 (remote).
+We have a few scholarships left but more importantly we are looking for a guest speaker on AI-LLM-Robotics for a 45 Q&amp;A with us.
+
+http://www.realworldrisk.com https://twitter.com/i/web/status/1668969663340871682");
+            Assert.AreEqual(tweet.Author.Acct, "nntaleb");
         }
 
         [TestMethod]

src/Tests/BirdsiteLive.Twitter.Tests/UsersTest.cs

@@ -39,6 +39,13 @@ namespace BirdsiteLive.ActivityPub.Tests
             Assert.AreEqual(user.Acct, "kobebryant");
         }
 
+        [TestMethod]
+        public async Task UserGrant()
+        {
+            var user = await _tweetService.GetUserAsync("grantimahara");
+            Assert.AreEqual(user.Name, "Grant Imahara");
+            Assert.AreEqual(user.Acct, "grantimahara");
+        }
 
     }
 }

src/Tests/dotMakeup.HackerNews.Tests/PostsTests.cs

@@ -0,0 +1,6 @@
+namespace dotMakeup.HackerNews.Tests;
+
+public class PostsTests
+{
+    
+}

src/Tests/dotMakeup.HackerNews.Tests/UsersTests.cs

@@ -0,0 +1,20 @@
+using dotMakeup.HackerNews;
+using System.Threading.Tasks;
+using Moq;
+
+namespace dotMakeup.HackerNews.Tests;
+
+[TestClass]
+public class UsersTests
+{
+    [TestMethod]
+    public async Task TestMethod1()
+    {
+        var httpFactory = new Mock<IHttpClientFactory>();
+        httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
+        var userService = new HNUserService(httpFactory.Object);
+        var user = await userService.GetUserAsync("dhouston");
+        
+        Assert.AreEqual(user.About, "Founder/CEO of Dropbox (http://www.dropbox.com ; yc summer '07)");
+    }
+}

src/Tests/dotMakeup.HackerNews.Tests/Usings.cs

@@ -0,0 +1 @@
+global using Microsoft.VisualStudio.TestTools.UnitTesting;

src/Tests/dotMakeup.HackerNews.Tests/dotMakeup.HackerNews.Tests.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
+        <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
+        <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
+        <PackageReference Include="coverlet.collector" Version="3.1.2" />
+        <PackageReference Include="moq" Version="4.16.1" />
+        <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\..\dotMakeup.HackerNews\dotMakeup.HackerNews.csproj" />
+    </ItemGroup>
+
+
+</Project>

src/dotMakeup.HackerNews/HNPostService.cs

@@ -0,0 +1,6 @@
+namespace dotMakeup.HackerNews;
+
+public class HNPostService
+{
+    
+}

src/dotMakeup.HackerNews/HNUserService.cs

@@ -0,0 +1,39 @@
+using System.Text.Json;
+using System.Web;
+using dotMakeup.HackerNews.Models;
+
+namespace dotMakeup.HackerNews;
+
+public class HNUserService
+{
+    private IHttpClientFactory _httpClientFactory;
+    public HNUserService(IHttpClientFactory httpClientFactory)
+    {
+            _httpClientFactory = httpClientFactory;
+        
+    }
+    public async Task<HNUser> GetUserAsync(string username)
+    {
+        string reqURL = "https://hacker-news.firebaseio.com/v0/user/dhouston.json";
+        reqURL = reqURL.Replace("dhouston", username);
+        
+        var client = _httpClientFactory.CreateClient();
+        var request = new HttpRequestMessage(new HttpMethod("GET"), reqURL);
+        
+        JsonDocument userDoc;
+        var httpResponse = await client.SendAsync(request);
+        httpResponse.EnsureSuccessStatusCode();
+        var c = await httpResponse.Content.ReadAsStringAsync();
+        userDoc = JsonDocument.Parse(c);
+
+        string about =
+            HttpUtility.HtmlDecode(userDoc.RootElement.GetProperty("about").GetString());
+        
+        var user = new HNUser()
+        {
+            Id = 0,
+            About = about,
+        };
+        return user;
+    }
+}

src/dotMakeup.HackerNews/Models/HNUser.cs

@@ -0,0 +1,7 @@
+namespace dotMakeup.HackerNews.Models;
+
+public class HNUser
+{
+    public long Id { get; set; }
+    public string About { get; set; }
+}

src/dotMakeup.HackerNews/dotMakeup.HackerNews.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+    
+    <ItemGroup>
+        <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
+    </ItemGroup>
+
+</Project>