README.md

@@ -8,26 +8,25 @@ Bird.makeup is a way to follow Twitter users from any ActivityPub service. The a
 
 Compared to BirdsiteLive, bird.makeup is:
 
-More scalable:
- - Twitter API calls are not rate-limited
- - It is possible to split the Twitter crawling to multiple servers
- - There are now integration tests for the non-official api
- - The core pipeline has been tweaked to remove bottlenecks. As of writing this, bird.makeup supports without problems more than 20k users. 
- - Twitter users with no followers on the fediverse will stop being fetched
-
-More native to the fediverse:
- - Retweets are propagated as boosts
- - Activities are now "unlisted" which means that they won't polute the public timeline, but they can still be boosted
- - WIP support for QT
-
 More modern:
  - Moved from .net core 3.1 to .net 6 which is still supported
  - Moved from postgres 9 to 15
  - Moved from Newtonsoft.Json to System.Text.Json
 
+More scalable:
+ - Twitter API calls are not rate-limited
+ - There are now integration tests for the non-official api
+ - The core pipeline has been tweaked to remove bottlenecks. As of writing this, bird.makeup supports without problems more than 10k users. 
+ - Twitter users with no followers on the fediverse will stop being fetched
+
+More native to the fediverse:
+ - Retweets are propagated as boosts
+ - Activities are now "unlisted" which means that they won't polute the public timeline
+ - WIP support for QT
+
 ## Official instance 
 
-You can find the official instance here: [bird.makeup](https://bird.makeup). If you are an instance admin that prefers to not have tweets federated to you, please block the entire instance. 
+You can find an official instance here: [bird.makeup](https://bird.makeup). If you are an instance admin that prefers to not have tweets federated to you, please block the entire instance. 
 
 Please consider if you really need another instance before spinning up a new one, as having multiple domain makes it harder for moderators to block twitter content. 
 

sql.md

@@ -29,15 +29,6 @@ SELECT COUNT(*), date_trunc('day', lastsync) FROM (SELECT unnest(followings) as
 SELECT COUNT(*), date_trunc('hour', lastsync) FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id GROUP BY date_trunc ORDER BY date_trunc;
 ```
 
-Lag by shards: 
-
-```SQL
-SELECT min(lastsync), mod(id, 100) FROM
-    (SELECT acct, id, lastsync FROM (SELECT unnest(followings) as follow FROM followers) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id) AS f
-GROUP BY mod
-ORDER BY min;
-```
-
 # Connections 
 
 ```SQL

src/BSLManager/App.cs

@@ -0,0 +1,252 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Moderation.Actions;
+using BSLManager.Domain;
+using BSLManager.Tools;
+using Terminal.Gui;
+
+namespace BSLManager
+{
+    public class App
+    {
+        private readonly IFollowersDal _followersDal;
+        private readonly IRemoveFollowerAction _removeFollowerAction;
+
+        private readonly FollowersListState _state = new FollowersListState();
+
+        #region Ctor
+        public App(IFollowersDal followersDal, IRemoveFollowerAction removeFollowerAction)
+        {
+            _followersDal = followersDal;
+            _removeFollowerAction = removeFollowerAction;
+        }
+        #endregion
+
+        public void Run()
+        {
+            Application.Init();
+            var top = Application.Top;
+
+            // Creates the top-level window to show
+            var win = new Window("BSL Manager")
+            {
+                X = 0,
+                Y = 1, // Leave one row for the toplevel menu
+
+                // By using Dim.Fill(), it will automatically resize without manual intervention
+                Width = Dim.Fill(),
+                Height = Dim.Fill()
+            };
+
+            top.Add(win);
+
+            // Creates a menubar, the item "New" has a help menu.
+            var menu = new MenuBar(new MenuBarItem[]
+            {
+                new MenuBarItem("_File", new MenuItem[]
+                {
+                    new MenuItem("_Quit", "", () =>
+                    {
+                        if (Quit()) top.Running = false;
+                    })
+                }),
+                //new MenuBarItem ("_Edit", new MenuItem [] {
+                //    new MenuItem ("_Copy", "", null),
+                //    new MenuItem ("C_ut", "", null),
+                //    new MenuItem ("_Paste", "", null)
+                //})
+            });
+            top.Add(menu);
+
+            static bool Quit()
+            {
+                var n = MessageBox.Query(50, 7, "Quit BSL Manager", "Are you sure you want to quit?", "Yes", "No");
+                return n == 0;
+            }
+            
+            RetrieveUserList();
+
+            var list = new ListView(_state.GetDisplayableList())
+            {
+                X = 1,
+                Y = 3,
+                Width = Dim.Fill(),
+                Height = Dim.Fill()
+            };
+
+            list.KeyDown += _ =>
+            {
+                if (_.KeyEvent.Key == Key.Enter)
+                {
+                    OpenFollowerDialog(list.SelectedItem);
+                } 
+                else if (_.KeyEvent.Key == Key.Delete
+                           || _.KeyEvent.Key == Key.DeleteChar
+                           || _.KeyEvent.Key == Key.Backspace
+                           || _.KeyEvent.Key == Key.D)
+                {
+                    OpenDeleteDialog(list.SelectedItem);
+                }
+            };
+
+            var listingFollowersLabel = new Label(1, 0, "Listing followers");
+            var filterLabel = new Label("Filter: ") { X = 1, Y = 1 };
+            var filterText = new TextField("")
+            {
+                X = Pos.Right(filterLabel),
+                Y = 1,
+                Width = 40
+            };
+
+            filterText.KeyDown += _ =>
+            {
+                var text = filterText.Text.ToString();
+                if (_.KeyEvent.Key == Key.Enter && !string.IsNullOrWhiteSpace(text))
+                {
+                    _state.FilterBy(text);
+                    ConsoleGui.RefreshUI();
+                }
+            };
+
+            win.Add(
+                listingFollowersLabel,
+                filterLabel,
+                filterText,
+                list
+            );
+
+            Application.Run();
+        }
+
+        private void OpenFollowerDialog(int selectedIndex)
+        {
+            var close = new Button(3, 14, "Close");
+            close.Clicked += () => Application.RequestStop();
+
+            var dialog = new Dialog("Info", 60, 18, close);
+
+            var follower = _state.GetElementAt(selectedIndex);
+
+            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
+            {
+                X = 1,
+                Y = 1,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var following = new Label($"Following Count: {follower.Followings.Count}")
+            {
+                X = 1,
+                Y = 3,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var errors = new Label($"Posting Errors: {follower.PostingErrorCount}")
+            {
+                X = 1,
+                Y = 4,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var inbox = new Label($"Inbox: {follower.InboxRoute}")
+            {
+                X = 1,
+                Y = 5,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var sharedInbox = new Label($"Shared Inbox: {follower.SharedInboxRoute}")
+            {
+                X = 1,
+                Y = 6,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+
+            dialog.Add(name);
+            dialog.Add(following);
+            dialog.Add(errors);
+            dialog.Add(inbox);
+            dialog.Add(sharedInbox);
+            dialog.Add(close);
+            Application.Run(dialog);
+        }
+
+        private void OpenDeleteDialog(int selectedIndex)
+        {
+            bool okpressed = false;
+            var ok = new Button(10, 14, "Yes");
+            ok.Clicked += () =>
+            {
+                Application.RequestStop();
+                okpressed = true;
+            };
+
+            var cancel = new Button(3, 14, "No");
+            cancel.Clicked += () => Application.RequestStop();
+
+            var dialog = new Dialog("Delete", 60, 18, cancel, ok);
+
+            var follower = _state.GetElementAt(selectedIndex);
+            var name = new Label($"User: @{follower.Acct}@{follower.Host}")
+            {
+                X = 1,
+                Y = 1,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            var entry = new Label("Delete user and remove all their followings?")
+            {
+                X = 1,
+                Y = 3,
+                Width = Dim.Fill(),
+                Height = 1
+            };
+            dialog.Add(name);
+            dialog.Add(entry);
+            Application.Run(dialog);
+
+            if (okpressed)
+            {
+                DeleteAndRemoveUser(selectedIndex);
+            }
+        }
+
+        private void DeleteAndRemoveUser(int el)
+        {
+            Application.MainLoop.Invoke(async () =>
+            {
+                try
+                {
+                    var userToDelete = _state.GetElementAt(el);
+                    
+                    BasicLogger.Log($"Delete {userToDelete.Acct}@{userToDelete.Host}");
+                    await _removeFollowerAction.ProcessAsync(userToDelete);
+                    BasicLogger.Log($"Remove user from list");
+                    _state.RemoveAt(el);
+                }
+                catch (Exception e)
+                {
+                    BasicLogger.Log(e.Message);
+                }
+
+                ConsoleGui.RefreshUI();
+            });
+        }
+
+        private void RetrieveUserList()
+        {
+            Application.MainLoop.Invoke(async () =>
+            {
+                var followers = await _followersDal.GetAllFollowersAsync();
+                _state.Load(followers.ToList());
+                ConsoleGui.RefreshUI();
+            });
+        }
+    }
+}

src/BSLManager/BSLManager.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Lamar" Version="5.0.3" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="5.0.0" />
+    <PackageReference Include="Terminal.Gui" Version="1.0.0-beta.11" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\BirdsiteLive.Common\BirdsiteLive.Common.csproj" />
+    <ProjectReference Include="..\BirdsiteLive.Moderation\BirdsiteLive.Moderation.csproj" />
+    <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="key.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

src/BSLManager/Bootstrapper.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Net.Http;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.Common.Structs;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Postgres.DataAccessLayers;
+using BirdsiteLive.DAL.Postgres.Settings;
+using Lamar;
+using Lamar.Scanning.Conventions;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace BSLManager
+{
+    public class Bootstrapper
+    {
+        private readonly DbSettings _dbSettings;
+        private readonly InstanceSettings _instanceSettings;
+
+        #region Ctor
+        public Bootstrapper(DbSettings dbSettings, InstanceSettings instanceSettings)
+        {
+            _dbSettings = dbSettings;
+            _instanceSettings = instanceSettings;
+        }
+        #endregion
+
+        public Container Init()
+        {
+            var container = new Container(x =>
+            {
+                x.For<DbSettings>().Use(x => _dbSettings);
+
+                x.For<InstanceSettings>().Use(x => _instanceSettings);
+
+                if (string.Equals(_dbSettings.Type, DbTypes.Postgres, StringComparison.OrdinalIgnoreCase))
+                {
+                    var connString = $"Host={_dbSettings.Host};Username={_dbSettings.User};Password={_dbSettings.Password};Database={_dbSettings.Name}";
+                    var postgresSettings = new PostgresSettings
+                    {
+                        ConnString = connString
+                    };
+                    x.For<PostgresSettings>().Use(x => postgresSettings);
+
+                    x.For<ITwitterUserDal>().Use<TwitterUserPostgresDal>().Singleton();
+                    x.For<IFollowersDal>().Use<FollowersPostgresDal>().Singleton();
+                    x.For<IDbInitializerDal>().Use<DbInitializerPostgresDal>().Singleton();
+                }
+                else
+                {
+                    throw new NotImplementedException($"{_dbSettings.Type} is not supported");
+                }
+
+                var serviceProvider = new ServiceCollection().AddHttpClient().BuildServiceProvider();
+                x.For<IHttpClientFactory>().Use(_ => serviceProvider.GetService<IHttpClientFactory>());
+
+                x.For(typeof(ILogger<>)).Use(typeof(DummyLogger<>));
+
+                x.Scan(_ =>
+                {
+                    _.Assembly("BirdsiteLive.Twitter");
+                    _.Assembly("BirdsiteLive.Domain");
+                    _.Assembly("BirdsiteLive.DAL");
+                    _.Assembly("BirdsiteLive.DAL.Postgres");
+                    _.Assembly("BirdsiteLive.Moderation");
+
+                    _.TheCallingAssembly();
+
+                    _.WithDefaultConventions();
+
+                    _.LookForRegistries();
+                });
+            });
+            return container;
+        }
+
+        public class DummyLogger<T> : ILogger<T>
+        {
+            public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
+            {
+            }
+
+            public bool IsEnabled(LogLevel logLevel)
+            {
+                return false;
+            }
+
+            public IDisposable BeginScope<TState>(TState state)
+            {
+                return null;
+            }
+        }
+    }
+}

src/BSLManager/Domain/FollowersListState.cs

@@ -0,0 +1,81 @@
+using System.Collections.Generic;
+using System.Linq;
+using BirdsiteLive.DAL.Models;
+
+namespace BSLManager.Domain
+{
+    public class FollowersListState
+    {
+        private readonly List<string> _filteredDisplayableUserList = new List<string>();
+
+        private List<Follower> _sourceUserList = new List<Follower>();
+        private List<Follower> _filteredSourceUserList = new List<Follower>();
+        
+        public void Load(List<Follower> followers)
+        {
+            _sourceUserList = followers.OrderByDescending(x => x.Followings.Count).ToList();
+           
+            ResetLists();
+        }
+
+        private void ResetLists()
+        {
+            _filteredSourceUserList = _sourceUserList.ToList();
+
+            _filteredDisplayableUserList.Clear();
+
+            foreach (var follower in _sourceUserList)
+            {
+                var displayedUser = $"{GetFullHandle(follower)}   ({follower.Followings.Count})  (err:{follower.PostingErrorCount})";
+                _filteredDisplayableUserList.Add(displayedUser);
+            }
+        }
+
+        public List<string> GetDisplayableList()
+        {
+            return _filteredDisplayableUserList;
+        }
+
+        public void FilterBy(string pattern)
+        {
+            ResetLists();
+
+            if (!string.IsNullOrWhiteSpace(pattern))
+            {
+                var elToRemove = _filteredSourceUserList
+                    .Where(x => !GetFullHandle(x).Contains(pattern))
+                    .Select(x => x)
+                    .ToList();
+
+                foreach (var el in elToRemove)
+                {
+                    _filteredSourceUserList.Remove(el);
+                    
+                    var dElToRemove = _filteredDisplayableUserList.First(x => x.Contains(GetFullHandle(el)));
+                    _filteredDisplayableUserList.Remove(dElToRemove);
+                }
+            }
+        }
+
+        private string GetFullHandle(Follower follower)
+        {
+            return $"@{follower.Acct}@{follower.Host}";
+        }
+
+        public void RemoveAt(int index)
+        {
+            var displayableUser = _filteredDisplayableUserList[index];
+            var sourceUser = _filteredSourceUserList[index];
+
+            _filteredDisplayableUserList.Remove(displayableUser);
+
+            _filteredSourceUserList.Remove(sourceUser);
+            _sourceUserList.Remove(sourceUser);
+        }
+
+        public Follower GetElementAt(int index)
+        {
+            return _filteredSourceUserList[index];
+        }
+    }
+}

src/BSLManager/Program.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+using BirdsiteLive.Common.Settings;
+using BirdsiteLive.DAL.Contracts;
+using BSLManager.Tools;
+using Microsoft.Extensions.Configuration;
+using NStack;
+using Terminal.Gui;
+
+namespace BSLManager
+{
+    class Program
+    {
+        static async Task Main(string[] args)
+        {
+            Console.OutputEncoding = Encoding.Default;
+
+            var settingsManager = new SettingsManager();
+            var settings = settingsManager.GetSettings();
+
+            //var builder = new ConfigurationBuilder()
+            //    .AddEnvironmentVariables();
+            //var configuration = builder.Build();
+
+            //var dbSettings = configuration.GetSection("Db").Get<DbSettings>();
+            //var instanceSettings = configuration.GetSection("Instance").Get<InstanceSettings>();
+
+            var bootstrapper = new Bootstrapper(settings.dbSettings, settings.instanceSettings);
+            var container = bootstrapper.Init();
+
+            var app = container.GetInstance<App>();
+            app.Run();
+        }
+    }
+}

src/BSLManager/Tools/BasicLogger.cs

@@ -0,0 +1,13 @@
+using System;
+using System.IO;
+
+namespace BSLManager.Tools
+{
+    public static class BasicLogger
+    {
+        public static void Log(string log)
+        {
+            File.AppendAllLines($"Log-{Guid.NewGuid()}.txt", new []{ log });
+        }
+    }
+}

src/BSLManager/Tools/ConsoleGui.cs

@@ -0,0 +1,15 @@
+using System.Reflection;
+using Terminal.Gui;
+
+namespace BSLManager.Tools
+{
+    public static class ConsoleGui
+    {
+        public static void RefreshUI()
+        {
+            typeof(Application)
+                .GetMethod("TerminalResized", BindingFlags.Static | BindingFlags.NonPublic)
+                .Invoke(null, null);
+        }
+    }
+}

src/BSLManager/Tools/SettingsManager.cs

@@ -0,0 +1,123 @@
+using System;
+using System.IO;
+using System.Text.Json;
+using System.Runtime.CompilerServices;
+using BirdsiteLive.Common.Settings;
+
+namespace BSLManager.Tools
+{
+    public class SettingsManager
+    {
+        private const string LocalFileName = "ManagerSettings.json";
+
+        public (DbSettings dbSettings, InstanceSettings instanceSettings) GetSettings()
+        {
+            var localSettingsData = GetLocalSettingsFile();
+            if (localSettingsData != null) return Convert(localSettingsData);
+
+            Console.WriteLine("We need to set up the manager");
+            Console.WriteLine("Please provide the following information as provided in the docker-compose file");
+
+            LocalSettingsData data;
+            do
+            {
+                data = GetDataFromUser();
+                Console.WriteLine();
+                Console.WriteLine("Please check if all is ok:");
+                Console.WriteLine();
+                Console.WriteLine($"Db Host: {data.DbHost}");
+                Console.WriteLine($"Db Name: {data.DbName}");
+                Console.WriteLine($"Db User: {data.DbUser}");
+                Console.WriteLine($"Db Password: {data.DbPassword}");
+                Console.WriteLine($"Instance Domain: {data.InstanceDomain}");
+                Console.WriteLine();
+
+                string resp;
+                do
+                {
+                    Console.WriteLine("Is it valid? (yes, no)");
+                    resp = Console.ReadLine()?.Trim().ToLowerInvariant();
+
+                    if (resp == "n" || resp == "no") data = null;
+
+                } while (resp != "y" && resp != "yes" && resp != "n" && resp != "no");
+
+            } while (data == null);
+            
+            SaveLocalSettings(data);
+            return Convert(data);
+        }
+
+        private LocalSettingsData GetDataFromUser()
+        {
+            var data = new LocalSettingsData();
+
+            Console.WriteLine("Db Host:");
+            data.DbHost = Console.ReadLine();
+
+            Console.WriteLine("Db Name:");
+            data.DbName = Console.ReadLine();
+
+            Console.WriteLine("Db User:");
+            data.DbUser = Console.ReadLine();
+
+            Console.WriteLine("Db Password:");
+            data.DbPassword = Console.ReadLine();
+
+            Console.WriteLine("Instance Domain:");
+            data.InstanceDomain = Console.ReadLine();
+
+            return data;
+        }
+
+        private (DbSettings dbSettings, InstanceSettings instanceSettings) Convert(LocalSettingsData data)
+        {
+            var dbSettings = new DbSettings
+            {
+                Type = data.DbType,
+                Host = data.DbHost,
+                Name = data.DbName,
+                User = data.DbUser,
+                Password = data.DbPassword
+            };
+            var instancesSettings = new InstanceSettings
+            {
+                Domain = data.InstanceDomain
+            };
+            return (dbSettings, instancesSettings);
+        }
+
+        private LocalSettingsData GetLocalSettingsFile()
+        {
+            try
+            {
+                if (!File.Exists(LocalFileName)) return null;
+
+                var jsonContent = File.ReadAllText(LocalFileName);
+                var content = JsonSerializer.Deserialize<LocalSettingsData>(jsonContent);
+                return content;
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+        }
+
+        private void SaveLocalSettings(LocalSettingsData data)
+        {
+            var jsonContent = JsonSerializer.Serialize(data);
+            File.WriteAllText(LocalFileName, jsonContent);
+        }
+    }
+
+    internal class LocalSettingsData
+    {
+        public string DbType { get; set; } = "postgres";
+        public string DbHost { get; set; }
+        public string DbName { get; set; }
+        public string DbUser { get; set; }
+        public string DbPassword { get; set; }
+
+        public string InstanceDomain { get; set; }
+    }
+}

src/BirdsiteLive.ActivityPub/BirdsiteLive.ActivityPub.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.ActivityPub/Models/Attachment.cs

@@ -1,14 +1,9 @@
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace BirdsiteLive.ActivityPub
+namespace BirdsiteLive.ActivityPub
 {
     public class Attachment
     {
         public string type { get; set; }
         public string mediaType { get; set; }
         public string url { get; set; }
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-        public string name { get; set; }
     }
 }

src/BirdsiteLive.Common/BirdsiteLive.Common.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
 </Project>

src/BirdsiteLive.Common/Regexes/HashtagRegexes.cs

@@ -5,6 +5,6 @@ namespace BirdsiteLive.Common.Regexes
     public class HashtagRegexes
     {
         public static readonly Regex HashtagName = new Regex(@"^[a-zA-Z0-9_]+$");
-        public static readonly Regex Hashtag = new Regex(@"(^|.?[ \n]+)#([a-zA-Z0-9_]+)(?=\s|$|[\[\]<>.,;:!?/|-])");
+        public static readonly Regex Hashtag = new Regex(@"(.?)#([a-zA-Z0-9_]+)(\s|$|[\[\]<>.,;:!?/|-])");
     }
 }

src/BirdsiteLive.Common/Regexes/UserRegexes.cs

@@ -5,6 +5,6 @@ namespace BirdsiteLive.Common.Regexes
     public class UserRegexes
     {
         public static readonly Regex TwitterAccount = new Regex(@"^[a-zA-Z0-9_]+$");
-        public static readonly Regex Mention = new Regex(@"(^|.?[ \n\.]+)@([a-zA-Z0-9_]+)(?=\s|$|[\[\]<>,;:'\.’!?/—\|-]|(. ))");
+        public static readonly Regex Mention = new Regex(@"(.?)@([a-zA-Z0-9_]+)(\s|$|[\[\]<>,;:'\.’!?/—\|-]|(. ))");
     }
 }

src/BirdsiteLive.Common/Settings/InstanceSettings.cs

@@ -18,9 +18,6 @@
         public int TweetCacheCapacity { get; set; } = 20_000;
         // "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
         public string TwitterBearerToken { get; set; } = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
-        public int m { get; set; } = 1;
-        public int n_start { get; set; } = 0;
-        public int n_end { get; set; } = 1;
         public int ParallelTwitterRequests { get; set; } = 10;
         public int ParallelFediversePosts { get; set; } = 10;
     }

src/BirdsiteLive.Cryptography/BirdsiteLive.Cryptography.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Domain/BusinessUseCases/ProcessFollowUser.cs

@@ -43,6 +43,9 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             if(!follower.Followings.Contains(twitterUserId))
                 follower.Followings.Add(twitterUserId);
 
+            if(!follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
+                follower.FollowingsSyncStatus.Add(twitterUserId, -1);
+            
             // Save Follower
             await _followerDal.UpdateFollowerAsync(follower);
         }

src/BirdsiteLive.Domain/BusinessUseCases/ProcessUnfollowUser.cs

@@ -36,6 +36,9 @@ namespace BirdsiteLive.Domain.BusinessUseCases
             if (follower.Followings.Contains(twitterUserId))
                 follower.Followings.Remove(twitterUserId);
 
+            if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
+                follower.FollowingsSyncStatus.Remove(twitterUserId);
+
             // Save or delete Follower
             if (follower.Followings.Any())
                 await _followerDal.UpdateFollowerAsync(follower);

src/BirdsiteLive.Domain/StatusService.cs

@@ -137,8 +137,7 @@ namespace BirdsiteLive.Domain
                 {
                     type = "Document",
                     url = x.Url,
-                    mediaType = x.MediaType,
-                    name = x.AltText
+                    mediaType = x.MediaType
                 };
             }).ToArray();
         }

src/BirdsiteLive.Domain/Tools/StatusExtractor.cs

@@ -32,6 +32,9 @@ namespace BirdsiteLive.Domain.Tools
         {
             var tags = new List<Tag>();
 
+            // Replace return lines
+            messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p>");
+            messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/>");
 
 
             // Extract Urls
@@ -122,10 +125,6 @@ namespace BirdsiteLive.Domain.Tools
                 }
             }
 
-            // Replace return lines
-            messageContent = Regex.Replace(messageContent, @"\r\n\r\n?|\n\n", "</p><p>");
-            messageContent = Regex.Replace(messageContent, @"\r\n?|\n", "<br/>");
-
             return (messageContent.Trim(), tags.ToArray());
         }
         

src/BirdsiteLive.Moderation/Actions/RemoveTwitterAccountAction.cs

@@ -41,6 +41,9 @@ namespace BirdsiteLive.Moderation.Actions
                 if (follower.Followings.Contains(twitterUserId))
                     follower.Followings.Remove(twitterUserId);
 
+                if (follower.FollowingsSyncStatus.ContainsKey(twitterUserId))
+                    follower.FollowingsSyncStatus.Remove(twitterUserId);
+
                 if (follower.Followings.Any())
                     await _followersDal.UpdateFollowerAsync(follower);
                 else

src/BirdsiteLive.Moderation/BirdsiteLive.Moderation.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
 

src/BirdsiteLive.Pipeline/Contracts/ISaveProgressionTask.cs

@@ -0,0 +1,11 @@
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.Pipeline.Models;
+
+namespace BirdsiteLive.Pipeline.Contracts
+{
+    public interface ISaveProgressionTask
+    {
+        Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct);
+    }
+}

src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs

@@ -61,25 +61,25 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
                         {
                             // skip the first time to avoid sending backlog of tweet
                             var tweetId = tweets.Last().Id;
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
                         }
                         else if (tweets.Length > 0 && user.LastTweetPostedId != -1)
                         {
                             userWtData.Tweets = tweets;
                             usersWtTweets.Add(userWtData);
                             var tweetId = tweets.Last().Id;
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, user.FetchingErrorCount, now);
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now);
                         }
                         else
                         {
-                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
+                            await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
                         }
 
                     } 
                     catch(Exception e)
                     {
                         _logger.LogError(e.Message);
-                        await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, now);
+                        await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now);
                     }
                 });
                 todo.Add(t);
@@ -104,7 +104,7 @@ namespace BirdsiteLive.Pipeline.Processors.SubTasks
                 if (user.LastTweetPostedId == -1)
                     tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct);
                 else
-                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetPostedId);
+                    tweets = await _twitterTweetsService.GetTimelineAsync(user.Acct, user.LastTweetSynchronizedForAllFollowersId);
             }
             catch (Exception e)
             {

src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -17,18 +16,16 @@ namespace BirdsiteLive.Pipeline.Processors
     {
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
-        private readonly InstanceSettings _instanceSettings;
         private readonly ILogger<RetrieveTwitterUsersProcessor> _logger;
         private static Random rng = new Random();
         
         public int WaitFactor = 1000 * 60; //1 min
 
         #region Ctor
-        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings, ILogger<RetrieveTwitterUsersProcessor> logger)
+        public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IFollowersDal followersDal, ILogger<RetrieveTwitterUsersProcessor> logger)
         {
             _twitterUserDal = twitterUserDal;
             _followersDal = followersDal;
-            _instanceSettings = instanceSettings;
             _logger = logger;
         }
         #endregion
@@ -39,38 +36,32 @@ namespace BirdsiteLive.Pipeline.Processors
             {
                 ct.ThrowIfCancellationRequested();
 
-                if (_instanceSettings.ParallelTwitterRequests == 0)
+                try
                 {
-                    while (true)
-                        await Task.Delay(10000);
-                }
+                    var users = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000);
 
-                var usersDal = await _twitterUserDal.GetAllTwitterUsersWithFollowersAsync(2000, _instanceSettings.n_start, _instanceSettings.n_end, _instanceSettings.m);
+                    var userCount = users.Any() ? Math.Min(users.Length, 200) : 1;
+                    var splitUsers = users.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
 
-                var userCount = usersDal.Any() ? Math.Min(usersDal.Length, 200) : 1;
-                var splitUsers = usersDal.OrderBy(a => rng.Next()).ToArray().Split(userCount).ToList();
-
-                foreach (var users in splitUsers)
+                    foreach (var u in splitUsers)
                     {
                         ct.ThrowIfCancellationRequested();
-                    List<UserWithDataToSync> toSync = new List<UserWithDataToSync>();
-                    foreach (var u in users)
-                    {
-                        var followers = await _followersDal.GetFollowersAsync(u.Id);
-                        toSync.Add( new UserWithDataToSync()
-                        {
-                            User = u,
-                            Followers = followers
-                        });
-                        
-                    }
-
-                    await twitterUsersBufferBlock.SendAsync(toSync.ToArray(), ct);
+                        UserWithDataToSync[] toSync = await Task.WhenAll(
+                            u.Select(async x => new UserWithDataToSync
+                                { User = x, Followers = await _followersDal.GetFollowersAsync(x.Id) } 
+                            )
+                        );
 
+                        await twitterUsersBufferBlock.SendAsync(toSync, ct);
                     }
 
                     await Task.Delay(10, ct); // this is somehow necessary
                 }
+                catch (Exception e)
+                {
+                    _logger.LogError(e, "Failing retrieving Twitter Users.");
+                }
+            }
         }
     }
 }

src/BirdsiteLive.Pipeline/Processors/SubTasks/SaveProgressionTask.cs

@@ -0,0 +1,61 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.Pipeline.Contracts;
+using BirdsiteLive.Pipeline.Models;
+using Microsoft.Extensions.Logging;
+
+namespace BirdsiteLive.Pipeline.Processors.SubTasks
+{
+    public class SaveProgressionTask : ISaveProgressionTask
+    {
+        private readonly ITwitterUserDal _twitterUserDal;
+        private readonly ILogger<SaveProgressionTask> _logger;
+
+        #region Ctor
+        public SaveProgressionTask(ITwitterUserDal twitterUserDal, ILogger<SaveProgressionTask> logger)
+        {
+            _twitterUserDal = twitterUserDal;
+            _logger = logger;
+        }
+        #endregion
+
+        public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, CancellationToken ct)
+        {
+            try
+            {
+                if (userWithTweetsToSync.Tweets.Length == 0)
+                {
+                    _logger.LogWarning("No tweets synchronized");
+                    return;
+                }
+                if(userWithTweetsToSync.Followers.Length == 0)
+                {
+                    _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct);
+                    return;
+                }
+            
+                var userId = userWithTweetsToSync.User.Id;
+                var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList();
+            
+                if (followingSyncStatuses.Count == 0)
+                {
+                    _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId);
+                    return;
+                }
+
+                var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max();
+                var minimumSync = followingSyncStatuses.Min();
+                var now = DateTime.UtcNow;
+                await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, "SaveProgressionProcessor.ProcessAsync() Exception");
+                throw;
+            }
+        }
+    }
+}

src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs

@@ -21,14 +21,16 @@ namespace BirdsiteLive.Pipeline
         private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor;
         private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor;
         private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor;
+        private readonly ISaveProgressionTask _saveProgressionTask;
         private readonly ILogger<StatusPublicationPipeline> _logger;
 
         #region Ctor
-        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger<StatusPublicationPipeline> logger)
+        public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionTask saveProgressionTask, ILogger<StatusPublicationPipeline> logger)
         {
             _retrieveTweetsProcessor = retrieveTweetsProcessor;
             _retrieveFollowersProcessor = retrieveFollowersProcessor;
             _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor;
+            _saveProgressionTask = saveProgressionTask;
             _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor;
 
             _logger = logger;

src/BirdsiteLive.Twitter/BirdsiteLive.Twitter.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/BirdsiteLive.Twitter/Models/ExtractedMedia.cs

@@ -4,6 +4,5 @@
     {
         public string MediaType { get; set; }
         public string Url { get; set; }
-        public string AltText { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs

@@ -17,6 +17,5 @@ namespace BirdsiteLive.Twitter.Models
         public string RetweetUrl { get; set; }
         public long RetweetId { get; set; }
         public TwitterUser OriginalAuthor { get; set; }
-        public TwitterUser Author { get; set; }
     }
 }

src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Threading;
 using System.Collections.Generic;
-using System.Collections.Concurrent;
 using System.Linq;
 using System.Threading.Tasks;
 using BirdsiteLive.Common.Settings;
@@ -27,9 +26,11 @@ namespace BirdsiteLive.Twitter.Tools
         private readonly ILogger<TwitterAuthenticationInitializer> _logger;
         private static bool _initialized;
         private readonly IHttpClientFactory _httpClientFactory;
-        private ConcurrentDictionary<String, String> _token2 = new ConcurrentDictionary<string, string>();
+        private List<HttpClient> _twitterClients = new List<HttpClient>();
+        private List<(String, String)> _tokens = new List<(string,string)>();
         static Random rnd = new Random();
         private RateLimiter _rateLimiter;
+        static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1, 1);
         private const int _targetClients = 3;
         private InstanceSettings _instanceSettings;
         private readonly (string, string)[] _apiKeys = new[]
@@ -39,17 +40,8 @@ namespace BirdsiteLive.Twitter.Tools
             ("CjulERsDeqhhjSme66ECg", "IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck"), // iPad
             ("3rJOl1ODzm9yZy63FACdg", "5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8"), // Mac
         };
-        
-        private readonly string[] _bTokens =  new[] 
-                {
-            // developer.twitter.com
-                    "AAAAAAAAAAAAAAAAAAAAACHguwAAAAAAaSlT0G31NDEyg%2BSnBN5JuyKjMCU%3Dlhg0gv0nE7KKyiJNEAojQbn8Y3wJm1xidDK7VnKGBP4ByJwHPb",
-            // tweetdeck new
-                    "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF",
-            // ipad -- TimelineSearch returns data in a different format, making nitter return empty results. on the other hand, it has high rate limits. build separate token pools per endpoint?
-                    "AAAAAAAAAAAAAAAAAAAAAGHtAgAAAAAA%2Bx7ILXNILCqkSGIzy6faIHZ9s3Q%3DQy97w6SIrzE7lQwPJEYQBsArEE2fC25caFwRBvAGi456G09vGR",
-        };
         public String BearerToken { 
+            //get { return "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"; }
             get
             {
                 return _instanceSettings.TwitterBearerToken;
@@ -62,10 +54,6 @@ namespace BirdsiteLive.Twitter.Tools
             _logger = logger;
             _instanceSettings = settings;
             _httpClientFactory = httpClientFactory;
-
-            var concuOpt = new ConcurrencyLimiterOptions();
-            concuOpt.PermitLimit = 1;
-            _rateLimiter = new ConcurrencyLimiter(concuOpt);
         }
         #endregion
 
@@ -74,9 +62,6 @@ namespace BirdsiteLive.Twitter.Tools
             var httpClient = _httpClientFactory.CreateClient();
             using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/oauth2/token?grant_type=client_credentials"))
             {
-                int r1 = rnd.Next(_bTokens.Length);
-                return _bTokens[r1];
-                
                 int r = rnd.Next(_apiKeys.Length);
                 var (login, password) = _apiKeys[r];
                 var authValue = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{login}:{password}")));
@@ -97,8 +82,20 @@ namespace BirdsiteLive.Twitter.Tools
         public async Task RefreshClient(HttpRequestMessage req)
         {
             string token = req.Headers.GetValues("x-guest-token").First();
+            string bearer = req.Headers.GetValues("Authorization").First().Replace("Bearer ", "");
 
-            _token2.TryRemove(token, out _);
+            var i = _tokens.IndexOf((bearer, token));
+
+            // this is prabably not thread safe but yolo
+            try
+            {
+                _twitterClients.RemoveAt(i);
+                _tokens.RemoveAt(i);
+            }
+            catch (IndexOutOfRangeException _)
+            {
+                _logger.LogError("Error refreshing twitter token");
+            }
 
             await RefreshCred();
             await Task.Delay(1000);
@@ -107,8 +104,21 @@ namespace BirdsiteLive.Twitter.Tools
 
         private async Task RefreshCred()
         {
+            
             (string bearer, string guest) = await GetCred();
-            _token2.TryAdd(guest, bearer);
+
+            HttpClient client = _httpClientFactory.CreateClient();
+            //HttpClient client = new HttpClient();
+
+            _twitterClients.Add(client);
+            _tokens.Add((bearer,guest));
+
+            if (_twitterClients.Count > _targetClients)
+            {
+                _twitterClients.RemoveAt(0);
+                _tokens.RemoveAt(0);
+            }
+
         }
 
         private async Task<(string, string)> GetCred()
@@ -116,26 +126,17 @@ namespace BirdsiteLive.Twitter.Tools
             string token;
             var httpClient = _httpClientFactory.CreateClient();
             string bearer = await GenerateBearerToken();
-            using RateLimitLease lease = await _rateLimiter.AcquireAsync(permitCount: 1);
-            using var request = new HttpRequestMessage(new HttpMethod("POST"),
-                "https://api.twitter.com/1.1/guest/activate.json");
-            request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer);
-            //request.Headers.Add("User-Agent",
-            //    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36");
-
-            HttpResponseMessage httpResponse;
-            do
+            using (var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.twitter.com/1.1/guest/activate.json"))
             {
-                httpResponse = await httpClient.SendAsync(request);
+                request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
+
+                var httpResponse = await httpClient.SendAsync(request);
 
                 var c = await httpResponse.Content.ReadAsStringAsync();
-                if (httpResponse.StatusCode == HttpStatusCode.TooManyRequests)
-                    await Task.Delay(1000);
                 httpResponse.EnsureSuccessStatusCode();
                 var doc = JsonDocument.Parse(c);
                 token = doc.RootElement.GetProperty("guest_token").GetString();
-
-            } while (httpResponse.StatusCode != HttpStatusCode.OK);
+            }
 
             return (bearer, token);
 
@@ -143,19 +144,19 @@ namespace BirdsiteLive.Twitter.Tools
 
         public async Task<HttpClient> MakeHttpClient()
         {
-            if (_token2.Count < _targetClients)
+            if (_twitterClients.Count < 2)
                 await RefreshCred();
-            return _httpClientFactory.CreateClient();
+            int r = rnd.Next(_twitterClients.Count);
+            return _twitterClients[r];
         }
         public HttpRequestMessage MakeHttpRequest(HttpMethod m, string endpoint, bool addToken)
         {
             var request = new HttpRequestMessage(m, endpoint);
-            (string token, string bearer) = _token2.MaxBy(x => rnd.Next());
+            int r = rnd.Next(_twitterClients.Count);
+            (string bearer, string token) = _tokens[r];
             request.Headers.TryAddWithoutValidation("Authorization", $"Bearer " + bearer); 
             request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");
             request.Headers.TryAddWithoutValidation("x-twitter-active-user", "yes");
-            //request.Headers.Add("User-Agent",
-            //    "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/113.0.5672.127 Safari/537.36");
             if (addToken)
                 request.Headers.TryAddWithoutValidation("x-guest-token", token);
             //request.Headers.TryAddWithoutValidation("Referer", "https://twitter.com/");

src/BirdsiteLive.Twitter/TwitterTweetsService.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.IO;
-using System.Net;
 using System.Net.Http;
 using System.Text.Json;
 using System.Text.Json.Nodes;
@@ -32,50 +31,6 @@ namespace BirdsiteLive.Twitter
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<TwitterTweetsService> _logger;
         private readonly InstanceSettings _instanceSettings;
-        private static string gqlFeatures = """
-        { 
-          "android_graphql_skip_api_media_color_palette": false,
-          "blue_business_profile_image_shape_enabled": false,
-          "creator_subscriptions_subscription_count_enabled": false,
-          "creator_subscriptions_tweet_preview_api_enabled": true,
-          "freedom_of_speech_not_reach_fetch_enabled": false,
-          "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
-          "hidden_profile_likes_enabled": false,
-          "highlights_tweets_tab_ui_enabled": false,
-          "interactive_text_enabled": false,
-          "longform_notetweets_consumption_enabled": true,
-          "longform_notetweets_inline_media_enabled": false,
-          "longform_notetweets_richtext_consumption_enabled": true,
-          "longform_notetweets_rich_text_read_enabled": false,
-          "responsive_web_edit_tweet_api_enabled": false,
-          "responsive_web_enhance_cards_enabled": false,
-          "responsive_web_graphql_exclude_directive_enabled": true,
-          "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
-          "responsive_web_graphql_timeline_navigation_enabled": false,
-          "responsive_web_media_download_video_enabled": false,
-          "responsive_web_text_conversations_enabled": false,
-          "responsive_web_twitter_article_tweet_consumption_enabled": false,
-          "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-          "rweb_lists_timeline_redesign_enabled": true,
-          "spaces_2022_h2_clipping": true,
-          "spaces_2022_h2_spaces_communities": true,
-          "standardized_nudges_misinfo": false,
-          "subscriptions_verification_info_enabled": true,
-          "subscriptions_verification_info_reason_enabled": true,
-          "subscriptions_verification_info_verified_since_enabled": true,
-          "super_follow_badge_privacy_enabled": false,
-          "super_follow_exclusive_tweet_notifications_enabled": false,
-          "super_follow_tweet_api_enabled": false,
-          "super_follow_user_api_enabled": false,
-          "tweet_awards_web_tipping_enabled": false,
-          "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
-          "tweetypie_unmention_optimization_enabled": false,
-          "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-          "verified_phone_label_enabled": false,
-          "vibe_api_enabled": false,
-          "view_counts_everywhere_api_enabled": false
-        }
-        """.Replace(" ", "").Replace("\n", "");
 
         #region Ctor
         public TwitterTweetsService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ICachedTwitterUserService twitterUserService, ITwitterUserDal twitterUserDal, InstanceSettings instanceSettings, ILogger<TwitterTweetsService> logger)
@@ -96,27 +51,21 @@ namespace BirdsiteLive.Twitter
             var client = await _twitterAuthenticationInitializer.MakeHttpClient();
 
 
-            // https://platform.twitter.com/embed/Tweet.html?id=1633788842770825216
             string reqURL =
-                "https://api.twitter.com/graphql/83h5UyHZ9wEKBVzALX8R_g/ConversationTimelineV2?variables={%22focalTweetId%22%3A%22"
+                "https://api.twitter.com/graphql/XjlydVWHFIDaAUny86oh2g/TweetDetail?variables=%7B%22focalTweetId%22%3A%22"
                 + statusId +
-                "%22,%22count%22:20,%22includeHasBirdwatchNotes%22:false}&features="+ gqlFeatures;
+                "%22,%22with_rux_injections%22%3Atrue,%22includePromotedContent%22%3Afalse,%22withCommunity%22%3Afalse,%22withQuickPromoteEligibilityTweetFields%22%3Afalse,%22withBirdwatchNotes%22%3Afalse,%22withSuperFollowsUserFields%22%3Afalse,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Afalse,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Afalse,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Atrue%7D";
             using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
             try
             {
                 JsonDocument tweet;
                 var httpResponse = await client.SendAsync(request);
-                if (httpResponse.StatusCode == HttpStatusCode.Unauthorized)
-                {
-                    _logger.LogError("Error retrieving tweet {statusId}; refreshing client", statusId);
-                    await _twitterAuthenticationInitializer.RefreshClient(request);
-                }
                 httpResponse.EnsureSuccessStatusCode();
                 var c = await httpResponse.Content.ReadAsStringAsync();
                 tweet = JsonDocument.Parse(c);
 
 
-                var timeline = tweet.RootElement.GetProperty("data").GetProperty("timeline_response")
+                var timeline = tweet.RootElement.GetProperty("data").GetProperty("threaded_conversation_with_injections_v2")
                     .GetProperty("instructions").EnumerateArray().First().GetProperty("entries").EnumerateArray();
 
                 var tweetInDoc = timeline.Where(x => x.GetProperty("entryId").GetString() == "tweet-" + statusId)
@@ -151,13 +100,8 @@ namespace BirdsiteLive.Twitter
 
 
             var reqURL =
-                "https://api.twitter.com/graphql/8IS8MaO-2EN6GZZZb8jF0g/UserWithProfileTweetsAndRepliesQueryV2?variables=%7B%22rest_id%22%3A%22" +
-                userId +
-                "%22,%22count%22%3A40,%22includeHasBirdwatchNotes%22%3Atrue}&features=" +
-                gqlFeatures;
-            //reqURL =
-            //    """https://twitter.com/i/api/graphql/rIIwMe1ObkGh_ByBtTCtRQ/UserTweets?variables={"userId":"44196397","count":20,"includePromotedContent":true,"withQuickPromoteEligibilityTweetFields":true,"withVoice":true,"withV2Timeline":true}&features={"rweb_lists_timeline_redesign_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":false,"tweet_awards_web_tipping_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_media_download_video_enabled":false,"responsive_web_enhance_cards_enabled":false}&fieldToggles={"withArticleRichContentState":false}""";
-            //reqURL = reqURL.Replace("44196397", userId.ToString());
+                "https://api.twitter.com/graphql/pNl8WjKAvaegIoVH--FuoQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" +
+                userId + "%22,%22count%22%3A40,%22includePromotedContent%22%3Atrue,%22withCommunity%22%3Atrue,%22withSuperFollowsUserFields%22%3Atrue,%22withDownvotePerspective%22%3Afalse,%22withReactionsMetadata%22%3Afalse,%22withReactionsPerspective%22%3Afalse,%22withSuperFollowsTweetFields%22%3Atrue,%22withVoice%22%3Atrue,%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue,%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue,%22verified_phone_label_enabled%22%3Afalse,%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue,%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse,%22tweetypie_unmention_optimization_enabled%22%3Atrue,%22vibe_api_enabled%22%3Atrue,%22responsive_web_edit_tweet_api_enabled%22%3Atrue,%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue,%22view_counts_everywhere_api_enabled%22%3Atrue,%22longform_notetweets_consumption_enabled%22%3Atrue,%22tweet_awards_web_tipping_enabled%22%3Afalse,%22freedom_of_speech_not_reach_fetch_enabled%22%3Afalse,%22standardized_nudges_misinfo%22%3Atrue,%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse,%22interactive_text_enabled%22%3Atrue,%22responsive_web_text_conversations_enabled%22%3Afalse,%22longform_notetweets_richtext_consumption_enabled%22%3Afalse,%22responsive_web_enhance_cards_enabled%22%3Afalse%7D";
             JsonDocument results;
             List<ExtractedTweet> extractedTweets = new List<ExtractedTweet>();
             using var request = _twitterAuthenticationInitializer.MakeHttpRequest(new HttpMethod("GET"), reqURL, true);
@@ -165,38 +109,48 @@ namespace BirdsiteLive.Twitter
             {
 
                 var httpResponse = await client.SendAsync(request);
-                var c = await httpResponse.Content.ReadAsStringAsync();
-                if (httpResponse.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden)
-                {
-                    _logger.LogError("Error retrieving timeline of {Username}; refreshing client", username);
-                    await _twitterAuthenticationInitializer.RefreshClient(request);
-                    return null;
-                }
                 httpResponse.EnsureSuccessStatusCode();
+                var c = await httpResponse.Content.ReadAsStringAsync();
                 results = JsonDocument.Parse(c);
 
                 _statisticsHandler.CalledTweetApi();
             }
+            catch (HttpRequestException e)
+            {
+                _logger.LogError(e, "Error retrieving timeline of {Username}; refreshing client", username);
+                //await _twitterAuthenticationInitializer.RefreshClient(request);
+                return null;
+            }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving timeline ", username);
                 return null;
             }
 
-            var timeline = results.RootElement.GetProperty("data").GetProperty("user_result").GetProperty("result")
-                .GetProperty("timeline_response").GetProperty("timeline").GetProperty("instructions").EnumerateArray();
+            var timeline = results.RootElement.GetProperty("data").GetProperty("user").GetProperty("result")
+                .GetProperty("timeline_v2").GetProperty("timeline").GetProperty("instructions").EnumerateArray();
 
             foreach (JsonElement timelineElement in timeline) 
             {
-                if (timelineElement.GetProperty("__typename").GetString() != "TimelineAddEntries")
+                if (timelineElement.GetProperty("type").GetString() != "TimelineAddEntries")
                     continue;
 
                 
                 foreach (JsonElement tweet in timelineElement.GetProperty("entries").EnumerateArray())
                 {
-                    if (tweet.GetProperty("content").GetProperty("__typename").GetString() != "TimelineTimelineItem")
+                    if (tweet.GetProperty("content").GetProperty("entryType").GetString() != "TimelineTimelineItem")
                         continue;
                     
+                    try 
+                    {
+                        JsonElement userDoc = tweet.GetProperty("content").GetProperty("itemContent")
+                                .GetProperty("tweet_results").GetProperty("core").GetProperty("user_results");
+
+                        TwitterUser tweetUser = _twitterUserService.Extract(userDoc);
+                        _twitterUserService.AddUser(tweetUser);
+                    }
+                    catch (Exception _)
+                    {}
 
                     try 
                     {   
@@ -225,45 +179,38 @@ namespace BirdsiteLive.Twitter
 
             JsonElement retweet;
             TwitterUser OriginalAuthor;
-            TwitterUser author = null;
             JsonElement inReplyToPostIdElement;
             JsonElement inReplyToUserElement;
             string inReplyToUser = null;
             long? inReplyToPostId = null;
             long retweetId = default;
 
-            string userName = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("core").GetProperty("user_result")
+            string userName = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("core").GetProperty("user_results")
                     .GetProperty("result").GetProperty("legacy").GetProperty("screen_name").GetString();
 
-            JsonElement userDoc = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("core")
-                    .GetProperty("user_result").GetProperty("result");
-
-            author = _twitterUserService.Extract(userDoc); 
-            
-            bool isReply = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+            bool isReply = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .TryGetProperty("in_reply_to_status_id_str", out inReplyToPostIdElement);
-            tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+            tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .TryGetProperty("in_reply_to_screen_name", out inReplyToUserElement);
             if (isReply) 
             {
                 inReplyToPostId = Int64.Parse(inReplyToPostIdElement.GetString());
                 inReplyToUser = inReplyToUserElement.GetString();
             }
-            bool isRetweet = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+            bool isRetweet = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .TryGetProperty("retweeted_status_result", out retweet);
             string MessageContent;
             if (!isRetweet)
             {
-                MessageContent = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+                MessageContent = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .GetProperty("full_text").GetString();
-                bool isNote = tweet.GetProperty("content").GetProperty("content")
-                        .GetProperty("tweetResult").GetProperty("result")
+                bool isNote = tweet.GetProperty("content").GetProperty("itemContent")
+                        .GetProperty("tweet_results").GetProperty("result")
                         .TryGetProperty("note_tweet", out var note);
                 if (isNote)
                 {
@@ -271,16 +218,15 @@ namespace BirdsiteLive.Twitter
                         .GetProperty("text").GetString();
                 }
                 OriginalAuthor = null;
-                
             }
             else 
             {
-                MessageContent = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+                MessageContent = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .GetProperty("retweeted_status_result").GetProperty("result")
                     .GetProperty("legacy").GetProperty("full_text").GetString();
-                bool isNote = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+                bool isNote = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .GetProperty("retweeted_status_result").GetProperty("result")
                     .TryGetProperty("note_tweet", out var note);
                 if (isNote)
@@ -288,34 +234,29 @@ namespace BirdsiteLive.Twitter
                     MessageContent = note.GetProperty("note_tweet_results").GetProperty("result")
                         .GetProperty("text").GetString();
                 }
-                string OriginalAuthorUsername = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+                string OriginalAuthorUsername = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .GetProperty("retweeted_status_result").GetProperty("result")
-                    .GetProperty("core").GetProperty("user_result").GetProperty("result")
+                    .GetProperty("core").GetProperty("user_results").GetProperty("result")
                     .GetProperty("legacy").GetProperty("screen_name").GetString();
-                JsonElement OriginalAuthorDoc = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                    .GetProperty("retweeted_status_result").GetProperty("result")
-                    .GetProperty("core").GetProperty("user_result").GetProperty("result");
-                OriginalAuthor = _twitterUserService.Extract(OriginalAuthorDoc); 
-                //OriginalAuthor = await _twitterUserService.GetUserAsync(OriginalAuthorUsername);
-                retweetId = Int64.Parse(tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+                OriginalAuthor = await _twitterUserService.GetUserAsync(OriginalAuthorUsername);
+                retweetId = Int64.Parse(tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .GetProperty("retweeted_status_result").GetProperty("result")
                     .GetProperty("rest_id").GetString());
             }
 
-            string creationTime = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+            string creationTime = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .GetProperty("created_at").GetString().Replace(" +0000", "");
 
             JsonElement extendedEntities;
-            bool hasMedia = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+            bool hasMedia = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .TryGetProperty("extended_entities", out extendedEntities);
 
-            JsonElement.ArrayEnumerator urls = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+            JsonElement.ArrayEnumerator urls = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .GetProperty("entities").GetProperty("urls").EnumerateArray();
             foreach (JsonElement url in urls)
             {
@@ -331,8 +272,7 @@ namespace BirdsiteLive.Twitter
                 {
                     var type = media.GetProperty("type").GetString();
                     string url = "";
-                    string altText = null;
-                    if (media.TryGetProperty("video_info", out _))
+                    if (type == "video" || type == "animated_gif")
                     {
                         var bitrate = -1;
                         foreach (JsonElement v in media.GetProperty("video_info").GetProperty("variants").EnumerateArray())
@@ -351,16 +291,10 @@ namespace BirdsiteLive.Twitter
                     {
                         url = media.GetProperty("media_url_https").GetString();
                     }
-
-                    if (media.TryGetProperty("ext_alt_text", out JsonElement altNode))
-                    {
-                        altText = altNode.GetString();
-                    }
                     var m = new ExtractedMedia
                     {
-                        MediaType = GetMediaType(type, url),
+                        MediaType = GetMediaType(type, media.GetProperty("media_url_https").GetString()),
                         Url = url,
-                        AltText = altText
                     };
                     Media.Add(m);
 
@@ -368,33 +302,20 @@ namespace BirdsiteLive.Twitter
                 }
             }
 
-            bool isQuoteTweet = tweet.GetProperty("content").GetProperty("content")
-                    .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
+            bool isQuoteTweet = tweet.GetProperty("content").GetProperty("itemContent")
+                    .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
                     .GetProperty("is_quote_status").GetBoolean();
 
             if (isQuoteTweet) 
             {
 
-                string quoteTweetId = tweet.GetProperty("content").GetProperty("content")
-                        .GetProperty("tweetResult").GetProperty("result").GetProperty("legacy")
-                        .GetProperty("quoted_status_id_str").GetString();
-                string quoteTweetAcct = tweet.GetProperty("content").GetProperty("content")
-                        .GetProperty("tweetResult").GetProperty("result")
-                        .GetProperty("quoted_status_result").GetProperty("result")
-                        .GetProperty("core").GetProperty("user_result").GetProperty("result")
-                        .GetProperty("legacy").GetProperty("screen_name").GetString();
-                //Uri test = new Uri(quoteTweetLink);
-                //string quoteTweetAcct = test.Segments[1].Replace("/", "");
-                //string quoteTweetId = test.Segments[3];
-                
-                string quoteTweetLink = $"https://{_instanceSettings.Domain}/@{quoteTweetAcct}/{quoteTweetId}";
-
-                //MessageContent.Replace($"https://twitter.com/i/web/status/{}", "");
-                MessageContent = MessageContent.Replace($"https://twitter.com/{quoteTweetAcct}/status/{quoteTweetId}", "");
-                
+                string quoteTweetLink = tweet.GetProperty("content").GetProperty("itemContent")
+                        .GetProperty("tweet_results").GetProperty("result").GetProperty("legacy")
+                        .GetProperty("quoted_status_permalink").GetProperty("expanded").GetString();
+                quoteTweetLink = quoteTweetLink.Replace("https://twitter.com/", $"https://{_instanceSettings.Domain}/users/");
+                quoteTweetLink = quoteTweetLink.Replace("/status/", "/statuses/");
                 MessageContent = MessageContent + "\n\n" + quoteTweetLink;
             }
-            
             var extractedTweet = new ExtractedTweet
             {
                 Id = Int64.Parse(tweet.GetProperty("entryId").GetString().Replace("tweet-", "")),
@@ -409,7 +330,6 @@ namespace BirdsiteLive.Twitter
                 RetweetUrl = "https://t.co/123",
                 RetweetId = retweetId,
                 OriginalAuthor = OriginalAuthor,
-                Author = author,
             };
        
             return extractedTweet;

src/BirdsiteLive.Twitter/TwitterUserService.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Linq;
-using System.Net;
 using System.Net.Http;
 using System.Text.Json;
 using System.Threading.Tasks;
@@ -25,53 +24,7 @@ namespace BirdsiteLive.Twitter
         private readonly ITwitterStatisticsHandler _statisticsHandler;
         private readonly ILogger<TwitterUserService> _logger;
         
-        private readonly string endpoint =
-            "https://api.twitter.com/graphql/pVrmNaXcxPjisIvKtLDMEA/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%7D&features=" + gqlFeatures;
-
-        private static string gqlFeatures = """
-        { 
-          "android_graphql_skip_api_media_color_palette": false,
-          "blue_business_profile_image_shape_enabled": false,
-          "creator_subscriptions_subscription_count_enabled": false,
-          "creator_subscriptions_tweet_preview_api_enabled": true,
-          "freedom_of_speech_not_reach_fetch_enabled": false,
-          "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
-          "hidden_profile_likes_enabled": false,
-          "highlights_tweets_tab_ui_enabled": false,
-          "interactive_text_enabled": false,
-          "longform_notetweets_consumption_enabled": true,
-          "longform_notetweets_inline_media_enabled": false,
-          "longform_notetweets_richtext_consumption_enabled": true,
-          "longform_notetweets_rich_text_read_enabled": false,
-          "responsive_web_edit_tweet_api_enabled": false,
-          "responsive_web_enhance_cards_enabled": false,
-          "responsive_web_graphql_exclude_directive_enabled": true,
-          "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
-          "responsive_web_graphql_timeline_navigation_enabled": false,
-          "responsive_web_media_download_video_enabled": false,
-          "responsive_web_text_conversations_enabled": false,
-          "responsive_web_twitter_article_tweet_consumption_enabled": false,
-          "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-          "rweb_lists_timeline_redesign_enabled": true,
-          "spaces_2022_h2_clipping": true,
-          "spaces_2022_h2_spaces_communities": true,
-          "standardized_nudges_misinfo": false,
-          "subscriptions_verification_info_enabled": true,
-          "subscriptions_verification_info_reason_enabled": true,
-          "subscriptions_verification_info_verified_since_enabled": true,
-          "super_follow_badge_privacy_enabled": false,
-          "super_follow_exclusive_tweet_notifications_enabled": false,
-          "super_follow_tweet_api_enabled": false,
-          "super_follow_user_api_enabled": false,
-          "tweet_awards_web_tipping_enabled": false,
-          "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
-          "tweetypie_unmention_optimization_enabled": false,
-          "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-          "verified_phone_label_enabled": false,
-          "vibe_api_enabled": false,
-          "view_counts_everywhere_api_enabled": false
-        }
-        """.Replace(" ", "").Replace("\n", "");
+        private readonly string endpoint = "https://twitter.com/i/api/graphql/4LB4fkCe3RDLDmOEEYtueg/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withSafetyModeUserFields%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_twitter_blue_new_verification_copy_is_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D";
 
         #region Ctor
         public TwitterUserService(ITwitterAuthenticationInitializer twitterAuthenticationInitializer, ITwitterStatisticsHandler statisticsHandler, ILogger<TwitterUserService> logger)
@@ -92,12 +45,6 @@ namespace BirdsiteLive.Twitter
             {
 
                 var httpResponse = await client.SendAsync(request);
-                if (httpResponse.StatusCode == HttpStatusCode.Unauthorized)
-                {
-                    _logger.LogError("Error retrieving user {Username}, Refreshing client", username);
-                    await _twitterAuthenticationInitializer.RefreshClient(request);
-                    return null;
-                }
                 httpResponse.EnsureSuccessStatusCode();
 
                 var c = await httpResponse.Content.ReadAsStringAsync();
@@ -121,6 +68,12 @@ namespace BirdsiteLive.Twitter
                 //    throw;
                 //}
             }
+            catch (HttpRequestException e)
+            {
+                _logger.LogError(e, "Error retrieving user {Username}, Refreshing client", username);
+                await _twitterAuthenticationInitializer.RefreshClient(request);
+                return null;
+            }
             catch (Exception e)
             {
                 _logger.LogError(e, "Error retrieving user {Username}", username);

src/BirdsiteLive.Wikidata/BirdsiteLive.Wikidata.csproj

@@ -1,18 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <OutputType>Exe</OutputType>
-        <TargetFramework>net7.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL.Postgres\BirdsiteLive.DAL.Postgres.csproj" />
-      <ProjectReference Include="..\DataAccessLayers\BirdsiteLive.DAL\BirdsiteLive.DAL.csproj" />
-      <Content Include="query.sparql">
-        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-      </Content>
-    </ItemGroup>
-
-</Project>

src/BirdsiteLive.Wikidata/Program.cs

@@ -1,48 +0,0 @@
-using System;
-using System.IO;
-using System.Net.Http;
-using System.Threading.Tasks;
-using System.Xml;
-using BirdsiteLive.DAL.Models;
-using BirdsiteLive.DAL.Postgres.DataAccessLayers;
-using BirdsiteLive.DAL.Postgres.Settings;
-
-var settings = new PostgresSettings()
-{
-    ConnString = System.Environment.GetEnvironmentVariable("ConnString"),
-};
-var dal = new TwitterUserPostgresDal(settings);
-
-var twitterUser = new HashSet<string>();
-var twitterUserQuery = await dal.GetAllTwitterUsersAsync();
-Console.WriteLine("Loading twitter users");
-foreach (SyncTwitterUser user in twitterUserQuery)
-{
-    twitterUser.Add(user.Acct);
-}
-Console.WriteLine("Done loading twitter users");
-
-Console.WriteLine("Hello, World!");
-var client = new HttpClient();
-string query = new StreamReader("query.sparql").ReadToEnd();
-
-client.DefaultRequestHeaders.Add("Accept", "text/csv");
-client.DefaultRequestHeaders.Add("User-Agent", "BirdMakeup/1.0 (https://bird.makeup; coolbot@example.org) BirdMakeup/1.0");
-var response = await client.GetAsync($"https://query.wikidata.org/sparql?query={Uri.EscapeDataString(query)}");
-var content = await response.Content.ReadAsStringAsync();
-
-// Console.WriteLine(content);
-
-foreach (string n in content.Split("\n"))
-{
-    var s = n.Split(",");
-    if (n.Length < 2)
-        continue;
-    
-    var acct = s[1].ToLower();
-    var fedi = "@" + s[2];
-    await dal.UpdateTwitterUserFediAcctAsync(acct, fedi);
-    if (twitterUser.Contains(acct))
-        Console.WriteLine(fedi);
-}
-

src/BirdsiteLive.Wikidata/README.md

@@ -1,33 +0,0 @@
-# Wikidata service
-
-Wikidata is the metadata community behind Wikipedia. See for example
-[Hank Green](https://www.wikidata.org/wiki/Q550996). In his page, there are 
-all the links to his wikipedia pages, and many facts about him. What is 
-particularly useful to us is the twitter username (P2002) and mastodon 
-username (P4033). 
-
-From this information, we can build a feature that suggests to follow the 
-native fediverse account of someone you are trying to follow from Twitter.
-
-The main downside is that those redirect are only for somewhat famous
-people/organisations.
-
-## Goals
-### Being reusable by others
-All this data can be useful to many other fediverse projects: tools 
-for finding interesting accounts to follow, "verified" badge powered by 
-Wikipedia, etc. I hope that by working on improving this dataset, we can
-help other projects thrive. 
-### Being independent of Twitter
-Bird.makeup has to build features in a way that can't be suddenly cut off.
-Building this feature with a "Log in with Twitter" is not viable.
-Wikipedia is independent and outside of Elon's reach.
-
-Also this system supports many other services: TikTok, Reddit, YouTube, etc. 
-Which is really useful to expend the scope of this project while reusing as 
-much work as possible
-### Having great moderation
-
-Wikipedia has many tools to help curate data and remove troll's submissions,
-far better than anything I can build. I much prefer contribute to what
-they are doing than try to compete

src/BirdsiteLive.Wikidata/query.sparql

@@ -1,9 +0,0 @@
-#Cats
-SELECT ?item ?username ?username2 ?linkcount ?itemLabel
-WHERE 
-{
-  ?item wdt:P2002 ?username. 
-  ?item wdt:P4033 ?username2. 
-        ?item wikibase:sitelinks ?linkcount .
-  SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } # Helps get the label in your language, if not, then en language
-} ORDER BY DESC(?linkcount) LIMIT 5000

src/BirdsiteLive.sln

@@ -47,16 +47,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Moderation.Tes
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Common.Tests", "Tests\BirdsiteLive.Common.Tests\BirdsiteLive.Common.Tests.csproj", "{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BSLManager", "BSLManager\BSLManager.csproj", "{4A84D351-E91B-4E58-8E20-211F0F4991D7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BSLManager.Tests", "Tests\BSLManager.Tests\BSLManager.Tests.csproj", "{D4457271-620E-465A-B08E-7FC63C99A2F6}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Twitter.Tests", "Tests\BirdsiteLive.Twitter.Tests\BirdsiteLive.Twitter.Tests.csproj", "{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.Wikidata", "BirdsiteLive.Wikidata\BirdsiteLive.Wikidata.csproj", "{EAB43087-359C-46BD-8796-5F7D9B473B39}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SocialNetworks", "SocialNetworks", "{7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dotMakeup.HackerNews", "dotMakeup.HackerNews\dotMakeup.HackerNews.csproj", "{060DE3F7-DB7E-45FD-B233-104C3C464F57}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "dotMakeup.HackerNews.Tests", "Tests\dotMakeup.HackerNews.Tests\dotMakeup.HackerNews.Tests.csproj", "{6D650384-7BDD-4628-A46C-2FE4A688DBA4}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -135,28 +131,25 @@ Global
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4A84D351-E91B-4E58-8E20-211F0F4991D7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D4457271-620E-465A-B08E-7FC63C99A2F6}.Release|Any CPU.Build.0 = Release|Any CPU
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C}.Release|Any CPU.Build.0 = Release|Any CPU
-		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{EAB43087-359C-46BD-8796-5F7D9B473B39}.Release|Any CPU.Build.0 = Release|Any CPU
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{160AD138-4E29-4706-8546-9826B529E9B2} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
+		{77C559D1-80A2-4B1C-A566-AE2D156944A4} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
 		{E64E7501-5DB8-4620-BA35-BA59FD746ABA} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
 		{155D46A4-2D05-47F2-8FFC-0B7C412A7652} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{D48450EE-D8BD-4228-9864-043AC88F7EE0} = {4FEAD6BC-3C8E-451A-8CA1-FF1AF47D26CC}
@@ -172,11 +165,8 @@ Global
 		{4BE541AC-8A93-4FA3-98AC-956CC2D5B748} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
 		{0A311BF3-4FD9-4303-940A-A3778890561C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{C69F7582-6050-44DC-BAAB-7C8F0BDA525C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
+		{D4457271-620E-465A-B08E-7FC63C99A2F6} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 		{2DFA0BFD-88F5-4434-A6E3-C93B5750E88C} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
-		{EAB43087-359C-46BD-8796-5F7D9B473B39} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C}
-		{060DE3F7-DB7E-45FD-B233-104C3C464F57} = {7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}
-		{77C559D1-80A2-4B1C-A566-AE2D156944A4} = {7ACCADEA-4B64-4ACB-A21D-0627674BBA9D}
-		{6D650384-7BDD-4628-A46C-2FE4A688DBA4} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE}

src/BirdsiteLive/BirdsiteLive.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <UserSecretsId>d21486de-a812-47eb-a419-05682bb68856</UserSecretsId>
     <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
     <Version>1.0</Version>

src/BirdsiteLive/Controllers/UsersController.cs

@@ -5,13 +5,12 @@ using System.Linq;
 using System.Text.Json;
 using System.Net.Mime;
 using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
 using BirdsiteLive.ActivityPub;
 using BirdsiteLive.ActivityPub.Models;
 using BirdsiteLive.Common.Regexes;
 using BirdsiteLive.Common.Settings;
-using BirdsiteLive.DAL.Contracts;
-using BirdsiteLive.DAL.Models;
 using BirdsiteLive.Domain;
 using BirdsiteLive.Models;
 using BirdsiteLive.Tools;
@@ -31,20 +30,16 @@ namespace BirdsiteLive.Controllers
         private readonly IUserService _userService;
         private readonly IStatusService _statusService;
         private readonly InstanceSettings _instanceSettings;
-        private readonly IFollowersDal _followersDal;
-        private readonly ITwitterUserDal _twitterUserDal;
         private readonly ILogger<UsersController> _logger;
 
         #region Ctor
-        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, IFollowersDal followersDal, ITwitterUserDal twitterUserDal, ILogger<UsersController> logger)
+        public UsersController(ICachedTwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ICachedTwitterTweetsService twitterTweetService, ILogger<UsersController> logger)
         {
             _twitterUserService = twitterUserService;
             _userService = userService;
             _statusService = statusService;
             _instanceSettings = instanceSettings;
             _twitterTweetService = twitterTweetService;
-            _followersDal = followersDal;
-            _twitterUserDal = twitterUserDal;
             _logger = logger;
         }
         #endregion
@@ -124,12 +119,6 @@ namespace BirdsiteLive.Controllers
             if (isSaturated) return View("ApiSaturated");
             if (notFound) return View("UserNotFound");
 
-            Follower[] followers = new Follower[] { };
-            
-            var userDal = await _twitterUserDal.GetTwitterUserAsync(user.Acct);
-            if (userDal != null)
-                followers = await _followersDal.GetFollowersAsync(userDal.Id);
-
             var displayableUser = new DisplayTwitterUser
             {
                 Name = user.Name,
@@ -138,9 +127,7 @@ namespace BirdsiteLive.Controllers
                 Url = user.Url,
                 ProfileImageUrl = user.ProfileImageUrl,
                 Protected = user.Protected,
-                FollowerCount = followers.Length,
-                MostPopularServer = followers.GroupBy(x => x.Host).OrderByDescending(x => x.Count()).Select(x => x.Key).FirstOrDefault("N/A"),
-                FediverseAccount = userDal.FediAcct,
+
                 InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}"
             };
             return View(displayableUser);
@@ -158,8 +145,7 @@ namespace BirdsiteLive.Controllers
             if (tweet == null)
                 return NotFound();
             
-            if (tweet.Author.Acct != id)
-                return NotFound();
+            var user = await _twitterUserService.GetUserAsync(id);
 
             var status = _statusService.GetStatus(id, tweet);
 
@@ -179,51 +165,12 @@ namespace BirdsiteLive.Controllers
             {
                 Text = tweet.MessageContent,
                 OgUrl = $"https://twitter.com/{id}/status/{statusId}",
-                UserProfileImage = tweet.Author.ProfileImageUrl,
-                UserName = tweet.Author.Name,
+                UserProfileImage = user.ProfileImageUrl,
+                UserName = user.Name,
             };
             return View(displayTweet);
         }
 
-        // Mastodon API for QT in some apps
-        [Route("/api/v1/statuses/{statusId}")]
-        public async Task<IActionResult> mastoApi(string id, string statusId)
-        {
-            if (!long.TryParse(statusId, out var parsedStatusId))
-                return NotFound();
-
-            var tweet = await _twitterTweetService.GetTweetAsync(parsedStatusId);
-            if (tweet == null)
-                return NotFound();
-            
-            var user = await _twitterUserService.GetUserAsync(tweet.Author.Acct);
-            var status = _statusService.GetActivity(tweet.Author.Acct, tweet);
-            var res = new MastodonPostApi()
-            {
-                id = parsedStatusId,
-                content = status.apObject.content,
-                created_at = status.published,
-                uri = $"https://{_instanceSettings.Domain}/users/{tweet.Author.Acct.ToLower()}/statuses/{tweet.Id}",
-                url = $"https://{_instanceSettings.Domain}/@{tweet.Author.Acct.ToLower()}/{tweet.Id}",
-                account = new MastodonUserApi()
-                {
-                    id = user.Id,
-                    username = user.Acct,
-                    acct = user.Acct,
-                    display_name = user.Name,
-                    note = user.Description,
-                    url = $"https://{_instanceSettings.Domain}/@{tweet.Author.Acct.ToLower()}",
-                    avatar = user.ProfileImageUrl,
-                    avatar_static = user.ProfileImageUrl,
-                    header = user.ProfileBannerURL,
-                    header_static = user.ProfileBannerURL,
-                }
-            };
-
-
-            var jsonApUser = JsonSerializer.Serialize(res);
-            return Content(jsonApUser, "application/activity+json; charset=utf-8");
-        }
         [Route("/users/{id}/statuses/{statusId}/activity")]
         public async Task<IActionResult> Activity(string id, string statusId)
         {
@@ -234,6 +181,8 @@ namespace BirdsiteLive.Controllers
             if (tweet == null)
                 return NotFound();
             
+            var user = await _twitterUserService.GetUserAsync(id);
+
             var status = _statusService.GetActivity(id, tweet);
 
             var jsonApUser = JsonSerializer.Serialize(status);

src/BirdsiteLive/Controllers/WellKnownController.cs

@@ -201,9 +201,6 @@ namespace BirdsiteLive.Controllers
             if (!string.IsNullOrWhiteSpace(domain) && domain != _settings.Domain)
                 return NotFound();
 
-            var user = await _twitterUserDal.GetTwitterUserAsync(name);
-            if (user is null)
-            {
             try
             {
                 await _twitterUserService.GetUserAsync(name);
@@ -225,7 +222,6 @@ namespace BirdsiteLive.Controllers
                 _logger.LogError(e, "Exception getting {Name}", name);
                 throw;
             }
-            }
 
             var actorUrl = UrlFactory.GetActorUrl(_settings.Domain, name);
 

src/BirdsiteLive/Models/DisplayTwitterUser.cs

@@ -8,10 +8,7 @@
         public string Url { get; set; }
         public string ProfileImageUrl { get; set; }
         public bool Protected { get; set; }
-        public int FollowerCount { get; set; }
-        public string MostPopularServer { get; set; }
 
         public string InstanceHandle { get; set; }
-        public string FediverseAccount { get; set; }
     }
 }

src/BirdsiteLive/Models/MastodonPostApi.cs

@@ -1,60 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace BirdsiteLive.Models;
-
-public class MastodonPostApi
-{
-    public long id { get; set; }
-    public string created_at { get; set; }
-    public long? in_reply_to_id { get; set; } = null;
-    public long? in_reply_to_account_id { get; set; } = null;
-    public bool sensitive { get; set; } = false;
-    public string spoiler_text { get; set; } = "";
-    public string visibility { get; set; } = "public";
-    public string language { get; set; } = "en";
-    public string uri { get; set; } 
-    public string url { get; set; } 
-    public int replies_count { get; set; } = 0;
-    public int reblogs_count { get; set; } = 0;
-    public int favorite_count { get; set; } = 0;
-    public string content { get; set; }
-    public MastodonUserApi account { get; set; }
-    public MastodonAppApi application { get; } = new MastodonAppApi();
-    
-    public List<MastodonAppApi> media_attachments { get; set; } = new List<MastodonAppApi>();
-    public List<MastodonAppApi> mentions { get; set; } = new List<MastodonAppApi>();
-    public List<MastodonAppApi> tags { get; set; } = new List<MastodonAppApi>();
-    public List<MastodonAppApi> emojis { get; set; } = new List<MastodonAppApi>();
-    public string card { get; set; }
-    public string poll { get; set; }
-    public string reblog { get; set; }
-}
-public class MastodonUserApi
-{
-    public long id { get; set; }
-    public string username { get; set; }
-    public string acct { get; set; }
-    public string display_name { get; set; }
-    public bool locked { get; set; } = false;
-    public bool bot { get; set; } = true;
-    public bool group { get; set; } = false;
-    public string note { get; set; }
-    public string url { get; set; }
-    public string avatar { get; set; }
-    public string avatar_static { get; set; }
-    public string header { get; set; }
-    public string header_static { get; set; }
-    public int followers_count { get; set; } = 0;
-    public int following_count { get; set; } = 0;
-    public int statuses_count { get; set; } = 0;
-
-    public List<MastodonAppApi> fields { get; set; } = new List<MastodonAppApi>();
-    public List<MastodonAppApi> emojis { get; set; } = new List<MastodonAppApi>();
-}
-
-public class MastodonAppApi
-{
-    public string name { get; set; } = "bird.makeup";
-    public string url { get; set; } = "https://bird.makeup/";
-}

src/BirdsiteLive/Services/CachedStatisticsService.cs

@@ -15,7 +15,7 @@ namespace BirdsiteLive.Services
         private readonly ITwitterUserDal _twitterUserDal;
         private readonly IFollowersDal _followersDal;
 
-        private static Task<CachedStatistics> _cachedStatistics;
+        private static CachedStatistics _cachedStatistics;
         private readonly InstanceSettings _instanceSettings;
 
         #region Ctor
@@ -24,36 +24,28 @@ namespace BirdsiteLive.Services
             _twitterUserDal = twitterUserDal;
             _instanceSettings = instanceSettings;
             _followersDal = followersDal;
-            _cachedStatistics = CreateStats();
         }
         #endregion
 
         public async Task<CachedStatistics> GetStatisticsAsync()
         {
-            var stats = await _cachedStatistics;
-            if ((DateTime.UtcNow - stats.RefreshedTime).TotalMinutes > 5)
-            {
-                _cachedStatistics = CreateStats();
-            }
-
-            return stats;
-        }
-
-        private async Task<CachedStatistics> CreateStats()
+            if (_cachedStatistics == null ||
+                (DateTime.UtcNow - _cachedStatistics.RefreshedTime).TotalMinutes > 15)
             {
                 var twitterUserCount = await _twitterUserDal.GetTwitterUsersCountAsync();
                 var twitterSyncLag = await _twitterUserDal.GetTwitterSyncLag();
                 var fediverseUsers = await _followersDal.GetFollowersCountAsync();
 
-            var stats = new CachedStatistics
+                _cachedStatistics = new CachedStatistics
                 {
                     RefreshedTime = DateTime.UtcNow,
                     SyncLag = twitterSyncLag,
                     TwitterUsers = twitterUserCount,
                     FediverseUsers = fediverseUsers
                 };
+            }
 
-            return stats;
+            return _cachedStatistics;
         }
     }
 

src/BirdsiteLive/Startup.cs

@@ -9,7 +9,6 @@ using BirdsiteLive.DAL.Contracts;
 using BirdsiteLive.DAL.Postgres.DataAccessLayers;
 using BirdsiteLive.DAL.Postgres.Settings;
 using BirdsiteLive.Models;
-using BirdsiteLive.Services;
 using BirdsiteLive.Twitter;
 using BirdsiteLive.Twitter.Tools;
 using Lamar;
@@ -91,8 +90,6 @@ namespace BirdsiteLive
 
             services.For<ITwitterAuthenticationInitializer>().Use<TwitterAuthenticationInitializer>().Singleton();
 
-            services.For<ICachedStatisticsService>().Use<CachedStatisticsService>().Singleton();
-
             services.Scan(_ =>
             {
                 _.Assembly("BirdsiteLive.Twitter");

src/BirdsiteLive/Views/Home/Index.cshtml

@@ -12,21 +12,18 @@
     </p>
     
     <form method="POST">
+        @*<div class="form-group">
+            <label for="exampleInputEmail1">Email address</label>
+            <input type="email" class="form-control" id="exampleInputEmail1" aria-describedby="emailHelp" placeholder="Enter email">
+            <small id="emailHelp" class="form-text text-muted">We'll never share your email with anyone else.</small>
+        </div>*@
         <div class="form-group">
+            @*<label for="exampleInputPassword1">Password</label>*@
             <input type="text" class="form-control col-8 col-sm-8 col-md-6 col-lg-4 mx-auto" id="handle" name="handle" autocomplete="off" placeholder="Twitter Handle">
         </div>
         <button type="submit" class="btn btn-primary">Show</button>
     </form>
 
-    <p style = "padding-top: 100px;">
-        <br />
-        bird.makeup is made with ❤️ by <a href="https://social.librem.one/@@vincent"> Vincent Cloutier</a> in 🇨🇦
-        <br />
-        <br />
-        Many thanks to our top Patreon supporters: <br/>
-        <a href="https://mstdn-social.com/@@fishcharlie">Charlie Fish</a>
-        
-    </p>
 
     @*@if (HtmlHelperExtensions.IsDebug())
     {

src/BirdsiteLive/Views/Users/Index.cshtml

@@ -28,9 +28,6 @@
         </div>
     </a>
     <br />
-    <div>
-        This account has @ViewData.Model.FollowerCount followers on the fediverse. The server with the most followers for this account is: @ViewData.Model.MostPopularServer
-    </div>
     <br />
 
     @if (ViewData.Model.Protected)
@@ -47,13 +44,4 @@
             <input type="text" name="textbox" value="@ViewData.Model.InstanceHandle" onclick="this.select()" class="form-control" readonly />
         </div>
     }
-    @if (ViewData.Model.FediverseAccount != null)
-    {
-        <br/>
-        <br/>
-        <div>
-            <p>There is a native fediverse account associated with this Twitter account:</p>
-            <input type="text" name="textbox" value="@ViewData.Model.FediverseAccount" onclick="this.select()" class="form-control" readonly />
-        </div>
-    }
 </div>

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/BirdsiteLive.DAL.Postgres.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs

@@ -23,7 +23,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
     public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal
     {
         private readonly PostgresTools _tools;
-        private readonly Version _currentVersion = new Version(3, 0);
+        private readonly Version _currentVersion = new Version(2, 5);
         private const string DbVersionType = "db-version";
 
         #region Ctor
@@ -136,8 +136,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 new Tuple<Version, Version>(new Version(2,1), new Version(2,2)),
                 new Tuple<Version, Version>(new Version(2,2), new Version(2,3)),
                 new Tuple<Version, Version>(new Version(2,3), new Version(2,4)),
-                new Tuple<Version, Version>(new Version(2,4), new Version(2,5)),
-                new Tuple<Version, Version>(new Version(2,5), new Version(3,0))
+                new Tuple<Version, Version>(new Version(2,4), new Version(2,5))
             };
         }
 
@@ -180,48 +179,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 await _tools.ExecuteRequestAsync(alterTwitterUserId);
 
             }
-            else if (from == new Version(2, 5) && to == new Version(3, 0))
-            {
-                var dropFollowingSyncStatus = $@"ALTER TABLE {_settings.FollowersTableName} DROP COLUMN followingssyncstatus";
-                await _tools.ExecuteRequestAsync(dropFollowingSyncStatus);
-
-                var dropLastTweet = $@"ALTER TABLE {_settings.TwitterUserTableName} DROP COLUMN lasttweetsynchronizedforallfollowersid";
-                await _tools.ExecuteRequestAsync(dropLastTweet);
-                
-                var addFediverseEquivalent = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD fediverseaccount text";
-                await _tools.ExecuteRequestAsync(addFediverseEquivalent);
-                
-                var createWorkers = $@"CREATE TABLE {_settings.WorkersTableName}
-                (
-                    id BIGINT PRIMARY KEY,
-                    rangeStart INTEGER,
-                    rangeEnd INTEGER,
-                    lastSeen TIMESTAMP (2) WITHOUT TIME ZONE,
-                    name text
-                );";
-                await _tools.ExecuteRequestAsync(createWorkers);
-                
-                var createWorkerInstance = $@" INSERT INTO {_settings.WorkersTableName} (id,rangeStart,rangeEnd) VALUES(0,0, 100) ";
-                await _tools.ExecuteRequestAsync(createWorkerInstance);
-                
-                var createInstagram = $@"CREATE TABLE {_settings.InstagramUserTableName}
-                (
-                    id SERIAL PRIMARY KEY,
-                    acct VARCHAR(20) UNIQUE, 
-                    
-                    data JSONB
-                );";
-                await _tools.ExecuteRequestAsync(createInstagram);
-                
-                var createInstagramPost = $@"CREATE TABLE {_settings.CachedInstaPostsTableName}
-                (
-                    id VARCHAR(30) PRIMARY KEY,
-                    acct VARCHAR(20) UNIQUE, 
-                    
-                    data JSONB
-                );";
-                await _tools.ExecuteRequestAsync(createInstagramPost);
-            }
             else
             {
                 throw new NotImplementedException();
@@ -250,10 +207,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 $@"DROP TABLE {_settings.DbVersionTableName};",
                 $@"DROP TABLE {_settings.TwitterUserTableName};",
                 $@"DROP TABLE {_settings.FollowersTableName};",
-                $@"DROP TABLE {_settings.CachedTweetsTableName};",
-                $@"DROP TABLE {_settings.InstagramUserTableName};",
-                $@"DROP TABLE {_settings.CachedInstaPostsTableName};",
-                $@"DROP TABLE {_settings.WorkersTableName};"
+                $@"DROP TABLE {_settings.CachedTweetsTableName};"
             };
 
             foreach (var r in dropsRequests)

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/FollowersPostgresDal.cs

@@ -21,9 +21,12 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
         }
         #endregion
 
-        public async Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null)
+        public async Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null, Dictionary<int, long> followingSyncStatus = null)
         {
             if(followings == null) followings = new int[0];
+            if(followingSyncStatus == null) followingSyncStatus = new Dictionary<int, long>();
+
+            var serializedDic = JsonSerializer.Serialize(followingSyncStatus);
 
             acct = acct.ToLowerInvariant();
             host = host.ToLowerInvariant();
@@ -31,8 +34,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             using (var dbConnection = Connection)
             {
                 await dbConnection.ExecuteAsync(
-                    $"INSERT INTO {_settings.FollowersTableName} (acct,host,inboxRoute,sharedInboxRoute,followings,actorId) VALUES(@acct,@host,@inboxRoute,@sharedInboxRoute,@followings,@actorId)",
-                    new { acct, host, inboxRoute, sharedInboxRoute, followings, actorId });
+                    $"INSERT INTO {_settings.FollowersTableName} (acct,host,inboxRoute,sharedInboxRoute,followings,followingsSyncStatus,actorId) VALUES(@acct,@host,@inboxRoute,@sharedInboxRoute,@followings,CAST(@followingsSyncStatus as json),@actorId)",
+                    new { acct, host, inboxRoute, sharedInboxRoute, followings, followingsSyncStatus = serializedDic, actorId });
             }
         }
 
@@ -75,10 +78,13 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             if (!await reader.ReadAsync())
                 return null;
 
+            string syncStatusString = reader["followingsSyncStatus"] as string;
+            var syncStatus = System.Text.Json.JsonSerializer.Deserialize<Dictionary<int, long>>(syncStatusString);
             return new Follower
             {
                 Id = reader["id"] as int? ?? default,
                 Followings = (reader["followings"] as int[] ?? new int[0]).ToList(),
+                FollowingsSyncStatus = syncStatus,
                 ActorId = reader["actorId"] as string,
                 Acct = reader["acct"] as string,
                 Host = reader["host"] as string,
@@ -105,10 +111,13 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             var followers = new List<Follower>();
             while (await reader.ReadAsync())
             {
+                string syncStatusString = reader["followingsSyncStatus"] as string;
+                var syncStatus = System.Text.Json.JsonSerializer.Deserialize<Dictionary<int, long>>(syncStatusString);
                 followers.Add(new Follower
                 {
                     Id = reader["id"] as int? ?? default,
                     Followings = (reader["followings"] as int[] ?? new int[0]).ToList(),
+                    FollowingsSyncStatus = syncStatus,
                     ActorId = reader["actorId"] as string,
                     Acct = reader["acct"] as string,
                     Host = reader["host"] as string,
@@ -138,12 +147,14 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             if (follower == default) throw new ArgumentException("follower");
             if (follower.Id == default) throw new ArgumentException("id");
 
-            var query = $"UPDATE {_settings.FollowersTableName} SET followings = $1, postingErrorCount = $2 WHERE id = $3";
+            var serializedDic = System.Text.Json.JsonSerializer.Serialize(follower.FollowingsSyncStatus);
+            var query = $"UPDATE {_settings.FollowersTableName} SET followings = $1, followingsSyncStatus = CAST($2 as json), postingErrorCount = $3 WHERE id = $4";
             await using var connection = DataSource.CreateConnection();
             await connection.OpenAsync();
             await using var command = new NpgsqlCommand(query, connection) {
                 Parameters = { 
                     new() { Value = follower.Followings}, 
+                    new() { Value = serializedDic}, 
                     new() { Value = follower.PostingErrorCount}, 
                     new() { Value = follower.Id}
                 }
@@ -193,6 +204,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 ActorId = follower.ActorId,
                 SharedInboxRoute = follower.SharedInboxRoute,
                 Followings = follower.Followings.ToList(),
+                FollowingsSyncStatus = JsonSerializer.Deserialize<Dictionary<int,long>>(follower.FollowingsSyncStatus),
                 PostingErrorCount = follower.PostingErrorCount
             };
         }
@@ -200,7 +212,10 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
     internal class SerializedFollower {
         public int Id { get; set; }
+
         public int[] Followings { get; set; }
+        public string FollowingsSyncStatus { get; set; }
+
         public string Acct { get; set; }
         public string Host { get; set; }
         public string InboxRoute { get; set; }

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs

@@ -27,8 +27,8 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             using (var dbConnection = Connection)
             {
                 await dbConnection.ExecuteAsync(
-                    $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId) VALUES(@acct,@lastTweetPostedId)",
-                    new { acct, lastTweetPostedId });
+                    $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)",
+                    new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId });
             }
         }
 
@@ -53,9 +53,9 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 Acct = reader["acct"] as string,
                 TwitterUserId = reader["twitterUserId"] as long? ?? default,
                 LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
+                LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
                 LastSync = reader["lastSync"] as DateTime? ?? default,
                 FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
-                FediAcct = reader["fediverseaccount"] as string,
             };
 
         }
@@ -79,6 +79,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                 Acct = reader["acct"] as string,
                 TwitterUserId = reader["twitterUserId"] as long? ?? default,
                 LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
+                LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
                 LastSync = reader["lastSync"] as DateTime? ?? default,
                 FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
             };
@@ -90,7 +91,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
             using (var dbConnection = Connection)
             {
-                var result = (await dbConnection.QueryAsync<TimeSpan?>(query)).FirstOrDefault() ?? TimeSpan.Zero;
+                var result = (await dbConnection.QueryAsync<TimeSpan>(query)).FirstOrDefault();
                 return result;
             }
         }
@@ -117,20 +118,14 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m)
+        public async Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber)
         {
-            var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id WHERE mod(id, $2) >= $3 AND mod(id, $2) <= $4 ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
+            var query = "SELECT * FROM (SELECT unnest(followings) as follow FROM followers GROUP BY follow) AS f INNER JOIN twitter_users ON f.follow=twitter_users.id  ORDER BY lastSync ASC NULLS FIRST LIMIT $1";
 
             await using var connection = DataSource.CreateConnection();
             await connection.OpenAsync();
             await using var command = new NpgsqlCommand(query, connection) {
-                Parameters =
-                {
-                    new() { Value = maxNumber},
-                    new() { Value = m},
-                    new() { Value = nStart},
-                    new() { Value = nEnd}
-                }
+                Parameters = { new() { Value = maxNumber}}
             };
             var reader = await command.ExecuteReaderAsync();
             var results = new List<SyncTwitterUser>();
@@ -142,6 +137,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
                         Acct = reader["acct"] as string,
                         TwitterUserId = reader["twitterUserId"] as long? ?? default,
                         LastTweetPostedId = reader["lastTweetPostedId"] as long? ?? default,
+                        LastTweetSynchronizedForAllFollowersId = reader["lastTweetSynchronizedForAllFollowersId"] as long? ?? default,
                         LastSync = reader["lastSync"] as DateTime? ?? default,
                         FetchingErrorCount = reader["fetchingErrorCount"] as int? ?? default,
                     }
@@ -173,19 +169,6 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
             }
         }
 
-        public async Task UpdateTwitterUserFediAcctAsync(string twitterUsername, string fediUsername)
-        {
-            if(twitterUsername == default) throw new ArgumentException("id");
-
-            var query = $"UPDATE {_settings.TwitterUserTableName} SET fediverseaccount = $1 WHERE acct = $2";
-            await using var connection = DataSource.CreateConnection();
-            await connection.OpenAsync();
-            await using var command = new NpgsqlCommand(query, connection) {
-                Parameters = { new() { Value = fediUsername}, new() { Value = twitterUsername}}
-            };
-
-            await command.ExecuteNonQueryAsync();
-        }
         public async Task UpdateTwitterUserIdAsync(string username, long twitterUserId)
         {
             if(username == default) throw new ArgumentException("id");
@@ -200,19 +183,21 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
             await command.ExecuteNonQueryAsync();
         }
-        public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, int fetchingErrorCount, DateTime lastSync)
+        public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync)
         {
             if(id == default) throw new ArgumentException("id");
             if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId");
+            if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId");
             if(lastSync == default) throw new ArgumentException("lastSync");
 
-            var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = $1, fetchingErrorCount = $2, lastSync = $3 WHERE id = $4";
+            var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = $1, lastTweetSynchronizedForAllFollowersId = $2, fetchingErrorCount = $3, lastSync = $4 WHERE id = $5";
 
             await using var connection = DataSource.CreateConnection();
             await connection.OpenAsync();
             await using var command = new NpgsqlCommand(query, connection) {
                 Parameters = { 
                     new() { Value = lastTweetPostedId}, 
+                    new() { Value = lastTweetSynchronizedForAllFollowersId},
                     new() { Value = fetchingErrorCount},
                     new() { Value = lastSync.ToUniversalTime()},
                     new() { Value = id},
@@ -224,7 +209,7 @@ namespace BirdsiteLive.DAL.Postgres.DataAccessLayers
 
         public async Task UpdateTwitterUserAsync(SyncTwitterUser user)
         {
-            await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.FetchingErrorCount, user.LastSync);
+            await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync);
         }
 
         public async Task DeleteTwitterUserAsync(string acct)

src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Settings/PostgresSettings.cs

@@ -6,9 +6,6 @@
 
         public string DbVersionTableName { get; set; } = "db_version";
         public string TwitterUserTableName { get; set; } = "twitter_users";
-        public string InstagramUserTableName { get; set; } = "instagram_users";
-        public string WorkersTableName { get; set; } = "workers";
-        public string CachedInstaPostsTableName { get; set; } = "cached_insta_posts";
         public string FollowersTableName { get; set; } = "followers";
         public string CachedTweetsTableName { get; set; } = "cached_tweets";
     }

src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IFollowersDal.cs

@@ -7,7 +7,8 @@ namespace BirdsiteLive.DAL.Contracts
     public interface IFollowersDal
     {
         Task<Follower> GetFollowerAsync(string acct, string host);
-        Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null);
+        Task CreateFollowerAsync(string acct, string host, string inboxRoute, string sharedInboxRoute, string actorId, int[] followings = null,
+            Dictionary<int, long> followingSyncStatus = null);
         Task<Follower[]> GetFollowersAsync(int followedUserId);
         Task<Follower[]> GetAllFollowersAsync();
         Task UpdateFollowerAsync(Follower follower);

src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs

@@ -9,10 +9,10 @@ namespace BirdsiteLive.DAL.Contracts
         Task CreateTwitterUserAsync(string acct, long lastTweetPostedId);
         Task<SyncTwitterUser> GetTwitterUserAsync(string acct);
         Task<SyncTwitterUser> GetTwitterUserAsync(int id);
-        Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber, int nStart, int nEnd, int m);
+        Task<SyncTwitterUser[]> GetAllTwitterUsersWithFollowersAsync(int maxNumber);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync(int maxNumber);
         Task<SyncTwitterUser[]> GetAllTwitterUsersAsync();
-        Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, int fetchingErrorCount, DateTime lastSync);
+        Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync);
         Task UpdateTwitterUserIdAsync(string username, long twitterUserId);
         Task UpdateTwitterUserAsync(SyncTwitterUser user);
         Task DeleteTwitterUserAsync(string acct);

src/DataAccessLayers/BirdsiteLive.DAL/Models/Follower.cs

@@ -7,6 +7,7 @@ namespace BirdsiteLive.DAL.Models
         public int Id { get; set; }
         
         public List<int> Followings { get; set; }
+        public Dictionary<int, long> FollowingsSyncStatus { get; set; }
 
         public string ActorId { get; set; }
         public string Acct { get; set; }

src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs

@@ -7,9 +7,9 @@ namespace BirdsiteLive.DAL.Models
         public int Id { get; set; }
         public long TwitterUserId { get; set; }
         public string Acct { get; set; }
-        public string FediAcct { get; set; }
 
         public long LastTweetPostedId { get; set; }
+        public long LastTweetSynchronizedForAllFollowersId { get; set; }
 
         public DateTime LastSync { get; set; }
 

src/Tests/BSLManager.Tests/BSLManager.Tests.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6</TargetFramework>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="2.1.1" />
+    <PackageReference Include="MSTest.TestFramework" Version="2.1.1" />
+    <PackageReference Include="coverlet.collector" Version="1.3.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\BSLManager\BSLManager.csproj" />
+  </ItemGroup>
+
+</Project>

src/Tests/BSLManager.Tests/Domain/FollowersListStateTests.cs

@@ -0,0 +1,307 @@
+using System.Collections.Generic;
+using System.Linq;
+using BirdsiteLive.DAL.Models;
+using BSLManager.Domain;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace BSLManager.Tests
+{
+    [TestClass]
+    public class FollowersListStateTests
+    {
+        [TestMethod]
+        public void FilterBy()
+        {
+            #region Stub
+            var followers = new List<Follower>
+            {
+                new Follower
+                {
+                    Id = 0,
+                    Acct = "test",
+                    Host = "host1",
+                    Followings = new List<int>()
+                }, 
+                new Follower
+                {
+                    Id = 1,
+                    Acct = "test",
+                    Host = "host2",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 2,
+                    Acct = "user1",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 3,
+                    Acct = "user2",
+                    Host = "host1",
+                    Followings = new List<int>()
+                }
+            };
+            #endregion
+
+            var state = new FollowersListState();
+            state.Load(followers);
+
+            state.FilterBy("test");
+            
+            #region Validate
+            Assert.AreEqual(2, state.GetDisplayableList().Count);
+            #endregion
+        }
+
+        [TestMethod]
+        public void FilterBy_GetElement()
+        {
+            #region Stub
+            var followers = new List<Follower>
+            {
+                new Follower
+                {
+                    Id = 0,
+                    Acct = "test",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 1,
+                    Acct = "test",
+                    Host = "host2",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 2,
+                    Acct = "user1",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 3,
+                    Acct = "user2",
+                    Host = "host1",
+                    Followings = new List<int>()
+                }
+            };
+            #endregion
+
+            var state = new FollowersListState();
+            state.Load(followers);
+
+            state.FilterBy("test");
+            var el = state.GetElementAt(1);
+
+            #region Validate
+            Assert.AreEqual(followers[1].Id, el.Id);
+            #endregion
+        }
+
+        [TestMethod]
+        public void GetElement()
+        {
+            #region Stub
+            var followers = new List<Follower>
+            {
+                new Follower
+                {
+                    Id = 0,
+                    Acct = "test",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 1,
+                    Acct = "test",
+                    Host = "host2",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 2,
+                    Acct = "user1",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 3,
+                    Acct = "user2",
+                    Host = "host1",
+                    Followings = new List<int>()
+                }
+            };
+            #endregion
+
+            var state = new FollowersListState();
+            state.Load(followers);
+
+            var el = state.GetElementAt(2);
+
+            #region Validate
+            Assert.AreEqual(followers[2].Id, el.Id);
+            #endregion
+        }
+
+        [TestMethod]
+        public void FilterBy_RemoveAt()
+        {
+            #region Stub
+            var followers = new List<Follower>
+            {
+                new Follower
+                {
+                    Id = 0,
+                    Acct = "test",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 1,
+                    Acct = "test",
+                    Host = "host2",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 2,
+                    Acct = "user1",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 3,
+                    Acct = "user2",
+                    Host = "host1",
+                    Followings = new List<int>()
+                }
+            };
+            #endregion
+
+            var state = new FollowersListState();
+            state.Load(followers.ToList());
+
+            state.FilterBy("test");
+            state.RemoveAt(1);
+
+            var list = state.GetDisplayableList();
+
+            #region Validate
+            Assert.AreEqual(1, list.Count);
+            Assert.IsTrue(list[0].Contains("@test@host1"));
+            #endregion
+        }
+
+        [TestMethod]
+        public void RemoveAt()
+        {
+            #region Stub
+            var followers = new List<Follower>
+            {
+                new Follower
+                {
+                    Id = 0,
+                    Acct = "test",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 1,
+                    Acct = "test",
+                    Host = "host2",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 2,
+                    Acct = "user1",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 3,
+                    Acct = "user2",
+                    Host = "host1",
+                    Followings = new List<int>()
+                }
+            };
+            #endregion
+
+            var state = new FollowersListState();
+            state.Load(followers.ToList());
+
+            state.RemoveAt(1);
+
+            var list = state.GetDisplayableList();
+
+            #region Validate
+            Assert.AreEqual(3, list.Count);
+            Assert.IsTrue(list[0].Contains("@test@host1"));
+            Assert.IsFalse(list[1].Contains("@test@host2"));
+            #endregion
+        }
+
+        [TestMethod]
+        public void FilterBy_ResetFilter()
+        {
+            #region Stub
+            var followers = new List<Follower>
+            {
+                new Follower
+                {
+                    Id = 0,
+                    Acct = "test",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 1,
+                    Acct = "test",
+                    Host = "host2",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 2,
+                    Acct = "user1",
+                    Host = "host1",
+                    Followings = new List<int>()
+                },
+                new Follower
+                {
+                    Id = 3,
+                    Acct = "user2",
+                    Host = "host1",
+                    Followings = new List<int>()
+                }
+            };
+            #endregion
+
+            var state = new FollowersListState();
+            state.Load(followers.ToList());
+            
+            #region Validate
+            state.FilterBy("data");
+            var list = state.GetDisplayableList();
+            Assert.AreEqual(0, list.Count);
+
+            state.FilterBy(string.Empty);
+            list = state.GetDisplayableList();
+            Assert.AreEqual(4, list.Count);
+            #endregion
+        }
+    }
+}

src/Tests/BirdsiteLive.ActivityPub.Tests/BirdsiteLive.ActivityPub.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
 

src/Tests/BirdsiteLive.Common.Tests/BirdsiteLive.Common.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Cryptography.Tests/BirdsiteLive.Cryptography.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.DAL.Postgres.Tests/BirdsiteLive.DAL.Postgres.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/Base/PostgresTestingBase.cs

@@ -19,9 +19,6 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers.Base
                 CachedTweetsTableName = "CachedTweetsTableName" + RandomGenerator.GetString(4),
                 FollowersTableName = "FollowersTableName" + RandomGenerator.GetString(4),
                 TwitterUserTableName = "TwitterUserTableName" + RandomGenerator.GetString(4),
-                InstagramUserTableName = "InstagramUserTableName" + RandomGenerator.GetString(4),
-                CachedInstaPostsTableName = "CachedInstaPosts" + RandomGenerator.GetString(4),
-                WorkersTableName = "workers" + RandomGenerator.GetString(4),
             };
             _tools = new PostgresTools(_settings);
         }

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs

@@ -44,7 +44,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             var result = await dal.GetFollowerAsync(acct, host);
 
@@ -57,6 +57,9 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(0, result.PostingErrorCount);
             Assert.AreEqual(following.Length, result.Followings.Count);
             Assert.AreEqual(following[0], result.Followings[0]);
+            Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count);
+            Assert.AreEqual(followingSync.First().Key, result.FollowingsSyncStatus.First().Key);
+            Assert.AreEqual(followingSync.First().Value, result.FollowingsSyncStatus.First().Value);
         }
 
         [TestMethod]
@@ -69,7 +72,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, null);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, null, null);
 
             var result = await dal.GetFollowerAsync(acct, host);
 
@@ -80,6 +83,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(inboxRoute, result.InboxRoute);
             Assert.AreEqual(sharedInboxRoute, result.SharedInboxRoute);
             Assert.AreEqual(0, result.Followings.Count);
+            Assert.AreEqual(0, result.FollowingsSyncStatus.Count);
             Assert.AreEqual(0, result.PostingErrorCount);
         }
 
@@ -108,7 +112,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             var result = await dal.GetFollowerAsync(acct, host);
 
@@ -120,6 +124,9 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.AreEqual(sharedInboxRoute, result.SharedInboxRoute);
             Assert.AreEqual(following.Length, result.Followings.Count);
             Assert.AreEqual(following[0], result.Followings[0]);
+            Assert.AreEqual(followingSync.Count, result.FollowingsSyncStatus.Count);
+            Assert.AreEqual(followingSync.First().Key, result.FollowingsSyncStatus.First().Key);
+            Assert.AreEqual(followingSync.First().Value, result.FollowingsSyncStatus.First().Value);
             Assert.AreEqual(0, result.PostingErrorCount);
         }
 
@@ -136,7 +143,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var inboxRoute = "/myhandle1/inbox";
             var sharedInboxRoute = "/inbox";
             var actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             //User 2 
             acct = "myhandle2";
@@ -145,7 +152,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle2/inbox";
             sharedInboxRoute = "/inbox2";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             //User 2 
             acct = "myhandle3";
@@ -154,7 +161,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle3/inbox";
             sharedInboxRoute = "/inbox3";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             var result = await dal.GetFollowersAsync(2);
             Assert.AreEqual(2, result.Length);
@@ -179,7 +186,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var inboxRoute = "/myhandle1/inbox";
             var sharedInboxRoute = "/inbox";
             var actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             //User 2 
             acct = "myhandle2";
@@ -188,7 +195,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle2/inbox";
             sharedInboxRoute = "/inbox2";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             //User 2 
             acct = "myhandle3";
@@ -197,7 +204,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle3/inbox";
             sharedInboxRoute = "/inbox3";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             var result = await dal.GetAllFollowersAsync();
             Assert.AreEqual(3, result.Length);
@@ -219,7 +226,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var inboxRoute = "/myhandle1/inbox";
             var sharedInboxRoute = "/inbox";
             var actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             //User 2 
             acct = "myhandle2";
@@ -228,7 +235,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle2/inbox";
             sharedInboxRoute = "/inbox2";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             //User 3
             acct = "myhandle3";
@@ -237,7 +244,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle3/inbox";
             sharedInboxRoute = "/inbox3";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             result = await dal.GetFollowersCountAsync();
             Assert.AreEqual(3, result);
@@ -259,7 +266,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var inboxRoute = "/myhandle1/inbox";
             var sharedInboxRoute = "/inbox";
             var actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             //User 2 
             acct = "myhandle2";
@@ -268,7 +275,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle2/inbox";
             sharedInboxRoute = "/inbox2";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             var follower = await dal.GetFollowerAsync(acct, host);
             follower.PostingErrorCount = 1;
@@ -281,7 +288,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             inboxRoute = "/myhandle3/inbox";
             sharedInboxRoute = "/inbox3";
             actorId = $"https://{host}/{acct}";
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
 
             follower = await dal.GetFollowerAsync(acct, host);
             follower.PostingErrorCount = 50;
@@ -308,7 +315,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
             var result = await dal.GetFollowerAsync(acct, host);
 
             var updatedFollowing = new List<int> { 12, 19, 23, 24 };
@@ -319,6 +326,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 {24, 173L}
             };
             result.Followings = updatedFollowing.ToList();
+            result.FollowingsSyncStatus = updatedFollowingSync;
             result.PostingErrorCount = 10;
             
             await dal.UpdateFollowerAsync(result);
@@ -326,6 +334,9 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(updatedFollowing.Count, result.Followings.Count);
             Assert.AreEqual(updatedFollowing[0], result.Followings[0]);
+            Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count);
+            Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key);
+            Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value);
             Assert.AreEqual(10, result.PostingErrorCount);
         }
 
@@ -346,7 +357,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
             var result = await dal.GetFollowerAsync(acct, host);
 
             var updatedFollowing = new List<int> { 12, 19, 23, 24 };
@@ -357,6 +368,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 {24, 173L}
             };
             result.Followings = updatedFollowing.ToList();
+            result.FollowingsSyncStatus = updatedFollowingSync;
             result.PostingErrorCount = 32768;
 
             await dal.UpdateFollowerAsync(result);
@@ -364,6 +376,9 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(updatedFollowing.Count, result.Followings.Count);
             Assert.AreEqual(updatedFollowing[0], result.Followings[0]);
+            Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count);
+            Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key);
+            Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value);
             Assert.AreEqual(32768, result.PostingErrorCount);
         }
 
@@ -384,7 +399,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
             var result = await dal.GetFollowerAsync(acct, host);
 
             var updatedFollowing = new[] { 12, 19 };
@@ -394,6 +409,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 {19, 171L}
             };
             result.Followings = updatedFollowing.ToList();
+            result.FollowingsSyncStatus = updatedFollowingSync;
             result.PostingErrorCount = 5;
 
             await dal.UpdateFollowerAsync(result);
@@ -401,6 +417,9 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(updatedFollowing.Length, result.Followings.Count);
             Assert.AreEqual(updatedFollowing[0], result.Followings[0]);
+            Assert.AreEqual(updatedFollowingSync.Count, result.FollowingsSyncStatus.Count);
+            Assert.AreEqual(updatedFollowingSync.First().Key, result.FollowingsSyncStatus.First().Key);
+            Assert.AreEqual(updatedFollowingSync.First().Value, result.FollowingsSyncStatus.First().Value);
             Assert.AreEqual(5, result.PostingErrorCount);
         }
 
@@ -421,7 +440,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
             var result = await dal.GetFollowerAsync(acct, host);
             Assert.AreEqual(0, result.PostingErrorCount);
 
@@ -476,7 +495,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
             var result = await dal.GetFollowerAsync(acct, host);
             Assert.IsNotNull(result);
 
@@ -503,7 +522,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var actorId = $"https://{host}/{acct}";
 
             var dal = new FollowersPostgresDal(_settings);
-            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following);
+            await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, actorId, following, followingSync);
             var result = await dal.GetFollowerAsync(acct, host);
             Assert.IsNotNull(result);
 

src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs

@@ -48,6 +48,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(lastTweetId, result.LastTweetPostedId);
+            Assert.AreEqual(lastTweetId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(0, result.FetchingErrorCount);
             Assert.IsTrue(result.Id > 0);
         }
@@ -66,6 +67,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(acct, resultById.Acct);
             Assert.AreEqual(lastTweetId, resultById.LastTweetPostedId);
+            Assert.AreEqual(lastTweetId, resultById.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(result.Id, resultById.Id);
         }
 
@@ -85,12 +87,13 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var updatedLastSyncId = 1550L;
             var now = DateTime.Now;
             var errors = 15;
-            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, errors, now);
+            await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now);
 
             result = await dal.GetTwitterUserAsync(acct);
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
+            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(errors, result.FetchingErrorCount);
             Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
         }
@@ -113,6 +116,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var errors = 15;
 
             result.LastTweetPostedId = updatedLastTweetId;
+            result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId;
             result.FetchingErrorCount = errors;
             result.LastSync = now;
             await dal.UpdateTwitterUserAsync(result);
@@ -121,6 +125,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
+            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(errors, result.FetchingErrorCount);
             Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
         }
@@ -143,6 +148,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             var errors = 32768;
 
             result.LastTweetPostedId = updatedLastTweetId;
+            result.LastTweetSynchronizedForAllFollowersId = updatedLastSyncId;
             result.FetchingErrorCount = errors;
             result.LastSync = now;
             await dal.UpdateTwitterUserAsync(result);
@@ -151,6 +157,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
 
             Assert.AreEqual(acct, result.Acct);
             Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId);
+            Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId);
             Assert.AreEqual(errors, result.FetchingErrorCount);
             Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100);
         }
@@ -160,7 +167,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(default, default, default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow);
         }
 
         [TestMethod]
@@ -168,16 +175,23 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
         public async Task Update_NoLastTweetPostedId()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, default,  default, DateTime.UtcNow);
+            await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow);
         }
 
+        [TestMethod]
+        [ExpectedException(typeof(ArgumentException))]
+        public async Task Update_NoLastTweetSynchronizedForAllFollowersId()
+        {
+            var dal = new TwitterUserPostgresDal(_settings);
+            await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow);
+        }
 
         [TestMethod]
         [ExpectedException(typeof(ArgumentException))]
         public async Task Update_NoLastSync()
         {
             var dal = new TwitterUserPostgresDal(_settings);
-            await dal.UpdateTwitterUserAsync(12, 9556, 65,  default);
+            await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default);
         }
 
         [TestMethod]
@@ -247,6 +261,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
             Assert.IsFalse(result[0].LastTweetPostedId == default);
+            Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
         }
 
         [TestMethod]
@@ -303,7 +318,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             {
                 var user = allUsers[i];
                 var date = i % 2 == 0 ? oldest : newest;
-                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, 0, date);
+                await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date);
             }
 
             var result = await dal.GetAllTwitterUsersAsync(10);
@@ -311,6 +326,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
             Assert.IsFalse(result[0].LastTweetPostedId == default);
+            Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
 
             foreach (var acc in result)
                 Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000);
@@ -333,6 +349,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
             Assert.IsFalse(result[0].Id == default);
             Assert.IsFalse(result[0].Acct == default);
             Assert.IsFalse(result[0].LastTweetPostedId == default);
+            Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default);
         }
 
         [TestMethod]
@@ -365,7 +382,7 @@ namespace BirdsiteLive.DAL.Postgres.Tests.DataAccessLayers
                 if (i == 0 || i == 2 || i == 3)
                 {
                     var t = await dal.GetTwitterUserAsync(acct);
-                    await dal.UpdateTwitterUserAsync(t.Id ,1L, 50+i*2, DateTime.Now);
+                    await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now);
                 }
             }
 

src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Domain.Tests/BirdsiteLive.Domain.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
     <LangVersion>11</LangVersion>
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs

@@ -30,6 +30,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 SharedInboxRoute = followerInbox,
                 InboxRoute = inbox,
                 Followings = new List<int>(),
+                FollowingsSyncStatus = new Dictionary<int, long>()
             };
 
             var twitterUser = new SyncTwitterUser
@@ -37,6 +38,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Id = 2,
                 Acct = twitterName,
                 LastTweetPostedId = -1,
+                LastTweetSynchronizedForAllFollowersId = -1
             };
             #endregion
 
@@ -54,12 +56,14 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                     It.Is<string>(y => y == followerInbox),
                     It.Is<string>(y => y == inbox),
                     It.Is<string>(y => y == actorId),
-                    null ))
+                    null,
+                    null))
                 .Returns(Task.CompletedTask);
 
             followersDalMock
                 .Setup(x => x.UpdateFollowerAsync(
-                    It.Is<Follower>(y => y.Followings.Contains(twitterUser.Id))
+                    It.Is<Follower>(y => y.Followings.Contains(twitterUser.Id)
+                                         && y.FollowingsSyncStatus[twitterUser.Id] == -1)
                 ))
                 .Returns(Task.CompletedTask);
 
@@ -104,6 +108,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 SharedInboxRoute = followerInbox,
                 InboxRoute = inbox,
                 Followings = new List<int>(),
+                FollowingsSyncStatus = new Dictionary<int, long>()
             };
 
             var twitterUser = new SyncTwitterUser
@@ -111,6 +116,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Id = 2,
                 Acct = twitterName,
                 LastTweetPostedId = -1,
+                LastTweetSynchronizedForAllFollowersId = -1
             };
             #endregion
 
@@ -122,7 +128,8 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
             
             followersDalMock
                 .Setup(x => x.UpdateFollowerAsync(
-                    It.Is<Follower>(y => y.Followings.Contains(twitterUser.Id) )
+                    It.Is<Follower>(y => y.Followings.Contains(twitterUser.Id)
+                                         && y.FollowingsSyncStatus[twitterUser.Id] == -1)
                 ))
                 .Returns(Task.CompletedTask);
 

src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessUnfollowUserTests.cs

@@ -52,6 +52,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Acct = username,
                 Host = domain,
                 Followings = new List<int>(),
+                FollowingsSyncStatus = new Dictionary<int, long>()
             };
             #endregion
 
@@ -90,6 +91,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Acct = username,
                 Host = domain,
                 Followings = new List<int> { 2, 3 },
+                FollowingsSyncStatus = new Dictionary<int, long> { { 2, 460 }, { 3, 563} }
             };
 
             var twitterUser = new SyncTwitterUser
@@ -97,6 +99,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Id = 2,
                 Acct = twitterName,
                 LastTweetPostedId = 460,
+                LastTweetSynchronizedForAllFollowersId = 460
             };
 
             var followerList = new List<Follower>
@@ -114,7 +117,8 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
 
             followersDalMock
                 .Setup(x => x.UpdateFollowerAsync(
-                    It.Is<Follower>(y => !y.Followings.Contains(twitterUser.Id) )
+                    It.Is<Follower>(y => !y.Followings.Contains(twitterUser.Id)
+                                         && !y.FollowingsSyncStatus.ContainsKey(twitterUser.Id))
                 ))
                 .Returns(Task.CompletedTask);
 
@@ -151,6 +155,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Acct = username,
                 Host = domain,
                 Followings = new List<int> { 2 },
+                FollowingsSyncStatus = new Dictionary<int, long> { { 2, 460 } }
             };
 
             var twitterUser = new SyncTwitterUser
@@ -158,6 +163,7 @@ namespace BirdsiteLive.Domain.Tests.BusinessUseCases
                 Id = 2,
                 Acct = twitterName,
                 LastTweetPostedId = 460,
+                LastTweetSynchronizedForAllFollowersId = 460
             };
 
             var followerList = new List<Follower>();

src/Tests/BirdsiteLive.Domain.Tests/Tools/StatusExtractorTests.cs

@@ -111,6 +111,7 @@ namespace BirdsiteLive.Domain.Tests.Tools
             Assert.IsTrue(result.content.Contains(@"<a href=""https://www.eff.org/deeplinks/2020/07/pact-act-not-solution-problem-harmful-online-content"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">https://www.</span><span class=""ellipsis"">eff.org/deeplinks/2020/07/pact</span><span class=""invisible"">-act-not-solution-problem-harmful-online-content</span></a>"));
             #endregion
         }
+        [Ignore]
         [TestMethod]
         public void Extract_FormatUrl_Long2_Test()
         {
@@ -127,29 +128,7 @@ namespace BirdsiteLive.Domain.Tests.Tools
 
             #region Validations
             logger.VerifyAll();
-            Assert.AreEqual(result.content, @"<a href=""https://twitterisgoinggreat.com/#twitters-first-dollar15bn-interest-payment-could-be-due-in-two-weeks"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">https://</span><span class=""ellipsis"">twitterisgoinggreat.com/#twitt</span><span class=""invisible"">ers-first-dollar15bn-interest-payment-could-be-due-in-two-weeks</span></a>");
-            Assert.AreEqual(0, result.tags.Length);
-
-            #endregion
-        }
-        
-        [TestMethod]
-        public void Extract_FormatUrl_Long3_Test()
-        {
-            #region Stubs
-            var message = $"https://domain.name/@WeekInEthNews/1668684659855880193";
-            #endregion
-
-            #region Mocks
-            var logger = new Mock<ILogger<StatusExtractor>>();
-            #endregion
-
-            var service = new StatusExtractor(_settings, logger.Object);
-            var result = service.Extract(message);
-
-            #region Validations
-            logger.VerifyAll();
-            Assert.AreEqual(result.content, @"<a href=""https://domain.name/@WeekInEthNews/1668684659855880193"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">https://</span><span class=""ellipsis"">domain.name/@WeekInEthNews/166</span><span class=""invisible"">8684659855880193</span></a>");
+            Assert.AreEqual(result.content, @"<a href=""https://twitterisgoinggreat.com/#twitters-first-dollar15bn-interest-payment-could-be-due-in-two-weeks"" rel=""nofollow noopener noreferrer"" target=""_blank""><span class=""invisible"">https://www.</span><span class=""ellipsis"">eff.org/deeplinks/2020/07/pact</span><span class=""invisible"">-act-not-solution-problem-harmful-online-content</span></a>");
             Assert.AreEqual(0, result.tags.Length);
 
             #endregion
@@ -654,7 +633,7 @@ namespace BirdsiteLive.Domain.Tests.Tools
         public void Extract_Emoji_Test()
         {
             #region Stubs
-            var message = $"😤 @mynickname 😎😍🤗🤩😘";
+            var message = $"😤@mynickname 😎😍🤗🤩😘";
             //var message = $"tests@mynickname";
             #endregion
 
@@ -669,13 +648,12 @@ namespace BirdsiteLive.Domain.Tests.Tools
             logger.VerifyAll();
             Assert.AreEqual(1, result.tags.Length);
             Assert.IsTrue(result.content.Contains(
-                @"😤 <span class=""h-card""><a href=""https://domain.name/users/mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
+                @"😤<span class=""h-card""><a href=""https://domain.name/users/mynickname"" class=""u-url mention"">@<span>mynickname</span></a></span>"));
 
             Assert.IsTrue(result.content.Contains(@"😎😍🤗🤩😘"));
             #endregion
         }
 
-        [Ignore]
         [TestMethod]
         public void Extract_Parenthesis_Test()
         {

src/Tests/BirdsiteLive.Moderation.Tests/Actions/RemoveTwitterAccountActionTests.cs

@@ -27,6 +27,7 @@ namespace BirdsiteLive.Moderation.Tests.Actions
                 {
                     Id = 48,
                     Followings = new List<int>{ 24 },
+                    FollowingsSyncStatus = new Dictionary<int, long> { { 24, 1024 } }
                 }
             };
             #endregion
@@ -83,6 +84,7 @@ namespace BirdsiteLive.Moderation.Tests.Actions
                 {
                     Id = 48,
                     Followings = new List<int>{ 24, 36 },
+                    FollowingsSyncStatus = new Dictionary<int, long> { { 24, 1024 }, { 36, 24 } }
                 }
             };
             #endregion
@@ -98,6 +100,7 @@ namespace BirdsiteLive.Moderation.Tests.Actions
                 .Setup(x => x.UpdateFollowerAsync(
                     It.Is<Follower>(y => y.Id == 48
                         && y.Followings.Count == 1
+                        && y.FollowingsSyncStatus.Count == 1
                     )))
                 .Returns(Task.CompletedTask);
 

src/Tests/BirdsiteLive.Moderation.Tests/BirdsiteLive.Moderation.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs

@@ -29,19 +29,12 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 new SyncTwitterUser(),
             };
             var maxUsers = 1000;
-            var instanceSettings = new InstanceSettings()
-            {
-                n_start = 1,
-            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users);
             
@@ -54,7 +47,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -79,19 +72,12 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 users.Add(new SyncTwitterUser());
 
             var maxUsers = 1000;
-            var instanceSettings = new InstanceSettings()
-            {
-                n_start = 1,
-            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -107,7 +93,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -132,19 +118,12 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 users.Add(new SyncTwitterUser());
 
             var maxUsers = 1000;
-            var instanceSettings = new InstanceSettings()
-            {
-                n_start = 1,
-            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .SetupSequence(x => x.GetAllTwitterUsersWithFollowersAsync(
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(users.ToArray())
                 .ReturnsAsync(new SyncTwitterUser[0])
@@ -160,7 +139,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 2;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -181,10 +160,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var buffer = new BufferBlock<UserWithDataToSync[]>();
 
             var maxUsers = 1000;
-            var instanceSettings = new InstanceSettings()
-            {
-                n_start = 1,
-            };
             #endregion
 
             #region Mocks
@@ -192,9 +167,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .ReturnsAsync(new SyncTwitterUser[0]);
 
@@ -206,7 +178,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 1;
             var t =processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -225,19 +197,12 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var buffer = new BufferBlock<UserWithDataToSync[]>();
 
             var maxUsers = 1000;
-            var instanceSettings = new InstanceSettings()
-            {
-                n_start = 1,
-            };
             #endregion
 
             #region Mocks
             var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
             twitterUserDalMock
                 .Setup(x => x.GetAllTwitterUsersWithFollowersAsync(
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
-                    It.Is<int>(y => true),
                     It.Is<int>(y => true)))
                 .Returns(async () => await DelayFaultedTask<SyncTwitterUser[]>(new Exception()));
 
@@ -249,7 +214,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 10;
             var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None);
 
@@ -271,10 +236,6 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             canTokenS.Cancel();
 
             var maxUsers = 1000;
-            var instanceSettings = new InstanceSettings()
-            {
-                n_start = 1,
-            };
             #endregion
 
             #region Mocks
@@ -288,7 +249,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var loggerMock = new Mock<ILogger<RetrieveTwitterUsersProcessor>>();
             #endregion
 
-            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, instanceSettings, loggerMock.Object);
+            var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, followersDalMock.Object, loggerMock.Object);
             processor.WaitFactor = 1;
             await processor.GetTwitterUsersAsync(buffer, canTokenS.Token);
         }

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs

@@ -0,0 +1,227 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using BirdsiteLive.DAL.Contracts;
+using BirdsiteLive.DAL.Models;
+using BirdsiteLive.Pipeline.Models;
+using BirdsiteLive.Pipeline.Processors.SubTasks;
+using BirdsiteLive.Twitter.Models;
+using Castle.DynamicProxy.Contributors;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Moq;
+
+namespace BirdsiteLive.Pipeline.Tests.Processors
+{
+    [TestClass]
+    public class SaveProgressionProcessorTests
+    {
+        [TestMethod]
+        public async Task ProcessAsync_Test()
+        {
+            #region Stubs
+            var user = new SyncTwitterUser
+            {
+                Id = 1
+            };
+            var tweet1 = new ExtractedTweet
+            {
+                Id = 36
+            };
+            var tweet2 = new ExtractedTweet
+            {
+                Id = 37
+            };
+            var follower1 = new Follower
+            {
+                FollowingsSyncStatus = new Dictionary<int, long>
+                {
+                    {1, 37}
+                }
+            };
+
+            var usersWithTweets = new UserWithDataToSync
+            {
+                Tweets = new []
+                {
+                    tweet1,
+                    tweet2
+                },
+                Followers = new []
+                {
+                    follower1
+                },
+                User = user
+            };
+
+            var loggerMock = new Mock<ILogger<SaveProgressionTask>>();
+            #endregion
+
+            #region Mocks
+            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+            twitterUserDalMock
+                .Setup(x => x.UpdateTwitterUserAsync(
+                    It.Is<int>(y => y == user.Id),
+                    It.Is<long>(y => y == tweet2.Id),
+                    It.Is<long>(y => y == tweet2.Id),
+                    It.Is<int>(y => y == 0),
+                    It.IsAny<DateTime>()
+                ))
+                .Returns(Task.CompletedTask);
+            #endregion
+
+            var processor = new SaveProgressionTask(twitterUserDalMock.Object, loggerMock.Object);
+            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
+
+            #region Validations
+            twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
+            #endregion
+        }
+
+        [TestMethod]
+        public async Task ProcessAsync_PartiallySynchronized_Test()
+        {
+            #region Stubs
+            var user = new SyncTwitterUser
+            {
+                Id = 1
+            };
+            var tweet1 = new ExtractedTweet
+            {
+                Id = 36
+            };
+            var tweet2 = new ExtractedTweet
+            {
+                Id = 37
+            };
+            var tweet3 = new ExtractedTweet
+            {
+                Id = 38
+            };
+            var follower1 = new Follower
+            {
+                FollowingsSyncStatus = new Dictionary<int, long>
+                {
+                    {1, 37}
+                }
+            };
+
+            var usersWithTweets = new UserWithDataToSync
+            {
+                Tweets = new[]
+                {
+                    tweet1,
+                    tweet2,
+                    tweet3
+                },
+                Followers = new[]
+                {
+                    follower1
+                },
+                User = user
+            };
+            #endregion
+
+            #region Mocks
+            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+            twitterUserDalMock
+                .Setup(x => x.UpdateTwitterUserAsync(
+                    It.Is<int>(y => y == user.Id),
+                    It.Is<long>(y => y == tweet3.Id),
+                    It.Is<long>(y => y == tweet2.Id),
+                    It.Is<int>(y => y == 0),
+                    It.IsAny<DateTime>()
+                ))
+                .Returns(Task.CompletedTask);
+
+            var loggerMock = new Mock<ILogger<SaveProgressionTask>>();
+            #endregion
+
+            var processor = new SaveProgressionTask(twitterUserDalMock.Object, loggerMock.Object);
+            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
+
+            #region Validations
+            twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
+            #endregion
+        }
+
+        [TestMethod]
+        public async Task ProcessAsync_PartiallySynchronized_MultiUsers_Test()
+        {
+            #region Stubs
+            var user = new SyncTwitterUser
+            {
+                Id = 1
+            };
+            var tweet1 = new ExtractedTweet
+            {
+                Id = 36
+            };
+            var tweet2 = new ExtractedTweet
+            {
+                Id = 37
+            };
+            var tweet3 = new ExtractedTweet
+            {
+                Id = 38
+            };
+            var follower1 = new Follower
+            {
+                FollowingsSyncStatus = new Dictionary<int, long>
+                {
+                    {1, 37}
+                }
+            };
+            var follower2 = new Follower
+            {
+                FollowingsSyncStatus = new Dictionary<int, long>
+                {
+                    {1, 38}
+                }
+            };
+
+            var usersWithTweets = new UserWithDataToSync
+            {
+                Tweets = new[]
+                {
+                    tweet1,
+                    tweet2,
+                    tweet3
+                },
+                Followers = new[]
+                {
+                    follower1,
+                    follower2
+                },
+                User = user
+            };
+            #endregion
+
+            #region Mocks
+            var twitterUserDalMock = new Mock<ITwitterUserDal>(MockBehavior.Strict);
+            twitterUserDalMock
+                .Setup(x => x.UpdateTwitterUserAsync(
+                    It.Is<int>(y => y == user.Id),
+                    It.Is<long>(y => y == tweet3.Id),
+                    It.Is<long>(y => y == tweet2.Id),
+                    It.Is<int>(y => y == 0),
+                    It.IsAny<DateTime>()
+                ))
+                .Returns(Task.CompletedTask);
+
+            var loggerMock = new Mock<ILogger<SaveProgressionTask>>();
+            #endregion
+
+            var processor = new SaveProgressionTask(twitterUserDalMock.Object, loggerMock.Object);
+            await processor.ProcessAsync(usersWithTweets, CancellationToken.None);
+
+            #region Validations
+            twitterUserDalMock.VerifyAll();
+            loggerMock.VerifyAll();
+            #endregion
+        }
+    }
+}

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SendTweetsToFollowersProcessorTests.cs

@@ -77,6 +77,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
             
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -164,6 +165,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 ParallelFediversePosts = 1
             };
 
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
             var removeFollowerMock = new Mock<IRemoveFollowerAction>(MockBehavior.Strict);
             #endregion
 
@@ -248,6 +250,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -340,6 +343,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
 
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
             var settings = new InstanceSettings
             {
                 ParallelFediversePosts = 1
@@ -436,6 +440,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
 
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
             var settings = new InstanceSettings
             {
                 ParallelFediversePosts = 1
@@ -514,6 +519,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -594,6 +600,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -682,6 +689,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -767,6 +775,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -856,6 +865,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
             var followersDalMock = new Mock<IFollowersDal>(MockBehavior.Strict);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -949,6 +959,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {
@@ -1043,6 +1054,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors
                 .Returns(Task.CompletedTask);
 
             var loggerMock = new Mock<ILogger<SendTweetsToFollowersProcessor>>();
+            var saveProgressMock = new Mock<ISaveProgressionTask>();
 
             var settings = new InstanceSettings
             {

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs

@@ -57,6 +57,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
+                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings
@@ -138,6 +139,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
+                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings { };
@@ -216,6 +218,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
+                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings
@@ -298,6 +301,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
+                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings
@@ -371,6 +375,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
+                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 10 } }
             };
 
             var settings = new InstanceSettings
@@ -451,6 +456,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
+                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 10 } }
             };
 
             var settings = new InstanceSettings
@@ -554,6 +560,7 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                 Id = 1,
                 Host = host,
                 InboxRoute = inbox,
+                FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
             };
 
             var settings = new InstanceSettings

src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs

@@ -61,18 +61,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 
@@ -158,18 +161,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 
@@ -256,18 +262,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 
@@ -341,18 +350,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 10}}
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 8}}
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 7}}
                 }
             };
 
@@ -435,18 +447,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 10}}
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 8}}
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> {{twitterUserId, 7}}
                 }
             };
 
@@ -553,18 +568,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 
@@ -630,18 +648,21 @@ namespace BirdsiteLive.Pipeline.Tests.Processors.SubTasks
                     Id = 1,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 9 } }
                 },
                 new Follower
                 {
                     Id = 2,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 8 } }
                 },
                 new Follower
                 {
                     Id = 3,
                     Host = host,
                     SharedInboxRoute = inbox,
+                    FollowingsSyncStatus = new Dictionary<int, long> { { twitterUserId, 7 } }
                 }
             };
 

src/Tests/BirdsiteLive.Pipeline.Tests/StatusPublicationPipelineTests.cs

@@ -30,16 +30,18 @@ namespace BirdsiteLive.Pipeline.Tests
             var retrieveTweetsProcessor = new Mock<IRetrieveTweetsProcessor>(MockBehavior.Strict);
             var retrieveFollowersProcessor = new Mock<IRetrieveFollowersProcessor>(MockBehavior.Strict);
             var sendTweetsToFollowersProcessor = new Mock<ISendTweetsToFollowersProcessor>(MockBehavior.Strict);
+            var saveProgressionProcessor = new Mock<ISaveProgressionTask>(MockBehavior.Strict);
             var logger = new Mock<ILogger<StatusPublicationPipeline>>();
             #endregion
 
-            var pipeline = new StatusPublicationPipeline(retrieveTweetsProcessor.Object, retrieveTwitterUserProcessor.Object, retrieveFollowersProcessor.Object, sendTweetsToFollowersProcessor.Object, logger.Object);
+            var pipeline = new StatusPublicationPipeline(retrieveTweetsProcessor.Object, retrieveTwitterUserProcessor.Object, retrieveFollowersProcessor.Object, sendTweetsToFollowersProcessor.Object, saveProgressionProcessor.Object, logger.Object);
             await pipeline.ExecuteAsync(ct.Token);
 
             #region Validations
             retrieveTweetsProcessor.VerifyAll();
             retrieveFollowersProcessor.VerifyAll();
             sendTweetsToFollowersProcessor.VerifyAll();
+            saveProgressionProcessor.VerifyAll();
             logger.VerifyAll();
             #endregion
         }

src/Tests/BirdsiteLive.Twitter.Tests/BirdsiteLive.Twitter.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net6</TargetFramework>
 
     <IsPackable>false</IsPackable>
   </PropertyGroup>

src/Tests/BirdsiteLive.Twitter.Tests/TimelineTests.cs

@@ -57,13 +57,12 @@ namespace BirdsiteLive.ActivityPub.Tests
         {
             var tweets = await _tweetService.GetTimelineAsync("grantimahara", default);
             Assert.IsTrue(tweets[0].IsReply);
-            Assert.IsTrue(tweets.Length > 10);
+            Assert.IsTrue(tweets.Length > 30);
 
             Assert.AreEqual(tweets[2].MessageContent, "Liftoff!");
             Assert.AreEqual(tweets[2].RetweetId, 1266812530833240064);
             Assert.AreEqual(tweets[2].Id, 1266813644626489345);
             Assert.AreEqual(tweets[2].OriginalAuthor.Acct, "SpaceX");
-            Assert.AreEqual(tweets[2].Author.Acct, "grantimahara");
             Assert.IsTrue(tweets[2].IsRetweet);
         }
 

src/Tests/BirdsiteLive.Twitter.Tests/TweetTests.cs

@@ -16,13 +16,10 @@ namespace BirdsiteLive.ActivityPub.Tests
     [TestClass]
     public class TweetTests
     {
-        private ITwitterTweetsService _tweetService = null;
-        
+        private ITwitterTweetsService _tweetService;
         [TestInitialize]
         public async Task TestInit()
         {
-            if (_tweetService != null)
-                return;
             
             var logger1 = new Mock<ILogger<TwitterAuthenticationInitializer>>(MockBehavior.Strict);
             var logger2 = new Mock<ILogger<TwitterUserService>>(MockBehavior.Strict);
@@ -42,7 +39,6 @@ namespace BirdsiteLive.ActivityPub.Tests
 
         }
 
-
         [TestMethod]
         public async Task SimpleTextTweet()
         {
@@ -61,7 +57,7 @@ namespace BirdsiteLive.ActivityPub.Tests
 
             Assert.AreEqual(tweet.Media[0].MediaType, "image/jpeg");
             Assert.AreEqual(tweet.Media.Length, 1);
-            Assert.AreEqual(tweet.Media[0].AltText, "President Obama with Speaker Nancy Pelosi in DC.");
+            // TODO test alt-text of images
         }
 
         [TestMethod]
@@ -79,18 +75,7 @@ namespace BirdsiteLive.ActivityPub.Tests
 
             Assert.AreEqual(tweet.Media.Length, 1);
             Assert.AreEqual(tweet.Media[0].MediaType, "video/mp4");
-            Assert.IsNull(tweet.Media[0].AltText);
             Assert.IsTrue(tweet.Media[0].Url.StartsWith("https://video.twimg.com/"));
-            
-            
-            var tweet2 = await _tweetService.GetTweetAsync(1657913781006258178);
-            Assert.AreEqual(tweet2.MessageContent,
-                "Coinbase has big international expansion plans\n\nTom Duff Gordon (@tomduffgordon), VP of International Policy @coinbase has the deets");
-
-            Assert.AreEqual(tweet2.Media.Length, 1);
-            Assert.AreEqual(tweet2.Media[0].MediaType, "video/mp4");
-            Assert.IsNull(tweet2.Media[0].AltText);
-            Assert.IsTrue(tweet2.Media[0].Url.StartsWith("https://video.twimg.com/"));
         }
 
         [Ignore]
@@ -110,30 +95,7 @@ namespace BirdsiteLive.ActivityPub.Tests
         {
             var tweet = await _tweetService.GetTweetAsync(1610807139089383427);
 
-            Assert.AreEqual(tweet.MessageContent, "When you gave them your keys you gave them your coins.\n\nhttps://domain.name/@kadhim/1610706613207285773");
-            Assert.AreEqual(tweet.Author.Acct, "RyanSAdams");
-        }
-        
-        [TestMethod]
-        public async Task QTandTextContainsLink()
-        {
-            var tweet = await _tweetService.GetTweetAsync(1668932525522305026);
-
-            Assert.AreEqual(tweet.MessageContent, @"https://domain.name/@WeekInEthNews/1668684659855880193");
-            Assert.AreEqual(tweet.Author.Acct, "WeekInEthNews");
-        }
-        
-        [Ignore]
-        [TestMethod]
-        public async Task QTandTextContainsWebLink()
-        {
-            var tweet = await _tweetService.GetTweetAsync(1668969663340871682);
-
-            Assert.AreEqual(tweet.MessageContent, @"Friends, our Real World Risk Workshop (now transformed into summer school) #RWRI (18th ed.) takes place July 10-21 (remote).
-We have a few scholarships left but more importantly we are looking for a guest speaker on AI-LLM-Robotics for a 45 Q&amp;A with us.
-
-http://www.realworldrisk.com https://twitter.com/i/web/status/1668969663340871682");
-            Assert.AreEqual(tweet.Author.Acct, "nntaleb");
+            Assert.AreEqual(tweet.MessageContent, "When you gave them your keys you gave them your coins.\n\nhttps://domain.name/users/kadhim/statuses/1610706613207285773");
         }
 
         [TestMethod]

src/Tests/BirdsiteLive.Twitter.Tests/UsersTest.cs

@@ -39,13 +39,6 @@ namespace BirdsiteLive.ActivityPub.Tests
             Assert.AreEqual(user.Acct, "kobebryant");
         }
 
-        [TestMethod]
-        public async Task UserGrant()
-        {
-            var user = await _tweetService.GetUserAsync("grantimahara");
-            Assert.AreEqual(user.Name, "Grant Imahara");
-            Assert.AreEqual(user.Acct, "grantimahara");
-        }
 
     }
 }

src/Tests/dotMakeup.HackerNews.Tests/PostsTests.cs

@@ -1,6 +0,0 @@
-namespace dotMakeup.HackerNews.Tests;
-
-public class PostsTests
-{
-    
-}

src/Tests/dotMakeup.HackerNews.Tests/UsersTests.cs

@@ -1,20 +0,0 @@
-using dotMakeup.HackerNews;
-using System.Threading.Tasks;
-using Moq;
-
-namespace dotMakeup.HackerNews.Tests;
-
-[TestClass]
-public class UsersTests
-{
-    [TestMethod]
-    public async Task TestMethod1()
-    {
-        var httpFactory = new Mock<IHttpClientFactory>();
-        httpFactory.Setup(_ => _.CreateClient(string.Empty)).Returns(new HttpClient());
-        var userService = new HNUserService(httpFactory.Object);
-        var user = await userService.GetUserAsync("dhouston");
-        
-        Assert.AreEqual(user.About, "Founder/CEO of Dropbox (http://www.dropbox.com ; yc summer '07)");
-    }
-}

src/Tests/dotMakeup.HackerNews.Tests/Usings.cs

@@ -1 +0,0 @@
-global using Microsoft.VisualStudio.TestTools.UnitTesting;

src/Tests/dotMakeup.HackerNews.Tests/dotMakeup.HackerNews.Tests.csproj

@@ -1,25 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net7.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-
-        <IsPackable>false</IsPackable>
-    </PropertyGroup>
-
-    <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
-        <PackageReference Include="MSTest.TestAdapter" Version="2.2.10" />
-        <PackageReference Include="MSTest.TestFramework" Version="2.2.10" />
-        <PackageReference Include="coverlet.collector" Version="3.1.2" />
-        <PackageReference Include="moq" Version="4.16.1" />
-        <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
-    </ItemGroup>
-
-    <ItemGroup>
-      <ProjectReference Include="..\..\dotMakeup.HackerNews\dotMakeup.HackerNews.csproj" />
-    </ItemGroup>
-
-
-</Project>

src/dotMakeup.HackerNews/HNPostService.cs

@@ -1,6 +0,0 @@
-namespace dotMakeup.HackerNews;
-
-public class HNPostService
-{
-    
-}

src/dotMakeup.HackerNews/HNUserService.cs

@@ -1,39 +0,0 @@
-using System.Text.Json;
-using System.Web;
-using dotMakeup.HackerNews.Models;
-
-namespace dotMakeup.HackerNews;
-
-public class HNUserService
-{
-    private IHttpClientFactory _httpClientFactory;
-    public HNUserService(IHttpClientFactory httpClientFactory)
-    {
-            _httpClientFactory = httpClientFactory;
-        
-    }
-    public async Task<HNUser> GetUserAsync(string username)
-    {
-        string reqURL = "https://hacker-news.firebaseio.com/v0/user/dhouston.json";
-        reqURL = reqURL.Replace("dhouston", username);
-        
-        var client = _httpClientFactory.CreateClient();
-        var request = new HttpRequestMessage(new HttpMethod("GET"), reqURL);
-        
-        JsonDocument userDoc;
-        var httpResponse = await client.SendAsync(request);
-        httpResponse.EnsureSuccessStatusCode();
-        var c = await httpResponse.Content.ReadAsStringAsync();
-        userDoc = JsonDocument.Parse(c);
-
-        string about =
-            HttpUtility.HtmlDecode(userDoc.RootElement.GetProperty("about").GetString());
-        
-        var user = new HNUser()
-        {
-            Id = 0,
-            About = about,
-        };
-        return user;
-    }
-}

src/dotMakeup.HackerNews/Models/HNUser.cs

@@ -1,7 +0,0 @@
-namespace dotMakeup.HackerNews.Models;
-
-public class HNUser
-{
-    public long Id { get; set; }
-    public string About { get; set; }
-}

src/dotMakeup.HackerNews/dotMakeup.HackerNews.csproj

@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-    <PropertyGroup>
-        <TargetFramework>net7.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-    </PropertyGroup>
-    
-    <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
-    </ItemGroup>
-
-</Project>